elasticsearch-java
elasticsearch-java copied to clipboard
elastic
Thread lock when unable to insert in Elasticsearch
Java API client version
8.15.1
Java version
openjdk version "21.0.5" 2024-10-15 LTS
Elasticsearch Version
8.15.1
Problem description
Insert data in Elasticsearch.
Important parameters:
throughput = 50 records/s
bulk concurrent requests = 2
bulk max actions = 10
bulk max size = 90 (never triggers insertions)
BulkIngester
bulkIngester = BulkIngester.of(b -> b
.client(client)
.maxOperations(bulkMaxActions)
.maxConcurrentRequests(bulkConcurrentRequests)
.maxSize(bulkMaxSize * 1024L * 1024)
.listener(esBulkListener));
BulkListener
public class EsBulkListener implements BulkListener<byte[]> {
private static final Logger LOG = LogManager.getLogger(EsBulkListener.class);
private AckCounter ackCounter;
private boolean acknowledgeRecords;
public EsBulkListener(boolean acknowledgeRecords) {
this.ackCounter = new AckCounter();
this.acknowledgeRecords = acknowledgeRecords;
}
@Override
public void beforeBulk(long executionId, BulkRequest request, List<byte[]> contexts) {
//LOG.info("BulkIngester Execution (" + executionId + ") - About to execute new bulk insert composed of " + request.operations().size() + " actions");
}
@Override
public void afterBulk(long executionId, BulkRequest request, List<byte[]> contexts, BulkResponse response) {
if (response.errors()) {
LOG.error("BulkIngester Bulk Response has failures: " + response.errors());
if (acknowledgeRecords) {
response.items().forEach(i -> {
if (i.error() != null) {
ackCounter.updateFailure(1L);
} else {
ackCounter.updateSuccess(1L);
}
});
}
} else {
//LOG.info("BulkIngester Execution (" + executionId + ") - Bulk insert composed of " + request.operations().size() +" actions, took " + response.took() + " ms");
if (acknowledgeRecords) {
ackCounter.updateSuccess(request.operations().size());
}
}
}
@Override
public void afterBulk(long executionId, BulkRequest request, List<byte[]> contexts, Throwable failure) {
LOG.error("BulkIngester (" + executionId + ") - Execution of bulk request failed: " + failure.getMessage());
if (acknowledgeRecords) {
ackCounter.updateFailure(request.operations().size());
}
}
AckCounter getAckCounter() {
return ackCounter;
}
}
Insertion:
public void send(Object record) {
Map<String, Object> esValues = (Map<String, Object>)record;
String docId = (String) esValues.remove(DOC_ID);
bulkIngester.add(op -> op
.index(ind -> ind
.id(docId)
.index(indexName)
.document(esValues)));
}
Set Elasticsearch master down to fail insertions in Elasticsearch.
You have errors: bulk request failed: Connection refused
Then fix Elasticsearch master to insert again.
Insertions are blocked.
If bulk concurrent requests = 1 there is no issue.
Note that same test does not fail with deprecated bulk processor
Here is the dump stack when having application blocked:
2025-01-30 13:34:09
Full thread dump OpenJDK 64-Bit Server VM (21.0.5+11-LTS mixed mode, sharing):
Threads class SMR info:
_java_thread_list=0x00007f405e4c7de0, length=16, elements={
0x00007f40b7b09010, 0x00007f40a529a060, 0x00007f40a529a860, 0x00007f40a5298090,
0x00007f40a5299080, 0x00007f40a5299880, 0x00007f40a5295fd0, 0x00007f40a52967e0,
0x00007f405f66a840, 0x00007f405f66b820, 0x00007f405ebe6f20, 0x00007f405ebe7fe0,
0x00007f405e5bb760, 0x00007f405e5bcf40, 0x00007f405e5bdec0, 0x00007f405e3577c0
}
"main" #1 [8] prio=5 os_prio=0 cpu=2702.36ms elapsed=279.64s tid=0x00007f40b7b09010 nid=8 waiting on condition [0x00007f40c0354000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park([email protected]/Native Method)
- parking to wait for <0x000000054d11a6a0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)
at java.util.concurrent.locks.LockSupport.park([email protected]/LockSupport.java:221)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire([email protected]/AbstractQueuedSynchronizer.java:754)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire([email protected]/AbstractQueuedSynchronizer.java:990)
at java.util.concurrent.locks.ReentrantLock$Sync.lock([email protected]/ReentrantLock.java:153)
at java.util.concurrent.locks.ReentrantLock.lock([email protected]/ReentrantLock.java:322)
at org.apache.http.nio.pool.AbstractNIOConnPool.lease(AbstractNIOConnPool.java:278)
at org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.requestConnection(PoolingNHttpClientConnectionManager.java:295)
at org.apache.http.impl.nio.client.AbstractClientExchangeHandler.requestConnection(AbstractClientExchangeHandler.java:381)
at org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.start(DefaultClientExchangeHandlerImpl.java:130)
at org.apache.http.impl.nio.client.CloseableHttpAsyncClientBase.execute(CloseableHttpAsyncClientBase.java:116)
at org.apache.http.impl.nio.client.InternalHttpAsyncClient.execute(InternalHttpAsyncClient.java:138)
at org.elasticsearch.client.RestClient.lambda$performRequestAsync$0(RestClient.java:397)
at org.elasticsearch.client.RestClient$$Lambda/0x00007f406529f590.run(Unknown Source)
at org.elasticsearch.client.Cancellable$RequestCancellable.runIfNotCancelled(Cancellable.java:90)
- locked <0x00000003fbf7f8a0> (a org.elasticsearch.client.Cancellable$RequestCancellable)
at org.elasticsearch.client.RestClient.performRequestAsync(RestClient.java:395)
at org.elasticsearch.client.RestClient.performRequestAsync(RestClient.java:382)
at co.elastic.clients.transport.rest_client.RestClientHttpClient.performRequestAsync(RestClientHttpClient.java:113)
at co.elastic.clients.transport.ElasticsearchTransportBase.performRequestAsync(ElasticsearchTransportBase.java:184)
at co.elastic.clients.elasticsearch.ElasticsearchAsyncClient.bulk(ElasticsearchAsyncClient.java:368)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.lambda$flush$2(BulkIngester.java:301)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester$$Lambda/0x00007f406529d680.get(Unknown Source)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition.whenReadyIf(FnCondition.java:89)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.flush(BulkIngester.java:276)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.lambda$add$7(BulkIngester.java:382)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester$$Lambda/0x00007f406529cbe0.run(Unknown Source)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition.lambda$whenReady$0(FnCondition.java:47)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition$$Lambda/0x00007f406529cdf8.get(Unknown Source)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition.whenReadyIf(FnCondition.java:89)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition.whenReady(FnCondition.java:46)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.add(BulkIngester.java:363)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.add(BulkIngester.java:395)
at com.dataproducer.output.elasticsearch.ElasticsearchOutputManager.send(ElasticsearchOutputManager.java:379)
at com.dataproducer.DataProducer.run(DataProducer.java:132)
at com.dataproducer.DataProducer.main(DataProducer.java:45)
"Reference Handler" #4 [11] daemon prio=10 os_prio=0 cpu=1.46ms elapsed=279.60s tid=0x00007f40a529a060 nid=11 waiting on condition [0x00007f40
5fefc000]
java.lang.Thread.State: RUNNABLE
at java.lang.ref.Reference.waitForReferencePendingList([email protected]/Native Method)
at java.lang.ref.Reference.processPendingReferences([email protected]/Reference.java:246)
at java.lang.ref.Reference$ReferenceHandler.run([email protected]/Reference.java:208)
"Finalizer" #5 [12] daemon prio=8 os_prio=0 cpu=0.30ms elapsed=279.60s tid=0x00007f40a529a860 nid=12 in Object.wait() [0x00007f405fdfb000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait0([email protected]/Native Method)
- waiting on <0x000000054c3ae028> (a java.lang.ref.NativeReferenceQueue$Lock)
at java.lang.Object.wait([email protected]/Object.java:366)
at java.lang.Object.wait([email protected]/Object.java:339)
at java.lang.ref.NativeReferenceQueue.await([email protected]/NativeReferenceQueue.java:48)
at java.lang.ref.ReferenceQueue.remove0([email protected]/ReferenceQueue.java:158)
at java.lang.ref.NativeReferenceQueue.remove([email protected]/NativeReferenceQueue.java:89)
- locked <0x000000054c3ae028> (a java.lang.ref.NativeReferenceQueue$Lock)
at java.lang.ref.Finalizer$FinalizerThread.run([email protected]/Finalizer.java:173)
"Signal Dispatcher" #6 [13] daemon prio=9 os_prio=0 cpu=0.47ms elapsed=279.60s tid=0x00007f40a5298090 nid=13 waiting on condition [0x000000000
0000000]
java.lang.Thread.State: RUNNABLE
"Service Thread" #7 [14] daemon prio=9 os_prio=0 cpu=2.24ms elapsed=279.60s tid=0x00007f40a5299080 nid=14 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Monitor Deflation Thread" #8 [15] daemon prio=9 os_prio=0 cpu=25.07ms elapsed=279.60s tid=0x00007f40a5299880 nid=15 runnable [0x0000000000000
000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" #9 [16] daemon prio=9 os_prio=0 cpu=4471.43ms elapsed=279.60s tid=0x00007f40a5295fd0 nid=16 waiting on condition [0x00000
00000000000]
java.lang.Thread.State: RUNNABLE
No compile task
"C1 CompilerThread0" #10 [17] daemon prio=9 os_prio=0 cpu=1325.90ms elapsed=279.60s tid=0x00007f40a52967e0 nid=17 waiting on condition [0x0000
000000000000]
java.lang.Thread.State: RUNNABLE
No compile task
"Common-Cleaner" #11 [18] daemon prio=8 os_prio=0 cpu=1.88ms elapsed=279.59s tid=0x00007f405f66a840 nid=18 waiting on condition [0x00007f405f6
37000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park([email protected]/Native Method)
- parking to wait for <0x000000054c3dfb18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos([email protected]/LockSupport.java:269)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await([email protected]/AbstractQueuedSynchronizer.java:1852)
at java.lang.ref.ReferenceQueue.await([email protected]/ReferenceQueue.java:71)
at java.lang.ref.ReferenceQueue.remove0([email protected]/ReferenceQueue.java:143)
at java.lang.ref.ReferenceQueue.remove([email protected]/ReferenceQueue.java:218)
at jdk.internal.ref.CleanerImpl.run([email protected]/CleanerImpl.java:140)
at java.lang.Thread.runWith([email protected]/Thread.java:1596)
at java.lang.Thread.run([email protected]/Thread.java:1583)
at jdk.internal.misc.InnocuousThread.run([email protected]/InnocuousThread.java:186)
"Notification Thread" #12 [19] daemon prio=9 os_prio=0 cpu=0.10ms elapsed=279.58s tid=0x00007f405f66b820 nid=19 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"elasticsearch-rest-client-0-thread-1" #14 [20] prio=5 os_prio=0 cpu=26.64ms elapsed=277.79s tid=0x00007f405ebe6f20 nid=20 waiting on condition
[0x00007f405e9a0000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park([email protected]/Native Method)
- parking to wait for <0x000000054d1dcbc8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)
at java.util.concurrent.locks.LockSupport.park([email protected]/LockSupport.java:221)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire([email protected]/AbstractQueuedSynchronizer.java:754)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire([email protected]/AbstractQueuedSynchronizer.java:990)
at java.util.concurrent.locks.ReentrantLock$Sync.lock([email protected]/ReentrantLock.java:153)
at java.util.concurrent.locks.ReentrantLock.lock([email protected]/ReentrantLock.java:322)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition.signalIfReadyAfter(FnCondition.java:118)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.lambda$flush$6(BulkIngester.java:347)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester$$Lambda/0x00007f406529fc18.apply(Unknown Source)
at java.util.concurrent.CompletableFuture.uniHandle([email protected]/CompletableFuture.java:934)
at java.util.concurrent.CompletableFuture$UniHandle.tryFire([email protected]/CompletableFuture.java:911)
at java.util.concurrent.CompletableFuture.postComplete([email protected]/CompletableFuture.java:510)
at java.util.concurrent.CompletableFuture.completeExceptionally([email protected]/CompletableFuture.java:2194)
at co.elastic.clients.transport.ElasticsearchTransportBase.lambda$performRequestAsync$0(ElasticsearchTransportBase.java:206)
at co.elastic.clients.transport.ElasticsearchTransportBase$$Lambda/0x00007f406529f9f0.apply(Unknown Source)
at java.util.concurrent.CompletableFuture.uniHandle([email protected]/CompletableFuture.java:934)
at java.util.concurrent.CompletableFuture$UniHandle.tryFire([email protected]/CompletableFuture.java:911)
at java.util.concurrent.CompletableFuture.postComplete([email protected]/CompletableFuture.java:510)
at java.util.concurrent.CompletableFuture.completeExceptionally([email protected]/CompletableFuture.java:2194)
at co.elastic.clients.transport.rest_client.RestClientHttpClient$1.onFailure(RestClientHttpClient.java:121)
at org.elasticsearch.client.RestClient$FailureTrackingResponseListener.onDefinitiveFailure(RestClient.java:688)
at org.elasticsearch.client.RestClient$1.failed(RestClient.java:426)
at org.apache.http.concurrent.BasicFuture.failed(BasicFuture.java:137)
at org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.executionFailed(DefaultClientExchangeHandlerImpl.java:101)
at org.apache.http.impl.nio.client.AbstractClientExchangeHandler.failed(AbstractClientExchangeHandler.java:432)
at org.apache.http.impl.nio.client.AbstractClientExchangeHandler.connectionRequestFailed(AbstractClientExchangeHandler.java:352)
at org.apache.http.impl.nio.client.AbstractClientExchangeHandler.access$100(AbstractClientExchangeHandler.java:64)
at org.apache.http.impl.nio.client.AbstractClientExchangeHandler$1.failed(AbstractClientExchangeHandler.java:396)
at org.apache.http.concurrent.BasicFuture.failed(BasicFuture.java:137)
at org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager$1.failed(PoolingNHttpClientConnectionManager.java:318)
at org.apache.http.concurrent.BasicFuture.failed(BasicFuture.java:137)
at org.apache.http.nio.pool.RouteSpecificPool.timeout(RouteSpecificPool.java:169)
at org.apache.http.nio.pool.AbstractNIOConnPool.requestTimeout(AbstractNIOConnPool.java:632)
at org.apache.http.nio.pool.AbstractNIOConnPool$InternalSessionRequestCallback.timeout(AbstractNIOConnPool.java:898)
at org.apache.http.impl.nio.reactor.SessionRequestImpl.timeout(SessionRequestImpl.java:198)
- locked <0x00000003fbf70820> (a org.apache.http.impl.nio.reactor.SessionRequestImpl)
at org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor.processTimeouts(DefaultConnectingIOReactor.java:213)
at org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor.processEvents(DefaultConnectingIOReactor.java:158)
at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.execute(AbstractMultiworkerIOReactor.java:351)
at org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.execute(PoolingNHttpClientConnectionManager.java:221)
at org.apache.http.impl.nio.client.CloseableHttpAsyncClientBase$1.run(CloseableHttpAsyncClientBase.java:64)
at java.lang.Thread.runWith([email protected]/Thread.java:1596)
at java.lang.Thread.run([email protected]/Thread.java:1583)
"elasticsearch-rest-client-0-thread-2" #15 [21] prio=5 os_prio=0 cpu=1815.30ms elapsed=277.79s tid=0x00007f405ebe7fe0 nid=21 runnable [0x00007
f405e8a1000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPoll.wait([email protected]/Native Method)
at sun.nio.ch.EPollSelectorImpl.doSelect([email protected]/EPollSelectorImpl.java:121)
at sun.nio.ch.SelectorImpl.lockAndDoSelect([email protected]/SelectorImpl.java:130)
- locked <0x000000054d154ea8> (a sun.nio.ch.Util$2)
- locked <0x000000054d154d40> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select([email protected]/SelectorImpl.java:142)
at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:255)
at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591)
at java.lang.Thread.runWith([email protected]/Thread.java:1596)
at java.lang.Thread.run([email protected]/Thread.java:1583)
"bulk-ingester-executor#1#17" #17 [22] daemon prio=5 os_prio=0 cpu=35.73ms elapsed=276.61s tid=0x00007f405e5bb760 nid=22 waiting on condition
[0x00007f405d2e2000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park([email protected]/Native Method)
- parking to wait for <0x000000054d1de7f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park([email protected]/LockSupport.java:371)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block([email protected]/AbstractQueuedSynchronizer.java:519)
at java.util.concurrent.ForkJoinPool.unmanagedBlock([email protected]/ForkJoinPool.java:3780)
at java.util.concurrent.ForkJoinPool.managedBlock([email protected]/ForkJoinPool.java:3725)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await([email protected]/AbstractQueuedSynchronizer.java:1712)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take([email protected]/ScheduledThreadPoolExecutor.java:1170)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take([email protected]/ScheduledThreadPoolExecutor.java:899)
at java.util.concurrent.ThreadPoolExecutor.getTask([email protected]/ThreadPoolExecutor.java:1070)
at java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1130)
at java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:642)
at java.lang.Thread.runWith([email protected]/Thread.java:1596)
at java.lang.Thread.run([email protected]/Thread.java:1583)
"bulk-ingester-executor#1#18" #18 [23] daemon prio=5 os_prio=0 cpu=36.53ms elapsed=276.50s tid=0x00007f405e5bcf40 nid=23 waiting on condition
[0x00007f405dfff000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park([email protected]/Native Method)
- parking to wait for <0x000000054d1de7f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park([email protected]/LockSupport.java:371)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block([email protected]/AbstractQueuedSynchronizer.java:519)
at java.util.concurrent.ForkJoinPool.unmanagedBlock([email protected]/ForkJoinPool.java:3780)
at java.util.concurrent.ForkJoinPool.managedBlock([email protected]/ForkJoinPool.java:3725)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await([email protected]/AbstractQueuedSynchronizer.java:1712)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take([email protected]/ScheduledThreadPoolExecutor.java:1170)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take([email protected]/ScheduledThreadPoolExecutor.java:899)
at java.util.concurrent.ThreadPoolExecutor.getTask([email protected]/ThreadPoolExecutor.java:1070)
at java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1130)
at java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:642)
at java.lang.Thread.runWith([email protected]/Thread.java:1596)
at java.lang.Thread.run([email protected]/Thread.java:1583)
"bulk-ingester-executor#1#19" #19 [24] daemon prio=5 os_prio=0 cpu=28.35ms elapsed=276.29s tid=0x00007f405e5bdec0 nid=24 waiting on condition
[0x00007f405d9a7000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park([email protected]/Native Method)
- parking to wait for <0x000000054d1dcbc8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)
at java.util.concurrent.locks.LockSupport.park([email protected]/LockSupport.java:221)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire([email protected]/AbstractQueuedSynchronizer.java:754)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire([email protected]/AbstractQueuedSynchronizer.java:990)
at java.util.concurrent.locks.ReentrantLock$Sync.lock([email protected]/ReentrantLock.java:153)
at java.util.concurrent.locks.ReentrantLock.lock([email protected]/ReentrantLock.java:322)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition.signalIfReady(FnCondition.java:96)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.lambda$flush$4(BulkIngester.java:340)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester$$Lambda/0x00007f40652aefb0.run(Unknown Source)
at java.util.concurrent.Executors$RunnableAdapter.call([email protected]/Executors.java:572)
at java.util.concurrent.FutureTask.run([email protected]/FutureTask.java:317)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run([email protected]/ScheduledThreadPoolExecutor.java:304)
at java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1144)
at java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:642)
at java.lang.Thread.runWith([email protected]/Thread.java:1596)
at java.lang.Thread.run([email protected]/Thread.java:1583)
"Attach Listener" #20 [45] daemon prio=9 os_prio=0 cpu=0.38ms elapsed=0.10s tid=0x00007f405e3577c0 nid=45 waiting on condition [0x000000000000
0000]
java.lang.Thread.State: RUNNABLE
"VM Thread" os_prio=0 cpu=111.16ms elapsed=279.61s tid=0x00007f40a52a8390 nid=10 runnable
"VM Periodic Task Thread" os_prio=0 cpu=186.71ms elapsed=279.62s tid=0x00007f40a52bdc10 nid=9 waiting on condition
JNI global refs: 15, weak refs: 0
Found one Java-level deadlock:
=============================
"main":
waiting for ownable synchronizer 0x000000054d11a6a0, (a java.util.concurrent.locks.ReentrantLock$NonfairSync),
which is held by "elasticsearch-rest-client-0-thread-1"
"elasticsearch-rest-client-0-thread-1":
waiting for ownable synchronizer 0x000000054d1dcbc8, (a java.util.concurrent.locks.ReentrantLock$NonfairSync),
which is held by "main"
Java stack information for the threads listed above:
===================================================
"main":
at jdk.internal.misc.Unsafe.park([email protected]/Native Method)
- parking to wait for <0x000000054d11a6a0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)
at java.util.concurrent.locks.LockSupport.park([email protected]/LockSupport.java:221)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire([email protected]/AbstractQueuedSynchronizer.java:754)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire([email protected]/AbstractQueuedSynchronizer.java:990)
at java.util.concurrent.locks.ReentrantLock$Sync.lock([email protected]/ReentrantLock.java:153)
at java.util.concurrent.locks.ReentrantLock.lock([email protected]/ReentrantLock.java:322)
at org.apache.http.nio.pool.AbstractNIOConnPool.lease(AbstractNIOConnPool.java:278)
at org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.requestConnection(PoolingNHttpClientConnectionManager.java:295)
at org.apache.http.impl.nio.client.AbstractClientExchangeHandler.requestConnection(AbstractClientExchangeHandler.java:381)
at org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.start(DefaultClientExchangeHandlerImpl.java:130)
at org.apache.http.impl.nio.client.CloseableHttpAsyncClientBase.execute(CloseableHttpAsyncClientBase.java:116)
at org.apache.http.impl.nio.client.InternalHttpAsyncClient.execute(InternalHttpAsyncClient.java:138)
at org.elasticsearch.client.RestClient.lambda$performRequestAsync$0(RestClient.java:397)
at org.elasticsearch.client.RestClient$$Lambda/0x00007f406529f590.run(Unknown Source)
at org.elasticsearch.client.Cancellable$RequestCancellable.runIfNotCancelled(Cancellable.java:90)
- locked <0x00000003fbf7f8a0> (a org.elasticsearch.client.Cancellable$RequestCancellable)
at org.elasticsearch.client.RestClient.performRequestAsync(RestClient.java:395)
at org.elasticsearch.client.RestClient.performRequestAsync(RestClient.java:382)
at co.elastic.clients.transport.rest_client.RestClientHttpClient.performRequestAsync(RestClientHttpClient.java:113)
at co.elastic.clients.transport.ElasticsearchTransportBase.performRequestAsync(ElasticsearchTransportBase.java:184)
at co.elastic.clients.elasticsearch.ElasticsearchAsyncClient.bulk(ElasticsearchAsyncClient.java:368)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.lambda$flush$2(BulkIngester.java:301)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester$$Lambda/0x00007f406529d680.get(Unknown Source)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition.whenReadyIf(FnCondition.java:89)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.flush(BulkIngester.java:276)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.lambda$add$7(BulkIngester.java:382)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester$$Lambda/0x00007f406529cbe0.run(Unknown Source)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition.lambda$whenReady$0(FnCondition.java:47)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition$$Lambda/0x00007f406529cdf8.get(Unknown Source)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition.whenReadyIf(FnCondition.java:89)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition.whenReady(FnCondition.java:46)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.add(BulkIngester.java:363)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.add(BulkIngester.java:395)
at com.dataproducer.output.elasticsearch.ElasticsearchOutputManager.send(ElasticsearchOutputManager.java:379)
at com.dataproducer.DataProducer.run(DataProducer.java:132)
at com.dataproducer.DataProducer.main(DataProducer.java:45)
"elasticsearch-rest-client-0-thread-1":
at jdk.internal.misc.Unsafe.park([email protected]/Native Method)
- parking to wait for <0x000000054d1dcbc8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)
at java.util.concurrent.locks.LockSupport.park([email protected]/LockSupport.java:221)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire([email protected]/AbstractQueuedSynchronizer.java:754)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire([email protected]/AbstractQueuedSynchronizer.java:990)
at java.util.concurrent.locks.ReentrantLock$Sync.lock([email protected]/ReentrantLock.java:153)
at java.util.concurrent.locks.ReentrantLock.lock([email protected]/ReentrantLock.java:322)
at co.elastic.clients.elasticsearch._helpers.bulk.FnCondition.signalIfReadyAfter(FnCondition.java:118)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester.lambda$flush$6(BulkIngester.java:347)
at co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester$$Lambda/0x00007f406529fc18.apply(Unknown Source)
at java.util.concurrent.CompletableFuture.uniHandle([email protected]/CompletableFuture.java:934)
at java.util.concurrent.CompletableFuture$UniHandle.tryFire([email protected]/CompletableFuture.java:911)
at java.util.concurrent.CompletableFuture.postComplete([email protected]/CompletableFuture.java:510)
at java.util.concurrent.CompletableFuture.completeExceptionally([email protected]/CompletableFuture.java:2194)
at co.elastic.clients.transport.ElasticsearchTransportBase.lambda$performRequestAsync$0(ElasticsearchTransportBase.java:206)
at co.elastic.clients.transport.ElasticsearchTransportBase$$Lambda/0x00007f406529f9f0.apply(Unknown Source)
at java.util.concurrent.CompletableFuture.uniHandle([email protected]/CompletableFuture.java:934)
at java.util.concurrent.CompletableFuture$UniHandle.tryFire([email protected]/CompletableFuture.java:911)
at java.util.concurrent.CompletableFuture.postComplete([email protected]/CompletableFuture.java:510)
at java.util.concurrent.CompletableFuture.completeExceptionally([email protected]/CompletableFuture.java:2194)
at co.elastic.clients.transport.rest_client.RestClientHttpClient$1.onFailure(RestClientHttpClient.java:121)
at org.elasticsearch.client.RestClient$FailureTrackingResponseListener.onDefinitiveFailure(RestClient.java:688)
at org.elasticsearch.client.RestClient$1.failed(RestClient.java:426)
at org.apache.http.concurrent.BasicFuture.failed(BasicFuture.java:137)
at org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.executionFailed(DefaultClientExchangeHandlerImpl.java:101)
at org.apache.http.impl.nio.client.AbstractClientExchangeHandler.failed(AbstractClientExchangeHandler.java:432)
at org.apache.http.impl.nio.client.AbstractClientExchangeHandler.connectionRequestFailed(AbstractClientExchangeHandler.java:352)
at org.apache.http.impl.nio.client.AbstractClientExchangeHandler.access$100(AbstractClientExchangeHandler.java:64)
at org.apache.http.impl.nio.client.AbstractClientExchangeHandler$1.failed(AbstractClientExchangeHandler.java:396)
at org.apache.http.concurrent.BasicFuture.failed(BasicFuture.java:137)
at org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager$1.failed(PoolingNHttpClientConnectionManager.java:318)
at org.apache.http.concurrent.BasicFuture.failed(BasicFuture.java:137)
at org.apache.http.nio.pool.RouteSpecificPool.timeout(RouteSpecificPool.java:169)
at org.apache.http.nio.pool.AbstractNIOConnPool.requestTimeout(AbstractNIOConnPool.java:632)
at org.apache.http.nio.pool.AbstractNIOConnPool$InternalSessionRequestCallback.timeout(AbstractNIOConnPool.java:898)
at org.apache.http.impl.nio.reactor.SessionRequestImpl.timeout(SessionRequestImpl.java:198)
- locked <0x00000003fbf70820> (a org.apache.http.impl.nio.reactor.SessionRequestImpl)
at org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor.processTimeouts(DefaultConnectingIOReactor.java:213)
at org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor.processEvents(DefaultConnectingIOReactor.java:158)
at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.execute(AbstractMultiworkerIOReactor.java:351)
at org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.execute(PoolingNHttpClientConnectionManager.java:221)
at org.apache.http.impl.nio.client.CloseableHttpAsyncClientBase$1.run(CloseableHttpAsyncClientBase.java:64)
at java.lang.Thread.runWith([email protected]/Thread.java:1596)
at java.lang.Thread.run([email protected]/Thread.java:1583)
Found 1 deadlock.
Hello, I'm seeing the same problem. Has there been any progress on this issue?
hey @FeetInAncientTime and @nicolasm35, is this still an issue? are you experiencing the same behavior with version 9.x of the client?
