Details
Description
It was noticed that during elastic search index updates, a considerably large number of threads were getting blocked as a result of contention for the org.elasticsearch.action.bulk.BulkProcessor's Mutual Exclusion lock. This causes progressively slower response times from the application.
We solve this problem by decoupling the request for search index updates from the processing of those requests. This ensures that requesting threads can return to their callers quickly thereby keeping the system responsive.
Example thread stacks which show the problem:
"event-7-thread-6669" id=22649 state=BLOCKED - waiting to lock <0x5b82b493> (a org.elasticsearch.action.bulk.BulkProcessor) owned by event-7-thread-6639 id=22616 at org.elasticsearch.action.bulk.BulkProcessor.internalAdd(BulkProcessor.java:283) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:268) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:264) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:250) at org.sonatype.nexus.repository.search.SearchServiceImpl.lambda$0(SearchServiceImpl.java:382) at org.sonatype.nexus.repository.search.SearchServiceImpl$$Lambda$341/838205662.accept(Unknown Source) at java.lang.Iterable.forEach(Iterable.java:75) at org.sonatype.nexus.repository.search.SearchServiceImpl.bulkPut(SearchServiceImpl.java:375) at org.sonatype.nexus.repository.search.SearchFacetImpl.bulkPut(SearchFacetImpl.java:182) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.CGLIB$bulkPut$5(<generated>) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0$$FastClassByGuice$$e617070f.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76) at org.sonatype.nexus.common.stateguard.MethodInvocationAction.run(MethodInvocationAction.java:39) at org.sonatype.nexus.common.stateguard.StateGuard$GuardImpl.run(StateGuard.java:272) at org.sonatype.nexus.common.stateguard.GuardedInterceptor.invoke(GuardedInterceptor.java:53) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:77) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:55) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.bulkPut(<generated>) at org.sonatype.nexus.repository.search.IndexRequest.bulkApply(IndexRequest.java:92) at org.sonatype.nexus.repository.search.IndexRequestProcessor.lambda$0(IndexRequestProcessor.java:139) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$484/1860009625.accept(Unknown Source) at java.util.Optional.ifPresent(Optional.java:159) at org.sonatype.nexus.repository.search.IndexRequestProcessor.doUpdateSearchIndex(IndexRequestProcessor.java:135) at org.sonatype.nexus.repository.search.IndexRequestProcessor.maybeUpdateSearchIndex(IndexRequestProcessor.java:115) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$483/1380218996.accept(Unknown Source) at java.util.HashMap.forEach(HashMap.java:1289) at org.sonatype.nexus.repository.search.IndexBatchRequest.apply(IndexBatchRequest.java:93) at org.sonatype.nexus.repository.search.IndexRequestProcessor.process(IndexRequestProcessor.java:99) at org.sonatype.nexus.repository.search.IndexRequestProcessor.on(IndexRequestProcessor.java:88) at sun.reflect.GeneratedMethodAccessor266.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.google.common.eventbus.Subscriber.invokeSubscriberMethod(Subscriber.java:87) at com.google.common.eventbus.Subscriber$1.run(Subscriber.java:72) at org.sonatype.nexus.internal.event.AffinityBarrier.lambda$1(AffinityBarrier.java:91) at org.sonatype.nexus.internal.event.AffinityBarrier$$Lambda$482/736097670.run(Unknown Source) at org.sonatype.nexus.thread.internal.MDCAwareRunnable.run(MDCAwareRunnable.java:40) at org.apache.shiro.subject.support.SubjectRunnable.doRun(SubjectRunnable.java:120) at org.apache.shiro.subject.support.SubjectRunnable.run(SubjectRunnable.java:108) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: count = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@4e7a94a8 "event-7-thread-6670" id=22650 state=BLOCKED - waiting to lock <0x5b82b493> (a org.elasticsearch.action.bulk.BulkProcessor) owned by event-7-thread-6639 id=22616 at org.elasticsearch.action.bulk.BulkProcessor.internalAdd(BulkProcessor.java:283) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:268) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:264) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:250) at org.sonatype.nexus.repository.search.SearchServiceImpl.lambda$0(SearchServiceImpl.java:382) at org.sonatype.nexus.repository.search.SearchServiceImpl$$Lambda$341/838205662.accept(Unknown Source) at java.lang.Iterable.forEach(Iterable.java:75) at org.sonatype.nexus.repository.search.SearchServiceImpl.bulkPut(SearchServiceImpl.java:375) at org.sonatype.nexus.repository.search.SearchFacetImpl.bulkPut(SearchFacetImpl.java:182) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.CGLIB$bulkPut$5(<generated>) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0$$FastClassByGuice$$e617070f.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76) at org.sonatype.nexus.common.stateguard.MethodInvocationAction.run(MethodInvocationAction.java:39) at org.sonatype.nexus.common.stateguard.StateGuard$GuardImpl.run(StateGuard.java:272) at org.sonatype.nexus.common.stateguard.GuardedInterceptor.invoke(GuardedInterceptor.java:53) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:77) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:55) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.bulkPut(<generated>) at org.sonatype.nexus.repository.search.IndexRequest.bulkApply(IndexRequest.java:92) at org.sonatype.nexus.repository.search.IndexRequestProcessor.lambda$0(IndexRequestProcessor.java:139) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$484/1860009625.accept(Unknown Source) at java.util.Optional.ifPresent(Optional.java:159) at org.sonatype.nexus.repository.search.IndexRequestProcessor.doUpdateSearchIndex(IndexRequestProcessor.java:135) at org.sonatype.nexus.repository.search.IndexRequestProcessor.maybeUpdateSearchIndex(IndexRequestProcessor.java:115) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$483/1380218996.accept(Unknown Source) at java.util.HashMap.forEach(HashMap.java:1289) at org.sonatype.nexus.repository.search.IndexBatchRequest.apply(IndexBatchRequest.java:93) at org.sonatype.nexus.repository.search.IndexRequestProcessor.process(IndexRequestProcessor.java:99) at org.sonatype.nexus.repository.search.IndexRequestProcessor.on(IndexRequestProcessor.java:88) at sun.reflect.GeneratedMethodAccessor266.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.google.common.eventbus.Subscriber.invokeSubscriberMethod(Subscriber.java:87) at com.google.common.eventbus.Subscriber$1.run(Subscriber.java:72) at org.sonatype.nexus.internal.event.AffinityBarrier.lambda$1(AffinityBarrier.java:91) at org.sonatype.nexus.internal.event.AffinityBarrier$$Lambda$482/736097670.run(Unknown Source) at org.sonatype.nexus.thread.internal.MDCAwareRunnable.run(MDCAwareRunnable.java:40) at org.apache.shiro.subject.support.SubjectRunnable.doRun(SubjectRunnable.java:120) at org.apache.shiro.subject.support.SubjectRunnable.run(SubjectRunnable.java:108) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: count = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@5792e71b "event-7-thread-6671" id=22651 state=BLOCKED - waiting to lock <0x5b82b493> (a org.elasticsearch.action.bulk.BulkProcessor) owned by event-7-thread-6639 id=22616 at org.elasticsearch.action.bulk.BulkProcessor.internalAdd(BulkProcessor.java:283) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:268) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:264) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:250) at org.sonatype.nexus.repository.search.SearchServiceImpl.lambda$0(SearchServiceImpl.java:382) at org.sonatype.nexus.repository.search.SearchServiceImpl$$Lambda$341/838205662.accept(Unknown Source) at java.lang.Iterable.forEach(Iterable.java:75) at org.sonatype.nexus.repository.search.SearchServiceImpl.bulkPut(SearchServiceImpl.java:375) at org.sonatype.nexus.repository.search.SearchFacetImpl.bulkPut(SearchFacetImpl.java:182) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.CGLIB$bulkPut$5(<generated>) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0$$FastClassByGuice$$e617070f.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76) at org.sonatype.nexus.common.stateguard.MethodInvocationAction.run(MethodInvocationAction.java:39) at org.sonatype.nexus.common.stateguard.StateGuard$GuardImpl.run(StateGuard.java:272) at org.sonatype.nexus.common.stateguard.GuardedInterceptor.invoke(GuardedInterceptor.java:53) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:77) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:55) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.bulkPut(<generated>) at org.sonatype.nexus.repository.search.IndexRequest.bulkApply(IndexRequest.java:92) at org.sonatype.nexus.repository.search.IndexRequestProcessor.lambda$0(IndexRequestProcessor.java:139) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$484/1860009625.accept(Unknown Source) at java.util.Optional.ifPresent(Optional.java:159) at org.sonatype.nexus.repository.search.IndexRequestProcessor.doUpdateSearchIndex(IndexRequestProcessor.java:135) at org.sonatype.nexus.repository.search.IndexRequestProcessor.maybeUpdateSearchIndex(IndexRequestProcessor.java:115) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$483/1380218996.accept(Unknown Source) at java.util.HashMap.forEach(HashMap.java:1289) at org.sonatype.nexus.repository.search.IndexBatchRequest.apply(IndexBatchRequest.java:93) at org.sonatype.nexus.repository.search.IndexRequestProcessor.process(IndexRequestProcessor.java:99) at org.sonatype.nexus.repository.search.IndexRequestProcessor.on(IndexRequestProcessor.java:88) at sun.reflect.GeneratedMethodAccessor266.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.google.common.eventbus.Subscriber.invokeSubscriberMethod(Subscriber.java:87) at com.google.common.eventbus.Subscriber$1.run(Subscriber.java:72) at org.sonatype.nexus.internal.event.AffinityBarrier.lambda$1(AffinityBarrier.java:91) at org.sonatype.nexus.internal.event.AffinityBarrier$$Lambda$482/736097670.run(Unknown Source) at org.sonatype.nexus.thread.internal.MDCAwareRunnable.run(MDCAwareRunnable.java:40) at org.apache.shiro.subject.support.SubjectRunnable.doRun(SubjectRunnable.java:120) at org.apache.shiro.subject.support.SubjectRunnable.run(SubjectRunnable.java:108) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: count = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@709bdc18 "event-7-thread-6672" id=22652 state=BLOCKED - waiting to lock <0x5b82b493> (a org.elasticsearch.action.bulk.BulkProcessor) owned by event-7-thread-6639 id=22616 at org.elasticsearch.action.bulk.BulkProcessor.internalAdd(BulkProcessor.java:283) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:268) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:264) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:250) at org.sonatype.nexus.repository.search.SearchServiceImpl.lambda$0(SearchServiceImpl.java:382) at org.sonatype.nexus.repository.search.SearchServiceImpl$$Lambda$341/838205662.accept(Unknown Source) at java.lang.Iterable.forEach(Iterable.java:75) at org.sonatype.nexus.repository.search.SearchServiceImpl.bulkPut(SearchServiceImpl.java:375) at org.sonatype.nexus.repository.search.SearchFacetImpl.bulkPut(SearchFacetImpl.java:182) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.CGLIB$bulkPut$5(<generated>) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0$$FastClassByGuice$$e617070f.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76) at org.sonatype.nexus.common.stateguard.MethodInvocationAction.run(MethodInvocationAction.java:39) at org.sonatype.nexus.common.stateguard.StateGuard$GuardImpl.run(StateGuard.java:272) at org.sonatype.nexus.common.stateguard.GuardedInterceptor.invoke(GuardedInterceptor.java:53) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:77) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:55) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.bulkPut(<generated>) at org.sonatype.nexus.repository.search.IndexRequest.bulkApply(IndexRequest.java:92) at org.sonatype.nexus.repository.search.IndexRequestProcessor.lambda$0(IndexRequestProcessor.java:139) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$484/1860009625.accept(Unknown Source) at java.util.Optional.ifPresent(Optional.java:159) at org.sonatype.nexus.repository.search.IndexRequestProcessor.doUpdateSearchIndex(IndexRequestProcessor.java:135) at org.sonatype.nexus.repository.search.IndexRequestProcessor.maybeUpdateSearchIndex(IndexRequestProcessor.java:115) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$483/1380218996.accept(Unknown Source) at java.util.HashMap.forEach(HashMap.java:1289) at org.sonatype.nexus.repository.search.IndexBatchRequest.apply(IndexBatchRequest.java:93) at org.sonatype.nexus.repository.search.IndexRequestProcessor.process(IndexRequestProcessor.java:99) at org.sonatype.nexus.repository.search.IndexRequestProcessor.on(IndexRequestProcessor.java:88) at sun.reflect.GeneratedMethodAccessor266.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.google.common.eventbus.Subscriber.invokeSubscriberMethod(Subscriber.java:87) at com.google.common.eventbus.Subscriber$1.run(Subscriber.java:72) at org.sonatype.nexus.internal.event.AffinityBarrier.lambda$1(AffinityBarrier.java:91) at org.sonatype.nexus.internal.event.AffinityBarrier$$Lambda$482/736097670.run(Unknown Source) at org.sonatype.nexus.thread.internal.MDCAwareRunnable.run(MDCAwareRunnable.java:40) at org.apache.shiro.subject.support.SubjectRunnable.doRun(SubjectRunnable.java:120) at org.apache.shiro.subject.support.SubjectRunnable.run(SubjectRunnable.java:108) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: count = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@2f60b52a "event-7-thread-6673" id=22653 state=BLOCKED - waiting to lock <0x5b82b493> (a org.elasticsearch.action.bulk.BulkProcessor) owned by event-7-thread-6639 id=22616 at org.elasticsearch.action.bulk.BulkProcessor.internalAdd(BulkProcessor.java:283) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:268) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:264) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:250) at org.sonatype.nexus.repository.search.SearchServiceImpl.lambda$0(SearchServiceImpl.java:382) at org.sonatype.nexus.repository.search.SearchServiceImpl$$Lambda$341/838205662.accept(Unknown Source) at java.lang.Iterable.forEach(Iterable.java:75) at org.sonatype.nexus.repository.search.SearchServiceImpl.bulkPut(SearchServiceImpl.java:375) at org.sonatype.nexus.repository.search.SearchFacetImpl.bulkPut(SearchFacetImpl.java:182) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.CGLIB$bulkPut$5(<generated>) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0$$FastClassByGuice$$e617070f.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76) at org.sonatype.nexus.common.stateguard.MethodInvocationAction.run(MethodInvocationAction.java:39) at org.sonatype.nexus.common.stateguard.StateGuard$GuardImpl.run(StateGuard.java:272) at org.sonatype.nexus.common.stateguard.GuardedInterceptor.invoke(GuardedInterceptor.java:53) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:77) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:55) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.bulkPut(<generated>) at org.sonatype.nexus.repository.search.IndexRequest.bulkApply(IndexRequest.java:92) at org.sonatype.nexus.repository.search.IndexRequestProcessor.lambda$0(IndexRequestProcessor.java:139) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$484/1860009625.accept(Unknown Source) at java.util.Optional.ifPresent(Optional.java:159) at org.sonatype.nexus.repository.search.IndexRequestProcessor.doUpdateSearchIndex(IndexRequestProcessor.java:135) at org.sonatype.nexus.repository.search.IndexRequestProcessor.maybeUpdateSearchIndex(IndexRequestProcessor.java:115) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$483/1380218996.accept(Unknown Source) at java.util.HashMap.forEach(HashMap.java:1289) at org.sonatype.nexus.repository.search.IndexBatchRequest.apply(IndexBatchRequest.java:93) at org.sonatype.nexus.repository.search.IndexRequestProcessor.process(IndexRequestProcessor.java:99) at org.sonatype.nexus.repository.search.IndexRequestProcessor.on(IndexRequestProcessor.java:88) at sun.reflect.GeneratedMethodAccessor266.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.google.common.eventbus.Subscriber.invokeSubscriberMethod(Subscriber.java:87) at com.google.common.eventbus.Subscriber$1.run(Subscriber.java:72) at org.sonatype.nexus.internal.event.AffinityBarrier.lambda$1(AffinityBarrier.java:91) at org.sonatype.nexus.internal.event.AffinityBarrier$$Lambda$482/736097670.run(Unknown Source) at org.sonatype.nexus.thread.internal.MDCAwareRunnable.run(MDCAwareRunnable.java:40) at org.apache.shiro.subject.support.SubjectRunnable.doRun(SubjectRunnable.java:120) at org.apache.shiro.subject.support.SubjectRunnable.run(SubjectRunnable.java:108) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: count = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@29121d1 "event-7-thread-6674" id=22654 state=BLOCKED - waiting to lock <0x5b82b493> (a org.elasticsearch.action.bulk.BulkProcessor) owned by event-7-thread-6639 id=22616 at org.elasticsearch.action.bulk.BulkProcessor.internalAdd(BulkProcessor.java:283) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:268) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:264) at org.elasticsearch.action.bulk.BulkProcessor.add(BulkProcessor.java:250) at org.sonatype.nexus.repository.search.SearchServiceImpl.lambda$0(SearchServiceImpl.java:382) at org.sonatype.nexus.repository.search.SearchServiceImpl$$Lambda$341/838205662.accept(Unknown Source) at java.lang.Iterable.forEach(Iterable.java:75) at org.sonatype.nexus.repository.search.SearchServiceImpl.bulkPut(SearchServiceImpl.java:375) at org.sonatype.nexus.repository.search.SearchFacetImpl.bulkPut(SearchFacetImpl.java:182) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.CGLIB$bulkPut$5(<generated>) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0$$FastClassByGuice$$e617070f.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76) at org.sonatype.nexus.common.stateguard.MethodInvocationAction.run(MethodInvocationAction.java:39) at org.sonatype.nexus.common.stateguard.StateGuard$GuardImpl.run(StateGuard.java:272) at org.sonatype.nexus.common.stateguard.GuardedInterceptor.invoke(GuardedInterceptor.java:53) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:77) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:55) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.bulkPut(<generated>) at org.sonatype.nexus.repository.search.IndexRequest.bulkApply(IndexRequest.java:92) at org.sonatype.nexus.repository.search.IndexRequestProcessor.lambda$0(IndexRequestProcessor.java:139) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$484/1860009625.accept(Unknown Source) at java.util.Optional.ifPresent(Optional.java:159) at org.sonatype.nexus.repository.search.IndexRequestProcessor.doUpdateSearchIndex(IndexRequestProcessor.java:135) at org.sonatype.nexus.repository.search.IndexRequestProcessor.maybeUpdateSearchIndex(IndexRequestProcessor.java:115) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$483/1380218996.accept(Unknown Source) at java.util.HashMap.forEach(HashMap.java:1289) at org.sonatype.nexus.repository.search.IndexBatchRequest.apply(IndexBatchRequest.java:93) at org.sonatype.nexus.repository.search.IndexRequestProcessor.process(IndexRequestProcessor.java:99) at org.sonatype.nexus.repository.search.IndexRequestProcessor.on(IndexRequestProcessor.java:88) at sun.reflect.GeneratedMethodAccessor266.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.google.common.eventbus.Subscriber.invokeSubscriberMethod(Subscriber.java:87) at com.google.common.eventbus.Subscriber$1.run(Subscriber.java:72) at org.sonatype.nexus.internal.event.AffinityBarrier.lambda$1(AffinityBarrier.java:91) at org.sonatype.nexus.internal.event.AffinityBarrier$$Lambda$482/736097670.run(Unknown Source) at org.sonatype.nexus.thread.internal.MDCAwareRunnable.run(MDCAwareRunnable.java:40) at org.apache.shiro.subject.support.SubjectRunnable.doRun(SubjectRunnable.java:120) at org.apache.shiro.subject.support.SubjectRunnable.run(SubjectRunnable.java:108) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: count = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@16ef32b
"event-7-thread-6636" id=22613 state=WAITING waiting on <0x76ee67cc> (a java.util.concurrent.Semaphore$NonfairSync) locked <0x76ee67cc> (a java.util.concurrent.Semaphore$NonfairSync) at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.elasticsearch.action.bulk.BulkRequestHandler$AsyncBulkRequestHandler.execute(BulkRequestHandler.java:119) at org.elasticsearch.action.bulk.BulkProcessor.execute(BulkProcessor.java:312) at org.elasticsearch.action.bulk.BulkProcessor.flush(BulkProcessor.java:331) at org.sonatype.nexus.repository.search.SearchServiceImpl.bulkPut(SearchServiceImpl.java:391) at org.sonatype.nexus.repository.search.SearchFacetImpl.bulkPut(SearchFacetImpl.java:182) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.CGLIB$bulkPut$5(<generated>) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0$$FastClassByGuice$$e617070f.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76) at org.sonatype.nexus.common.stateguard.MethodInvocationAction.run(MethodInvocationAction.java:39) at org.sonatype.nexus.common.stateguard.StateGuard$GuardImpl.run(StateGuard.java:272) at org.sonatype.nexus.common.stateguard.GuardedInterceptor.invoke(GuardedInterceptor.java:53) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:77) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:55) at org.sonatype.nexus.repository.search.SearchFacetImpl$$EnhancerByGuice$$e810e9f0.bulkPut(<generated>) at org.sonatype.nexus.repository.search.IndexRequest.bulkApply(IndexRequest.java:92) at org.sonatype.nexus.repository.search.IndexRequestProcessor.lambda$0(IndexRequestProcessor.java:139) at org.sonatype.nexus.repository.search.IndexRequestProcessor$$Lambda$484/1860009625.accept(Unknown Source) at java.util.Optional.ifPresent(Optional.java:159)