Details
Description
in large docker repositories we have seen long-running queries hold database connections to the point of thread exhaustion and connection pool exhaustion. An example thread:
/nexus/repository/docker-hosted/v2/.../manifests/... Transaction type: Web Transaction name: /nexus/repository/docker-hosted/v2/.../.../manifests/... Duration: 1,839,304.0 milliseconds CPU time: 98,223.1 milliseconds "qtp190480402-899 <command>sql.select from asset where bucket = :bucket and attributes[docker][content_digest] = :propValue</command>" #899 java.lang.Thread.State: WAITING at sun.misc.Unsafe.park(Native Method) - waiting on java.util.concurrent.locks.ReentrantLock$NonfairSync@55a195a1 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:209) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:285) at com.orientechnologies.common.collection.closabledictionary.OClosableEntry.acquireStateLock(OClosableEntry.java:84) at com.orientechnologies.common.collection.closabledictionary.OClosableLinkedContainer.acquire(OClosableLinkedContainer.java:292) at com.orientechnologies.orient.core.storage.cache.local.OWOWCache.getFilledUpTo(OWOWCache.java:959) at com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurableComponent.getFilledUpTo(ODurableComponent.java:132) at com.orientechnologies.orient.core.storage.impl.local.paginated.OPaginatedCluster.readFullEntry(OPaginatedCluster.java:1889) at com.orientechnologies.orient.core.storage.impl.local.paginated.OPaginatedCluster.readRecordBuffer(OPaginatedCluster.java:779) at com.orientechnologies.orient.core.storage.impl.local.paginated.OPaginatedCluster.readRecord(OPaginatedCluster.java:742) at com.orientechnologies.orient.core.storage.impl.local.paginated.OPaginatedCluster.readRecord(OPaginatedCluster.java:721) at com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.doReadRecord(OAbstractPaginatedStorage.java:4220) at com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.readRecord(OAbstractPaginatedStorage.java:3807) at com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.readRecord(OAbstractPaginatedStorage.java:1410) at com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx$SimpleRecordReader.readRecord(ODatabaseDocumentTx.java:3395) at com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx.executeReadRecord(ODatabaseDocumentTx.java:2008) at com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx.load(ODatabaseDocumentTx.java:656) at com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx.load(ODatabaseDocumentTx.java:103) at com.orientechnologies.orient.core.sql.OCommandExecutorSQLSelect.executeSearchRecord(OCommandExecutorSQLSelect.java:585) at com.orientechnologies.orient.core.sql.OCommandExecutorSQLSelect.serialIterator(OCommandExecutorSQLSelect.java:1638) at com.orientechnologies.orient.core.sql.OCommandExecutorSQLSelect.fetchFromTarget(OCommandExecutorSQLSelect.java:1585) at com.orientechnologies.orient.core.sql.OCommandExecutorSQLSelect.fetchValuesFromIndexCursor(OCommandExecutorSQLSelect.java:2466) at com.orientechnologies.orient.core.sql.OCommandExecutorSQLSelect.searchForIndexes(OCommandExecutorSQLSelect.java:2280) at com.orientechnologies.orient.core.sql.OCommandExecutorSQLSelect.searchInClasses(OCommandExecutorSQLSelect.java:1017) at com.orientechnologies.orient.core.sql.OCommandExecutorSQLResultsetAbstract.assignTarget(OCommandExecutorSQLResultsetAbstract.java:203) at com.orientechnologies.orient.core.sql.OCommandExecutorSQLSelect.assignTarget(OCommandExecutorSQLSelect.java:527) at com.orientechnologies.orient.core.sql.OCommandExecutorSQLSelect.executeSearch(OCommandExecutorSQLSelect.java:509) at com.orientechnologies.orient.core.sql.OCommandExecutorSQLSelect.execute(OCommandExecutorSQLSelect.java:485) at com.orientechnologies.orient.core.sql.OCommandExecutorSQLDelegate.execute(OCommandExecutorSQLDelegate.java:70) at com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.executeCommand(OAbstractPaginatedStorage.java:3400) at com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.command(OAbstractPaginatedStorage.java:3318) at com.orientechnologies.orient.core.command.OCommandRequestTextAbstract.execute(OCommandRequestTextAbstract.java:69) at org.sonatype.nexus.repository.storage.MetadataNodeEntityAdapter.findByProperty(MetadataNodeEntityAdapter.java:148) at org.sonatype.nexus.repository.storage.StorageTxImpl.findAssetWithProperty(StorageTxImpl.java:346) at sun.reflect.GeneratedMethodAccessor226.invoke() at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.sonatype.nexus.common.stateguard.SimpleMethodInvocation.proceed(SimpleMethodInvocation.java:53) at org.sonatype.nexus.common.stateguard.MethodInvocationAction.run(MethodInvocationAction.java:39) at org.sonatype.nexus.common.stateguard.StateGuard$GuardImpl.run(StateGuard.java:272) at org.sonatype.nexus.common.stateguard.GuardedInterceptor.invoke(GuardedInterceptor.java:53) at org.sonatype.nexus.common.stateguard.StateGuardAspect$1.invoke(StateGuardAspect.java:63) at com.sun.proxy.$Proxy241.findAssetWithProperty() at org.sonatype.nexus.repository.docker.internal.DockerFacetUtils.findAssetByContentDigest(DockerFacetUtils.java:175) at sun.reflect.GeneratedMethodAccessor460.invoke() at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:98) at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite$StaticMetaMethodSiteNoUnwrapNoCoerce.invoke(StaticMetaMethodSite.java:151) at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite.call(StaticMetaMethodSite.java:91) at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:144) at org.sonatype.nexus.repository.docker.internal.V2ManifestUtilImpl.insertEmptyLayerIfNeeded(V2ManifestUtilImpl.groovy:200) at org.sonatype.nexus.repository.docker.internal.V2ManifestUtilImpl$insertEmptyLayerIfNeeded$1.callStatic() at org.sonatype.nexus.repository.docker.internal.V2ManifestUtilImpl.mayDowngrade(V2ManifestUtilImpl.groovy:139) at org.sonatype.nexus.repository.docker.internal.V2ManifestUtilImpl$$EnhancerByGuice$$d2636a8e.CGLIB$mayDowngrade$8(<generated>) at org.sonatype.nexus.repository.docker.internal.V2ManifestUtilImpl$$EnhancerByGuice$$d2636a8e$$FastClassByGuice$$2667923b.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76) at org.sonatype.nexus.transaction.TransactionInterceptor.invoke(TransactionInterceptor.java:49) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:77) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:55) at org.sonatype.nexus.repository.docker.internal.V2ManifestUtilImpl$$EnhancerByGuice$$d2636a8e.mayDowngrade(<generated>) at org.sonatype.nexus.repository.docker.internal.DockerHostedFacetImpl.getManifestByTag(DockerHostedFacetImpl.java:774) at org.sonatype.nexus.repository.docker.internal.DockerHostedFacetImpl$$EnhancerByGuice$$8b6a7319.CGLIB$getManifestByTag$14(<generated>) at org.sonatype.nexus.repository.docker.internal.DockerHostedFacetImpl$$EnhancerByGuice$$8b6a7319$$FastClassByGuice$$bb884868.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76) at org.sonatype.nexus.transaction.TransactionalWrapper.proceedWithTransaction(TransactionalWrapper.java:57) at org.sonatype.nexus.transaction.TransactionInterceptor.proceedWithTransaction(TransactionInterceptor.java:66) at org.sonatype.nexus.transaction.TransactionInterceptor.invoke(TransactionInterceptor.java:55) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:77) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:55) at org.sonatype.nexus.repository.docker.internal.DockerHostedFacetImpl$$EnhancerByGuice$$8b6a7319.getManifestByTag(<generated>) at org.sonatype.nexus.repository.docker.internal.DockerHostedFacet$getManifestByTag$0.call() at org.sonatype.nexus.repository.docker.internal.V2Handlers$_closure14.doCall(V2Handlers.groovy:254) at sun.reflect.GeneratedMethodAccessor268.invoke() at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:98) at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325) at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:264) at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1034) at groovy.lang.Closure.call(Closure.java:418) at org.codehaus.groovy.runtime.ConvertedClosure.invokeCustom(ConvertedClosure.java:54) at org.codehaus.groovy.runtime.ConversionHandler.invoke(ConversionHandler.java:124) at com.sun.proxy.$Proxy223.handle() at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.view.handlers.LastDownloadedHandler.handle(LastDownloadedHandler.java:63) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.storage.UnitOfWorkHandler.handle(UnitOfWorkHandler.java:39) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.view.handlers.ContentHeadersHandler.handle(ContentHeadersHandler.java:46) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.view.handlers.ConditionalRequestHandler.handle(ConditionalRequestHandler.java:72) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.http.PartialFetchHandler.handle(PartialFetchHandler.java:59) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.security.SecurityHandler.handle(SecurityHandler.java:52) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.view.Context$proceed.call() at org.sonatype.nexus.repository.docker.internal.V2Handlers$_closure18.doCall(V2Handlers.groovy:298) at sun.reflect.GeneratedMethodAccessor222.invoke() at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:98) at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325) at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:264) at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1034) at groovy.lang.Closure.call(Closure.java:418) at org.codehaus.groovy.runtime.ConvertedClosure.invokeCustom(ConvertedClosure.java:54) at org.codehaus.groovy.runtime.ConversionHandler.invoke(ConversionHandler.java:124) at com.sun.proxy.$Proxy223.handle() at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.view.Context$proceed.call() at org.sonatype.nexus.repository.docker.internal.V2Handlers$_closure16.doCall(V2Handlers.groovy:273) at sun.reflect.GeneratedMethodAccessor224.invoke() at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:98) at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325) at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:264) at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1034) at groovy.lang.Closure.call(Closure.java:418) at org.codehaus.groovy.runtime.ConvertedClosure.invokeCustom(ConvertedClosure.java:54) at org.codehaus.groovy.runtime.ConversionHandler.invoke(ConversionHandler.java:124) at com.sun.proxy.$Proxy223.handle() at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.view.Context$proceed.call() at org.sonatype.nexus.repository.docker.internal.V2Handlers$_closure1.doCall(V2Handlers.groovy:90) at sun.reflect.GeneratedMethodAccessor223.invoke() at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:98) at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325) at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:264) at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1034) at groovy.lang.Closure.call(Closure.java:418) at org.codehaus.groovy.runtime.ConvertedClosure.invokeCustom(ConvertedClosure.java:54) at org.codehaus.groovy.runtime.ConversionHandler.invoke(ConversionHandler.java:124) at com.sun.proxy.$Proxy223.handle() at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at com.sonatype.analytics.internal.handler.AnalyticsMeteringHandler.handle(AnalyticsMeteringHandler.java:69) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.view.handlers.TimingHandler.handle(TimingHandler.java:58) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.view.Context.start(Context.java:114) at org.sonatype.nexus.repository.view.Router.dispatch(Router.java:64) at org.sonatype.nexus.repository.view.ConfigurableViewFacet.dispatch(ConfigurableViewFacet.java:52) at org.sonatype.nexus.repository.view.ConfigurableViewFacet.dispatch(ConfigurableViewFacet.java:43) at org.sonatype.nexus.repository.httpbridge.internal.ViewServlet.dispatchAndSend(ViewServlet.java:212) at org.sonatype.nexus.repository.httpbridge.internal.ViewServlet.doService(ViewServlet.java:174) at org.sonatype.nexus.repository.httpbridge.internal.ViewServlet.service(ViewServlet.java:126) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:286) at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:276) at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:181) at com.google.inject.servlet.DynamicServletPipeline.service(DynamicServletPipeline.java:71) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85) at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:112) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82) at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:61) at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108) at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137) at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125) at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66) at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108) at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137) at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125) at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66) at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108) at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137) at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125) at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66) at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108) at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137) at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125) at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66) at org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449) at org.sonatype.nexus.security.SecurityFilter.executeChain(SecurityFilter.java:85) at org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365) at org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90) at org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83) at org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383) at org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362) at org.sonatype.nexus.security.SecurityFilter.doFilterInternal(SecurityFilter.java:101) at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82) at org.sonatype.nexus.repository.httpbridge.internal.ExhaustRequestFilter.doFilter(ExhaustRequestFilter.java:80) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82) at com.sonatype.nexus.licensing.internal.LicensingRedirectFilter.doFilter(LicensingRedirectFilter.java:114) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82) at com.codahale.metrics.servlet.AbstractInstrumentedFilter.doFilter(AbstractInstrumentedFilter.java:112) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82) at org.sonatype.nexus.internal.web.ErrorPageFilter.doFilter(ErrorPageFilter.java:79) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82) at org.sonatype.nexus.internal.web.EnvironmentFilter.doFilter(EnvironmentFilter.java:101) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82) at org.sonatype.nexus.internal.web.HeaderPatternFilter.doFilter(HeaderPatternFilter.java:98) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82) at com.google.inject.servlet.DynamicFilterPipeline.dispatch(DynamicFilterPipeline.java:104) at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:135) at org.sonatype.nexus.bootstrap.osgi.DelegatingFilter.doFilter(DelegatingFilter.java:73) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1700) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1667) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at com.codahale.metrics.jetty9.InstrumentedHandler.handle(InstrumentedHandler.java:239) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:152) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:505) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:427) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:321) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:698) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:804) at java.lang.Thread.run(Thread.java:745)