Critical Make a RubyGems group. ( gem-group )
Put a RubyGems proxy to https://rubygems.org in the group as the only member.
Make a HEAD request for http://localhost:8081/repository/gem-group/specs.4.8.gz - HEAD with return no content as expected, status 200.
Make a GET request for http://localhost:8081/repository/gem-group/specs.4.8.gz - GET will return content and status 200.
A local test on Nexus with no load showed consistent 6-7 second response times infinitely for both types of requests with a single user making the request.
The time required for Nexus to begin a HTTP response is identical for both HTTP methods and much slower than sending the same request to rubygems.org.
Repeating the requests over and over even though no content has changed, does not improve the response time.
A thread dump while a HEAD request is active shows the following work being performed:
"qtp421378397-200" #200 prio=5 os_prio=31 tid=0x00007fd55066b800 nid=0x6207 runnable [0x000070000f81f000]
java.lang.Thread.State: RUNNABLE
at java.util.zip.Inflater.inflateBytes(Native Method)
at java.util.zip.Inflater.inflate(Inflater.java:259)
- locked <0x00000007ba428078> (a java.util.zip.ZStreamRef)
at java.util.zip.InflaterInputStream.read(InflaterInputStream.java:152)
at java.util.zip.GZIPInputStream.read(GZIPInputStream.java:117)
at java.util.zip.InflaterInputStream.read(InflaterInputStream.java:122)
at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad.loadByte(MarshalLoad.java:123)
at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad.loadObject(MarshalLoad.java:108)
at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad.access$4(MarshalLoad.java:107)
at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad$ListLoadVisitor.accept(MarshalLoad.java:324)
at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad$ListLoadVisitor.accept(MarshalLoad.java:1)
at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad.loadObject(MarshalLoad.java:113)
at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad.access$4(MarshalLoad.java:107)
at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad$ListLoadVisitor.accept(MarshalLoad.java:324)
at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad$ListLoadVisitor.accept(MarshalLoad.java:1)
at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad.loadObject(MarshalLoad.java:113)
at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad.fromBytes(MarshalLoad.java:104)
at org.sonatype.nexus.repository.rubygems.marshal.Marshal.load(Marshal.java:72)
at org.sonatype.nexus.repository.rubygems.internal.RubygemsGroupHandler.handleSpecsIndex(RubygemsGroupHandler.java:131)
at org.sonatype.nexus.repository.rubygems.internal.RubygemsGroupHandler.doGet(RubygemsGroupHandler.java:70)
at org.sonatype.nexus.repository.group.GroupHandler.handle(GroupHandler.java:81)
at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
at com.sonatype.nexus.plugins.healthcheck.downloads.AssetDownloadCountContributedHandler.handle(AssetDownloadCountContributedHandler.java:47)
at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
at com.sonatype.nexus.clm.internal.QuarantineContributedHandler.handle(QuarantineContributedHandler.java:62)
at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
at com.sonatype.nexus.usertoken.plugin.internal.UserTokenHandler.handle(UserTokenHandler.java:70)
at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
at org.sonatype.nexus.repository.view.handlers.HandlerContributor.handle(HandlerContributor.java:67)
at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
at org.sonatype.nexus.repository.view.handlers.ExceptionHandler.handle(ExceptionHandler.java:43)
at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
at org.sonatype.nexus.repository.security.SecurityHandler.handle(SecurityHandler.java:52)
at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
at org.sonatype.nexus.repository.view.handlers.TimingHandler.handle(TimingHandler.java:46)
at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80)
at org.sonatype.nexus.repository.view.Context.start(Context.java:114)
at org.sonatype.nexus.repository.view.Router.dispatch(Router.java:63)
at org.sonatype.nexus.repository.view.ConfigurableViewFacet.dispatch(ConfigurableViewFacet.java:52)
at org.sonatype.nexus.repository.view.ConfigurableViewFacet.dispatch(ConfigurableViewFacet.java:43)
at org.sonatype.nexus.repository.httpbridge.internal.ViewServlet.dispatchAndSend(ViewServlet.java:198)
at org.sonatype.nexus.repository.httpbridge.internal.ViewServlet.doService(ViewServlet.java:160)
at org.sonatype.nexus.repository.httpbridge.internal.ViewServlet.service(ViewServlet.java:117)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:286)
at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:276)
at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:181)
at com.google.inject.servlet.DynamicServletPipeline.service(DynamicServletPipeline.java:71)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85)
at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:112)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:61)
at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108)
at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137)
at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66)
at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108)
at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137)
at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66)
at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108)
at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137)
at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66)
at org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449)
at org.sonatype.nexus.security.SecurityFilter.executeChain(SecurityFilter.java:85)
at org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365)
at org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90)
at org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83)
at org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383)
at org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362)
at org.sonatype.nexus.security.SecurityFilter.doFilterInternal(SecurityFilter.java:101)
at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
at org.sonatype.nexus.repository.httpbridge.internal.ExhaustRequestFilter.doFilter(ExhaustRequestFilter.java:71)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
at com.sonatype.nexus.licensing.internal.LicensingRedirectFilter.doFilter(LicensingRedirectFilter.java:108)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
at com.codahale.metrics.servlet.AbstractInstrumentedFilter.doFilter(AbstractInstrumentedFilter.java:97)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
at org.sonatype.nexus.internal.web.ErrorPageFilter.doFilter(ErrorPageFilter.java:68)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
at org.sonatype.nexus.internal.web.EnvironmentFilter.doFilter(EnvironmentFilter.java:102)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
at org.sonatype.nexus.internal.web.HeaderPatternFilter.doFilter(HeaderPatternFilter.java:98)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
at com.google.inject.servlet.DynamicFilterPipeline.dispatch(DynamicFilterPipeline.java:104)
at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:135)
at org.sonatype.nexus.bootstrap.osgi.DelegatingFilter.doFilter(DelegatingFilter.java:73)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1668)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1158)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1090)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:119)
at com.codahale.metrics.jetty9.InstrumentedHandler.handle(InstrumentedHandler.java:175)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:109)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:119)
at org.eclipse.jetty.server.Server.handle(Server.java:517)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:242)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:75)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:213)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
Locked ownable synchronizers:
- <0x00000007ba1104a0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)
As load increases ( increase concurrent threads requesting these files ), the time to respond increases exponentially.
Further it is very easy to trigger OutOfMemory when multiple threads request this file at the same time using either method type.
2017-08-25 16:37:29,329-0300 ERROR [qtp421378397-267] *SYSTEM org.sonatype.nexus.internal.web.ErrorPageServlet - Unexpected exception java.lang.OutOfMemoryError: Java heap space at java.util.IdentityHashMap.resize(IdentityHashMap.java:471) at java.util.IdentityHashMap.put(IdentityHashMap.java:440) at org.sonatype.nexus.repository.rubygems.marshal.ObjectCache.add(ObjectCache.java:65) at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad$ListLoadVisitor.accept(MarshalLoad.java:322) at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad$ListLoadVisitor.accept(MarshalLoad.java:1) at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad.loadObject(MarshalLoad.java:113) at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad.access$4(MarshalLoad.java:107) at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad$ListLoadVisitor.accept(MarshalLoad.java:324) at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad$ListLoadVisitor.accept(MarshalLoad.java:1) at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad.loadObject(MarshalLoad.java:113) at org.sonatype.nexus.repository.rubygems.marshal.MarshalLoad.fromBytes(MarshalLoad.java:104) at org.sonatype.nexus.repository.rubygems.marshal.Marshal.load(Marshal.java:72) at org.sonatype.nexus.repository.rubygems.internal.RubygemsGroupHandler.handleSpecsIndex(RubygemsGroupHandler.java:131) at org.sonatype.nexus.repository.rubygems.internal.RubygemsGroupHandler.doGet(RubygemsGroupHandler.java:70) at org.sonatype.nexus.repository.group.GroupHandler.handle(GroupHandler.java:81) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at com.sonatype.nexus.plugins.healthcheck.downloads.AssetDownloadCountContributedHandler.handle(AssetDownloadCountContributedHandler.java:47) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at com.sonatype.nexus.clm.internal.QuarantineContributedHandler.handle(QuarantineContributedHandler.java:62) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at com.sonatype.nexus.usertoken.plugin.internal.UserTokenHandler.handle(UserTokenHandler.java:70) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.view.handlers.HandlerContributor.handle(HandlerContributor.java:67) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.view.handlers.ExceptionHandler.handle(ExceptionHandler.java:43) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.security.SecurityHandler.handle(SecurityHandler.java:52) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.view.handlers.TimingHandler.handle(TimingHandler.java:46) at org.sonatype.nexus.repository.view.Context.proceed(Context.java:80) at org.sonatype.nexus.repository.view.Context.start(Context.java:114) at org.sonatype.nexus.repository.view.Router.dispatch(Router.java:63) 2017-08-25 16:38:43,398-0300 ERROR [fileinstall-/app/nexus-testing/3.5.0-02/nexus-installer-3.5.0-02-mac-archive/nexus-3.5.0-02/deploy] *SYSTEM org.apache.felix.fileinstall - In main loop, we have serious trouble java.lang.OutOfMemoryError: GC overhead limit exceeded at java.util.Arrays.copyOfRange(Arrays.java:3664) at java.lang.String.<init>(String.java:207) at java.lang.String.substring(String.java:1933) at java.io.File.getName(File.java:456) at org.apache.felix.fileinstall.internal.BundleTransformer.canHandle(BundleTransformer.java:41) at org.apache.felix.fileinstall.internal.DirectoryWatcher.findListener(DirectoryWatcher.java:524) at org.apache.felix.fileinstall.internal.DirectoryWatcher.doProcess(DirectoryWatcher.java:456) at org.apache.felix.fileinstall.internal.DirectoryWatcher.process(DirectoryWatcher.java:361) at org.apache.felix.fileinstall.internal.DirectoryWatcher.run(DirectoryWatcher.java:312)
