From 9240375cb73b47e43ad5e9a5d77f33dfef0f6925 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sun, 16 Aug 2020 12:04:13 -0400 Subject: [PATCH] call System.exit when OOME encountered --- .../OutOfMemoryErrorMapper.java | 27 ++++++++++ .../ExitOnOutOfMemoryErrorThreadFactory.java | 50 +++++++++++++++++++ .../wizard/server/RestServerBase.java | 15 ++++-- .../server/config/HttpConfiguration.java | 1 + 4 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 wizard-server/src/main/java/org/cobbzilla/wizard/exceptionmappers/OutOfMemoryErrorMapper.java create mode 100644 wizard-server/src/main/java/org/cobbzilla/wizard/server/ExitOnOutOfMemoryErrorThreadFactory.java diff --git a/wizard-server/src/main/java/org/cobbzilla/wizard/exceptionmappers/OutOfMemoryErrorMapper.java b/wizard-server/src/main/java/org/cobbzilla/wizard/exceptionmappers/OutOfMemoryErrorMapper.java new file mode 100644 index 0000000..d920b96 --- /dev/null +++ b/wizard-server/src/main/java/org/cobbzilla/wizard/exceptionmappers/OutOfMemoryErrorMapper.java @@ -0,0 +1,27 @@ +package org.cobbzilla.wizard.exceptionmappers; + +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.wizard.server.config.RestServerConfiguration; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; + +import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; +import static org.cobbzilla.util.system.OutOfMemoryErrorUncaughtExceptionHandler.EXIT_ON_OOME; +import static org.cobbzilla.wizard.resources.ResourceUtil.serverError; + +@Slf4j +public class OutOfMemoryErrorMapper implements ExceptionMapper { + + @Autowired private RestServerConfiguration configuration; + + @Override public Response toResponse(OutOfMemoryError e) { + if (configuration.getHttp().isExitOnOutOfMemoryError()) { + EXIT_ON_OOME.uncaughtException(Thread.currentThread(), e); + } + log.error("!!!!! OutOfMemoryError: "+shortError(e), e); + return serverError(); + } + +} diff --git a/wizard-server/src/main/java/org/cobbzilla/wizard/server/ExitOnOutOfMemoryErrorThreadFactory.java b/wizard-server/src/main/java/org/cobbzilla/wizard/server/ExitOnOutOfMemoryErrorThreadFactory.java new file mode 100644 index 0000000..6c8c3d6 --- /dev/null +++ b/wizard-server/src/main/java/org/cobbzilla/wizard/server/ExitOnOutOfMemoryErrorThreadFactory.java @@ -0,0 +1,50 @@ +package org.cobbzilla.wizard.server; + +import org.glassfish.grizzly.Grizzly; +import org.glassfish.grizzly.memory.MemoryManager; +import org.glassfish.grizzly.memory.ThreadLocalPoolProvider; +import org.glassfish.grizzly.threadpool.DefaultWorkerThread; +import org.glassfish.grizzly.threadpool.ThreadPoolConfig; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.cobbzilla.util.system.OutOfMemoryErrorUncaughtExceptionHandler.EXIT_ON_OOME; + +public class ExitOnOutOfMemoryErrorThreadFactory implements ThreadFactory { + + private final ThreadPoolConfig config; + private final AtomicInteger counter = new AtomicInteger(0); + + public ExitOnOutOfMemoryErrorThreadFactory(ThreadPoolConfig config) { + this.config = config; + } + + @Override public Thread newThread(Runnable r) { + final MemoryManager mm = config.getMemoryManager(); + final ThreadLocalPoolProvider threadLocalPoolProvider; + + if (mm instanceof ThreadLocalPoolProvider) { + threadLocalPoolProvider = (ThreadLocalPoolProvider) mm; + } else { + threadLocalPoolProvider = null; + } + + final DefaultWorkerThread thread = + new DefaultWorkerThread(Grizzly.DEFAULT_ATTRIBUTE_BUILDER, + config.getPoolName() + '(' + counter.incrementAndGet() + ')', + ((threadLocalPoolProvider != null) ? threadLocalPoolProvider.createThreadLocalPool() : null), + r); + + thread.setUncaughtExceptionHandler(EXIT_ON_OOME); + thread.setPriority(config.getPriority()); + thread.setDaemon(config.isDaemon()); + final ClassLoader initial = config.getInitialClassLoader(); + if (initial != null) { + thread.setContextClassLoader(initial); + } + + return thread; + } + +} diff --git a/wizard-server/src/main/java/org/cobbzilla/wizard/server/RestServerBase.java b/wizard-server/src/main/java/org/cobbzilla/wizard/server/RestServerBase.java index f7a09b8..13a1152 100644 --- a/wizard-server/src/main/java/org/cobbzilla/wizard/server/RestServerBase.java +++ b/wizard-server/src/main/java/org/cobbzilla/wizard/server/RestServerBase.java @@ -23,6 +23,8 @@ import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.glassfish.grizzly.http.server.*; +import org.glassfish.grizzly.nio.transport.TCPNIOTransport; +import org.glassfish.grizzly.threadpool.ThreadPoolConfig; import org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpContainer; import org.glassfish.jersey.media.multipart.MultiPartFeature; import org.glassfish.jersey.message.internal.StreamingOutputProvider; @@ -190,17 +192,22 @@ public abstract class RestServerBase implemen final HttpServer httpServer = new HttpServer(); final NetworkListener listener = new NetworkListener("grizzly-"+serverName, getListenAddress(), httpConfig.getPort()); + final TCPNIOTransport transport = listener.getTransport(); if (httpConfig.hasSelectorThreads()) { log.info("buildServer: using "+httpConfig.getSelectorThreads()+" selector threads"); - listener.getTransport().setSelectorRunnersCount(httpConfig.getSelectorThreads()); + transport.setSelectorRunnersCount(httpConfig.getSelectorThreads()); } else { - log.info("buildServer: using "+listener.getTransport().getSelectorRunnersCount()+" selector threads"); + log.info("buildServer: using "+ transport.getSelectorRunnersCount()+" selector threads"); } + final ThreadPoolConfig workerThreadPoolConfig = transport.getWorkerThreadPoolConfig(); if (httpConfig.hasWorkerThreads()) { log.info("buildServer: using "+httpConfig.getWorkerThreads()+" worker threads"); - listener.getTransport().getWorkerThreadPoolConfig().setMaxPoolSize(httpConfig.getWorkerThreads()); + workerThreadPoolConfig.setMaxPoolSize(httpConfig.getWorkerThreads()); } else { - log.info("buildServer: using "+listener.getTransport().getWorkerThreadPoolConfig().getMaxPoolSize()+" worker threads"); + log.info("buildServer: using "+ workerThreadPoolConfig.getMaxPoolSize()+" worker threads"); + } + if (httpConfig.isExitOnOutOfMemoryError()) { + workerThreadPoolConfig.setThreadFactory(new ExitOnOutOfMemoryErrorThreadFactory(workerThreadPoolConfig)); } httpServer.addListener(listener); diff --git a/wizard-server/src/main/java/org/cobbzilla/wizard/server/config/HttpConfiguration.java b/wizard-server/src/main/java/org/cobbzilla/wizard/server/config/HttpConfiguration.java index 9cc0b85..1f96bf9 100644 --- a/wizard-server/src/main/java/org/cobbzilla/wizard/server/config/HttpConfiguration.java +++ b/wizard-server/src/main/java/org/cobbzilla/wizard/server/config/HttpConfiguration.java @@ -26,4 +26,5 @@ public class HttpConfiguration { @Getter @Setter private Integer workerThreads; public boolean hasWorkerThreads () { return workerThreads != null; } + @Getter @Setter private boolean exitOnOutOfMemoryError = true; }