fix for file descriptor leak

This commit is contained in:
Jörg Prante 2020-01-13 14:16:44 +01:00
parent b980dc9b9c
commit 8b50868c16
13 changed files with 106 additions and 83 deletions

View file

@ -1,6 +1,6 @@
group = org.xbib group = org.xbib
name = netty-http name = netty-http
version = 4.1.44.1 version = 4.1.44.2
# netty # netty
netty.version = 4.1.44.Final netty.version = 4.1.44.Final

View file

@ -4,11 +4,10 @@ dependencies {
compile project(":netty-http-client-api") compile project(":netty-http-client-api")
compile "io.netty:netty-handler-proxy:${project.property('netty.version')}" compile "io.netty:netty-handler-proxy:${project.property('netty.version')}"
runtime "org.bouncycastle:bcpkix-jdk15on:${project.property('bouncycastle.version')}" runtime "org.bouncycastle:bcpkix-jdk15on:${project.property('bouncycastle.version')}"
// we are an Mac OS X 10.12, but tcnative > 2.0.26 is for 10.13+
if (Os.isFamily(Os.FAMILY_MAC)) { if (Os.isFamily(Os.FAMILY_MAC)) {
runtime "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative-legacy-macosx.version')}" runtime "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative-legacy-macosx.version')}"
runtime project(':netty-http-kqueue') //runtime project(':netty-http-kqueue')
} else { } else if (Os.isFamily(Os.FAMILY_UNIX)) {
runtime "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}" runtime "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}"
runtime project(':netty-http-epoll') runtime project(':netty-http-epoll')
} }

View file

@ -22,6 +22,7 @@ import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslHandler; import io.netty.handler.ssl.SslHandler;
import io.netty.handler.ssl.SslProvider; import io.netty.handler.ssl.SslProvider;
import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
import io.netty.util.concurrent.Future;
import org.xbib.netty.http.client.api.HttpChannelInitializer; import org.xbib.netty.http.client.api.HttpChannelInitializer;
import org.xbib.netty.http.client.api.ProtocolProvider; import org.xbib.netty.http.client.api.ProtocolProvider;
import org.xbib.netty.http.client.api.Request; import org.xbib.netty.http.client.api.Request;
@ -120,43 +121,40 @@ public final class Client implements AutoCloseable {
for (ProtocolProvider<HttpChannelInitializer, Transport> provider : ServiceLoader.load(ProtocolProvider.class)) { for (ProtocolProvider<HttpChannelInitializer, Transport> provider : ServiceLoader.load(ProtocolProvider.class)) {
protocolProviders.add(provider); protocolProviders.add(provider);
if (logger.isLoggable(Level.FINEST)) { if (logger.isLoggable(Level.FINEST)) {
logger.log(Level.FINEST, "protocol provider up: " + provider.transportClass()); logger.log(Level.FINEST, "protocol provider: " + provider.transportClass());
} }
} }
initializeTrustManagerFactory(clientConfig); initializeTrustManagerFactory(clientConfig);
this.byteBufAllocator = byteBufAllocator != null ? byteBufAllocator : ByteBufAllocator.DEFAULT; this.byteBufAllocator = byteBufAllocator != null ? byteBufAllocator : ByteBufAllocator.DEFAULT;
if (eventLoopGroup != null) { if (eventLoopGroup != null) {
this.eventLoopGroup = eventLoopGroup; this.eventLoopGroup = eventLoopGroup;
} else { }
ServiceLoader<TransportProvider> transportProviders = ServiceLoader.load(TransportProvider.class); if (socketChannelClass != null) {
for (TransportProvider transportProvider : transportProviders) { this.socketChannelClass = socketChannelClass;
if (clientConfig.getTransportProviderName() == null || clientConfig.getTransportProviderName().equals(transportProvider.getClass().getName())) { }
this.eventLoopGroup = transportProvider.createEventLoopGroup(clientConfig.getThreadCount(), new HttpClientThreadFactory()); ServiceLoader<TransportProvider> transportProviders = ServiceLoader.load(TransportProvider.class);
if (logger.isLoggable(Level.FINEST)) { for (TransportProvider transportProvider : transportProviders) {
logger.log(Level.FINEST, "transport provider event loop group: " + this.eventLoopGroup.getClass().getName()); if (this.eventLoopGroup == null &&
} (clientConfig.getTransportProviderName() == null || clientConfig.getTransportProviderName().equals(transportProvider.getClass().getName()))) {
} this.eventLoopGroup = transportProvider.createEventLoopGroup(clientConfig.getThreadCount(), new HttpClientThreadFactory());
}
if (this.socketChannelClass == null &&
(clientConfig.getTransportProviderName() == null || clientConfig.getTransportProviderName().equals(transportProvider.getClass().getName()))) {
this.socketChannelClass = transportProvider.createSocketChannelClass();
} }
} }
if (this.eventLoopGroup == null) { if (this.eventLoopGroup == null) {
this.eventLoopGroup = new NioEventLoopGroup(clientConfig.getThreadCount(), new HttpClientThreadFactory()); this.eventLoopGroup = new NioEventLoopGroup(clientConfig.getThreadCount(), new HttpClientThreadFactory());
} }
if (socketChannelClass != null) { if (logger.isLoggable(Level.FINEST)) {
this.socketChannelClass = socketChannelClass; logger.log(Level.FINEST, "event loop group class: " + this.eventLoopGroup.getClass().getName());
} else {
ServiceLoader<TransportProvider> transportProviders = ServiceLoader.load(TransportProvider.class);
for (TransportProvider transportProvider : transportProviders) {
if (clientConfig.getTransportProviderName() == null || clientConfig.getTransportProviderName().equals(transportProvider.getClass().getName())) {
this.socketChannelClass = transportProvider.createSocketChannelClass();
if (logger.isLoggable(Level.FINEST)) {
logger.log(Level.FINEST, "transport provider channel: " + this.socketChannelClass.getName());
}
}
}
} }
if (this.socketChannelClass == null) { if (this.socketChannelClass == null) {
this.socketChannelClass = NioSocketChannel.class; this.socketChannelClass = NioSocketChannel.class;
} }
if (logger.isLoggable(Level.FINEST)) {
logger.log(Level.FINEST, "socket channel class: " + this.socketChannelClass.getName());
}
this.bootstrap = new Bootstrap() this.bootstrap = new Bootstrap()
.group(this.eventLoopGroup) .group(this.eventLoopGroup)
.channel(this.socketChannelClass) .channel(this.socketChannelClass)
@ -392,8 +390,9 @@ public final class Client implements AutoCloseable {
if (hasPooledConnections()) { if (hasPooledConnections()) {
pool.close(); pool.close();
} }
eventLoopGroup.shutdownGracefully(1L, amount, timeUnit); Future<?> future = eventLoopGroup.shutdownGracefully(1L, amount, timeUnit);
eventLoopGroup.awaitTermination(amount, timeUnit); eventLoopGroup.awaitTermination(amount, timeUnit);
future.sync();
} catch (Exception e) { } catch (Exception e) {
throw new IOException(e); throw new IOException(e);
} }

View file

@ -13,12 +13,13 @@ public class HttpResponseHandler extends SimpleChannelInboundHandler<FullHttpRes
public void channelRead0(ChannelHandlerContext ctx, FullHttpResponse fullHttpResponse) throws Exception { public void channelRead0(ChannelHandlerContext ctx, FullHttpResponse fullHttpResponse) throws Exception {
Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get();
transport.responseReceived(ctx.channel(), null, fullHttpResponse); transport.responseReceived(ctx.channel(), null, fullHttpResponse);
// do not close ctx here
} }
@Override @Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get();
transport.fail(cause); transport.fail(cause);
ctx.channel().close(); // do not close ctx here
} }
} }

View file

@ -15,12 +15,13 @@ public class Http2ResponseHandler extends SimpleChannelInboundHandler<FullHttpRe
Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get();
Integer streamId = httpResponse.headers().getInt(HttpConversionUtil.ExtensionHeaderNames.STREAM_ID.text()); Integer streamId = httpResponse.headers().getInt(HttpConversionUtil.ExtensionHeaderNames.STREAM_ID.text());
transport.responseReceived(ctx.channel(), streamId, httpResponse); transport.responseReceived(ctx.channel(), streamId, httpResponse);
// do not close ctx here
} }
@Override @Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get();
transport.fail(cause); transport.fail(cause);
ctx.channel().close(); // do not close ctx here
} }
} }

View file

@ -102,15 +102,11 @@ public abstract class BaseTransport implements Transport {
@Override @Override
public void close() { public void close() {
// channels are present, maybe forgot a get() to receive responses?
if (!channels.isEmpty()) { if (!channels.isEmpty()) {
get(); get();
} }
for (Flow flow : flowMap.values()) { cancel();
flow.close();
}
channels.clear();
httpDataFactory.cleanAllHttpData();
// do not clear requests
} }
@Override @Override
@ -160,12 +156,7 @@ public abstract class BaseTransport implements Transport {
flow.get(key).get(value, timeUnit); flow.get(key).get(value, timeUnit);
completeRequest(requestKey); completeRequest(requestKey);
} catch (Exception e) { } catch (Exception e) {
if (requestKey != null) { completeRequestExceptionally(requestKey, e);
Request request = requests.get(requestKey);
if (request != null && request.getCompletableFuture() != null) {
request.getCompletableFuture().completeExceptionally(e);
}
}
flow.fail(e); flow.fail(e);
} finally { } finally {
flow.remove(key); flow.remove(key);
@ -217,6 +208,7 @@ public abstract class BaseTransport implements Transport {
flowMap.clear(); flowMap.clear();
channels.clear(); channels.clear();
requests.clear(); requests.clear();
httpDataFactory.cleanAllHttpData();
} }
@Override @Override

View file

@ -1,6 +1,7 @@
package org.xbib.netty.http.client.transport; package org.xbib.netty.http.client.transport;
import io.netty.channel.Channel; import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import io.netty.handler.codec.http.DefaultFullHttpRequest; import io.netty.handler.codec.http.DefaultFullHttpRequest;
import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.FullHttpResponse; import io.netty.handler.codec.http.FullHttpResponse;

View file

@ -0,0 +1,38 @@
package org.xbib.netty.http.client.test.http1;
import com.sun.management.UnixOperatingSystemMXBean;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.xbib.netty.http.client.Client;
import org.xbib.netty.http.client.api.Request;
import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
import java.util.logging.Level;
import java.util.logging.Logger;
@Disabled
class FileDescriptorLeakTest {
private static final Logger logger = Logger.getLogger(FileDescriptorLeakTest.class.getName());
@Test
void testFileLeak() throws Exception {
OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean();
for (int i = 0; i< 10; i++) {
if (os instanceof UnixOperatingSystemMXBean) {
logger.info("before: number of open file descriptor : " + ((UnixOperatingSystemMXBean) os).getOpenFileDescriptorCount());
}
try (Client client = Client.builder().setThreadCount(1).build()) {
Request request = Request.get().url("http://xbib.org")
.setResponseListener(resp -> {
logger.log(Level.INFO, "status = " + resp.getStatus());
})
.build();
client.execute(request);
}
if (os instanceof UnixOperatingSystemMXBean){
logger.info("after: number of open file descriptor : " + ((UnixOperatingSystemMXBean) os).getOpenFileDescriptorCount());
}
}
}
}

View file

@ -3,13 +3,12 @@ import org.apache.tools.ant.taskdefs.condition.Os
dependencies { dependencies {
compile project(":netty-http-common") compile project(":netty-http-common")
compile project(":netty-http-server-api") compile project(":netty-http-server-api")
// we are an Mac OS X 10.12, but tcnative > 2.0.26 is for 10.13+
if (Os.isFamily(Os.FAMILY_MAC)) { if (Os.isFamily(Os.FAMILY_MAC)) {
runtime "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative-legacy-macosx.version')}" testRuntime "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative-legacy-macosx.version')}"
runtime project(':netty-http-kqueue') //runtime project(':netty-http-kqueue')
} else { } else if (Os.isFamily(Os.FAMILY_UNIX)) {
runtime "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}" testRuntime "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}"
runtime project(':netty-http-epoll') testRuntime project(':netty-http-epoll')
} }
testCompile project(":netty-http-client") testCompile project(":netty-http-client")
testRuntime project(":netty-http-bouncycastle") testRuntime project(":netty-http-bouncycastle")

View file

@ -63,9 +63,9 @@ public final class Server implements AutoCloseable {
} }
} }
private static final AtomicLong requestCounter = new AtomicLong(); private static final AtomicLong requestCounter = new AtomicLong(0);
private static final AtomicLong responseCounter = new AtomicLong(); private static final AtomicLong responseCounter = new AtomicLong(0);
private final ServerConfig serverConfig; private final ServerConfig serverConfig;

View file

@ -68,9 +68,6 @@ public class HttpServerRequest implements ServerRequest {
} }
void handleParameters() { void handleParameters() {
if (logger.isLoggable(Level.FINER)) {
logger.log(Level.FINER, () -> "request = " + httpRequest);
}
Charset charset = HttpUtil.getCharset(httpRequest, StandardCharsets.UTF_8); Charset charset = HttpUtil.getCharset(httpRequest, StandardCharsets.UTF_8);
this.url = URL.builder() this.url = URL.builder()
.charset(charset, CodingErrorAction.REPLACE) .charset(charset, CodingErrorAction.REPLACE)
@ -79,13 +76,6 @@ public class HttpServerRequest implements ServerRequest {
QueryParameters queryParameters = url.getQueryParams(); QueryParameters queryParameters = url.getQueryParams();
CharSequence mimeType = HttpUtil.getMimeType(httpRequest); CharSequence mimeType = HttpUtil.getMimeType(httpRequest);
ByteBuf byteBuf = httpRequest.content(); ByteBuf byteBuf = httpRequest.content();
if (logger.isLoggable(Level.FINER)) {
logger.log(Level.FINER, () -> "url = " + url +
" charset = " + charset +
" mime type = " + mimeType +
" queryParameters = " + queryParameters +
" body exists = " + (byteBuf != null));
}
if (byteBuf != null) { if (byteBuf != null) {
if (httpRequest.method().equals(HttpMethod.POST)) { if (httpRequest.method().equals(HttpMethod.POST)) {
String params; String params;

View file

@ -108,9 +108,9 @@ class CleartextTest {
} }
@Test @Test
void testMultithreadedPooledClearTextHttp1() throws Exception { void testMultithreadPooledClearTextHttp1() throws Exception {
int threads = 8; int threads = 2;
int loop = 1000; int loop = 1024;
HttpAddress httpAddress = HttpAddress.http1("localhost", 8008); HttpAddress httpAddress = HttpAddress.http1("localhost", 8008);
Domain domain = Domain.builder(httpAddress) Domain domain = Domain.builder(httpAddress)
.singleEndpoint("/**", (request, response) -> .singleEndpoint("/**", (request, response) ->
@ -123,10 +123,9 @@ class CleartextTest {
.addPoolNode(httpAddress) .addPoolNode(httpAddress)
.setPoolNodeConnectionLimit(threads) .setPoolNodeConnectionLimit(threads)
.build(); .build();
AtomicInteger counter = new AtomicInteger(); AtomicInteger counter = new AtomicInteger(0);
final ResponseListener<HttpResponse> responseListener = resp -> { final ResponseListener<HttpResponse> responseListener = resp -> {
if (resp.getStatus().getCode() == HttpResponseStatus.OK.code()) { if (resp.getStatus().getCode() == HttpResponseStatus.OK.code()) {
logger.log(Level.FINE, resp.getBodyAsString(StandardCharsets.UTF_8));
counter.incrementAndGet(); counter.incrementAndGet();
} }
}; };
@ -138,34 +137,41 @@ class CleartextTest {
try { try {
for (int i = 0; i < loop; i++) { for (int i = 0; i < loop; i++) {
String payload = t + "/" + i; String payload = t + "/" + i;
Request request = Request.get().setVersion(HttpVersion.HTTP_1_1) Request request = Request.get()
.setVersion(HttpVersion.HTTP_1_1)
.url(server.getServerConfig().getAddress().base()) .url(server.getServerConfig().getAddress().base())
.content(payload, "text/plain") .content(payload, "text/plain")
.setResponseListener(responseListener) .setResponseListener(responseListener)
.build(); .build();
// note: a new transport is created per execution // note: in HTTP 1, a new transport is created per execution
Transport transport = client.newTransport(); Transport transport = client.newTransport();
transport.execute(request); transport.execute(request);
if (transport.isFailed()) { if (transport.isFailed()) {
logger.log(Level.WARNING, "transport failed: " + transport.getFailure().getMessage(), transport.getFailure()); logger.log(Level.WARNING, "transport failed: " + transport.getFailure().getMessage(), transport.getFailure());
break; break;
} }
transport.get(); transport.get(20L, TimeUnit.SECONDS);
} }
} catch (Exception e) { } catch (Throwable e) {
logger.log(Level.SEVERE, e.getMessage(), e); logger.log(Level.SEVERE, e.getMessage(), e);
} }
}); });
} }
executorService.shutdown(); executorService.shutdown();
boolean terminated = executorService.awaitTermination(30, TimeUnit.SECONDS); boolean terminated = executorService.awaitTermination(20L, TimeUnit.SECONDS);
executorService.shutdownNow();
logger.log(Level.INFO, "terminated = " + terminated + ", now waiting for transport to complete"); logger.log(Level.INFO, "terminated = " + terminated + ", now waiting for transport to complete");
} catch (Exception e) { } catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e); logger.log(Level.SEVERE, e.getMessage(), e);
} finally { } finally {
client.shutdownGracefully(); server.shutdownGracefully(20L, TimeUnit.SECONDS);
server.shutdownGracefully(); client.shutdownGracefully(20L, TimeUnit.SECONDS);
} }
logger.log(Level.INFO, "server requests = " + server.getRequestCounter() +
" server responses = " + server.getResponseCounter());
logger.log(Level.INFO, "client requests = " + client.getRequestCounter() +
" client responses = " + client.getResponseCounter());
logger.log(Level.INFO, "expected=" + (threads * loop) + " counter=" + counter.get());
assertEquals(threads * loop, counter.get()); assertEquals(threads * loop, counter.get());
} }
} }

View file

@ -131,29 +131,24 @@ class CleartextTest {
int loop = 1024; int loop = 1024;
HttpAddress httpAddress = HttpAddress.http2("localhost", 8008); HttpAddress httpAddress = HttpAddress.http2("localhost", 8008);
Domain domain = Domain.builder(httpAddress) Domain domain = Domain.builder(httpAddress)
.singleEndpoint("/", (request, response) -> .singleEndpoint("/**", (request, response) ->
ServerResponse.write(response, HttpResponseStatus.OK, "text/plain", ServerResponse.write(response, HttpResponseStatus.OK, "text/plain",
request.getContent().retain())) request.getContent().toString(StandardCharsets.UTF_8)))
.build();
Server server = Server.builder(domain)
.build(); .build();
Server server = Server.builder(domain).build();
server.accept(); server.accept();
Client client = Client.builder() Client client = Client.builder()
.addPoolNode(httpAddress) .addPoolNode(httpAddress)
.setPoolNodeConnectionLimit(threads) .setPoolNodeConnectionLimit(threads)
.build(); .build();
AtomicInteger counter = new AtomicInteger(); AtomicInteger counter = new AtomicInteger(0);
// a HTTP/2 listener always receives responses out-of-order
final ResponseListener<HttpResponse> responseListener = resp -> { final ResponseListener<HttpResponse> responseListener = resp -> {
if (resp.getStatus().getCode() == HttpResponseStatus.OK.code()) { if (resp.getStatus().getCode() == HttpResponseStatus.OK.code()) {
counter.incrementAndGet(); counter.incrementAndGet();
} else {
logger.log(Level.INFO, "response listener: headers = " + resp.getHeaders() +
" response body = " + resp.getBodyAsString(StandardCharsets.UTF_8));
} }
}; };
try { try {
// note: for HTTP/2 only, we use a single shared transport // note: for HTTP/2 only, we can use a single shared transport
final Transport transport = client.newTransport(); final Transport transport = client.newTransport();
ExecutorService executorService = Executors.newFixedThreadPool(threads); ExecutorService executorService = Executors.newFixedThreadPool(threads);
for (int n = 0; n < threads; n++) { for (int n = 0; n < threads; n++) {
@ -162,7 +157,8 @@ class CleartextTest {
try { try {
for (int i = 0; i < loop; i++) { for (int i = 0; i < loop; i++) {
String payload = t + "/" + i; String payload = t + "/" + i;
Request request = Request.get().setVersion("HTTP/2.0") Request request = Request.get()
.setVersion("HTTP/2.0")
.url(server.getServerConfig().getAddress().base()) .url(server.getServerConfig().getAddress().base())
.content(payload, "text/plain") .content(payload, "text/plain")
.setResponseListener(responseListener) .setResponseListener(responseListener)
@ -183,6 +179,7 @@ class CleartextTest {
executorService.shutdownNow(); executorService.shutdownNow();
logger.log(Level.INFO, "terminated = " + terminated + ", now waiting for transport to complete"); logger.log(Level.INFO, "terminated = " + terminated + ", now waiting for transport to complete");
transport.get(20L, TimeUnit.SECONDS); transport.get(20L, TimeUnit.SECONDS);
logger.log(Level.INFO, "transport complete");
} finally { } finally {
server.shutdownGracefully(20L, TimeUnit.SECONDS); server.shutdownGracefully(20L, TimeUnit.SECONDS);
client.shutdownGracefully(20L, TimeUnit.SECONDS); client.shutdownGracefully(20L, TimeUnit.SECONDS);