From 35d67784bb0797d5c26ffec273ad5dd8f7736fe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Prante?= Date: Tue, 18 Apr 2023 15:10:53 +0200 Subject: [PATCH] check byte order mark, enable duplicate values in form parameters --- gradle.properties | 2 +- .../http/server/netty/HttpRequestBuilder.java | 13 +- .../NettyHttpServerByteOrderMarkTest.java | 115 ++++++++++++++++++ .../http/server/BaseHttpServerContext.java | 8 +- 4 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 net-http-server-netty/src/test/java/org/xbib/net/http/netty/test/NettyHttpServerByteOrderMarkTest.java diff --git a/gradle.properties b/gradle.properties index f0e6625..36234ce 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group = org.xbib name = net-http -version = 3.3.1 +version = 3.3.2 org.gradle.warning.mode = ALL diff --git a/net-http-server-netty/src/main/java/org/xbib/net/http/server/netty/HttpRequestBuilder.java b/net-http-server-netty/src/main/java/org/xbib/net/http/server/netty/HttpRequestBuilder.java index 4e9d539..9925eef 100644 --- a/net-http-server-netty/src/main/java/org/xbib/net/http/server/netty/HttpRequestBuilder.java +++ b/net-http-server-netty/src/main/java/org/xbib/net/http/server/netty/HttpRequestBuilder.java @@ -1,5 +1,6 @@ package org.xbib.net.http.server.netty; +import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.multipart.FileUpload; @@ -27,6 +28,8 @@ public class HttpRequestBuilder extends BaseHttpRequestBuilder { protected ByteBuffer byteBuffer; + protected CharBuffer charBuffer; + protected HttpRequestBuilder() { } @@ -47,9 +50,10 @@ public class HttpRequestBuilder extends BaseHttpRequestBuilder { setMethod(HttpMethod.valueOf(fullHttpRequest.method().name())); setRequestURI(fullHttpRequest.uri()); fullHttpRequest.headers().entries().forEach(e -> addHeader(e.getKey(), e.getValue())); - // read all bytes from request into a JDK ByteBuffer. This might be expensive. if (fullHttpRequest.content() != null) { - byteBuffer = ByteBuffer.wrap(ByteBufUtil.getBytes(fullHttpRequest.content())); + ByteBuf byteBuf = fullHttpRequest.content(); + byte[] bytes = ByteBufUtil.getBytes(byteBuf); + byteBuffer = ByteBuffer.wrap(bytes); } } return this; @@ -62,7 +66,10 @@ public class HttpRequestBuilder extends BaseHttpRequestBuilder { @Override public CharBuffer getBodyAsChars(Charset charset) { - return byteBuffer != null ? charset.decode(byteBuffer) : null; + if (charBuffer == null) { + charBuffer = byteBuffer != null ? charset.decode(byteBuffer) : null; + } + return charBuffer; } @Override diff --git a/net-http-server-netty/src/test/java/org/xbib/net/http/netty/test/NettyHttpServerByteOrderMarkTest.java b/net-http-server-netty/src/test/java/org/xbib/net/http/netty/test/NettyHttpServerByteOrderMarkTest.java new file mode 100644 index 0000000..6253cc7 --- /dev/null +++ b/net-http-server-netty/src/test/java/org/xbib/net/http/netty/test/NettyHttpServerByteOrderMarkTest.java @@ -0,0 +1,115 @@ +package org.xbib.net.http.netty.test; + +import io.netty.bootstrap.Bootstrap; +import org.junit.jupiter.api.Test; +import org.xbib.net.NetworkClass; +import org.xbib.net.Parameter; +import org.xbib.net.URL; +import org.xbib.net.http.HttpAddress; +import org.xbib.net.http.HttpHeaderNames; +import org.xbib.net.http.HttpHeaderValues; +import org.xbib.net.http.HttpMethod; +import org.xbib.net.http.HttpResponseStatus; +import org.xbib.net.http.client.netty.HttpRequest; +import org.xbib.net.http.client.netty.NettyHttpClient; +import org.xbib.net.http.client.netty.NettyHttpClientConfig; +import org.xbib.net.http.server.application.BaseApplication; +import org.xbib.net.http.server.domain.BaseHttpDomain; +import org.xbib.net.http.server.executor.BaseExecutor; +import org.xbib.net.http.server.executor.Executor; +import org.xbib.net.http.server.netty.NettyHttpServer; +import org.xbib.net.http.server.netty.NettyHttpServerConfig; +import org.xbib.net.http.server.route.BaseHttpRouter; +import org.xbib.net.http.server.route.HttpRouter; +import org.xbib.net.http.server.service.BaseHttpService; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +class NettyHttpServerByteOrderMarkTest { + + private static final Logger logger = Logger.getLogger(NettyHttpServerByteOrderMarkTest.class.getName()); + + @Test + void testJsonPostRequest() throws Exception { + + String body = "{\"syntax\":\"CQL\",\"offset\":0,\"size\":10,\"service\":[\"hbz\"],\"op\":[\"\",\"@and\",\"@and\"],\"key\":[\"bib.any\",\"bib.title\",\"bib.identifierISSN\"],\"query\":[\"linux einführung\",\"\",\"\"]}"; + + URL url = URL.from("http://localhost:8008/"); + HttpAddress httpAddress1 = HttpAddress.http1(url); + NettyHttpServerConfig nettyHttpServerConfig = new NettyHttpServerConfig(); + nettyHttpServerConfig.setServerName("NettyHttpServer", + Bootstrap.class.getPackage().getImplementationVersion()); + nettyHttpServerConfig.setNetworkClass(NetworkClass.LOCAL); + nettyHttpServerConfig.setDebug(true); + nettyHttpServerConfig.setChunkWriteEnabled(true); + + HttpRouter router = BaseHttpRouter.builder() + .addDomain(BaseHttpDomain.builder() + .setHttpAddress(httpAddress1) + .addService(BaseHttpService.builder() + .setPath("/") + .setMethod(HttpMethod.POST) + .setHandler(ctx -> { + logger.log(Level.FINEST, "handler starting"); + String content = ctx.request().getBodyAsChars(StandardCharsets.UTF_8).toString(); + logger.log(Level.FINEST, "got content = " + content); + logger.log(Level.FINEST, "got FORM params op = " + ctx.httpRequest().getParameter().getAll("op", Parameter.Domain.FORM)); + logger.log(Level.FINEST, "got FORM params key = " + ctx.httpRequest().getParameter().getAll("key", Parameter.Domain.FORM)); + logger.log(Level.FINEST, "got FORM params query = " + ctx.httpRequest().getParameter().getAll("query", Parameter.Domain.FORM)); + ctx.response() + .setResponseStatus(HttpResponseStatus.OK) + .setHeader(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.TEXT_PLAIN) + .setCharset(StandardCharsets.UTF_8); + ctx.write("parameter = " + ctx.httpRequest().getParameter().allToString() + + " local address = " + ctx.httpRequest().getLocalAddress() + + " remote address = " + ctx.httpRequest().getRemoteAddress() + + " attributes = " + ctx.getAttributes() + + " content = " + content + ); + }) + .build()) + .build()) + .build(); + + Executor executor = BaseExecutor.builder() + .build(); + + try (NettyHttpServer server = NettyHttpServer.builder() + .setHttpServerConfig(nettyHttpServerConfig) + .setApplication(BaseApplication.builder() + .setExecutor(executor) + .setRouter(router) + .build()) + .build()) { + server.bind(); + NettyHttpClientConfig config = new NettyHttpClientConfig() + .setGzipEnabled(true) + .setChunkWriteEnabled(true) + .setObjectAggregationEnabled(true) + .setDebug(true); + AtomicBoolean received = new AtomicBoolean(); + try (NettyHttpClient client = NettyHttpClient.builder() + .setConfig(config) + .build()) { + HttpRequest request = HttpRequest.post() + .setURL(url) + .content(body, "application/json", StandardCharsets.UTF_8) + .setResponseListener(resp -> { + logger.log(Level.INFO, " status = " + resp.getStatus() + + " got response headers = " + resp.getHeaders() + + " got response body = " + resp.getBodyAsChars(StandardCharsets.UTF_8)); + received.set(true); + }) + .build(); + client.execute(request).get().close(); + } + + assertTrue(received.get()); + } + } +} diff --git a/net-http-server/src/main/java/org/xbib/net/http/server/BaseHttpServerContext.java b/net-http-server/src/main/java/org/xbib/net/http/server/BaseHttpServerContext.java index 78b07b5..82b06cd 100644 --- a/net-http-server/src/main/java/org/xbib/net/http/server/BaseHttpServerContext.java +++ b/net-http-server/src/main/java/org/xbib/net/http/server/BaseHttpServerContext.java @@ -38,6 +38,8 @@ public class BaseHttpServerContext implements HttpServerContext { private static final String PATH_SEPARATOR = "/"; + private static final String BOM = "\uffff"; + private final Application application; private final HttpRequestBuilder httpRequestBuilder; @@ -220,7 +222,8 @@ public class BaseHttpServerContext implements HttpServerContext { .charset(charset, CodingErrorAction.REPLACE) .path(httpRequestBuilder.getRequestURI()) .build(); - ParameterBuilder formParameterBuilder = Parameter.builder().domain(Parameter.Domain.FORM); + ParameterBuilder formParameterBuilder = Parameter.builder().domain(Parameter.Domain.FORM) + .enableDuplicates(); // https://www.w3.org/TR/html4/interact/forms.html#h-17.13.4 if (HttpMethod.POST.equals(httpRequestBuilder.getMethod()) && (mimeType != null && mimeType.contains(HttpHeaderValues.APPLICATION_X_WWW_FORM_URLENCODED))) { @@ -234,6 +237,9 @@ public class BaseHttpServerContext implements HttpServerContext { if (contentType != null && contentType.contains(HttpHeaderValues.APPLICATION_JSON)) { String content = httpRequestBuilder.getBodyAsChars(StandardCharsets.UTF_8).toString(); try { + if (content.startsWith(BOM)) { + content = content.substring(BOM.length()); + } Map map = Json.toMap(content); for (Map.Entry entry : map.entrySet()) { if (entry.getValue() instanceof Iterable iterable) {