check byte order mark, enable duplicate values in form parameters

gradle.properties

@@ -1,5 +1,5 @@
 group = org.xbib
 name = net-http
-version = 3.3.1
+version = 3.3.2
 
 org.gradle.warning.mode = ALL

net-http-server-netty/src/main/java/org/xbib/net/http/server/netty/HttpRequestBuilder.java

@@ -1,5 +1,6 @@
 package org.xbib.net.http.server.netty;
 
+import io.netty.buffer.ByteBuf;
 import io.netty.buffer.ByteBufUtil;
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.multipart.FileUpload;
@@ -27,6 +28,8 @@ public class HttpRequestBuilder extends BaseHttpRequestBuilder {
 
     protected ByteBuffer byteBuffer;
 
+    protected CharBuffer charBuffer;
+
     protected HttpRequestBuilder() {
     }
 
@@ -47,9 +50,10 @@ public class HttpRequestBuilder extends BaseHttpRequestBuilder {
             setMethod(HttpMethod.valueOf(fullHttpRequest.method().name()));
             setRequestURI(fullHttpRequest.uri());
             fullHttpRequest.headers().entries().forEach(e -> addHeader(e.getKey(), e.getValue()));
-            // read all bytes from request into a JDK ByteBuffer. This might be expensive.
             if (fullHttpRequest.content() != null) {
-                byteBuffer = ByteBuffer.wrap(ByteBufUtil.getBytes(fullHttpRequest.content()));
+                ByteBuf byteBuf = fullHttpRequest.content();
+                byte[] bytes = ByteBufUtil.getBytes(byteBuf);
+                byteBuffer = ByteBuffer.wrap(bytes);
             }
         }
         return this;
@@ -62,7 +66,10 @@ public class HttpRequestBuilder extends BaseHttpRequestBuilder {
 
     @Override
     public CharBuffer getBodyAsChars(Charset charset) {
-        return byteBuffer != null ? charset.decode(byteBuffer) : null;
+        if (charBuffer == null) {
+            charBuffer = byteBuffer != null ? charset.decode(byteBuffer) : null;
+        }
+        return charBuffer;
     }
 
     @Override

net-http-server-netty/src/test/java/org/xbib/net/http/netty/test/NettyHttpServerByteOrderMarkTest.java

@@ -0,0 +1,115 @@
+package org.xbib.net.http.netty.test;
+
+import io.netty.bootstrap.Bootstrap;
+import org.junit.jupiter.api.Test;
+import org.xbib.net.NetworkClass;
+import org.xbib.net.Parameter;
+import org.xbib.net.URL;
+import org.xbib.net.http.HttpAddress;
+import org.xbib.net.http.HttpHeaderNames;
+import org.xbib.net.http.HttpHeaderValues;
+import org.xbib.net.http.HttpMethod;
+import org.xbib.net.http.HttpResponseStatus;
+import org.xbib.net.http.client.netty.HttpRequest;
+import org.xbib.net.http.client.netty.NettyHttpClient;
+import org.xbib.net.http.client.netty.NettyHttpClientConfig;
+import org.xbib.net.http.server.application.BaseApplication;
+import org.xbib.net.http.server.domain.BaseHttpDomain;
+import org.xbib.net.http.server.executor.BaseExecutor;
+import org.xbib.net.http.server.executor.Executor;
+import org.xbib.net.http.server.netty.NettyHttpServer;
+import org.xbib.net.http.server.netty.NettyHttpServerConfig;
+import org.xbib.net.http.server.route.BaseHttpRouter;
+import org.xbib.net.http.server.route.HttpRouter;
+import org.xbib.net.http.server.service.BaseHttpService;
+
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class NettyHttpServerByteOrderMarkTest {
+
+    private static final Logger logger = Logger.getLogger(NettyHttpServerByteOrderMarkTest.class.getName());
+
+    @Test
+    void testJsonPostRequest() throws Exception {
+
+        String body = "{\"syntax\":\"CQL\",\"offset\":0,\"size\":10,\"service\":[\"hbz\"],\"op\":[\"\",\"@and\",\"@and\"],\"key\":[\"bib.any\",\"bib.title\",\"bib.identifierISSN\"],\"query\":[\"linux einführung\",\"\",\"\"]}";
+
+        URL url = URL.from("http://localhost:8008/");
+        HttpAddress httpAddress1 = HttpAddress.http1(url);
+        NettyHttpServerConfig nettyHttpServerConfig = new NettyHttpServerConfig();
+        nettyHttpServerConfig.setServerName("NettyHttpServer",
+                Bootstrap.class.getPackage().getImplementationVersion());
+        nettyHttpServerConfig.setNetworkClass(NetworkClass.LOCAL);
+        nettyHttpServerConfig.setDebug(true);
+        nettyHttpServerConfig.setChunkWriteEnabled(true);
+
+        HttpRouter router = BaseHttpRouter.builder()
+                .addDomain(BaseHttpDomain.builder()
+                        .setHttpAddress(httpAddress1)
+                        .addService(BaseHttpService.builder()
+                                .setPath("/")
+                                .setMethod(HttpMethod.POST)
+                                .setHandler(ctx -> {
+                                    logger.log(Level.FINEST, "handler starting");
+                                    String content = ctx.request().getBodyAsChars(StandardCharsets.UTF_8).toString();
+                                    logger.log(Level.FINEST, "got content = " + content);
+                                    logger.log(Level.FINEST, "got FORM params op = " + ctx.httpRequest().getParameter().getAll("op", Parameter.Domain.FORM));
+                                    logger.log(Level.FINEST, "got FORM params key = " + ctx.httpRequest().getParameter().getAll("key", Parameter.Domain.FORM));
+                                    logger.log(Level.FINEST, "got FORM params query = " + ctx.httpRequest().getParameter().getAll("query", Parameter.Domain.FORM));
+                                    ctx.response()
+                                            .setResponseStatus(HttpResponseStatus.OK)
+                                            .setHeader(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.TEXT_PLAIN)
+                                            .setCharset(StandardCharsets.UTF_8);
+                                    ctx.write("parameter = " + ctx.httpRequest().getParameter().allToString() +
+                                            " local address = " + ctx.httpRequest().getLocalAddress() +
+                                            " remote address = " + ctx.httpRequest().getRemoteAddress() +
+                                            " attributes = " + ctx.getAttributes() +
+                                            " content = " + content
+                                    );
+                                })
+                                .build())
+                        .build())
+                .build();
+
+        Executor executor = BaseExecutor.builder()
+                .build();
+
+        try (NettyHttpServer server = NettyHttpServer.builder()
+                .setHttpServerConfig(nettyHttpServerConfig)
+                .setApplication(BaseApplication.builder()
+                        .setExecutor(executor)
+                        .setRouter(router)
+                        .build())
+                .build()) {
+            server.bind();
+            NettyHttpClientConfig config = new NettyHttpClientConfig()
+                    .setGzipEnabled(true)
+                    .setChunkWriteEnabled(true)
+                    .setObjectAggregationEnabled(true)
+                    .setDebug(true);
+            AtomicBoolean received = new AtomicBoolean();
+            try (NettyHttpClient client = NettyHttpClient.builder()
+                    .setConfig(config)
+                    .build()) {
+                HttpRequest request = HttpRequest.post()
+                        .setURL(url)
+                        .content(body, "application/json", StandardCharsets.UTF_8)
+                        .setResponseListener(resp -> {
+                            logger.log(Level.INFO, " status = " + resp.getStatus() +
+                                    " got response headers = " + resp.getHeaders() +
+                                    " got response body = " + resp.getBodyAsChars(StandardCharsets.UTF_8));
+                            received.set(true);
+                        })
+                        .build();
+                client.execute(request).get().close();
+            }
+
+            assertTrue(received.get());
+        }
+    }
+}

net-http-server/src/main/java/org/xbib/net/http/server/BaseHttpServerContext.java

@@ -38,6 +38,8 @@ public class BaseHttpServerContext implements HttpServerContext {
 
     private static final String PATH_SEPARATOR = "/";
 
+    private static final String BOM = "\uffff";
+
     private final Application application;
 
     private final HttpRequestBuilder httpRequestBuilder;
@@ -220,7 +222,8 @@ public class BaseHttpServerContext implements HttpServerContext {
                 .charset(charset, CodingErrorAction.REPLACE)
                 .path(httpRequestBuilder.getRequestURI())
                 .build();
-        ParameterBuilder formParameterBuilder = Parameter.builder().domain(Parameter.Domain.FORM);
+        ParameterBuilder formParameterBuilder = Parameter.builder().domain(Parameter.Domain.FORM)
+                .enableDuplicates();
         // https://www.w3.org/TR/html4/interact/forms.html#h-17.13.4
         if (HttpMethod.POST.equals(httpRequestBuilder.getMethod()) &&
                 (mimeType != null && mimeType.contains(HttpHeaderValues.APPLICATION_X_WWW_FORM_URLENCODED))) {
@@ -234,6 +237,9 @@ public class BaseHttpServerContext implements HttpServerContext {
         if (contentType != null && contentType.contains(HttpHeaderValues.APPLICATION_JSON)) {
             String content = httpRequestBuilder.getBodyAsChars(StandardCharsets.UTF_8).toString();
             try {
+                if (content.startsWith(BOM)) {
+                    content = content.substring(BOM.length());
+                }
                 Map<String, Object> map = Json.toMap(content);
                 for (Map.Entry<String, Object> entry : map.entrySet()) {
                     if (entry.getValue() instanceof Iterable<?> iterable) {