From 233973596689a9a8cac289a41c405ed526acba5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Fri, 4 May 2018 11:32:53 +0200 Subject: [PATCH] more work on HTTP transports with pooling and multithreading, update to Netty 4.1.24, update to Gradle 4.7 --- build.gradle | 35 +- gradle.properties | 5 +- gradle/wrapper/gradle-wrapper.jar | Bin 54329 -> 54413 bytes gradle/wrapper/gradle-wrapper.properties | 4 +- .../org/xbib/netty/http/client/Client.java | 137 ++--- .../xbib/netty/http/client/ClientBuilder.java | 5 + .../xbib/netty/http/client/ClientConfig.java | 13 + .../org/xbib/netty/http/client/Request.java | 29 +- .../netty/http/client/RequestBuilder.java | 3 +- .../HttpChannelInitializer.java | 53 +- .../HttpChunkContentCompressor.java | 2 +- .../{http1 => http}/HttpResponseHandler.java | 8 +- .../TrafficLoggingHandler.java | 2 +- .../client/handler/http/package-info.java | 4 + .../client/handler/http1/package-info.java | 4 - .../http2/Http2ChannelInitializer.java | 139 ++--- .../http2/Http2PushPromiseHandler.java | 24 - .../handler/http2/Http2ResponseHandler.java | 14 +- .../handler/http2/Http2SettingsHandler.java | 18 - .../Http2StreamFrameToHttpObjectCodec.java | 227 ++++++++ .../client/listener/HttpHeadersListener.java | 9 - ...nseListener.java => ResponseListener.java} | 2 +- .../http/client/pool/BoundedChannelPool.java | 155 +++--- .../org/xbib/netty/http/client/pool/Pool.java | 7 +- .../netty/http/client/rest/RestClient.java | 5 - .../xbib/netty/http/client/retry/BackOff.java | 4 +- .../http/client/retry/ExponentialBackOff.java | 99 ++-- .../http/client/transport/BaseTransport.java | 238 +++++---- .../netty/http/client/transport/Flow.java | 72 +++ .../http/client/transport/Http1Transport.java | 152 ------ .../http/client/transport/Http2Transport.java | 260 +++++----- .../http/client/transport/HttpTransport.java | 141 +++++ .../http/client/transport/Transport.java | 24 +- .../test/LoggingBase.java => TestBase.java} | 15 +- .../client/test/CompletableFutureTest.java | 2 +- .../netty/http/client/test/ConscryptTest.java | 4 +- .../client/test/CookieSetterHttpBinTest.java | 4 +- .../http/client/test/ElasticsearchTest.java | 5 +- .../netty/http/client/test/Http1Test.java | 3 +- .../netty/http/client/test/Http2Test.java | 3 +- .../http/client/test/SecureHttp1Test.java | 3 +- .../{LeakTest.java => ThreadLeakTest.java} | 17 +- .../xbib/netty/http/client/test/XbibTest.java | 3 +- .../http/client/test/pool/EpollTest.java | 55 +- .../client/test/pool/MockEpollServer.java | 64 --- .../http/client/test/pool/MockNioServer.java | 62 --- .../netty/http/client/test/pool/NioTest.java | 54 +- .../netty/http/client/test/pool/PoolTest.java | 47 +- .../test/{ => pool}/PooledClientTest.java | 22 +- .../simple => hacks}/Http2FramesTest.java | 6 +- .../simple => hacks}/SimpleHttp1Test.java | 11 +- .../simple => hacks}/SimpleHttp2Test.java | 40 +- .../xbib/netty/http/common/HttpAddress.java | 10 +- .../org/xbib/netty/http/server/Server.java | 24 +- .../handler/HttpServerChannelInitializer.java | 220 -------- .../{http1 => }/IdleTimeoutHandler.java | 2 +- .../{http1 => }/TrafficLoggingHandler.java | 2 +- .../HttpChannelInitializer.java | 57 ++- .../handler/http/HttpPipelinedRequest.java | 23 + .../handler/http/HttpPipelinedResponse.java | 34 ++ .../handler/http/HttpPipeliningHandler.java | 75 +++ .../server/handler/http1/HttpHandler.java | 55 -- .../server/handler/http2/DummyHandler.java | 17 - .../server/handler/http2/FrameListener.java | 33 -- .../handler/http2/HelloWorldHttp1Handler.java | 71 --- .../handler/http2/HelloWorldHttp2Handler.java | 87 ---- .../http2/Http2ChannelInitializer.java | 236 +++------ .../server/handler/http2/Http2Handler.java | 483 ------------------ .../handler/http2/Http2RequestHandler.java | 33 -- .../handler/http2/Http2SettingsHandler.java | 18 - .../Http2StreamFrameToHttpObjectCodec.java | 228 +++++++++ .../server/handler/http2/UserEventLogger.java | 23 - .../internal/ClosedSessionException.java | 7 - .../server/internal/Http1ObjectEncoder.java | 192 ------- .../server/internal/Http2ObjectEncoder.java | 167 ------ .../server/internal/HttpObjectEncoder.java | 77 --- .../http/server/internal/package-info.java | 4 - .../server/transport/BaseServerTransport.java | 14 +- .../server/transport/Http2ServerResponse.java | 175 +++++++ .../transport/Http2ServerTransport.java | 19 +- ...rResponse.java => HttpServerResponse.java} | 30 +- ...ransport.java => HttpServerTransport.java} | 18 +- .../http/server/transport/ServerRequest.java | 11 +- .../http/server/transport/ServerResponse.java | 4 +- .../server/transport/ServerTransport.java | 3 + .../test/LoggingBase.java => TestBase.java} | 16 +- .../simple => hacks}/CleartextHttp2Test.java | 44 +- .../http/hacks/HttpPipeliningHandlerTest.java | 207 ++++++++ .../MultiplexCodecCleartextHttp2Test.java | 37 +- .../MultithreadedCleartextHttp2Test.java | 50 +- ...eadedMultiplexCodecCleartextHttp2Test.java | 98 ++-- .../xbib/netty/http/hacks/package-info.java | 4 + .../http/server/test/CleartextHttp1Test.java | 138 ++++- .../http/server/test/CleartextHttp2Test.java | 244 +++++++-- .../test/MultithreadedCleartextHttp2Test.java | 92 ---- .../server/test/PooledCleartextHttp1Test.java | 63 --- .../http/server/test/SecureHttp1Test.java | 168 +++++- .../http/server/test/SecureHttp2Test.java | 185 +++++-- .../test/SelfSignedCertificateTest.java | 2 - .../netty/http/server/test/ServerTest.java | 2 + .../http/server/test/ThreadLeakTest.java | 41 ++ 101 files changed, 2980 insertions(+), 3177 deletions(-) rename netty-http-client/src/main/java/org/xbib/netty/http/client/handler/{http1 => http}/HttpChannelInitializer.java (50%) rename netty-http-client/src/main/java/org/xbib/netty/http/client/handler/{http1 => http}/HttpChunkContentCompressor.java (94%) rename netty-http-client/src/main/java/org/xbib/netty/http/client/handler/{http1 => http}/HttpResponseHandler.java (71%) rename netty-http-client/src/main/java/org/xbib/netty/http/client/handler/{http1 => http}/TrafficLoggingHandler.java (95%) create mode 100644 netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/package-info.java delete mode 100644 netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/package-info.java delete mode 100644 netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2PushPromiseHandler.java delete mode 100644 netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2SettingsHandler.java create mode 100644 netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2StreamFrameToHttpObjectCodec.java delete mode 100644 netty-http-client/src/main/java/org/xbib/netty/http/client/listener/HttpHeadersListener.java rename netty-http-client/src/main/java/org/xbib/netty/http/client/listener/{HttpResponseListener.java => ResponseListener.java} (81%) create mode 100644 netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Flow.java delete mode 100644 netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Http1Transport.java create mode 100644 netty-http-client/src/main/java/org/xbib/netty/http/client/transport/HttpTransport.java rename netty-http-client/src/test/java/org/xbib/{netty/http/client/test/LoggingBase.java => TestBase.java} (62%) rename netty-http-client/src/test/java/org/xbib/netty/http/client/test/{LeakTest.java => ThreadLeakTest.java} (80%) delete mode 100644 netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/MockEpollServer.java delete mode 100644 netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/MockNioServer.java rename netty-http-client/src/test/java/org/xbib/netty/http/client/test/{ => pool}/PooledClientTest.java (75%) rename netty-http-client/src/test/java/org/xbib/netty/http/{client/test/simple => hacks}/Http2FramesTest.java (98%) rename netty-http-client/src/test/java/org/xbib/netty/http/{client/test/simple => hacks}/SimpleHttp1Test.java (98%) rename netty-http-client/src/test/java/org/xbib/netty/http/{client/test/simple => hacks}/SimpleHttp2Test.java (94%) delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/HttpServerChannelInitializer.java rename netty-http-server/src/main/java/org/xbib/netty/http/server/handler/{http1 => }/IdleTimeoutHandler.java (94%) rename netty-http-server/src/main/java/org/xbib/netty/http/server/handler/{http1 => }/TrafficLoggingHandler.java (95%) rename netty-http-server/src/main/java/org/xbib/netty/http/server/handler/{http1 => http}/HttpChannelInitializer.java (54%) create mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpPipelinedRequest.java create mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpPipelinedResponse.java create mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpPipeliningHandler.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/HttpHandler.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/DummyHandler.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/FrameListener.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/HelloWorldHttp1Handler.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/HelloWorldHttp2Handler.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2Handler.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2RequestHandler.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2SettingsHandler.java create mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2StreamFrameToHttpObjectCodec.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/UserEventLogger.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/internal/ClosedSessionException.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/internal/Http1ObjectEncoder.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/internal/Http2ObjectEncoder.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/internal/HttpObjectEncoder.java delete mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/internal/package-info.java create mode 100644 netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http2ServerResponse.java rename netty-http-server/src/main/java/org/xbib/netty/http/server/transport/{Http1ServerResponse.java => HttpServerResponse.java} (84%) rename netty-http-server/src/main/java/org/xbib/netty/http/server/transport/{Http1ServerTransport.java => HttpServerTransport.java} (70%) rename netty-http-server/src/test/java/org/xbib/{netty/http/server/test/LoggingBase.java => TestBase.java} (62%) rename netty-http-server/src/test/java/org/xbib/netty/http/{server/test/simple => hacks}/CleartextHttp2Test.java (86%) create mode 100644 netty-http-server/src/test/java/org/xbib/netty/http/hacks/HttpPipeliningHandlerTest.java rename netty-http-server/src/test/java/org/xbib/netty/http/{server/test/simple => hacks}/MultiplexCodecCleartextHttp2Test.java (90%) rename netty-http-server/src/test/java/org/xbib/netty/http/{server/test/multithread => hacks}/MultithreadedCleartextHttp2Test.java (84%) rename netty-http-server/src/test/java/org/xbib/netty/http/{server/test/multithread => hacks}/MultithreadedMultiplexCodecCleartextHttp2Test.java (72%) create mode 100644 netty-http-server/src/test/java/org/xbib/netty/http/hacks/package-info.java delete mode 100644 netty-http-server/src/test/java/org/xbib/netty/http/server/test/MultithreadedCleartextHttp2Test.java delete mode 100644 netty-http-server/src/test/java/org/xbib/netty/http/server/test/PooledCleartextHttp1Test.java create mode 100644 netty-http-server/src/test/java/org/xbib/netty/http/server/test/ThreadLeakTest.java diff --git a/build.gradle b/build.gradle index 48a49ed..5f0c322 100644 --- a/build.gradle +++ b/build.gradle @@ -38,7 +38,6 @@ subprojects { dependencies { alpnagent "org.mortbay.jetty.alpn:jetty-alpn-agent:${project.property('alpnagent.version')}" - asciidoclet "org.xbib:asciidoclet:${project.property('asciidoclet.version')}" wagon "org.apache.maven.wagon:wagon-ssh:${project.property('wagon.version')}" } @@ -61,7 +60,7 @@ subprojects { jvmArgs "-javaagent:" + configurations.alpnagent.asPath } testLogging { - showStandardStreams = false + showStandardStreams = true exceptionFormat = 'full' } } @@ -82,7 +81,7 @@ subprojects { 'source-highlighter': 'coderay' } - javadoc { + /*javadoc { options.docletpath = configurations.asciidoclet.files.asType(List) options.doclet = "org.xbib.asciidoclet.Asciidoclet" options.overview = "src/docs/asciidoclet/overview.adoc" @@ -92,7 +91,7 @@ subprojects { configure(options) { noTimestamp = true } - } + }*/ task javadocJar(type: Jar, dependsOn: classes) { from javadoc @@ -116,7 +115,6 @@ subprojects { } } - ext { user = 'jprante' name = 'netty-http-client' @@ -126,7 +124,6 @@ subprojects { scmDeveloperConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git' } - task xbibUpload(type: Upload) { group = 'publish' configuration = configurations.archives @@ -195,18 +192,20 @@ subprojects { } } - -} - -spotbugs { - effort = "max" - reportLevel = "low" - //includeFilter = file("findbugs-exclude.xml") -} - -tasks.withType(com.github.spotbugs.SpotBugsTask) { - ignoreFailures = true - reports { + spotbugs { + toolVersion = '3.1.3' + sourceSets = [sourceSets.main] + ignoreFailures = true + effort = "max" + reportLevel = "high" + // includeFilter = file("config/findbugs/findbugs-include.xml") + // excludeFilter = file("config/findbugs/findbugs-excludes.xml") + } + spotbugsMain.reports { + xml.enabled = false + html.enabled = true + } + spotbugsTest.reports { xml.enabled = false html.enabled = true } diff --git a/gradle.properties b/gradle.properties index 9a3a20d..4ebacba 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,8 +1,8 @@ group = org.xbib name = netty-http-client -version = 4.1.22.2 +version = 4.1.24.0 -netty.version = 4.1.22.Final +netty.version = 4.1.24.Final tcnative.version = 2.0.7.Final conscrypt.version = 1.0.1 bouncycastle.version = 1.57 @@ -10,7 +10,6 @@ xbib-net-url.version = 1.1.0 alpnagent.version = 2.0.7 junit.version = 4.12 jackson.version = 2.8.11.1 -asciidoclet.version = 1.6.0.0 wagon.version = 3.0.0 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index f6b961fd5a86aa5fbfe90f707c3138408be7c718..91ca28c8b802289c3a438766657a5e98f20eff03 100644 GIT binary patch delta 7397 zcmY+JWmFVEyv3IV=@jV(>F!z@2@w$KmS%wkX_s20k!ES6J0t|8k&cC>kxuDOX?#B4 zJ1^$b@7_Ce=FH5Ong2AGqQ;b=#*3n*Sr_*uNE)JFxShG70OBcY>n*sk%p!xW6fgjQ zBRDM&7tC8npX5nn+ckXXnY>Y{cCI=kWN6%){zd#LVaB+K7p4${BAbJYEe{+=^g7o2 z__WmJ>zD&~hr|ATWSj$-S%Gb#e5Sm?p<;&4WZ3+X?4hqH!1wr#Ksqkbh>`eCw*T+> zdr4oIU5+5{&z^U6jHJ3~r{01DW22Zt>hf#9IBE|C7t*QCvC|Q0*{0hOPzIC%= zQ~86R@Nx1*(OD6DAAhX2>zDp?^3l(m3<+(5VctWD-UDZ}ZTc@y9Q*FiP_%Axap|1< z!cW)r{Ltu)8r7|NIH$V8}$P&4CK|U7pkjTPh7{DVY2-*0E<=ux<`7XB~s7k>OnJOT0zW_N)}E z+g1`<4W42#8BC;7L%Z2UF=_s>L?z*|w|n&6dGuDALeY@xl%#dE8tsB$VoFb#-B1$; z?l2EYZZ3fYzSNvjt^GTfLZ5Ck_rS zb=+}lXfza2Kg4%j^mWuVtR-m0-PGJ-<{|A1_w>f(M2+>%gZeS4Q=RPJtsOa~wu^M{;luOUT~@`NyIiV`8gW1%!)1FBg69GkONB9Yy5ArTOk<9HJqOLnS; z_ha%bf9_T|HUSg07;>0*`)&S?nJd$ zX35$0rU2z3=pctjJUdOtFihi$tv5`)U{}Il=z(>9A?GY3_6KH&9M;c_J`G%m=(d@~ zQT|Z%r%J9eROrRbV?3?ln9u_v;qBZ?{YznP790&%Huf27@X93tC6JT;OaECsKUiea zzsl(|B)p_fa=uWSn^5<*HOAGAd>a^5NZyd3eI-Dg=$i<;SI>EGT@Vi9(jt~1bqU#R)(|KcV?Vnrdv z>od_H7eYVBMcSVUI!l{l9kdDQIF4F^RZM2ct(;m%MiI7=QXq_GefpEOfWhE~h@&Qi zRz{RPm*_pzQ9)$~R-*KwZ_rHOcl@xjB5G}jfbd-AAJN0GJ{*^1n+xVki0+BFneb56 z_OQA3_N?{?Hz7?YG`|5^|5vRjWbL!d@@)$E4_73(1!$ zexzL2VHAi_;ULOV67s}an3{HylUZb&MWy2J)T{QZ?vY-?fi9f|8RN-eyeop^y0-dz#>hdNrDDENnCbbIPam0LCGW!8rk8}OVsk1$ zeMXB>B6zg+;t#aLLHu1m6=DSY@hk^^P(jT7X>k(@nj-`AW#tb6^2l9Q32_x>)@d%N ze{vs?DtgQbR>Vh4e+Ut4j@2UYV`daonL6yRRonz7?d$hAB>k9>C(#fyKz=7vb#s?& zPG?|(#U;du3%CZEfRtIJHF(Tk5}UsyGiOkK8cqJw9cUEg?;v%o$G%i?=bA&=FK0Ab z%73UIXQ?6;in>pMTS3N`Z`RLn#euKic?WCsA~;jZYU@HfIlFrBRi8CJn`NeUig`p%kI zm6yJA34!)1L5d2pz+PDrXItk)oVs2~Z+f|kpFsvkmdi%^V4p4n`6EW96&6If%`F<;#pM$`8!{M*kv2ftii=> z)VSk{*&GKY?4ac_K_Ccj$4-qst|QIVS$F$}BArRSw%hKRJ!u^NsR<9(TkcfEm;e}2 zR7T#HY*O^0A!mmjWUj}BIc{Rc_ABb4&7ea~aj6I<;O!O2av>lyJLnBXsa|sjwk3|? z<$7m#H=r0{o23~ut7i#a$>+(jRsO!X75JvKt`y39dJ!7nocO5$MItV<$dGL9$}gdt z>V5ShA%*FMWseR0UV;WF4Eq13E>ak{0|v#w<*yH^ADvR0B--JYm8-5H1K zVCtf~$hfY>fx)s;iX+Lc6=IOnik?}#FSTyZ0Y046r_yQ{q5cUUYA`aXv#IbZ82a)o=5&_lnAI}NuapU)+> zvraAS9lK`~t!C;2wqJmXeNw#MXd-l4gqXa}R^QZnbTOJK;;?@I@rm%&)XT!SNUM{(Lnx-WhRxPu7{fnlcuStU#%j%GVhvSQyi=Oa>s@|Bti%&f;g2~wYZL4OB}Mxk z+b?F2QRYl8SEH$^Gq|gKB#haIz z!t;iN69p}snp1u63`$`ywSPlVpl8E$$3d3G`1YD}`;<3C3`sekPxy0hxMmFyo{Al` zjUy|SyiSr2y5f=5&mgP6XF%K}*ytW8IoHpLli1}$X7!wnQbBvW;f3szBEF}rR? zY`y}KF6S!8;D}f}))rrPH$rXQNJnS4R74kLLc$Xlfi3HZv7(c;kW~xC!l%tIi8<%%fuw4h?eLli`gI6Kc1dj;G?dZU%Rmk>0 zlI*fs%~LpWeDKVqjhU}l1BezIGOxA@6Fy1dK%$X2@wi`9-Dk2rw!iAbE@yVE;^ikoLwA7OmE8`D`Eh`+;V~q8G+M=nR8w0D3rUpl=QIqW6cR#W zY;H=fZDK3v(JfVx;PyNoNGh$v?A6D?Ny}xo_u*btdiuJqOFV-}gTehtE+HTLD#vCi z(P;=v&BQ)l*JOM5u$++%C0KV4T1u0cFKqo1W9t@;MmdYl7p_>lNlO)kT)JCW9VtPHV=LvN7Cmu!15P~= zF$c_e{H7EUclHaLPVW}W^}+i_m3k%^T=!F`!E7jtgD#J*Vrk%!gD!bR`;?-cQw6^s zMs?OLY~y(CPr6ho2v{&%;&-<-BtQw`Ip1zs{2 zlJHij73LF3dT9AFz7*)QEA_3p!}WYQ0?UXmjQacL??6~-aXabU#l!MXHFclTo1S(a{$9+Ou=$cZDW!05dm|L@K!HX_WTu+?pi z$kqV%mHXt^bQ5$ho@BSoFJ?IYgt;hefLyzkEix$*J}QA5*Dk9&DFAaCBd*-o7VNTW zlvG=i*DqY|*|1z;)URPCs(YsgPe~S0Zam|}Y9`eqd2n!l{gFqusPv4n+&W+Fk)xbm zdzXM7H=IUdvNTq4Kthztjw5VUA;BmZ#T_a} zXfCQ{AXqeAM&}`GrRdZ)&TYsdzqA9lM`6MdV_3Bc&Q<-{go7O~E^ zB+1H5?E^JZhaahb98T-Q=;L^R_V?-q9=(nTHYUTIvVQs?(oce8pj#_agXmxlPa_bK zau%s)QijfkTYKXL^IMmNlo8?0wTk{5qF^^Za9AiYC4CJlE5;ez-EDZhq=WtV+9|=J zdrjxAPf6+VIY|9?-N@&u?um#2njz=;MmR!S(Qmtp^<%eV{n8&8ahAq!gPbab^ic`s zyJ0~wcc99P7n5uLIQeEmYkzWh5Vrh@;jC-W2ZwsPj>On| z(^XgybF&i7-OamAiBIwSlZlSR#Vdp}R)ZsekYoaci@43^_HDzOFN#Ygfn8V?j~7T|6QKR zmkHh^#w|>z0HL?8tL4 z#j>Jk@asSr=>&I8HHPo-uHK!5P$s$*!T6DZaAxro&F<|k)!b3vsAMfhs+KaHbFTqF z@E*H_x@c_yC1XKvMRO`cO0DhJz=Ysa#sj0v%Cb_Lfw z__Tj&^5blO$_^Vs9$%Defz%#eJ>{M>`+`?KO3+@yNN&l@ph=^p?3kpN6HX^mLL8;h znX}4v-_c1ZGNxK)$1vi4km%f%ejMkj0D~6P=zkXprF=)BTwIrrvDK197V+Vd)W1H7wH&?}1G!1${&? zji^*OwX5nDnXEZsh`aiRBDWb5%DiG%R*%%4a7zk2KHuRt{-opP?bG6RQ@_Jrjwlcc z1c?I{V!mStTm$P+uMqeQJ+G&A-Bogkd9n^ zow6ZV5GhtRsc+n5Vpm!gL~@IgZ%M6SEirNsUCuhF?$hN(;FV7cQV-0WmV+mE&j1UM z*5gsoF|(+ckJqjHl>N;;_7d$6)awK+so|DP85m4m6cAC}YjgzQewH$pvS#KLNV=gt zWpcapyxLARfGB_63|p6Ui?{UbnZJMi1E7pVZdJRUG0y0E8+C|I2lvjlSp5_*-9ppR z9Q?Y|hjxxf;g3l!AMqNq?7hENyaue$&EQ)6FbDWW#-+fS((exFbIWL=*R@bj_n6DF zGCb@vMge`!wA!G4&-nEB;*{lqQ0^)M{2B->b;pmtJ_V>dbMl(ZYwlESZKFt3Bd@t; zzC*F~yF5cZKTtpDuTE;>X#^O@JfP2J!20&Rf0;m2)H9@oGAu5WhS2UQ705k;s>y`Sd{YI7DBC zBW86YCiE2dyyhHO54}mazQYIr&G>9TXvbTK9y3uYjZq3E0q|CoB)hdt{WX#^W6{9b z&nmv5v0eF#uJwL#X3rXVGUQea5`B~e}uKLq6{Vekx6e!b8jq)3LJg-M=&0@ZdMnC*XKnKQY#v0E4a#+rkFQG4PF7MDnu$0{d@2)I(fA!1#X({jHj zKz%G%jhilKF2nq6|EC<~QQ!(T#~2+bAc{|YD0lVv+2a^p7)&X3?Y#g)mTJl)9^m8s z*)qZWHG$;QO3&fF5Pr+Eyk+V;JgzHaiSU4Mr<=IkOW*N3%Xx*Cn8({WVH}|VjK0f+x=;||J9}N#t+mw ze-&XLYz)9#H4ID`K0E~8)~)&PQ>%`k2OI7G{A4i>C{zExYTMuX+yA!D!tkzMlE1|_ zJb1|f2XI2=pR7{Bg%1rA!qEmPf!1pOEH!mJxP}@z+-Q&k=&b(F${dseX6XFGlR;&m zkKsR5A5sMBnfy0mz^jMEfL1neP8()8K7s?K8!nCj0ncpU%{GGY_$fko3xX2G4?csi z2Qon5av{`k<6${qqw_y(>mc~oXMUIxNcQ2Mb?<@;ry1b@(t7=sOmL?Wejv8@KfLmy zgfIO~a(wy2|{?vC?!fAymk}} ztd0Gb$0wk}UyaEEixU1@lEOR3IDoB5|C3fG5&at~GERxo1lJmW56nvcpD;H4Kl>BH zffJNKuP^^>xd}-iGW#DkWz+s0$^pd5|0@~cc$0jXlaLj2=bn>46Jlg*Nq_8n4 delta 7284 zcmZ9Rbxa&UyY*p_;_gLisRLH+^J!mIhcBl0_*qd%+} z)z^`2t@btAW%fOwIG?H%1_Gy}x}Ohmufbm)bnYro1zK{}9Me&D!8f@=>j4?J0qgJA zg}{&N4ZX;wm8$Z$}t~&p!qz zM5;2nN)8r$+-$is#4E@!h2DJts4|?vM8xO@h=*PjKYn;=j4y+ja!IgSxH&u(A7W1;N33(Q2q!A~a znNEU3e@F;(X)!lnU|J{3)s6K=skjw?AmpuVeqrer_x^Qs?U&(N%rWdSjeR~{n!e@(0V97}K*+nu2Gq5=j6o^hv|lm0T9Bb8WMS zgKYy1bsTHu6|8vQ?<5DpPa|m~{O6AS^c_47VPgZ=b4afD;E_7_Xb%RrlN{zQ?;(Li zk{4cli|tG6EF)i*N7s||M~56IUUx4Wa+^=RB_F3rZ&~^KuljFRS0H}eL+JeqEg{}H zg-#~J!arLXb*9*e!eH!7QguKjMMk{*#Re%DD+RoOV+Vv3c? z%Hl`dk8~MmbT;yavPj;fG|3rJJ+TeqjV+kDyskGI#1j|bu@EVctW@VIiwMv498cGt z3~9Jq|71$c-rJ`=M1*Kk?gj!{PPI+l6LXxK(Zko3FVB}enFMsJ1scd_yis6sE&zQV z59eLQhBr$;2q zENT4}bGlE%Z?&PZn(k*cvMTfMGPUc5GPg3dMG`Tb0Zo2bZAMsZT_(XqJ>j;N*gs~4 zR>e9`XjM2!Z6K2Yxgoa9`a?fa&&8sA7C>E^p#7RnEM_e?n>+Wsc2$`Mvp~cu+x_GE z7~xu>RZ*PooqG`+#i9C*yjbD{(Tgkg2!SM$=r+WTC`k>s69#Eb^wiktBAd-Qh;air zLF@d4xu)Ou>3pZ0NWd;T%63rQ$E7_U_E0B8J)=a)1Mz8322(^MxmkEDO&G4fPmVRh zTB7+LNZ}xKL#M2AOk3b?IY$o$QAd@J7Vm9S$zmCqrLC6^tv9`W{fhGvVfLBo&B8ll z1#Vwm%#Fh8-?(1zQ45zOS{$2=eyEAawONDQ4`4WKE|t{uoO8~C%{xXt-q0{hEU@Ml zNMAHY4sxe$ra?)cEw!y9c|5|DMP;_#=9;OLz&*_DcUp05!Dk$KI!v;`#GtM)+;x*& zH1Z)5*(AA9Aib*i^aAfH;d|jk*M=bO{uk_#uzSQ`Z@Y5_-(;elL9~XZdG8)M)D#gA z3E<#RP~hO;B;oSVJ0$4w;oxq7aB$Rr9T#^SXbm3?gmCAxIOFZ^)Z}C%(uek1I=oRT zZ(b&{6zV1#YBMhCO?sb{_*I6fJVm@C?YG+c$HiW4gg%vJS|B`L=2ovCKlSTc7Z+_D z0*`9~8(a?j8(e^=5Nl^==M>4W+Zis_y*st99=(@=;A_T4f zT=>NdWLvRhY_A`22k9OX&n+qq*rz!@D zNbOF8e6^EoqI|I&&EqiihS_L#%$FUp zHH5}E+Yx9SDWq{yyagHZuXEHK&^>qDTTfERdpcCH46v7BktjXLcO zEgF3+W0D$Zxp6jfQKLRnF8Ma!+sTcIO@m87O}y-OXLWH53tCQ zH{cQxfsdM`o0Hu#8|VbiX&()wUuZWF-u>!ihJpn!7h_h%IcYdO z1{mQ3T*WVT&dT|1tT)3LYfQ8`eYtZ1?&vn65$iDAV9AR9=oCMKS;Jn<2$f9%8m>2X zdTY)PQ(k{!BNyPU{|ReyGrCN;{!9+}!AZz#RD3#DQ{va%(mB*R;k$=c?&cNaJGZ!X zV=bB;%C5tOR2(yQjVGV1(#t=B*R4##m~9a`dJierXH{O(Jeb?t<{e|;m(H&RD4cP#>8G zk4lf(3s8vck@u|0b~#DsCYvrq5P;^pH8~NE^8T9B)6KI38RZi z&4Vu<@bl`AJ!m~vI+IHtkf`XwycZm=JgN(~{GkQpIn?OSmxYd~B;i^hPW`Y7b&)|v zoQfS5)YY$YCK2~n_@<2LfX6jMx)-#-V9q6)z@*tkYgRYu`(4SIYilW^@N+|{CkufR zeU6u27BN?G^RebJPyAnYB)1jI@_M;SL|&}a&;3EfNQ zgMG>KWF9*(qMrOSVb@5 z6={LFVXask#>{ck;vqwfIXWJG#MmBs?+mqbVO=dHR?!O=&_!*cHo=2cneOa-9fo<3 zU%s*00RrT~KkdOzqO4$^1LT*TuYN%G@II}+oTKszMYMX3suww4wYoo5rtn;*Q>ARp zv#!5Ot2g~i<%L(b+y=+!NQPE#zKhyDO8`Rao&^h^OA*dv^_a<4gH(hgChjDggFTCS zDN{hnn83LJ&i7^fW^+R6eWVB%?R%nLt(}!S?=;Bi@At|2-44hRryBP;Pi$$YyFJY7 z1Gfqu1!K2y$k>%n)IDKFroQKqpjCbg*>?s!MgjFDN$EyRwwH0x%g^%t^7iN9bwy7w zFP{2?b1|is#^rbPIa+p@2YEUz`1)^}+pVw6p$Rv3=scGX)oK>*{tKYb6)(FZE;VVOs6uz?KPbMtFx9tVzX4 zEK01{Tn!42gf>7)zK6MA!D87dH;jKE+M$)|C>_(_i0in%db3?K$$)F}WX_2f0e-qeNH7EQ7=tIh-L}XuaF?; zDZHQy9hTD)`<4iNY9CXL-okbhZ#~?A={e~66>O`dz1gQs!@h(vi!-={osm8(B|#|c zdA1Hv2<}8at=;P)O`$Ade*e{RY}AApi?e=Frl=JZd5e-}qb1fq4SWhrgHW5Yl94;u z#p=hIA(%;h&RrBWoUo%rt&xT9KNj^**E|7E-^Tvr(xRApqI!hAFTaR;N|C>A=52^? zR4X6Tyw~~_W-j?e({%N72A2FidkoSr4qGlTZsbb)&Y&?Hjej`0#S~rV7q~D34dqc* z>BED%3e!&gaphbp8F7jkCfG%JnZs?V!i|ymNMEyAc?^2N{Ze$6sP%&SrRqZUo-O{m z-KucR<#HMn6tymcd8qeTdG-FKqyLNAy{O~}$Ne*nVq(=S=!Gy~;tSEA2@1$@ycb<; zo82-{$Y1{HETkRDVCL-seuOyaULfGp*q5D^WP`*W#=;xQ!{j8n>$d%E)lsQ@uhonl zo`o@ul5}4FF^?BiW0XSsL0k{|=NN{v5*NhsPuZzn!}%JU;OfPM3ex!mXb1eg(hpRm zl0k13EYj;pmRX!#D`z5a$S$PWru4gtuF$C;YhlGS_vidK_-~e%&{s6S-MMvjV)cIv zPiWtA70_W4+|p#{b~TNBNC-Oj-v2VQWoqn~o0Bw3b%1_=k{E63zEV9WYnytQu-}?0 zzVB*A7F{ZHmXSJ!Ia->WHQK>2(T493$)Sj*NG5tt^wq6w{=|J#!Z!wzx8o^PQtJHY zY2sO7sLAU(n+k?6K@ z!e%G8h>!0=7%r3V8&wBIE{!Jt6#dvWYqFBAB(2#>bI$7y{Uc=sC;reVV_ER}Ln?ct z<&R~eb7qM_Z?SuW<<=i+IR3lCm60(R-K4Tdv9fZVlFSEj*HqA38mQq$$S(+rYEBo=nYPVmu zdE~*fYRvN|YL?{?&eSSg$^x4JC`X_Bb>kbE?#6a-ximH&Bibi3s$~xmFjFOU{4X!c z5tItqJ!jT~&h^V!Cc_k`(Bxa5h3;7L94Aug8 zI)503EUDZw^Naj}408$aeP8A$aYOfF?N&N3W*~IsrIDRHmb4bwCpaw8pdBmYG*Z#Q znNPbLE2^oKm|FDZND6Dc9K>A)Z+6G;Sx&4;nS6M(3NXS%3liB&!&Ea~rjF1@zhIml zM<}H#cC*@vk_XlZN{2L$N%6iE4s7_r=7GW9?ArFc5h@W7j7wX7!mVT8PIx*icI5}O zz_f2*{H!G~ewBN!K=OFhuZGzlvvhM5&AA4JT zF3W*zCn_FfXO8p=vy&9`0T%8Y5*Vm!Alo)aIpc_Y)eSxAukGN4_Qtfq1)|qH>w*Oq ze|H}q4t&~2H{1RhY8M6U_D(8qOUIsxw_fLdEseSIY<$-?Q_zxOtsUv{Xt&D^DSACA zfz+eyJ)jkB*9OXoSMOIW^Ub)c9KqS~#k%MMEu{fWlGBpg!KR3WWk;X@_{$H zgR+mlL|qJw4I({z9;P?a8eO*j!Mf-zaS2ZgVy5CBx6cm00YtAU;M=kkHes@NC+*I7 zXR8wTslDI#v=0-a^<*c7g$VDPAJRiR_cb~ZXL#UXnHK$6Ouk&&>*#blvtU^2Ny#IS zNdaOr;m0NlM@M7ccqr+IqM=l11WUXbs6|RGb+L$e%UgWK8TiK92(|vr8PQh{hQN&8 zHv(4VcesAMn3)w^v}z9QVMZP~ECr?WBx2e8@|Ona3QyB&b~O#fJDl)qJ93=*A)saf zQ9~iWrCWNf9W^qEURF3gTWHcUvaJuim?#AFh8zQ-N2tWE1p&kR7gj&5kZwmLRmq6t zDe6^q0%7SMj$gCaVMPc`W%?_i|8voWOf^dlN#P+Gq^vNgFAqi%tlM5@n!Fip{A=z| zZe%lkadj+xQDTXsdSP2kRuNHE@j1$F*>z&dE7u)~#8~D&dIT8#jkEaN69s}b*Z_HR zi%16ErWZe6f^ILobOzyc!Z%|cH6koHD&9M?cax3{CvB43ij($u6Pn_-!6K!pgv^XjtvU~E$ACJ zx(81gk57hqsyO^6t#-O5tPMRvJNzwp*U)PfOt$*eN_LM~|Nd4*m=INkv=qX}scq2* ze!dcNF8Z1K&00xVs|9lhGF%cjk`MNlaUOOPBcV(rp`; z*pXNr_ou(W7C01FewCk)?7LebGPBcL>o=NKhB;GU2lV&(y7d#NHo(DwlMa< zuf)w~H{Ayua|;|-5dd{B_U~t`M!a=zgj~_+Vd4V@-Di78agKSG-|bP>vo*oyIIFCF zpNyk29Bdh%qkjcwyd7SBJFhz9^9}4jA96E7It@`0_Yx7$`qh7Gln@DVnOnhvd2DmD z^td?>Wqb|t>q{8$Jhv#l)ilrqOZtn;xKkAxzQV7EDD4vpiX^~^PtSr1oygSa$dtT* zrQjOg{9j)E%yQ1-^8r-hsiBiQJ1Qce4YS(ox?HUg<=?5?Z;T0Bh~P~SwlgrdRmQF< zmyV&jeG3r-AKK4f@h~7)+`2uXroM#l{4s%lyy%~FY;g&Qi_(LQxU75RzJxDr;Zmi{ z1nPQ_O}w(mZSRnJuvM76-yKV?dAX|pfvkduD1fOE^~WcVy}6TKG$$orx!bUxb6AD= zWcGcfJa1J}B8lfgg`nEQA~gft`N<-#1%l@RoE{Tgf6#$KBmx4&5`9(k(KZ+T38X93 ze7Zl`@0sC=oY&@h_D#|jMoFHzL+`Z$j!eQXWADV`oAoMb{pQ2lwlvc@XCL|>=i}7ps@ZPpD8dj;6XJ(M&K*eW#d|q{Fe}?HjA8R zHtjbOZEs9pNLnKxb=*WfF)uVd5vz#VfUo4|J5%WJs}MM1$h+}Y^QXk$JJ!obvy7&| z{B1d}SwrKP>SC{~S9UTJ^@9Zh(Z_@r)W&@okqX%ikJr$f^`aXSwS^~Qw5~57_FtzY z0Mb0PL{fdv3DHAp=X5q1Ia`0$#Nf)KUzSH}NpTsG52gztC$USxLdA_5+6i@Rf`+Mx z6rns6k=gGfUic=}iXe(B(jn2!b=KabIIrl>a>BaR=c#%fDaG9%Yfca|c!^#aJt?~c zUCrZO(GWV~9Oe$Fe-qIDs#$ZzwuGUz>xWFe&b<1~p1&3RDEul1QOp60*3x}sMBypn zzM;G--A?C_?ZYUeX6ktK!}0dMy{4w5>}XKM=vVo3Fh$|QUvM(j)x3&Y>x%W*9pCp0 zmS~1CW~R;-<#BMSf;=VJ?*Vb%s;b1hdxdYAmFM$ALx59Le1mczmU;=r8BqP#f@-lC zKtudrZ71&$Ig0-q#r;j={yWI|pc})4(324XK%o3T6#2#vz)}2%1 transports; - private TransportListener transportListener; - private BoundedChannelPool pool; public Client() { @@ -126,7 +108,7 @@ public final class Client { this.bootstrap = new Bootstrap() .group(this.eventLoopGroup) .channel(this.socketChannelClass) - //.option(ChannelOption.ALLOCATOR, byteBufAllocator) + .option(ChannelOption.ALLOCATOR, byteBufAllocator) .option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNodelay()) .option(ChannelOption.SO_KEEPALIVE, clientConfig.isKeepAlive()) .option(ChannelOption.SO_REUSEADDR, clientConfig.isReuseAddr()) @@ -134,9 +116,6 @@ public final class Client { .option(ChannelOption.SO_RCVBUF, clientConfig.getTcpReceiveBufferSize()) .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis()) .option(ChannelOption.WRITE_BUFFER_WATER_MARK, clientConfig.getWriteBufferWaterMark()); - this.httpResponseHandler = new HttpResponseHandler(); - this.http2SettingsHandler = new Http2SettingsHandler(); - this.http2ResponseHandler = new Http2ResponseHandler(); this.transports = new CopyOnWriteArrayList<>(); if (!clientConfig.getPoolNodes().isEmpty()) { List nodes = clientConfig.getPoolNodes(); @@ -151,7 +130,8 @@ public final class Client { } ClientChannelPoolHandler clientChannelPoolHandler = new ClientChannelPoolHandler(); this.pool = new BoundedChannelPool<>(semaphore, clientConfig.getPoolVersion(), - clientConfig.isPoolSecure(), nodes, bootstrap, clientChannelPoolHandler, retries); + nodes, bootstrap, clientChannelPoolHandler, retries, + BoundedChannelPool.PoolKeySelectorType.ROUNDROBIN); Integer nodeConnectionLimit = clientConfig.getPoolNodeConnectionLimit(); if (nodeConnectionLimit == null || nodeConnectionLimit == 0) { nodeConnectionLimit = nodes.size(); @@ -176,22 +156,10 @@ public final class Client { return byteBufAllocator; } - public EventLoopGroup getEventLoopGroup() { - return eventLoopGroup; - } - - public void setTransportListener(TransportListener transportListener) { - this.transportListener = transportListener; - } - public boolean hasPooledConnections() { return pool != null && !clientConfig.getPoolNodes().isEmpty(); } - public BoundedChannelPool getPool() { - return pool; - } - public void logDiagnostics(Level level) { logger.log(level, () -> "OpenSSL available: " + OpenSsl.isAvailable() + " OpenSSL ALPN support: " + OpenSsl.isAlpnSupported() + @@ -206,29 +174,22 @@ public final class Client { return newTransport(null); } - public Transport newTransport(URL url, HttpVersion httpVersion) { - return newTransport(HttpAddress.of(url, httpVersion)); - } - public Transport newTransport(HttpAddress httpAddress) { - Transport transport = null; + Transport transport; if (httpAddress != null) { if (httpAddress.getVersion().majorVersion() == 1) { - transport = new Http1Transport(this, httpAddress); + transport = new HttpTransport(this, httpAddress); } else { transport = new Http2Transport(this, httpAddress); } } else if (hasPooledConnections()) { if (pool.getVersion().majorVersion() == 1) { - transport = new Http1Transport(this, null); + transport = new HttpTransport(this, null); } else { transport = new Http2Transport(this, null); } } else { - throw new IllegalStateException(); - } - if (transportListener != null) { - transportListener.onOpen(transport); + throw new IllegalStateException("no address given to connect to"); } transports.add(transport); return transport; @@ -238,14 +199,13 @@ public final class Client { Channel channel; if (httpAddress != null) { HttpVersion httpVersion = httpAddress.getVersion(); - ChannelInitializer initializer; + ChannelInitializer initializer; SslHandler sslHandler = newSslHandler(clientConfig, byteBufAllocator, httpAddress); if (httpVersion.majorVersion() == 1) { - initializer = new HttpChannelInitializer(clientConfig, httpAddress, - sslHandler, httpResponseHandler); + initializer = new HttpChannelInitializer(clientConfig, httpAddress, sslHandler, + new Http2ChannelInitializer(clientConfig, httpAddress, sslHandler)); } else { - initializer = new Http2ChannelInitializer(clientConfig, httpAddress, - sslHandler, http2SettingsHandler, http2ResponseHandler); + initializer = new Http2ChannelInitializer(clientConfig, httpAddress, sslHandler); } try { channel = bootstrap.handler(initializer) @@ -267,21 +227,18 @@ public final class Client { return channel; } - public Channel newChannel() throws IOException { - return newChannel(null); - } - - public void releaseChannel(Channel channel) throws IOException{ - if (channel != null) { - if (hasPooledConnections()) { - try { - pool.release(channel); - } catch (Exception e) { - throw new IOException(e); - } - } else { - channel.close(); + public void releaseChannel(Channel channel, boolean close) throws IOException{ + if (channel == null) { + return; + } + if (hasPooledConnections()) { + try { + pool.release(channel, close); + } catch (Exception e) { + throw new IOException(e); } + } else if (close) { + channel.close(); } } @@ -293,19 +250,15 @@ public final class Client { public CompletableFuture execute(Request request, Function supplier) throws IOException { - return newTransport(HttpAddress.of(request.url(), request.httpVersion())).execute(request, supplier); - } - - public Transport pooledExecute(Request request) throws IOException { - Transport transport = newTransport(); - transport.execute(request); - return transport; + return newTransport(HttpAddress.of(request.url(), request.httpVersion())) + .execute(request, supplier); } /** * For following redirects, construct a new transport. * @param transport the previous transport * @param request the new request for continuing the request. + * @throws IOException if continuation fails */ public void continuation(Transport transport, Request request) throws IOException { Transport nextTransport = newTransport(HttpAddress.of(request.url(), request.httpVersion())); @@ -328,14 +281,7 @@ public final class Client { close(transport); } - public Transport prepareRequest(Request request) { - return newTransport(HttpAddress.of(request.url(), request.httpVersion())); - } - public void close(Transport transport) throws IOException { - if (transportListener != null) { - transportListener.onClose(transport); - } transport.close(); transports.remove(transport); } @@ -344,12 +290,13 @@ public final class Client { for (Transport transport : transports) { close(transport); } - } - - public void shutdownGracefully() throws IOException { + // how to wait for all responses for the pool? if (hasPooledConnections()) { pool.close(); } + } + + public void shutdownGracefully() throws IOException { close(); shutdown(); } @@ -439,13 +386,6 @@ public final class Client { ApplicationProtocolNames.HTTP_2); } - public interface TransportListener { - - void onOpen(Transport transport); - - void onClose(Transport transport); - } - static class HttpClientThreadFactory implements ThreadFactory { private int number = 0; @@ -474,17 +414,12 @@ public final class Client { HttpVersion httpVersion = httpAddress.getVersion(); SslHandler sslHandler = newSslHandler(clientConfig, byteBufAllocator, httpAddress); if (httpVersion.majorVersion() == 1) { - HttpChannelInitializer initializer = new HttpChannelInitializer(clientConfig, httpAddress, - sslHandler, httpResponseHandler); - if (channel instanceof SocketChannel) { - initializer.initChannel((SocketChannel) channel); - } + HttpChannelInitializer initializer = new HttpChannelInitializer(clientConfig, httpAddress, sslHandler, + new Http2ChannelInitializer(clientConfig, httpAddress, sslHandler)); + initializer.initChannel(channel); } else { - Http2ChannelInitializer initializer = new Http2ChannelInitializer(clientConfig, httpAddress, - sslHandler, http2SettingsHandler, http2ResponseHandler); - if (channel instanceof SocketChannel) { - initializer.initChannel((SocketChannel) channel); - } + Http2ChannelInitializer initializer = new Http2ChannelInitializer(clientConfig, httpAddress, sslHandler); + initializer.initChannel(channel); } } } diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/ClientBuilder.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/ClientBuilder.java index ee73632..3782cd9 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/ClientBuilder.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/ClientBuilder.java @@ -222,6 +222,11 @@ public class ClientBuilder { return this; } + public ClientBuilder enableNegotiation(boolean enableNegotiation) { + clientConfig.setEnableNegotiation(enableNegotiation); + return this; + } + public Client build() { return new Client(clientConfig, byteBufAllocator, eventLoopGroup, socketChannelClass); } diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/ClientConfig.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/ClientConfig.java index 855315a..43bfe9b 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/ClientConfig.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/ClientConfig.java @@ -163,6 +163,8 @@ public class ClientConfig { * Default for backoff. */ BackOff BACK_OFF = BackOff.ZERO_BACKOFF; + + Boolean ENABLE_NEGOTIATION = false; } private static TrustManagerFactory TRUST_MANAGER_FACTORY; @@ -249,6 +251,8 @@ public class ClientConfig { private BackOff backOff = Defaults.BACK_OFF; + private boolean enableNegotiation = Defaults.ENABLE_NEGOTIATION; + public ClientConfig setDebug(boolean debug) { this.debug = debug; return this; @@ -611,6 +615,15 @@ public class ClientConfig { return backOff; } + public ClientConfig setEnableNegotiation(boolean enableNegotiation) { + this.enableNegotiation = enableNegotiation; + return this; + } + + public boolean isEnableNegotiation() { + return enableNegotiation; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/Request.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/Request.java index b52381a..9ea25e5 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/Request.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/Request.java @@ -10,8 +10,7 @@ import io.netty.handler.codec.http.cookie.Cookie; import org.xbib.net.URL; import org.xbib.netty.http.client.listener.CookieListener; -import org.xbib.netty.http.client.listener.HttpHeadersListener; -import org.xbib.netty.http.client.listener.HttpResponseListener; +import org.xbib.netty.http.client.listener.ResponseListener; import org.xbib.netty.http.client.retry.BackOff; import java.nio.charset.StandardCharsets; @@ -51,9 +50,7 @@ public class Request { private CompletableFuture completableFuture; - private HttpResponseListener responseListener; - - private HttpHeadersListener headersListener; + private ResponseListener responseListener; private CookieListener cookieListener; @@ -136,6 +133,12 @@ public class Request { return true; } + public void release() { + if (content != null) { + content.release(); + } + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -160,14 +163,6 @@ public class Request { return completableFuture; } - public Request setHeadersListener(HttpHeadersListener httpHeadersListener) { - this.headersListener = httpHeadersListener; - return this; - } - - public HttpHeadersListener getHeadersListener() { - return headersListener; - } public Request setCookieListener(CookieListener cookieListener) { this.cookieListener = cookieListener; @@ -178,12 +173,12 @@ public class Request { return cookieListener; } - public Request setResponseListener(HttpResponseListener httpResponseListener) { - this.responseListener = httpResponseListener; + public Request setResponseListener(ResponseListener responseListener) { + this.responseListener = responseListener; return this; } - public HttpResponseListener getResponseListener() { + public ResponseListener getResponseListener() { return responseListener; } @@ -224,7 +219,7 @@ public class Request { } public static RequestBuilder builder(HttpMethod httpMethod) { - return new RequestBuilder(PooledByteBufAllocator.DEFAULT).setMethod(httpMethod); + return builder(PooledByteBufAllocator.DEFAULT, httpMethod); } public static RequestBuilder builder(ByteBufAllocator allocator, HttpMethod httpMethod) { diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/RequestBuilder.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/RequestBuilder.java index 42624f0..39070d9 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/RequestBuilder.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/RequestBuilder.java @@ -350,7 +350,8 @@ public class RequestBuilder { } private void content(byte[] buf, AsciiString contentType) { - content(allocator.buffer().writeBytes(buf), contentType); + ByteBuf byteBuf = allocator.buffer(); + content(byteBuf.writeBytes(buf), contentType); } private void content(ByteBuf body, AsciiString contentType) { diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/HttpChannelInitializer.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/HttpChannelInitializer.java similarity index 50% rename from netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/HttpChannelInitializer.java rename to netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/HttpChannelInitializer.java index 45ab3a8..81f3d91 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/HttpChannelInitializer.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/HttpChannelInitializer.java @@ -1,20 +1,24 @@ -package org.xbib.netty.http.client.handler.http1; +package org.xbib.netty.http.client.handler.http; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelPipeline; -import io.netty.channel.socket.SocketChannel; import io.netty.handler.codec.http.HttpClientCodec; import io.netty.handler.codec.http.HttpContentDecompressor; import io.netty.handler.codec.http.HttpObjectAggregator; import io.netty.handler.logging.LogLevel; +import io.netty.handler.ssl.ApplicationProtocolNames; +import io.netty.handler.ssl.ApplicationProtocolNegotiationHandler; import io.netty.handler.ssl.SslHandler; import org.xbib.netty.http.client.ClientConfig; +import org.xbib.netty.http.client.handler.http2.Http2ChannelInitializer; import org.xbib.netty.http.common.HttpAddress; import java.util.logging.Level; import java.util.logging.Logger; -public class HttpChannelInitializer extends ChannelInitializer { +public class HttpChannelInitializer extends ChannelInitializer { private static final Logger logger = Logger.getLogger(HttpChannelInitializer.class.getName()); @@ -26,18 +30,21 @@ public class HttpChannelInitializer extends ChannelInitializer { private final HttpResponseHandler httpResponseHandler; + private final Http2ChannelInitializer http2ChannelInitializer; + public HttpChannelInitializer(ClientConfig clientConfig, HttpAddress httpAddress, SslHandler sslHandler, - HttpResponseHandler httpResponseHandler) { + Http2ChannelInitializer http2ChannelInitializer) { this.clientConfig = clientConfig; this.httpAddress = httpAddress; this.sslHandler = sslHandler; - this.httpResponseHandler = httpResponseHandler; + this.http2ChannelInitializer = http2ChannelInitializer; + this.httpResponseHandler = new HttpResponseHandler(); } @Override - public void initChannel(SocketChannel channel) { + public void initChannel(Channel channel) { if (clientConfig.isDebug()) { channel.pipeline().addLast(new TrafficLoggingHandler(LogLevel.DEBUG)); } @@ -47,17 +54,43 @@ public class HttpChannelInitializer extends ChannelInitializer { configureCleartext(channel); } if (clientConfig.isDebug()) { - logger.log(Level.FINE, "HTTP 1 channel initialized: " + channel.pipeline().names()); + logger.log(Level.FINE, "HTTP 1.1 client channel initialized: " + channel.pipeline().names()); } } - private void configureEncrypted(SocketChannel channel) { + private void configureEncrypted(Channel channel) { ChannelPipeline pipeline = channel.pipeline(); pipeline.addLast(sslHandler); - configureCleartext(channel); + if (clientConfig.isEnableNegotiation()) { + ApplicationProtocolNegotiationHandler negotiationHandler = + new ApplicationProtocolNegotiationHandler(ApplicationProtocolNames.HTTP_1_1) { + @Override + protected void configurePipeline(ChannelHandlerContext ctx, String protocol) { + if (ApplicationProtocolNames.HTTP_2.equals(protocol)) { + http2ChannelInitializer.configureCleartext(ctx.channel()); + if (clientConfig.isDebug()) { + logger.log(Level.FINE, "after negotiation to HTTP/2: " + ctx.pipeline().names()); + } + return; + } + if (ApplicationProtocolNames.HTTP_1_1.equals(protocol)) { + configureCleartext(ctx.channel()); + if (clientConfig.isDebug()) { + logger.log(Level.FINE, "after negotiation to HTTP 1.1: " + ctx.pipeline().names()); + } + return; + } + ctx.close(); + throw new IllegalStateException("protocol not accepted: " + protocol); + } + }; + channel.pipeline().addLast(negotiationHandler); + } else { + configureCleartext(channel); + } } - private void configureCleartext(SocketChannel channel) { + private void configureCleartext(Channel channel) { ChannelPipeline pipeline = channel.pipeline(); pipeline.addLast(new HttpClientCodec(clientConfig.getMaxInitialLineLength(), clientConfig.getMaxHeadersSize(), clientConfig.getMaxChunkSize())); diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/HttpChunkContentCompressor.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/HttpChunkContentCompressor.java similarity index 94% rename from netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/HttpChunkContentCompressor.java rename to netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/HttpChunkContentCompressor.java index 1643365..5da36fe 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/HttpChunkContentCompressor.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/HttpChunkContentCompressor.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.client.handler.http1; +package org.xbib.netty.http.client.handler.http; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/HttpResponseHandler.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/HttpResponseHandler.java similarity index 71% rename from netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/HttpResponseHandler.java rename to netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/HttpResponseHandler.java index ead69a5..6ee0e15 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/HttpResponseHandler.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/HttpResponseHandler.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.client.handler.http1; +package org.xbib.netty.http.client.handler.http; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; @@ -10,11 +10,9 @@ import org.xbib.netty.http.client.transport.Transport; public class HttpResponseHandler extends SimpleChannelInboundHandler { @Override - protected void channelRead0(ChannelHandlerContext ctx, FullHttpResponse httpResponse) { + public void channelRead0(ChannelHandlerContext ctx, FullHttpResponse httpResponse) throws Exception { Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); - transport.headersReceived(null, httpResponse.headers()); - transport.responseReceived(null, httpResponse); - transport.success(); + transport.responseReceived(ctx.channel(),null, httpResponse); } @Override diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/TrafficLoggingHandler.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/TrafficLoggingHandler.java similarity index 95% rename from netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/TrafficLoggingHandler.java rename to netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/TrafficLoggingHandler.java index 333a2f0..b4bfcd9 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/TrafficLoggingHandler.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/TrafficLoggingHandler.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.client.handler.http1; +package org.xbib.netty.http.client.handler.http; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandler; diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/package-info.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/package-info.java new file mode 100644 index 0000000..fe6afdc --- /dev/null +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http/package-info.java @@ -0,0 +1,4 @@ +/** + * HTTP handlers for Netty HTTP client. + */ +package org.xbib.netty.http.client.handler.http; diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/package-info.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/package-info.java deleted file mode 100644 index 4cb6b26..0000000 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http1/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * HTTP handlers for Netty HTTP client. - */ -package org.xbib.netty.http.client.handler.http1; diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2ChannelInitializer.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2ChannelInitializer.java index 4d3f904..a16f869 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2ChannelInitializer.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2ChannelInitializer.java @@ -1,26 +1,35 @@ package org.xbib.netty.http.client.handler.http2; +import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.channel.ChannelInitializer; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.codec.http2.DefaultHttp2Connection; -import io.netty.handler.codec.http2.Http2Connection; +import io.netty.channel.ChannelPipeline; +import io.netty.handler.codec.http2.DefaultHttp2SettingsFrame; +import io.netty.handler.codec.http2.Http2ConnectionAdapter; +import io.netty.handler.codec.http2.Http2ConnectionDecoder; +import io.netty.handler.codec.http2.Http2ConnectionEncoder; import io.netty.handler.codec.http2.Http2ConnectionHandler; +import io.netty.handler.codec.http2.Http2ConnectionPrefaceAndSettingsFrameWrittenEvent; +import io.netty.handler.codec.http2.Http2Exception; +import io.netty.handler.codec.http2.Http2FrameAdapter; +import io.netty.handler.codec.http2.Http2FrameCodec; import io.netty.handler.codec.http2.Http2FrameLogger; -import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandlerBuilder; -import io.netty.handler.codec.http2.InboundHttp2ToHttpAdapterBuilder; +import io.netty.handler.codec.http2.Http2Headers; +import io.netty.handler.codec.http2.Http2MultiplexCodec; +import io.netty.handler.codec.http2.Http2MultiplexCodecBuilder; +import io.netty.handler.codec.http2.Http2Settings; import io.netty.handler.logging.LogLevel; -import io.netty.handler.ssl.ApplicationProtocolNames; -import io.netty.handler.ssl.ApplicationProtocolNegotiationHandler; import io.netty.handler.ssl.SslHandler; import org.xbib.netty.http.client.ClientConfig; -import org.xbib.netty.http.client.handler.http1.TrafficLoggingHandler; +import org.xbib.netty.http.client.handler.http.TrafficLoggingHandler; +import org.xbib.netty.http.client.transport.Transport; import org.xbib.netty.http.common.HttpAddress; import java.util.logging.Level; import java.util.logging.Logger; -public class Http2ChannelInitializer extends ChannelInitializer { +public class Http2ChannelInitializer extends ChannelInitializer { private static final Logger logger = Logger.getLogger(Http2ChannelInitializer.class.getName()); @@ -30,29 +39,16 @@ public class Http2ChannelInitializer extends ChannelInitializer { private final SslHandler sslHandler; - private final Http2SettingsHandler http2SettingsHandler; - - private final Http2ResponseHandler http2ResponseHandler; - public Http2ChannelInitializer(ClientConfig clientConfig, HttpAddress httpAddress, - SslHandler sslHandler, - Http2SettingsHandler http2SettingsHandler, - Http2ResponseHandler http2ResponseHandler) { + SslHandler sslHandler) { this.clientConfig = clientConfig; this.httpAddress = httpAddress; this.sslHandler = sslHandler; - this.http2SettingsHandler = http2SettingsHandler; - this.http2ResponseHandler = http2ResponseHandler; } - /** - * The channel initialization for HTTP/2. - * - * @param channel socket channel - */ @Override - public void initChannel(SocketChannel channel) { + public void initChannel(Channel channel) { if (clientConfig.isDebug()) { channel.pipeline().addLast(new TrafficLoggingHandler(LogLevel.DEBUG)); } @@ -66,44 +62,73 @@ public class Http2ChannelInitializer extends ChannelInitializer { } } - private void configureEncrypted(SocketChannel channel) { + private void configureEncrypted(Channel channel) { channel.pipeline().addLast(sslHandler); - ApplicationProtocolNegotiationHandler negotiationHandler = new ApplicationProtocolNegotiationHandler("") { + configureCleartext(channel); + } + + public void configureCleartext(Channel ch) { + ChannelInitializer initializer = new ChannelInitializer() { @Override - protected void configurePipeline(ChannelHandlerContext ctx, String protocol) { - if (ApplicationProtocolNames.HTTP_2.equals(protocol)) { - ctx.pipeline().addLast(newConnectionHandler(), http2SettingsHandler, http2ResponseHandler); - if (clientConfig.isDebug()) { - logger.log(Level.FINE, "after negotiation: " + ctx.pipeline().names()); - } - return; - } - // we do not fall back to HTTP1 - ctx.close(); - throw new IllegalStateException("protocol not accepted: " + protocol); + protected void initChannel(Channel ch) { + throw new IllegalStateException(); } }; - channel.pipeline().addLast(negotiationHandler); -} - - private void configureCleartext(SocketChannel ch) { - ch.pipeline().addLast(newConnectionHandler(), http2SettingsHandler, http2ResponseHandler); - } - - private Http2ConnectionHandler newConnectionHandler() { - Http2Connection http2Connection = new DefaultHttp2Connection(false); - HttpToHttp2ConnectionHandlerBuilder http2ConnectionHandlerBuilder = new HttpToHttp2ConnectionHandlerBuilder() - .initialSettings(clientConfig.getHttp2Settings()) - .connection(http2Connection) - .frameListener(new Http2PushPromiseHandler(http2Connection, - new InboundHttp2ToHttpAdapterBuilder(http2Connection) - .maxContentLength(clientConfig.getMaxContentLength()) - .propagateSettings(true) - .build())); + Http2MultiplexCodecBuilder clientMultiplexCodecBuilder = Http2MultiplexCodecBuilder.forClient(initializer) + .initialSettings(clientConfig.getHttp2Settings()); if (clientConfig.isDebug()) { - Http2FrameLogger http2FrameLogger = new Http2FrameLogger(clientConfig.getDebugLogLevel(), "client"); - http2ConnectionHandlerBuilder.frameLogger(http2FrameLogger); + clientMultiplexCodecBuilder.frameLogger(new Http2FrameLogger(LogLevel.DEBUG, "client")); + } + Http2MultiplexCodec http2MultiplexCodec = clientMultiplexCodecBuilder.build(); + ChannelPipeline p = ch.pipeline(); + p.addLast("client-codec", http2MultiplexCodec); + //p.addLast("client-push-promise", new PushPromiseHandler()); + p.addLast("client-messages", new ClientMessages()); + } + + class ClientMessages extends ChannelInboundHandlerAdapter { + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (msg instanceof DefaultHttp2SettingsFrame) { + DefaultHttp2SettingsFrame settingsFrame = (DefaultHttp2SettingsFrame) msg; + Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); + if (transport != null) { + transport.settingsReceived(settingsFrame.settings()); + } + } + } + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof Http2ConnectionPrefaceAndSettingsFrameWrittenEvent) { + Http2ConnectionPrefaceAndSettingsFrameWrittenEvent event = + (Http2ConnectionPrefaceAndSettingsFrameWrittenEvent)evt; + Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); + if (transport != null) { + transport.settingsReceived(null); + } + } + ctx.fireUserEventTriggered(evt); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); + if (transport != null) { + transport.fail(cause); + } + } + } + + class PushPromiseHandler extends Http2FrameAdapter { + + @Override + public void onPushPromiseRead(ChannelHandlerContext ctx, int streamId, int promisedStreamId, + Http2Headers headers, int padding) throws Http2Exception { + super.onPushPromiseRead(ctx, streamId, promisedStreamId, headers, padding); + Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); + transport.pushPromiseReceived(ctx.channel(), streamId, promisedStreamId, headers); } - return http2ConnectionHandlerBuilder.build(); } } diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2PushPromiseHandler.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2PushPromiseHandler.java deleted file mode 100644 index 10f98bf..0000000 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2PushPromiseHandler.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.xbib.netty.http.client.handler.http2; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http2.DelegatingDecompressorFrameListener; -import io.netty.handler.codec.http2.Http2Connection; -import io.netty.handler.codec.http2.Http2Exception; -import io.netty.handler.codec.http2.Http2FrameListener; -import io.netty.handler.codec.http2.Http2Headers; -import org.xbib.netty.http.client.transport.Transport; - -public class Http2PushPromiseHandler extends DelegatingDecompressorFrameListener { - - public Http2PushPromiseHandler(Http2Connection connection, Http2FrameListener listener) { - super(connection, listener); - } - - @Override - public void onPushPromiseRead(ChannelHandlerContext ctx, int streamId, int promisedStreamId, - Http2Headers headers, int padding) throws Http2Exception { - super.onPushPromiseRead(ctx, streamId, promisedStreamId, headers, padding); - Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); - transport.pushPromiseReceived(streamId, promisedStreamId, headers); - } -} diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2ResponseHandler.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2ResponseHandler.java index 412b776..918422a 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2ResponseHandler.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2ResponseHandler.java @@ -7,24 +7,14 @@ import io.netty.handler.codec.http.FullHttpResponse; import io.netty.handler.codec.http2.HttpConversionUtil; import org.xbib.netty.http.client.transport.Transport; -import java.io.IOException; - @ChannelHandler.Sharable public class Http2ResponseHandler extends SimpleChannelInboundHandler { @Override - protected void channelRead0(ChannelHandlerContext ctx, FullHttpResponse httpResponse) { + protected void channelRead0(ChannelHandlerContext ctx, FullHttpResponse httpResponse) throws Exception { Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); Integer streamId = httpResponse.headers().getInt(HttpConversionUtil.ExtensionHeaderNames.STREAM_ID.text()); - transport.headersReceived(streamId, httpResponse.headers()); - transport.responseReceived(streamId, httpResponse); - } - - @Override - public void channelInactive(ChannelHandlerContext ctx) { - ctx.fireChannelInactive(); - Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); - transport.fail(new IOException("channel closed")); + transport.responseReceived(ctx.channel(), streamId, httpResponse); } @Override diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2SettingsHandler.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2SettingsHandler.java deleted file mode 100644 index e9fb6ab..0000000 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2SettingsHandler.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.xbib.netty.http.client.handler.http2; - -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.handler.codec.http2.Http2Settings; -import org.xbib.netty.http.client.transport.Transport; - -@ChannelHandler.Sharable -public class Http2SettingsHandler extends SimpleChannelInboundHandler { - - @Override - protected void channelRead0(ChannelHandlerContext ctx, Http2Settings http2Settings) { - Transport transport = ctx.channel().attr(Transport.TRANSPORT_ATTRIBUTE_KEY).get(); - transport.settingsReceived(ctx.channel(), http2Settings); - ctx.pipeline().remove(this); - } -} diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2StreamFrameToHttpObjectCodec.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2StreamFrameToHttpObjectCodec.java new file mode 100644 index 0000000..d908b74 --- /dev/null +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/handler/http2/Http2StreamFrameToHttpObjectCodec.java @@ -0,0 +1,227 @@ +package org.xbib.netty.http.client.handler.http2; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandler.Sharable; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.EncoderException; +import io.netty.handler.codec.MessageToMessageCodec; +import io.netty.handler.codec.http.DefaultHttpContent; +import io.netty.handler.codec.http.DefaultLastHttpContent; +import io.netty.handler.codec.http.FullHttpMessage; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaderValues; +import io.netty.handler.codec.http.HttpMessage; +import io.netty.handler.codec.http.HttpObject; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpScheme; +import io.netty.handler.codec.http.HttpUtil; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.codec.http2.DefaultHttp2DataFrame; +import io.netty.handler.codec.http2.DefaultHttp2HeadersFrame; +import io.netty.handler.codec.http2.Http2DataFrame; +import io.netty.handler.codec.http2.Http2Exception; +import io.netty.handler.codec.http2.Http2Headers; +import io.netty.handler.codec.http2.Http2HeadersFrame; +import io.netty.handler.codec.http2.Http2MultiplexCodec; +import io.netty.handler.codec.http2.Http2StreamChannel; +import io.netty.handler.codec.http2.Http2StreamFrame; +import io.netty.handler.codec.http2.HttpConversionUtil; +import io.netty.handler.ssl.SslHandler; +import io.netty.util.internal.UnstableApi; + +import java.util.List; + +/** + * This handler converts from {@link Http2StreamFrame} to {@link HttpObject}, + * and back. It can be used as an adapter in conjunction with {@link + * Http2MultiplexCodec} to make http/2 connections backward-compatible with + * {@link ChannelHandler}s expecting {@link HttpObject}. + * + * For simplicity, it converts to chunked encoding unless the entire stream + * is a single header. + * + * Patched version of original Netty's Http2StreamFrameToHttpObjectCodec. + * This one is using the streamId from {@code frame.stream().id()}. + */ +@UnstableApi +@Sharable +public class Http2StreamFrameToHttpObjectCodec extends MessageToMessageCodec { + + private final boolean isServer; + + private final boolean validateHeaders; + + private HttpScheme scheme; + + public Http2StreamFrameToHttpObjectCodec(final boolean isServer, + final boolean validateHeaders) { + this.isServer = isServer; + this.validateHeaders = validateHeaders; + scheme = HttpScheme.HTTP; + } + + public Http2StreamFrameToHttpObjectCodec(final boolean isServer) { + this(isServer, true); + } + + @Override + public boolean acceptInboundMessage(Object msg) throws Exception { + return (msg instanceof Http2HeadersFrame) || (msg instanceof Http2DataFrame); + } + + @Override + protected void decode(ChannelHandlerContext ctx, Http2StreamFrame frame, List out) throws Exception { + if (frame instanceof Http2HeadersFrame) { + int id = frame.stream() != null ? frame.stream().id() : -1; + Http2HeadersFrame headersFrame = (Http2HeadersFrame) frame; + Http2Headers headers = headersFrame.headers(); + + final CharSequence status = headers.status(); + + // 100-continue response is a special case where Http2HeadersFrame#isEndStream=false + // but we need to decode it as a FullHttpResponse to play nice with HttpObjectAggregator. + if (null != status && HttpResponseStatus.CONTINUE.codeAsText().contentEquals(status)) { + final FullHttpMessage fullMsg = newFullMessage(id, headers, ctx.alloc()); + out.add(fullMsg); + return; + } + + if (headersFrame.isEndStream()) { + if (headers.method() == null && status == null) { + LastHttpContent last = new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER, validateHeaders); + HttpConversionUtil.addHttp2ToHttpHeaders(id, headers, last.trailingHeaders(), + HttpVersion.HTTP_1_1, true, true); + out.add(last); + } else { + FullHttpMessage full = newFullMessage(id, headers, ctx.alloc()); + out.add(full); + } + } else { + HttpMessage req = newMessage(id, headers); + if (!HttpUtil.isContentLengthSet(req)) { + req.headers().add(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED); + } + out.add(req); + } + } else if (frame instanceof Http2DataFrame) { + Http2DataFrame dataFrame = (Http2DataFrame) frame; + if (dataFrame.isEndStream()) { + out.add(new DefaultLastHttpContent(dataFrame.content().retain(), validateHeaders)); + } else { + out.add(new DefaultHttpContent(dataFrame.content().retain())); + } + } + } + + private void encodeLastContent(LastHttpContent last, List out) { + boolean needFiller = !(last instanceof FullHttpMessage) && last.trailingHeaders().isEmpty(); + if (last.content().isReadable() || needFiller) { + out.add(new DefaultHttp2DataFrame(last.content().retain(), last.trailingHeaders().isEmpty())); + } + if (!last.trailingHeaders().isEmpty()) { + Http2Headers headers = HttpConversionUtil.toHttp2Headers(last.trailingHeaders(), validateHeaders); + out.add(new DefaultHttp2HeadersFrame(headers, true)); + } + } + + /** + * Encode from an {@link HttpObject} to an {@link Http2StreamFrame}. This method will + * be called for each written message that can be handled by this encoder. + * + * NOTE: 100-Continue responses that are NOT {@link FullHttpResponse} will be rejected. + * + * @param ctx the {@link ChannelHandlerContext} which this handler belongs to + * @param obj the {@link HttpObject} message to encode + * @param out the {@link List} into which the encoded msg should be added + * needs to do some kind of aggregation + * @throws Exception is thrown if an error occurs + */ + @Override + protected void encode(ChannelHandlerContext ctx, HttpObject obj, List out) throws Exception { + // 100-continue is typically a FullHttpResponse, but the decoded + // Http2HeadersFrame should not be marked as endStream=true + if (obj instanceof HttpResponse) { + final HttpResponse res = (HttpResponse) obj; + if (res.status().equals(HttpResponseStatus.CONTINUE)) { + if (res instanceof FullHttpResponse) { + final Http2Headers headers = toHttp2Headers(res); + out.add(new DefaultHttp2HeadersFrame(headers, false)); + return; + } else { + throw new EncoderException( + HttpResponseStatus.CONTINUE.toString() + " must be a FullHttpResponse"); + } + } + } + + if (obj instanceof HttpMessage) { + Http2Headers headers = toHttp2Headers((HttpMessage) obj); + boolean noMoreFrames = false; + if (obj instanceof FullHttpMessage) { + FullHttpMessage full = (FullHttpMessage) obj; + noMoreFrames = !full.content().isReadable() && full.trailingHeaders().isEmpty(); + } + + out.add(new DefaultHttp2HeadersFrame(headers, noMoreFrames)); + } + + if (obj instanceof LastHttpContent) { + LastHttpContent last = (LastHttpContent) obj; + encodeLastContent(last, out); + } else if (obj instanceof HttpContent) { + HttpContent cont = (HttpContent) obj; + out.add(new DefaultHttp2DataFrame(cont.content().retain(), false)); + } + } + + private Http2Headers toHttp2Headers(final HttpMessage msg) { + if (msg instanceof HttpRequest) { + msg.headers().set( + HttpConversionUtil.ExtensionHeaderNames.SCHEME.text(), + scheme.name()); + } + + return HttpConversionUtil.toHttp2Headers(msg, validateHeaders); + } + + private HttpMessage newMessage(final int id, + final Http2Headers headers) throws Http2Exception { + return isServer ? + HttpConversionUtil.toHttpRequest(id, headers, validateHeaders) : + HttpConversionUtil.toHttpResponse(id, headers, validateHeaders); + } + + private FullHttpMessage newFullMessage(final int id, + final Http2Headers headers, + final ByteBufAllocator alloc) throws Http2Exception { + return isServer ? + HttpConversionUtil.toFullHttpRequest(id, headers, alloc, validateHeaders) : + HttpConversionUtil.toFullHttpResponse(id, headers, alloc, validateHeaders); + } + + @Override + public void handlerAdded(final ChannelHandlerContext ctx) throws Exception { + super.handlerAdded(ctx); + + // this handler is typically used on an Http2StreamChannel. at this + // stage, ssl handshake should've been established. checking for the + // presence of SslHandler in the parent's channel pipeline to + // determine the HTTP scheme should suffice, even for the case where + // SniHandler is used. + scheme = isSsl(ctx) ? HttpScheme.HTTPS : HttpScheme.HTTP; + } + + protected boolean isSsl(final ChannelHandlerContext ctx) { + final Channel ch = ctx.channel(); + final Channel connChannel = (ch instanceof Http2StreamChannel) ? ch.parent() : ch; + return null != connChannel.pipeline().get(SslHandler.class); + } +} diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/listener/HttpHeadersListener.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/listener/HttpHeadersListener.java deleted file mode 100644 index 311436c..0000000 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/listener/HttpHeadersListener.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.xbib.netty.http.client.listener; - -import io.netty.handler.codec.http.HttpHeaders; - -@FunctionalInterface -public interface HttpHeadersListener { - - void onHeaders(HttpHeaders httpHeaders); -} diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/listener/HttpResponseListener.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/listener/ResponseListener.java similarity index 81% rename from netty-http-client/src/main/java/org/xbib/netty/http/client/listener/HttpResponseListener.java rename to netty-http-client/src/main/java/org/xbib/netty/http/client/listener/ResponseListener.java index f06b1ee..5acd2a2 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/listener/HttpResponseListener.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/listener/ResponseListener.java @@ -3,7 +3,7 @@ package org.xbib.netty.http.client.listener; import io.netty.handler.codec.http.FullHttpResponse; @FunctionalInterface -public interface HttpResponseListener { +public interface ResponseListener { void onResponse(FullHttpResponse fullHttpResponse); } diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/pool/BoundedChannelPool.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/pool/BoundedChannelPool.java index 3181322..b66d368 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/pool/BoundedChannelPool.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/pool/BoundedChannelPool.java @@ -5,12 +5,15 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPromise; import io.netty.channel.pool.ChannelPoolHandler; import io.netty.channel.socket.SocketChannel; import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http2.DefaultHttp2GoAwayFrame; import io.netty.util.AttributeKey; import org.xbib.netty.http.common.PoolKey; +import java.io.IOException; import java.net.ConnectException; import java.util.ArrayList; import java.util.HashMap; @@ -21,6 +24,7 @@ import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.locks.Lock; @@ -36,8 +40,6 @@ public class BoundedChannelPool implements Pool { private final HttpVersion httpVersion; - private final boolean isSecure; - private final ChannelPoolHandler channelPoolhandler; private final List nodes; @@ -60,26 +62,36 @@ public class BoundedChannelPool implements Pool { private final AttributeKey attributeKey; + private PoolKeySelector poolKeySelector; + /** * @param semaphore the concurrency level * @param httpVersion the HTTP version of the pool connections - * @param isSecure if this pool has secure connections * @param nodes the endpoint nodes, any element may contain the port (followed after ":") * to override the defaultPort argument * @param bootstrap bootstrap instance * @param channelPoolHandler channel pool handler being notified upon new connection is created * @param retriesPerNode the max count of the subsequent connection failures to the node before * the node will be excluded from the pool. If set to 0, the value is ignored. + * @param poolKeySelectorType pool key selector type */ - public BoundedChannelPool(Semaphore semaphore, HttpVersion httpVersion, boolean isSecure, + public BoundedChannelPool(Semaphore semaphore, HttpVersion httpVersion, List nodes, Bootstrap bootstrap, - ChannelPoolHandler channelPoolHandler, int retriesPerNode) { + ChannelPoolHandler channelPoolHandler, int retriesPerNode, + PoolKeySelectorType poolKeySelectorType) { this.semaphore = semaphore; this.httpVersion = httpVersion; - this.isSecure = isSecure; this.channelPoolhandler = channelPoolHandler; this.nodes = nodes; this.retriesPerNode = retriesPerNode; + switch (poolKeySelectorType) { + case RANDOM: + this.poolKeySelector = new RandomPoolKeySelector(); + break; + case ROUNDROBIN: + this.poolKeySelector = new RoundRobinKeySelector(); + break; + } this.lock = new ReentrantLock(); this.attributeKey = AttributeKey.valueOf("poolKey"); if (nodes == null || nodes.isEmpty()) { @@ -105,10 +117,6 @@ public class BoundedChannelPool implements Pool { return httpVersion; } - public boolean isSecure() { - return isSecure; - } - public AttributeKey getAttributeKey() { return attributeKey; } @@ -121,7 +129,7 @@ public class BoundedChannelPool implements Pool { for (int i = 0; i < channelCount; i++) { Channel channel = newConnection(); if (channel == null) { - throw new ConnectException("failed to prepare"); + throw new ConnectException("failed to prepare channels"); } K key = channel.attr(attributeKey).get(); if (channel.isActive()) { @@ -133,7 +141,7 @@ public class BoundedChannelPool implements Pool { channel.close(); } } - logger.log(Level.FINE,"prepared " + channelCount + " channels"); + logger.log(Level.FINE,"prepared " + channelCount + " channels: " + availableChannels); } @Override @@ -156,35 +164,7 @@ public class BoundedChannelPool implements Pool { } @Override - public int acquire(List channels, int maxCount) throws Exception { - int availableCount = semaphore.drainPermits(); - if (availableCount == 0) { - return availableCount; - } - if (availableCount > maxCount) { - semaphore.release(availableCount - maxCount); - availableCount = maxCount; - } - Channel channel; - for (int i = 0; i < availableCount; i ++) { - if ((channel = poll()) == null) { - channel = newConnection(); - } - if (channel == null) { - semaphore.release(availableCount - i); - throw new ConnectException(); - } else { - if (channelPoolhandler != null) { - channelPoolhandler.channelAcquired(channel); - } - channels.add(channel); - } - } - return availableCount; - } - - @Override - public void release(Channel channel) throws Exception { + public void release(Channel channel, boolean close) throws Exception { try { if (channel != null) { if (channel.isActive()) { @@ -193,10 +173,9 @@ public class BoundedChannelPool implements Pool { if (channelQueue != null) { channelQueue.add(channel); } - } else if (channel.isOpen()) { + } else if (channel.isOpen() && close) { + logger.log(Level.FINE, "trying to close channel " + channel); channel.close(); - } else { - logger.log(Level.WARNING, "channel not active or open while release"); } if (channelPoolhandler != null) { channelPoolhandler.channelReleased(channel); @@ -208,16 +187,10 @@ public class BoundedChannelPool implements Pool { } @Override - public void release(List channels) throws Exception { - for (Channel channel : channels) { - release(channel); - } - } - - @Override - public void close() { + public void close() throws IOException { lock.lock(); try { + logger.log(Level.FINE, "closing pool"); int count = 0; Set channelSet = new HashSet<>(); for (Map.Entry> entry : availableChannels.entrySet()) { @@ -228,7 +201,21 @@ public class BoundedChannelPool implements Pool { } for (Channel channel : channelSet) { if (channel != null && channel.isOpen()) { - logger.log(Level.FINE, "closing channel " + channel); + logger.log(Level.FINE, "trying to abort channel " + channel); + if (httpVersion.majorVersion() == 2) { + // be polite, send a go away frame + DefaultHttp2GoAwayFrame goAwayFrame = new DefaultHttp2GoAwayFrame(0); + ChannelPromise channelPromise = channel.newPromise(); + channel.writeAndFlush(goAwayFrame, channelPromise); + try { + channelPromise.get(); + logger.log(Level.FINE, "goaway frame sent to " + channel); + } catch (ExecutionException e) { + // ignore error if goaway can not be sent + } catch (InterruptedException e) { + throw new IOException(e); + } + } channel.close(); count++; } @@ -237,7 +224,7 @@ public class BoundedChannelPool implements Pool { channels.clear(); bootstraps.clear(); counts.clear(); - logger.log(Level.FINE, "closed " + count + " connections"); + logger.log(Level.FINE, "closed pool (found " + count + " connections open)"); } finally { lock.unlock(); } @@ -246,17 +233,13 @@ public class BoundedChannelPool implements Pool { private Channel newConnection() throws ConnectException { Channel channel = null; K key = null; - K nextKey; - int min = Integer.MAX_VALUE; - int next; - int i = ThreadLocalRandom.current().nextInt(numberOfNodes); - for (int j = i; j < numberOfNodes; j ++) { - nextKey = nodes.get(j % numberOfNodes); - if (counts == null) { - throw new ConnectException("strange"); - } + Integer min = Integer.MAX_VALUE; + Integer next; + //int r = ThreadLocalRandom.current().nextInt(numberOfNodes); + for (int j = 0; j < numberOfNodes; j++) { + K nextKey = poolKeySelector.key(); //nodes.get(j % numberOfNodes); next = counts.get(nextKey); - if (next == 0) { + if (next == null || next == 0) { key = nextKey; break; } else if (next < min) { @@ -305,7 +288,6 @@ public class BoundedChannelPool implements Pool { if (retriesPerNode > 0) { failedCounts.put(key, 0); } - logger.log(Level.FINE,"new connection to " + key + " created"); } return channel; } @@ -319,19 +301,11 @@ public class BoundedChannelPool implements Pool { } private Channel poll() { - int i = ThreadLocalRandom.current().nextInt(numberOfNodes); Queue channelQueue; Channel channel; - for(int j = i; j < i + numberOfNodes; j ++) { - K key = nodes.get(j % numberOfNodes); - // for HTTP/2, use channel list - logger.log(Level.FINE, "pool version = " + httpVersion); - if (httpVersion.majorVersion() == 2) { - List list = channels.get(key); - if (!list.isEmpty()) { - logger.log(Level.INFO, "we have a channel " + list); - } - } + //int r = ThreadLocalRandom.current().nextInt(numberOfNodes); + for (int j = 0; j < numberOfNodes; j++) { + K key = poolKeySelector.key(); //nodes.get(j % numberOfNodes); channelQueue = availableChannels.get(key); if (channelQueue != null) { channel = channelQueue.poll(); @@ -339,12 +313,39 @@ public class BoundedChannelPool implements Pool { return channel; } } else { - logger.log(Level.FINE, "channelqueue is null"); + logger.log(Level.WARNING, "channel queue is null?"); } } return null; } + public enum PoolKeySelectorType { + RANDOM, ROUNDROBIN + } + + private interface PoolKeySelector { + K key(); + } + + private class RandomPoolKeySelector implements PoolKeySelector { + + @Override + public K key() { + int r = ThreadLocalRandom.current().nextInt(numberOfNodes); + return nodes.get(r % numberOfNodes); + } + } + + private class RoundRobinKeySelector implements PoolKeySelector { + + int r = 0; + + @Override + public K key() { + return nodes.get(r++ % numberOfNodes); + } + } + private class CloseChannelListener implements ChannelFutureListener { private final K key; diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/pool/Pool.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/pool/Pool.java index 041759c..a100dc2 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/pool/Pool.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/pool/Pool.java @@ -1,7 +1,6 @@ package org.xbib.netty.http.client.pool; import java.io.Closeable; -import java.util.List; public interface Pool extends Closeable { @@ -9,9 +8,5 @@ public interface Pool extends Closeable { T acquire() throws Exception; - int acquire(List list, int maxCount) throws Exception; - - void release(T t) throws Exception; - - void release(List list) throws Exception; + void release(T t, boolean close) throws Exception; } diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/rest/RestClient.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/rest/RestClient.java index e5375ad..92f44d7 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/rest/RestClient.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/rest/RestClient.java @@ -13,12 +13,9 @@ import org.xbib.netty.http.client.transport.Transport; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.logging.Logger; public class RestClient { - private static final Logger logger = Logger.getLogger(RestClient.class.getName()); - private Client client; private Transport transport; @@ -70,6 +67,4 @@ public class RestClient { transport.execute(requestBuilder.build().setResponseListener(restClient::setResponse)).get(); return restClient; } - - } diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/retry/BackOff.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/retry/BackOff.java index b346438..bd1ca97 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/retry/BackOff.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/retry/BackOff.java @@ -1,7 +1,5 @@ package org.xbib.netty.http.client.retry; -import java.io.IOException; - /** * Back-off policy when retrying an operation. */ @@ -20,6 +18,8 @@ public interface BackOff { * Gets the number of milliseconds to wait before retrying the operation or {@link #STOP} to * indicate that no retries should be made. * + * @return milliseconds before operation retry + * *

* Example usage: *

diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/retry/ExponentialBackOff.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/retry/ExponentialBackOff.java index 8f20b8c..7a64ce6 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/retry/ExponentialBackOff.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/retry/ExponentialBackOff.java @@ -185,6 +185,10 @@ public class ExponentialBackOff implements BackOff { /** * Returns a random value from the interval [randomizationFactor * currentInterval, * randomizationFactor * currentInterval]. + * @param randomizationFactor the randomization factor + * @param random scaling factor + * @param currentIntervalMillis milliseconds + * @return random value */ public static int getRandomValueFromInterval(double randomizationFactor, double random, int currentIntervalMillis) { double delta = randomizationFactor * currentIntervalMillis; @@ -196,14 +200,17 @@ public class ExponentialBackOff implements BackOff { return (int) (minInterval + (random * (maxInterval - minInterval + 1))); } - /** Returns the initial retry interval in milliseconds. */ + /** + * Returns the initial retry interval in milliseconds. + * @return interval milliseconds + */ public final int getInitialIntervalMillis() { return initialIntervalMillis; } /** * Returns the randomization factor to use for creating a range around the retry interval. - * + * @return randomization factor *

* A randomization factor of 0.5 results in a random period ranging between 50% below and 50% * above the retry interval. @@ -215,6 +222,7 @@ public class ExponentialBackOff implements BackOff { /** * Returns the current retry interval in milliseconds. + * @return current interval in milliseconds */ public final int getCurrentIntervalMillis() { return currentIntervalMillis; @@ -222,6 +230,7 @@ public class ExponentialBackOff implements BackOff { /** * Returns the value to multiply the current interval with for each retry attempt. + * @return multiplier */ public final double getMultiplier() { return multiplier; @@ -230,6 +239,7 @@ public class ExponentialBackOff implements BackOff { /** * Returns the maximum value of the back off period in milliseconds. Once the current interval * reaches this value it stops increasing. + * @return maximum interval value in milliseconds */ public final int getMaxIntervalMillis() { return maxIntervalMillis; @@ -237,7 +247,7 @@ public class ExponentialBackOff implements BackOff { /** * Returns the maximum elapsed time in milliseconds. - * + * @return maximum elapsed time in milliseconds *

* If the time elapsed since an {@link ExponentialBackOff} instance is created goes past the * max_elapsed_time then the method {@link #nextBackOffMillis()} starts returning @@ -251,7 +261,7 @@ public class ExponentialBackOff implements BackOff { /** * Returns the elapsed time in milliseconds since an {@link ExponentialBackOff} instance is * created and is reset when {@link #reset()} is called. - * + * @return the elapsed time in milliseconds *

* The elapsed time is computed using {@link System#nanoTime()}. *

@@ -286,6 +296,7 @@ public class ExponentialBackOff implements BackOff { /** * Returns the current value of the most precise available system timer, in nanoseconds for use to * measure elapsed time, to match the behavior of {@link System#nanoTime()}. + * @return value of timer in nanoseconds */ long nanoTime(); @@ -345,7 +356,8 @@ public class ExponentialBackOff implements BackOff { /** * Builds a new instance of {@link ExponentialBackOff}. - * */ + * @return an {@link ExponentialBackOff} instance + */ public ExponentialBackOff build() { if (initialIntervalMillis <= 0) { throw new IllegalArgumentException(); @@ -365,17 +377,11 @@ public class ExponentialBackOff implements BackOff { return new ExponentialBackOff(this); } - /** - * Returns the initial retry interval in milliseconds. The default value is - * {@link #DEFAULT_INITIAL_INTERVAL_MILLIS}. - */ - public final int getInitialIntervalMillis() { - return initialIntervalMillis; - } - /** * Sets the initial retry interval in milliseconds. The default value is * {@link #DEFAULT_INITIAL_INTERVAL_MILLIS}. Must be {@code > 0}. + * @param initialIntervalMillis interval milliseconds + * @return the builder * *

* Overriding is only supported for the purpose of calling the super implementation and changing @@ -387,28 +393,12 @@ public class ExponentialBackOff implements BackOff { return this; } - /** - * Returns the randomization factor to use for creating a range around the retry interval. The - * default value is {@link #DEFAULT_RANDOMIZATION_FACTOR}. - * - *

- * A randomization factor of 0.5 results in a random period ranging between 50% below and 50% - * above the retry interval. - *

- * - *

- * Overriding is only supported for the purpose of calling the super implementation and changing - * the return type, but nothing else. - *

- */ - public final double getRandomizationFactor() { - return randomizationFactor; - } - /** * Sets the randomization factor to use for creating a range around the retry interval. The * default value is {@link #DEFAULT_RANDOMIZATION_FACTOR}. Must fall in the range * {@code 0 <= randomizationFactor < 1}. + * @param randomizationFactor the randomization factor + * @return the builder * *

* A randomization factor of 0.5 results in a random period ranging between 50% below and 50% @@ -425,17 +415,11 @@ public class ExponentialBackOff implements BackOff { return this; } - /** - * Returns the value to multiply the current interval with for each retry attempt. The default - * value is {@link #DEFAULT_MULTIPLIER}. - */ - public final double getMultiplier() { - return multiplier; - } - /** * Sets the value to multiply the current interval with for each retry attempt. The default * value is {@link #DEFAULT_MULTIPLIER}. Must be {@code >= 1}. + * @param multiplier the multiplier + * @return the builder * *

* Overriding is only supported for the purpose of calling the super implementation and changing @@ -447,19 +431,12 @@ public class ExponentialBackOff implements BackOff { return this; } - /** - * Returns the maximum value of the back off period in milliseconds. Once the current interval - * reaches this value it stops increasing. The default value is - * {@link #DEFAULT_MAX_INTERVAL_MILLIS}. Must be {@code >= initialInterval}. - */ - public final int getMaxIntervalMillis() { - return maxIntervalMillis; - } - /** * Sets the maximum value of the back off period in milliseconds. Once the current interval * reaches this value it stops increasing. The default value is * {@link #DEFAULT_MAX_INTERVAL_MILLIS}. + * @param maxIntervalMillis maximum interval in miliseconds + * @return the builder * *

* Overriding is only supported for the purpose of calling the super implementation and changing @@ -471,23 +448,11 @@ public class ExponentialBackOff implements BackOff { return this; } - /** - * Returns the maximum elapsed time in milliseconds. The default value is - * {@link #DEFAULT_MAX_ELAPSED_TIME_MILLIS}. - * - *

- * If the time elapsed since an {@link ExponentialBackOff} instance is created goes past the - * max_elapsed_time then the method {@link #nextBackOffMillis()} starts returning - * {@link BackOff#STOP}. The elapsed time can be reset by calling {@link #reset()}. - *

- */ - public final int getMaxElapsedTimeMillis() { - return maxElapsedTimeMillis; - } - /** * Sets the maximum elapsed time in milliseconds. The default value is * {@link #DEFAULT_MAX_ELAPSED_TIME_MILLIS}. Must be {@code > 0}. + * @param maxElapsedTimeMillis maximum elapsed time millis + * @return the builder * *

* If the time elapsed since an {@link ExponentialBackOff} instance is created goes past the @@ -505,16 +470,10 @@ public class ExponentialBackOff implements BackOff { return this; } - /** - * Returns the nano clock. - */ - public final NanoClock getNanoClock() { - return nanoClock; - } - /** * Sets the nano clock ({@link NanoClock#SYSTEM} by default). - * + * @param nanoClock the nano clock + * @return the builder *

* Overriding is only supported for the purpose of calling the super implementation and changing * the return type, but nothing else. diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/BaseTransport.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/BaseTransport.java index fe9e9d9..cb38236 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/BaseTransport.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/BaseTransport.java @@ -1,16 +1,10 @@ package org.xbib.netty.http.client.transport; import io.netty.channel.Channel; -import io.netty.handler.codec.http.DefaultFullHttpRequest; -import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.FullHttpResponse; import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.cookie.ClientCookieDecoder; -import io.netty.handler.codec.http.cookie.ClientCookieEncoder; import io.netty.handler.codec.http.cookie.Cookie; -import io.netty.handler.codec.http2.HttpConversionUtil; import org.xbib.net.PercentDecoder; import org.xbib.net.URL; import org.xbib.net.URLSyntaxException; @@ -18,8 +12,6 @@ import org.xbib.netty.http.client.Client; import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.client.Request; import org.xbib.netty.http.client.RequestBuilder; -import org.xbib.netty.http.client.listener.CookieListener; -import org.xbib.netty.http.client.listener.HttpHeadersListener; import org.xbib.netty.http.client.retry.BackOff; import java.io.IOException; @@ -27,15 +19,15 @@ import java.net.ConnectException; import java.nio.charset.MalformedInputException; import java.nio.charset.StandardCharsets; import java.nio.charset.UnmappableCharacterException; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.SortedMap; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; @@ -49,64 +41,26 @@ abstract class BaseTransport implements Transport { protected final HttpAddress httpAddress; - protected Channel channel; - - protected SortedMap requests; - protected Throwable throwable; + private static final Request DUMMY = Request.builder(HttpMethod.GET).build(); + + private final Map channels; + + final Map channelFlowMap; + + final SortedMap requests; + private Map cookieBox; BaseTransport(Client client, HttpAddress httpAddress) { this.client = client; this.httpAddress = httpAddress; + this.channels = new ConcurrentHashMap<>(); + this.channelFlowMap = new ConcurrentHashMap<>(); this.requests = new ConcurrentSkipListMap<>(); } - @Override - public Transport execute(Request request) throws IOException { - ensureConnect(); - if (throwable != null) { - return this; - } - // Some HTTP 1 servers do not understand URIs in HTTP command line in spite of RFC 7230. - // The "origin form" requires a "Host" header. - // Our algorithm is: use always "origin form" for HTTP 1, use absolute form for HTTP 2. - // The reason is that Netty derives the HTTP/2 scheme header from the absolute form. - String uri = request.httpVersion().majorVersion() == 1 ? - request.url().relativeReference() : request.url().toString(); - FullHttpRequest fullHttpRequest = request.content() == null ? - new DefaultFullHttpRequest(request.httpVersion(), request.httpMethod(), uri) : - new DefaultFullHttpRequest(request.httpVersion(), request.httpMethod(), uri, - request.content()); - Integer streamId = nextStream(); - if (streamId != null && streamId > 0) { - request.headers().set(HttpConversionUtil.ExtensionHeaderNames.STREAM_ID.text(), Integer.toString(streamId)); - } else { - if (request.httpVersion().majorVersion() == 2) { - logger.log(Level.WARNING, "no streamId but HTTP/2 request. Strange!!! " + getClass().getName()); - } - } - // add matching cookies from box (previous requests) and new cookies from request builder - Collection cookies = new ArrayList<>(); - cookies.addAll(matchCookiesFromBox(request)); - cookies.addAll(matchCookies(request)); - if (!cookies.isEmpty()) { - request.headers().set(HttpHeaderNames.COOKIE, ClientCookieEncoder.STRICT.encode(cookies)); - } - // add stream-id and cookie headers - fullHttpRequest.headers().set(request.headers()); - if (streamId != null) { - requests.put(streamId, request); - } - // flush after putting request into requests map - if (channel.isWritable()) { - channel.writeAndFlush(fullHttpRequest); - - } - return this; - } - /** * Experimental method for executing in a wrapping completable future. * @param request request @@ -124,9 +78,8 @@ abstract class BaseTransport implements Transport { } @Override - public synchronized void close() throws IOException { + public synchronized void close() { get(); - client.releaseChannel(channel); } @Override @@ -139,53 +92,130 @@ abstract class BaseTransport implements Transport { return throwable; } + /** + * The underlying network layer failed, not possible to know the request. + * So we fail all (open) promises. + * @param throwable the exception + */ @Override - public void headersReceived(Integer streamId, HttpHeaders httpHeaders) { - Request request = fromStreamId(streamId); - if (request != null) { - HttpHeadersListener httpHeadersListener = request.getHeadersListener(); - if (httpHeadersListener != null) { - httpHeadersListener.onHeaders(httpHeaders); - } - for (String cookieString : httpHeaders.getAll(HttpHeaderNames.SET_COOKIE)) { - Cookie cookie = ClientCookieDecoder.STRICT.decode(cookieString); - addCookie(cookie); - CookieListener cookieListener = request.getCookieListener(); - if (cookieListener != null) { - cookieListener.onCookie(cookie); - } - } + public void fail(Throwable throwable) { + // do not fail more than once + if (this.throwable != null) { + return; + } + logger.log(Level.SEVERE, "failing: " + throwable.getMessage(), throwable); + this.throwable = throwable; + for (Flow flow : channelFlowMap.values()) { + flow.fail(throwable); } } - private void ensureConnect() throws IOException { - if (channel == null) { - channel = client.newChannel(httpAddress); - if (channel != null) { - channel.attr(TRANSPORT_ATTRIBUTE_KEY).set(this); - awaitSettings(); + @Override + public Transport get() { + return get(client.getClientConfig().getReadTimeoutMillis(), TimeUnit.MILLISECONDS); + } + + @Override + public Transport get(long value, TimeUnit timeUnit) { + for (Map.Entry entry : channelFlowMap.entrySet()) { + Flow flow = entry.getValue(); + for (Integer key : flow.keys()) { + try { + flow.get(key).get(value, timeUnit); + } catch (Exception e) { + String requestKey = getRequestKey(entry.getKey(), key); + Request request = requests.get(requestKey); + if (request != null && request.getCompletableFuture() != null) { + request.getCompletableFuture().completeExceptionally(e); + } + flow.fail(e); + } finally { + flow.remove(key); + } + } + flow.close(); + } + channels.values().forEach(channel -> { + try { + client.releaseChannel(channel, true); + } catch (IOException e) { + logger.log(Level.WARNING, e.getMessage(), e); + } + }); + channelFlowMap.clear(); + channels.clear(); + requests.clear(); + return this; + } + + @Override + public void cancel() { + for (Map.Entry entry : channelFlowMap.entrySet()) { + Flow flow = entry.getValue(); + for (Integer key : flow.keys()) { + try { + flow.get(key).cancel(true); + } catch (Exception e) { + String requestKey = getRequestKey(entry.getKey(), key); + Request request = requests.get(requestKey); + if (request != null && request.getCompletableFuture() != null) { + request.getCompletableFuture().completeExceptionally(e); + } + flow.fail(e); + } finally { + flow.remove(key); + } + } + flow.close(); + } + channels.values().forEach(channel -> { + try { + client.releaseChannel(channel, true); + } catch (IOException e) { + logger.log(Level.WARNING, e.getMessage(), e); + } + }); + channelFlowMap.clear(); + channels.clear(); + requests.clear(); + } + + protected abstract String getRequestKey(String channelId, Integer streamId); + + Channel mapChannel(Request request) throws IOException { + Channel channel; + if (!client.hasPooledConnections()) { + channel = channels.get(DUMMY); + if (channel == null) { + channel = switchNextChannel(); + } + channels.put(DUMMY, channel); + } else { + channel = switchNextChannel(); + channels.put(request, channel); + } + return channel; + } + + private Channel switchNextChannel() throws IOException { + Channel channel = client.newChannel(httpAddress); + if (channel != null) { + channel.attr(TRANSPORT_ATTRIBUTE_KEY).set(this); + waitForSettings(); + } else { + ConnectException connectException; + if (httpAddress != null) { + connectException = new ConnectException("unable to connect to " + httpAddress); + } else if (client.hasPooledConnections()) { + connectException = new ConnectException("unable to get channel from pool"); } else { - ConnectException connectException; - if (httpAddress != null) { - connectException = new ConnectException("unable to connect to " + httpAddress); - } else if (client.hasPooledConnections()){ - connectException = new ConnectException("unable to get channel from pool"); - } else { - // if API misuse - connectException = new ConnectException("unable to get channel"); - } - this.throwable = connectException; - this.channel = null; - throw connectException; + // API misuse + connectException = new ConnectException("unable to get channel"); } + this.throwable = connectException; + throw connectException; } - } - - protected Request fromStreamId(Integer streamId) { - if (streamId == null) { - streamId = requests.lastKey(); - } - return requests.get(streamId); + return channel; } protected Request continuation(Request request, FullHttpResponse httpResponse) throws URLSyntaxException { @@ -224,7 +254,6 @@ abstract class BaseTransport implements Transport { request.cookies().forEach(newHttpRequestBuilder::addCookie); Request newHttpRequest = newHttpRequestBuilder.build(); newHttpRequest.setResponseListener(request.getResponseListener()); - newHttpRequest.setHeadersListener(request.getHeadersListener()); newHttpRequest.setCookieListener(request.getCookieListener()); StringBuilder hostAndPort = new StringBuilder(); hostAndPort.append(redirUrl.getHost()); @@ -235,6 +264,7 @@ abstract class BaseTransport implements Transport { logger.log(Level.FINE, "redirect url: " + redirUrl + " old request: " + request.toString() + " new request: " + newHttpRequest.toString()); + request.release(); return newHttpRequest; } break; @@ -297,20 +327,20 @@ abstract class BaseTransport implements Transport { return cookieBox; } - private void addCookie(Cookie cookie) { + void addCookie(Cookie cookie) { if (cookieBox == null) { this.cookieBox = Collections.synchronizedMap(new LRUCache(32)); } cookieBox.put(cookie, true); } - private List matchCookiesFromBox(Request request) { + List matchCookiesFromBox(Request request) { return cookieBox == null ? Collections.emptyList() : cookieBox.keySet().stream().filter(cookie -> matchCookie(request.url(), cookie) ).collect(Collectors.toList()); } - private List matchCookies(Request request) { + List matchCookies(Request request) { return request.cookies().stream().filter(cookie -> matchCookie(request.url(), cookie) ).collect(Collectors.toList()); diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Flow.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Flow.java new file mode 100644 index 0000000..6147e31 --- /dev/null +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Flow.java @@ -0,0 +1,72 @@ +package org.xbib.netty.http.client.transport; + +import java.util.Set; +import java.util.SortedMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicInteger; + +class Flow { + + private final AtomicInteger counter; + + private final SortedMap> map; + + Flow() { + this.counter = new AtomicInteger(3); + this.map = new ConcurrentSkipListMap<>(); + } + + CompletableFuture get(Integer key) { + return map.get(key); + } + + Set keys() { + return map.keySet(); + } + + Integer firstKey() { + return map.firstKey(); + } + + Integer lastKey() { + return map.lastKey(); + } + + void put(Integer key, CompletableFuture promise) { + map.put(key, promise); + } + + void remove(Integer key) { + if (key != null) { + map.remove(key); + } + } + + Integer nextStreamId() { + Integer streamId = counter.getAndAdd(2); + if (streamId == Integer.MIN_VALUE) { + // reset if overflow, Java wraps atomic integers to Integer.MIN_VALUE + // should we send a GOAWAY? + counter.set(3); + streamId = 3; + } + map.put(streamId, new CompletableFuture<>()); + return streamId; + } + + void fail(Throwable throwable) { + for (CompletableFuture promise : map.values()) { + promise.completeExceptionally(throwable); + } + } + + public void close() { + map.clear(); + } + + @Override + public String toString() { + return "[next=" + counter + ", " + map + "]"; + } +} diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Http1Transport.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Http1Transport.java deleted file mode 100644 index 1da4eb9..0000000 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Http1Transport.java +++ /dev/null @@ -1,152 +0,0 @@ -package org.xbib.netty.http.client.transport; - -import io.netty.channel.Channel; -import io.netty.handler.codec.http.FullHttpResponse; -import io.netty.handler.codec.http2.Http2Headers; -import io.netty.handler.codec.http2.Http2Settings; -import org.xbib.net.URLSyntaxException; -import org.xbib.netty.http.client.Client; -import org.xbib.netty.http.common.HttpAddress; -import org.xbib.netty.http.client.Request; -import org.xbib.netty.http.client.listener.HttpResponseListener; - -import java.io.IOException; -import java.util.SortedMap; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class Http1Transport extends BaseTransport { - - private static final Logger logger = Logger.getLogger(Http1Transport.class.getName()); - - private final AtomicInteger sequentialCounter; - - private SortedMap> sequentialPromiseMap; - - public Http1Transport(Client client, HttpAddress httpAddress) { - super(client, httpAddress); - this.sequentialCounter = new AtomicInteger(); - this.sequentialPromiseMap = new ConcurrentSkipListMap<>(); - } - - @Override - public Integer nextStream() { - Integer streamId = sequentialCounter.getAndIncrement(); - if (streamId == Integer.MIN_VALUE) { - // reset if overflow, Java wraps atomic integers to Integer.MIN_VALUE - sequentialCounter.set(0); - streamId = 0; - } - sequentialPromiseMap.put(streamId, new CompletableFuture<>()); - return streamId; - } - - @Override - public void settingsReceived(Channel channel, Http2Settings http2Settings) { - } - - @Override - public void awaitSettings() { - } - - @Override - public void responseReceived(Integer streamId, FullHttpResponse fullHttpResponse) { - Request request = fromStreamId(streamId); - if (request != null) { - HttpResponseListener responseListener = request.getResponseListener(); - if (responseListener != null) { - responseListener.onResponse(fullHttpResponse); - } - } - try { - Request retryRequest = retry(request, fullHttpResponse); - if (retryRequest != null) { - // retry transport, wait for completion - client.retry(this, retryRequest); - } else { - Request continueRequest = continuation(request, fullHttpResponse); - if (continueRequest != null) { - // continue with new transport, synchronous call here, wait for completion - client.continuation(this, continueRequest); - } - } - } catch (URLSyntaxException | IOException e) { - logger.log(Level.WARNING, e.getMessage(), e); - } - if (!sequentialPromiseMap.isEmpty()) { - CompletableFuture promise = sequentialPromiseMap.get(sequentialPromiseMap.firstKey()); - if (promise != null) { - promise.complete(true); - } - } - } - - @Override - public void pushPromiseReceived(Integer streamId, Integer promisedStreamId, Http2Headers headers) { - } - - @Override - public void awaitResponse(Integer streamId) throws IOException, TimeoutException { - if (streamId == null) { - return; - } - if (throwable != null) { - return; - } - CompletableFuture promise = sequentialPromiseMap.get(streamId); - if (promise != null) { - long millis = client.getClientConfig().getReadTimeoutMillis(); - Request request = fromStreamId(streamId); - if (request != null && request.getTimeoutInMillis() > 0) { - millis = request.getTimeoutInMillis(); - } - try { - promise.get(millis, TimeUnit.MILLISECONDS); - } catch (TimeoutException e) { - this.throwable = e; - throw new TimeoutException("timeout of " + millis + " milliseconds exceeded"); - } catch (InterruptedException | ExecutionException e) { - this.throwable = e; - throw new IOException(e); - } finally { - sequentialPromiseMap.remove(streamId); - } - } - } - - @Override - public Transport get() { - try { - for (Integer streamId : sequentialPromiseMap.keySet()) { - awaitResponse(streamId); - client.releaseChannel(channel); - } - } catch (IOException | TimeoutException e) { - logger.log(Level.WARNING, e.getMessage(), e); - } finally { - sequentialPromiseMap.clear(); - } - return this; - } - - @Override - public void success() { - for (CompletableFuture promise : sequentialPromiseMap.values()) { - promise.complete(true); - } - } - - @Override - public void fail(Throwable throwable) { - this.throwable = throwable; - for (CompletableFuture promise : sequentialPromiseMap.values()) { - promise.completeExceptionally(throwable); - } - } -} diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Http2Transport.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Http2Transport.java index 0fe1ae1..b066a71 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Http2Transport.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Http2Transport.java @@ -1,23 +1,38 @@ package org.xbib.netty.http.client.transport; import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.cookie.ClientCookieDecoder; +import io.netty.handler.codec.http.cookie.ClientCookieEncoder; +import io.netty.handler.codec.http.cookie.Cookie; +import io.netty.handler.codec.http2.DefaultHttp2DataFrame; +import io.netty.handler.codec.http2.DefaultHttp2Headers; +import io.netty.handler.codec.http2.DefaultHttp2HeadersFrame; import io.netty.handler.codec.http2.Http2Headers; import io.netty.handler.codec.http2.Http2Settings; +import io.netty.handler.codec.http2.Http2StreamChannel; +import io.netty.handler.codec.http2.Http2StreamChannelBootstrap; +import io.netty.handler.codec.http2.HttpConversionUtil; import org.xbib.net.URLSyntaxException; import org.xbib.netty.http.client.Client; +import org.xbib.netty.http.client.handler.http2.Http2ResponseHandler; +import org.xbib.netty.http.client.handler.http2.Http2StreamFrameToHttpObjectCodec; +import org.xbib.netty.http.client.listener.CookieListener; import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.client.Request; -import org.xbib.netty.http.client.listener.HttpResponseListener; +import org.xbib.netty.http.client.listener.ResponseListener; import java.io.IOException; -import java.util.SortedMap; +import java.util.ArrayList; +import java.util.Collection; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; @@ -27,33 +42,76 @@ public class Http2Transport extends BaseTransport { private CompletableFuture settingsPromise; - private final AtomicInteger streamIdCounter; - - private SortedMap> streamidPromiseMap; + private final ChannelInitializer initializer; public Http2Transport(Client client, HttpAddress httpAddress) { super(client, httpAddress); - streamIdCounter = new AtomicInteger(3); - streamidPromiseMap = new ConcurrentSkipListMap<>(); - settingsPromise = (httpAddress != null /*&& httpAddress.isSecure() */) || - (client.hasPooledConnections() && client.getPool().isSecure()) ? - new CompletableFuture<>() : null; + this.settingsPromise = httpAddress != null ? new CompletableFuture<>() : null; + final Transport transport = this; + this.initializer = new ChannelInitializer() { + @Override + protected void initChannel(Channel ch) { + ch.attr(TRANSPORT_ATTRIBUTE_KEY).set(transport); + ChannelPipeline p = ch.pipeline(); + p.addLast("child-client-frame-converter", + new Http2StreamFrameToHttpObjectCodec(false)); + p.addLast("child-client-chunk-aggregator", + new HttpObjectAggregator(client.getClientConfig().getMaxContentLength())); + p.addLast("child-client-response-handler", + new Http2ResponseHandler()); + } + }; } @Override - public Integer nextStream() { - Integer streamId = streamIdCounter.getAndAdd(2); - if (streamId == Integer.MIN_VALUE) { - // reset if overflow, Java wraps atomic integers to Integer.MIN_VALUE - streamIdCounter.set(3); - streamId = 3; + public Transport execute(Request request) throws IOException { + Channel channel = mapChannel(request); + if (throwable != null) { + return this; } - streamidPromiseMap.put(streamId, new CompletableFuture<>()); - return streamId; + final String channelId = channel.id().toString(); + channelFlowMap.putIfAbsent(channelId, new Flow()); + Http2StreamChannel childChannel = new Http2StreamChannelBootstrap(channel) + .handler(initializer).open().syncUninterruptibly().getNow(); + String authority = request.url().getHost() + (request.url().getPort() != null ? ":" + request.url().getPort() : ""); + String path = request.url().getPath() != null && !request.url().getPath().isEmpty() ? + request.url().getPath() : "/"; + Http2Headers http2Headers = new DefaultHttp2Headers() + .method(request.httpMethod().asciiName()) + .scheme(request.url().getScheme()) + .authority(authority) + .path(path); + final Integer streamId = channelFlowMap.get(channelId).nextStreamId(); + if (streamId == null) { + throw new IllegalStateException(); + } + requests.put(getRequestKey(channelId, streamId), request); + http2Headers.setInt(HttpConversionUtil.ExtensionHeaderNames.STREAM_ID.text(), streamId); + // add matching cookies from box (previous requests) and new cookies from request builder + Collection cookies = new ArrayList<>(); + cookies.addAll(matchCookiesFromBox(request)); + cookies.addAll(matchCookies(request)); + if (!cookies.isEmpty()) { + request.headers().set(HttpHeaderNames.COOKIE, ClientCookieEncoder.STRICT.encode(cookies)); + } + // add stream-id and cookie headers + HttpConversionUtil.toHttp2Headers(request.headers(), http2Headers); + boolean hasContent = request.content() != null && request.content().readableBytes() > 0; + DefaultHttp2HeadersFrame headersFrame = new DefaultHttp2HeadersFrame(http2Headers, !hasContent); + childChannel.write(headersFrame); + if (hasContent) { + DefaultHttp2DataFrame dataFrame = new DefaultHttp2DataFrame(request.content(), true); + childChannel.write(dataFrame); + } + childChannel.flush(); + if (client.hasPooledConnections()) { + client.releaseChannel(channel, false); + } + return this; } @Override - public void settingsReceived(Channel channel, Http2Settings http2Settings) { + public void settingsReceived(Http2Settings http2Settings) { if (settingsPromise != null) { settingsPromise.complete(true); } else { @@ -62,138 +120,88 @@ public class Http2Transport extends BaseTransport { } @Override - public void awaitSettings() { + public void waitForSettings() { if (settingsPromise != null) { try { - logger.log(Level.FINE, "waiting for settings"); settingsPromise.get(client.getClientConfig().getReadTimeoutMillis(), TimeUnit.MILLISECONDS); } catch (TimeoutException e) { - logger.log(Level.WARNING, "settings timeout"); + logger.log(Level.WARNING, "timeout in client while waiting for settings"); settingsPromise.completeExceptionally(e); } catch (InterruptedException | ExecutionException e) { settingsPromise.completeExceptionally(e); } - } else { - logger.log(Level.WARNING, "settings promise is null"); } } @Override - public void responseReceived(Integer streamId, FullHttpResponse fullHttpResponse) { + public void responseReceived(Channel channel, Integer streamId, FullHttpResponse fullHttpResponse) { + if (throwable != null) { + logger.log(Level.WARNING, "throwable not null for response " + fullHttpResponse, throwable); + return; + } if (streamId == null) { - logger.log(Level.WARNING, "no stream ID, unexpected message received: " + fullHttpResponse); + logger.log(Level.WARNING, "stream ID is null for response " + fullHttpResponse); return; } - CompletableFuture promise = streamidPromiseMap.get(streamId); - if (promise == null) { - logger.log(Level.WARNING, "response received for stream ID " + streamId + " but found no promise"); + // format of childchan channel ID is "/" + String channelId = channel.id().toString(); + int pos = channelId.indexOf('/'); + channelId = pos > 0 ? channelId.substring(0, pos) : channelId; + Flow flow = channelFlowMap.get(channelId); + if (flow == null) { return; } - Request request = fromStreamId(streamId); - if (request != null) { - HttpResponseListener responseListener = request.getResponseListener(); - if (responseListener != null) { - responseListener.onResponse(fullHttpResponse); - } - try { - Request retryRequest = retry(request, fullHttpResponse); - if (retryRequest != null) { - // retry transport, wait for completion - client.retry(this, retryRequest); - } else { - Request continueRequest = continuation(request, fullHttpResponse); - if (continueRequest != null) { - // continue with new transport, synchronous call here, wait for completion - client.continuation(this, continueRequest); + String requestKey = getRequestKey(channelId, streamId); + CompletableFuture promise = flow.get(streamId); + if (promise != null) { + Request request = requests.get(requestKey); + if (request == null) { + promise.completeExceptionally(new IllegalStateException()); + } else { + for (String cookieString : fullHttpResponse.headers().getAll(HttpHeaderNames.SET_COOKIE)) { + Cookie cookie = ClientCookieDecoder.STRICT.decode(cookieString); + addCookie(cookie); + CookieListener cookieListener = request.getCookieListener(); + if (cookieListener != null) { + cookieListener.onCookie(cookie); } } - } catch (URLSyntaxException | IOException e) { - logger.log(Level.WARNING, e.getMessage(), e); - } - } - promise.complete(true); - } - - @Override - public void pushPromiseReceived(Integer streamId, Integer promisedStreamId, Http2Headers headers) { - streamidPromiseMap.put(promisedStreamId, new CompletableFuture<>()); - requests.put(promisedStreamId, fromStreamId(streamId)); - } - - @Override - public void awaitResponse(Integer streamId) throws IOException { - if (streamId == null) { - return; - } - if (throwable != null) { - return; - } - CompletableFuture promise = streamidPromiseMap.get(streamId); - if (promise != null) { - try { - long millis = client.getClientConfig().getReadTimeoutMillis(); - Request request = fromStreamId(streamId); - if (request != null && request.getTimeoutInMillis() > 0) { - millis = request.getTimeoutInMillis(); + ResponseListener responseListener = request.getResponseListener(); + if (responseListener != null) { + responseListener.onResponse(fullHttpResponse); + } + try { + Request retryRequest = retry(request, fullHttpResponse); + if (retryRequest != null) { + // retry transport, wait for completion + client.retry(this, retryRequest); + } else { + Request continueRequest = continuation(request, fullHttpResponse); + if (continueRequest != null) { + // continue with new transport, synchronous call here, wait for completion + client.continuation(this, continueRequest); + } + } + promise.complete(true); + } catch (URLSyntaxException | IOException e) { + promise.completeExceptionally(e); } - promise.get(millis, TimeUnit.MILLISECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - this.throwable = e; - throw new IOException(e); - } finally { - streamidPromiseMap.remove(streamId); } } + channelFlowMap.get(channelId).remove(streamId); + requests.remove(requestKey); } @Override - public Transport get() { - for (Integer streamId : streamidPromiseMap.keySet()) { - try { - awaitResponse(streamId); - } catch (IOException e) { - notifyRequest(streamId, e); - } - } - if (throwable != null) { - streamidPromiseMap.clear(); - } - return this; + public void pushPromiseReceived(Channel channel, Integer streamId, Integer promisedStreamId, Http2Headers headers) { + String channelId = channel.id().toString(); + channelFlowMap.get(channelId).put(promisedStreamId, new CompletableFuture<>()); + String requestKey = getRequestKey(channel.id().toString(), promisedStreamId); + requests.put(requestKey, requests.get(requestKey)); } @Override - public void success() { - for (CompletableFuture promise : streamidPromiseMap.values()) { - promise.complete(true); - } - } - - /** - * The underlying network layer failed, not possible to know the request. - * So we fail all (open) promises. - * @param throwable the exception - */ - @Override - public void fail(Throwable throwable) { - // fail fast, do not fail more than once - if (this.throwable != null) { - return; - } - this.throwable = throwable; - for (CompletableFuture promise : streamidPromiseMap.values()) { - promise.completeExceptionally(throwable); - } - } - - /** - * Try to notify request about failure. - * @param streamId stream ID - * @param throwable the exception - */ - private void notifyRequest(Integer streamId, Throwable throwable) { - Request request = fromStreamId(streamId); - if (request != null && request.getCompletableFuture() != null) { - request.getCompletableFuture().completeExceptionally(throwable); - } + protected String getRequestKey(String channelId, Integer streamId) { + return channelId + "#" + streamId; } } diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/HttpTransport.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/HttpTransport.java new file mode 100644 index 0000000..982ce85 --- /dev/null +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/HttpTransport.java @@ -0,0 +1,141 @@ +package org.xbib.netty.http.client.transport; + +import io.netty.channel.Channel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.cookie.ClientCookieDecoder; +import io.netty.handler.codec.http.cookie.ClientCookieEncoder; +import io.netty.handler.codec.http.cookie.Cookie; +import io.netty.handler.codec.http2.Http2Headers; +import io.netty.handler.codec.http2.Http2Settings; +import io.netty.handler.codec.http2.HttpConversionUtil; +import org.xbib.net.URLSyntaxException; +import org.xbib.netty.http.client.Client; +import org.xbib.netty.http.client.listener.CookieListener; +import org.xbib.netty.http.common.HttpAddress; +import org.xbib.netty.http.client.Request; +import org.xbib.netty.http.client.listener.ResponseListener; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.CompletableFuture; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class HttpTransport extends BaseTransport { + + private static final Logger logger = Logger.getLogger(HttpTransport.class.getName()); + + public HttpTransport(Client client, HttpAddress httpAddress) { + super(client, httpAddress); + } + + @Override + public Transport execute(Request request) throws IOException { + Channel channel = mapChannel(request); + if (throwable != null) { + return this; + } + final String channelId = channel.id().toString(); + channelFlowMap.putIfAbsent(channelId, new Flow()); + // Some HTTP 1 servers do not understand URIs in HTTP command line in spite of RFC 7230. + // The "origin form" requires a "Host" header. + // Our algorithm is: use always "origin form" for HTTP 1, use absolute form for HTTP 2. + // The reason is that Netty derives the HTTP/2 scheme header from the absolute form. + String uri = request.httpVersion().majorVersion() == 1 ? + request.url().relativeReference() : request.url().toString(); + FullHttpRequest fullHttpRequest = request.content() == null ? + new DefaultFullHttpRequest(request.httpVersion(), request.httpMethod(), uri) : + new DefaultFullHttpRequest(request.httpVersion(), request.httpMethod(), uri, request.content()); + final Integer streamId = channelFlowMap.get(channelId).nextStreamId(); + if (streamId == null) { + throw new IllegalStateException(); + } + String requestKey = channelId + "#" + streamId; + requests.put(requestKey, request); + // do we need the stream ID here in HTTP 1 header? + request.headers().set(HttpConversionUtil.ExtensionHeaderNames.STREAM_ID.text(), Integer.toString(streamId)); + // add matching cookies from box (previous requests) and new cookies from request builder + Collection cookies = new ArrayList<>(); + cookies.addAll(matchCookiesFromBox(request)); + cookies.addAll(matchCookies(request)); + if (!cookies.isEmpty()) { + request.headers().set(HttpHeaderNames.COOKIE, ClientCookieEncoder.STRICT.encode(cookies)); + } + // add stream-id and cookie headers + fullHttpRequest.headers().set(request.headers()); + // flush after putting request into requests map + if (channel.isWritable()) { + channel.writeAndFlush(fullHttpRequest); + } + return this; + } + + @Override + public void responseReceived(Channel channel, Integer streamId, FullHttpResponse fullHttpResponse) { + if (throwable != null) { + logger.log(Level.WARNING, "throwable not null for response " + fullHttpResponse, throwable); + return; + } + // streamID is expected to be null, last request on memory is expected to be current, remove request from memory + Request request = requests.remove(requests.lastKey()); + if (request != null) { + for (String cookieString : fullHttpResponse.headers().getAll(HttpHeaderNames.SET_COOKIE)) { + Cookie cookie = ClientCookieDecoder.STRICT.decode(cookieString); + addCookie(cookie); + CookieListener cookieListener = request.getCookieListener(); + if (cookieListener != null) { + cookieListener.onCookie(cookie); + } + } + ResponseListener responseListener = request.getResponseListener(); + if (responseListener != null) { + responseListener.onResponse(fullHttpResponse); + } + } + try { + Request retryRequest = retry(request, fullHttpResponse); + if (retryRequest != null) { + // retry transport, wait for completion + client.retry(this, retryRequest); + } else { + Request continueRequest = continuation(request, fullHttpResponse); + if (continueRequest != null) { + // continue with new transport, synchronous call here, wait for completion + client.continuation(this, continueRequest); + } + } + } catch (URLSyntaxException | IOException e) { + logger.log(Level.WARNING, e.getMessage(), e); + } + String channelId = channel.id().toString(); + Flow flow = channelFlowMap.get(channelId); + if (flow == null) { + return; + } + CompletableFuture promise = flow.get(flow.lastKey()); + if (promise != null) { + promise.complete(true); + } + } + + @Override + public void settingsReceived(Http2Settings http2Settings) { + } + + @Override + public void waitForSettings() { + } + + @Override + public void pushPromiseReceived(Channel channel, Integer streamId, Integer promisedStreamId, Http2Headers headers) { + } + + @Override + protected String getRequestKey(String channelId, Integer streamId) { + return requests.lastKey(); + } +} diff --git a/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Transport.java b/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Transport.java index 2d3d395..3888871 100644 --- a/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Transport.java +++ b/netty-http-client/src/main/java/org/xbib/netty/http/client/transport/Transport.java @@ -2,8 +2,8 @@ package org.xbib.netty.http.client.transport; import io.netty.channel.Channel; import io.netty.handler.codec.http.FullHttpResponse; -import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.cookie.Cookie; +import io.netty.handler.codec.http2.Http2Exception; import io.netty.handler.codec.http2.Http2Headers; import io.netty.handler.codec.http2.Http2Settings; import io.netty.util.AttributeKey; @@ -12,7 +12,7 @@ import org.xbib.netty.http.client.Request; import java.io.IOException; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.TimeUnit; import java.util.function.Function; public interface Transport { @@ -23,27 +23,23 @@ public interface Transport { CompletableFuture execute(Request request, Function supplier) throws IOException; - Integer nextStream(); + void waitForSettings(); - void settingsReceived(Channel channel, Http2Settings http2Settings); + void settingsReceived(Http2Settings http2Settings) throws IOException; - void awaitSettings(); + void responseReceived(Channel channel, Integer streamId, FullHttpResponse fullHttpResponse) throws IOException; + + void pushPromiseReceived(Channel channel, Integer streamId, Integer promisedStreamId, Http2Headers headers) throws Http2Exception; void setCookieBox(Map cookieBox); Map getCookieBox(); - void responseReceived(Integer streamId, FullHttpResponse fullHttpResponse); - - void headersReceived(Integer streamId, HttpHeaders httpHeaders); - - void pushPromiseReceived(Integer streamId, Integer promisedStreamId, Http2Headers headers); - - void awaitResponse(Integer streamId) throws IOException, TimeoutException; - Transport get(); - void success(); + Transport get(long value, TimeUnit timeUnit); + + void cancel(); void fail(Throwable throwable); diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/LoggingBase.java b/netty-http-client/src/test/java/org/xbib/TestBase.java similarity index 62% rename from netty-http-client/src/test/java/org/xbib/netty/http/client/test/LoggingBase.java rename to netty-http-client/src/test/java/org/xbib/TestBase.java index e734d15..1449c1e 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/LoggingBase.java +++ b/netty-http-client/src/test/java/org/xbib/TestBase.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.client.test; +package org.xbib; import java.util.logging.ConsoleHandler; import java.util.logging.Handler; @@ -7,20 +7,25 @@ import java.util.logging.LogManager; import java.util.logging.Logger; import java.util.logging.SimpleFormatter; -public class LoggingBase { +public class TestBase { static { + System.setProperty("io.netty.noUnsafe", Boolean.toString(true)); + System.setProperty("io.netty.noKeySetOptimization", Boolean.toString(true)); + //System.setProperty("io.netty.recycler.maxCapacity", Integer.toString(0)); + //System.setProperty("io.netty.leakDetection.level", "paranoid"); + System.setProperty("java.util.logging.SimpleFormatter.format", - "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %2$s %5$s %6$s%n"); + "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %5$s %6$s%n"); LogManager.getLogManager().reset(); Logger rootLogger = LogManager.getLogManager().getLogger(""); Handler handler = new ConsoleHandler(); handler.setFormatter(new SimpleFormatter()); rootLogger.addHandler(handler); - rootLogger.setLevel(Level.ALL); + rootLogger.setLevel(Level.FINE); for (Handler h : rootLogger.getHandlers()) { handler.setFormatter(new SimpleFormatter()); - h.setLevel(Level.ALL); + h.setLevel(Level.FINE); } } } diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/CompletableFutureTest.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/CompletableFutureTest.java index e4d2edc..253e122 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/CompletableFutureTest.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/CompletableFutureTest.java @@ -32,7 +32,7 @@ public class CompletableFutureTest { .exceptionally(Throwable::getMessage) .thenCompose(content -> { logger.log(Level.INFO, content); - // POST is not allowed, we don't care + // POST is not allowed, will give a 405. We don't care try { return client.execute(Request.post() .url("http://google.com/") diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/ConscryptTest.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/ConscryptTest.java index 3c0b287..92f4c7a 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/ConscryptTest.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/ConscryptTest.java @@ -1,8 +1,8 @@ package org.xbib.netty.http.client.test; import org.conscrypt.Conscrypt; -import org.junit.Ignore; import org.junit.Test; +import org.xbib.TestBase; import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Request; @@ -11,7 +11,7 @@ import java.nio.charset.StandardCharsets; import java.util.logging.Level; import java.util.logging.Logger; -public class ConscryptTest extends LoggingBase { +public class ConscryptTest extends TestBase { private static final Logger logger = Logger.getLogger(""); diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/CookieSetterHttpBinTest.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/CookieSetterHttpBinTest.java index 5f2e788..1065076 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/CookieSetterHttpBinTest.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/CookieSetterHttpBinTest.java @@ -1,6 +1,7 @@ package org.xbib.netty.http.client.test; import org.junit.Test; +import org.xbib.TestBase; import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Request; @@ -11,7 +12,7 @@ import java.util.logging.Logger; /** */ -public class CookieSetterHttpBinTest extends LoggingBase { +public class CookieSetterHttpBinTest extends TestBase { private static final Logger logger = Logger.getLogger(CookieSetterHttpBinTest.class.getName()); @@ -36,7 +37,6 @@ public class CookieSetterHttpBinTest extends LoggingBase { .url("http://httpbin.org/cookies/set?name=value") .build() .setCookieListener(cookie -> logger.log(Level.INFO, "this is the cookie: " + cookie.toString())) - .setHeadersListener(headers -> logger.log(Level.INFO, "headers = " + headers.entries().toString())) .setResponseListener(fullHttpResponse -> { String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); logger.log(Level.INFO, "status = " + fullHttpResponse.status() + " response body = " + response); diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/ElasticsearchTest.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/ElasticsearchTest.java index e4055d9..5570e23 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/ElasticsearchTest.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/ElasticsearchTest.java @@ -2,6 +2,7 @@ package org.xbib.netty.http.client.test; import org.junit.Ignore; import org.junit.Test; +import org.xbib.TestBase; import org.xbib.netty.http.client.Client; import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.client.Request; @@ -20,7 +21,7 @@ import java.util.logging.Logger; import static org.junit.Assert.assertEquals; @Ignore -public class ElasticsearchTest extends LoggingBase { +public class ElasticsearchTest extends TestBase { private static final Logger logger = Logger.getLogger(ElasticsearchTest.class.getName()); @@ -103,7 +104,7 @@ public class ElasticsearchTest extends LoggingBase { } try { for (int i = 0; i < max; i++) { - client.pooledExecute(queries.get(i)).get(); + client.newTransport().execute(queries.get(i)).get(); } } catch (IOException e) { logger.log(Level.WARNING, e.getMessage(), e); diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/Http1Test.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/Http1Test.java index 18a3853..09a9eeb 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/Http1Test.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/Http1Test.java @@ -2,6 +2,7 @@ package org.xbib.netty.http.client.test; import io.netty.handler.codec.http.HttpMethod; import org.junit.Test; +import org.xbib.TestBase; import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Request; @@ -10,7 +11,7 @@ import java.nio.charset.StandardCharsets; import java.util.logging.Level; import java.util.logging.Logger; -public class Http1Test extends LoggingBase { +public class Http1Test extends TestBase { private static final Logger logger = Logger.getLogger(Http1Test.class.getName()); diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/Http2Test.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/Http2Test.java index 4b45574..3569ebf 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/Http2Test.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/Http2Test.java @@ -3,6 +3,7 @@ package org.xbib.netty.http.client.test; import io.netty.handler.codec.http.HttpMethod; import org.junit.Ignore; import org.junit.Test; +import org.xbib.TestBase; import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Request; @@ -11,7 +12,7 @@ import java.nio.charset.StandardCharsets; import java.util.logging.Level; import java.util.logging.Logger; -public class Http2Test extends LoggingBase { +public class Http2Test extends TestBase { private static final Logger logger = Logger.getLogger(Http2Test.class.getName()); diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/SecureHttp1Test.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/SecureHttp1Test.java index f3c40ac..a31b560 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/SecureHttp1Test.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/SecureHttp1Test.java @@ -3,6 +3,7 @@ package org.xbib.netty.http.client.test; import io.netty.handler.codec.http.HttpMethod; import org.junit.Ignore; import org.junit.Test; +import org.xbib.TestBase; import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Request; @@ -11,7 +12,7 @@ import java.nio.charset.StandardCharsets; import java.util.logging.Level; import java.util.logging.Logger; -public class SecureHttp1Test extends LoggingBase { +public class SecureHttp1Test extends TestBase { private static final Logger logger = Logger.getLogger(SecureHttp1Test.class.getName()); diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/LeakTest.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/ThreadLeakTest.java similarity index 80% rename from netty-http-client/src/test/java/org/xbib/netty/http/client/test/LeakTest.java rename to netty-http-client/src/test/java/org/xbib/netty/http/client/test/ThreadLeakTest.java index fefb811..0e70876 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/LeakTest.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/ThreadLeakTest.java @@ -2,6 +2,7 @@ package org.xbib.netty.http.client.test; import org.junit.After; import org.junit.Test; +import org.xbib.TestBase; import org.xbib.netty.http.client.Client; import java.io.IOException; @@ -9,9 +10,15 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -public class LeakTest { +public class ThreadLeakTest extends TestBase { - private static final Logger logger = Logger.getLogger(LeakTest.class.getName()); + private static final Logger logger = Logger.getLogger(ThreadLeakTest.class.getName()); + + @Test + public void testForLeaks() throws IOException { + Client client = new Client(); + client.shutdownGracefully(); + } @After public void checkThreads() { @@ -19,10 +26,4 @@ public class LeakTest { logger.log(Level.INFO, "threads = " + threadSet.size() ); threadSet.forEach( thread -> logger.log(Level.INFO, thread.toString())); } - - @Test - public void testForLeaks() throws IOException { - Client client = new Client(); - client.shutdownGracefully(); - } } diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/XbibTest.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/XbibTest.java index cb011cb..5c3b2a8 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/XbibTest.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/XbibTest.java @@ -3,6 +3,7 @@ package org.xbib.netty.http.client.test; import io.netty.handler.codec.http.FullHttpResponse; import io.netty.handler.proxy.HttpProxyHandler; import org.junit.Test; +import org.xbib.TestBase; import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Request; @@ -14,7 +15,7 @@ import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; -public class XbibTest extends LoggingBase { +public class XbibTest extends TestBase { private static final Logger logger = Logger.getLogger(""); diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/EpollTest.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/EpollTest.java index 73c19c4..72fc753 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/EpollTest.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/EpollTest.java @@ -1,15 +1,18 @@ package org.xbib.netty.http.client.test.pool; import io.netty.bootstrap.Bootstrap; +import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; import io.netty.channel.EventLoopGroup; import io.netty.channel.SimpleChannelInboundHandler; import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.epoll.EpollServerSocketChannel; import io.netty.channel.epoll.EpollSocketChannel; import io.netty.channel.socket.SocketChannel; @@ -25,12 +28,14 @@ import org.xbib.netty.http.client.pool.BoundedChannelPool; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.Closeable; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import java.util.logging.Level; import java.util.logging.Logger; @@ -76,8 +81,8 @@ public class EpollTest { .option(ChannelOption.SO_KEEPALIVE, true) .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.TCP_NODELAY, true); - channelPool = new BoundedChannelPool<>(semaphore, HttpVersion.HTTP_1_1,false, - NODES, bootstrap, null, 0); + channelPool = new BoundedChannelPool<>(semaphore, HttpVersion.HTTP_1_1, + NODES, bootstrap, null, 0, BoundedChannelPool.PoolKeySelectorType.ROUNDROBIN); channelPool.prepare(CONCURRENCY); } @@ -101,7 +106,7 @@ public class EpollTest { Thread.sleep(1); // very short? } channel.writeAndFlush(PAYLOAD.retain()).sync(); - channelPool.release(channel); + channelPool.release(channel, false); longAdder.increment(); } catch (InterruptedException e) { break; @@ -131,4 +136,48 @@ public class EpollTest { } } + class MockEpollServer implements Closeable { + + private final EventLoopGroup dispatchGroup; + + private final EventLoopGroup workerGroup; + + private final ChannelFuture bindFuture; + + private final AtomicLong reqCounter; + + public MockEpollServer(int port, int dropEveryRequest) throws InterruptedException { + dispatchGroup = new EpollEventLoopGroup(); + workerGroup = new EpollEventLoopGroup(); + reqCounter = new AtomicLong(0); + ServerBootstrap bootstrap = new ServerBootstrap() + .group(dispatchGroup, workerGroup) + .channel(EpollServerSocketChannel.class) + .childHandler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) { + if (dropEveryRequest > 0) { + ch.pipeline().addLast(new SimpleChannelInboundHandler() { + @Override + protected void channelRead0(ChannelHandlerContext ctx, Object msg) { + if (reqCounter.incrementAndGet() % dropEveryRequest == 0) { + Channel channel = ctx.channel(); + logger.log(Level.INFO,"dropping the connection " + channel); + channel.close(); + } + } + }); + } + } + }); + bindFuture = bootstrap.bind(port).sync(); + } + + @Override + public void close() { + bindFuture.channel().close(); + workerGroup.shutdownGracefully(); + dispatchGroup.shutdownGracefully(); + } + } } diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/MockEpollServer.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/MockEpollServer.java deleted file mode 100644 index 04c3891..0000000 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/MockEpollServer.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.xbib.netty.http.client.test.pool; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.channel.epoll.EpollEventLoopGroup; -import io.netty.channel.epoll.EpollServerSocketChannel; -import io.netty.channel.socket.SocketChannel; - -import java.io.Closeable; -import java.util.concurrent.atomic.AtomicLong; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class MockEpollServer implements Closeable { - - private static final Logger logger = Logger.getLogger(MockEpollServer.class.getName()); - - private final EventLoopGroup dispatchGroup; - - private final EventLoopGroup workerGroup; - - private final ChannelFuture bindFuture; - - private final AtomicLong reqCounter; - - public MockEpollServer(int port, int dropEveryRequest) throws InterruptedException { - dispatchGroup = new EpollEventLoopGroup(); - workerGroup = new EpollEventLoopGroup(); - reqCounter = new AtomicLong(0); - ServerBootstrap bootstrap = new ServerBootstrap() - .group(dispatchGroup, workerGroup) - .channel(EpollServerSocketChannel.class) - .childHandler(new ChannelInitializer() { - @Override - public void initChannel(SocketChannel ch) { - if (dropEveryRequest > 0) { - ch.pipeline().addLast(new SimpleChannelInboundHandler() { - @Override - protected void channelRead0(ChannelHandlerContext ctx, Object msg) { - if (reqCounter.incrementAndGet() % dropEveryRequest == 0) { - Channel channel = ctx.channel(); - logger.log(Level.INFO,"dropping the connection " + channel); - channel.close(); - } - } - }); - } - } - }); - bindFuture = bootstrap.bind(port).sync(); - } - - @Override - public void close() { - bindFuture.channel().close(); - workerGroup.shutdownGracefully(); - dispatchGroup.shutdownGracefully(); - } -} diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/MockNioServer.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/MockNioServer.java deleted file mode 100644 index 3bc5a82..0000000 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/MockNioServer.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.xbib.netty.http.client.test.pool; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; - -import java.io.Closeable; -import java.util.concurrent.atomic.AtomicLong; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class MockNioServer implements Closeable { - - private static final Logger logger = Logger.getLogger(MockNioServer.class.getName()); - - private final EventLoopGroup dispatchGroup; - - private final EventLoopGroup workerGroup; - - private final ChannelFuture bindFuture; - - private final AtomicLong reqCounter; - - public MockNioServer(int port, int dropEveryRequest) throws InterruptedException { - dispatchGroup = new NioEventLoopGroup(); - workerGroup = new NioEventLoopGroup(); - reqCounter = new AtomicLong(0); - ServerBootstrap bootstrap = new ServerBootstrap() - .group(dispatchGroup, workerGroup) - .channel(NioServerSocketChannel.class) - .childHandler(new ChannelInitializer() { - @Override - public void initChannel(SocketChannel ch) { - ch.pipeline().addLast(new SimpleChannelInboundHandler() { - @Override - protected void channelRead0(ChannelHandlerContext ctx, Object msg) { - if (reqCounter.incrementAndGet() % dropEveryRequest == 0) { - Channel channel = ctx.channel(); - logger.log(Level.INFO, "dropping the connection " + channel); - channel.close(); - } - } - }); - } - }); - bindFuture = bootstrap.bind(port).sync(); - } - - @Override - public void close() { - bindFuture.channel().close(); - workerGroup.shutdownGracefully(); - dispatchGroup.shutdownGracefully(); - } -} diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/NioTest.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/NioTest.java index 6fbc2ea..5b1b1b0 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/NioTest.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/NioTest.java @@ -1,9 +1,11 @@ package org.xbib.netty.http.client.test.pool; import io.netty.bootstrap.Bootstrap; +import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; @@ -11,6 +13,7 @@ import io.netty.channel.EventLoopGroup; import io.netty.channel.SimpleChannelInboundHandler; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.http.HttpVersion; import org.junit.After; @@ -20,12 +23,14 @@ import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.client.pool.Pool; import org.xbib.netty.http.client.pool.BoundedChannelPool; +import java.io.Closeable; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import java.util.logging.Level; import java.util.logging.Logger; @@ -73,8 +78,8 @@ public class NioTest { .option(ChannelOption.SO_KEEPALIVE, true) .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.TCP_NODELAY, true); - channelPool = new BoundedChannelPool<>(semaphore, HttpVersion.HTTP_1_1,false, - NODES, bootstrap, null, 0); + channelPool = new BoundedChannelPool<>(semaphore, HttpVersion.HTTP_1_1, + NODES, bootstrap, null, 0, BoundedChannelPool.PoolKeySelectorType.ROUNDROBIN); channelPool.prepare(CONCURRENCY); } @@ -98,7 +103,7 @@ public class NioTest { Thread.sleep(1); } channel.writeAndFlush(PAYLOAD.retain()).sync(); - channelPool.release(channel); + channelPool.release(channel, false); longAdder.increment(); } catch (InterruptedException e) { break; @@ -127,4 +132,47 @@ public class NioTest { logger.log(Level.WARNING, cause.getMessage(), cause); } } + public class MockNioServer implements Closeable { + + private final EventLoopGroup dispatchGroup; + + private final EventLoopGroup workerGroup; + + private final ChannelFuture bindFuture; + + private final AtomicLong reqCounter; + + public MockNioServer(int port, int dropEveryRequest) throws InterruptedException { + dispatchGroup = new NioEventLoopGroup(); + workerGroup = new NioEventLoopGroup(); + reqCounter = new AtomicLong(0); + ServerBootstrap bootstrap = new ServerBootstrap() + .group(dispatchGroup, workerGroup) + .channel(NioServerSocketChannel.class) + .childHandler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) { + ch.pipeline().addLast(new SimpleChannelInboundHandler() { + @Override + protected void channelRead0(ChannelHandlerContext ctx, Object msg) { + if (reqCounter.incrementAndGet() % dropEveryRequest == 0) { + Channel channel = ctx.channel(); + logger.log(Level.INFO, "dropping the connection " + channel); + channel.close(); + } + } + }); + } + }); + bindFuture = bootstrap.bind(port).sync(); + } + + @Override + public void close() { + bindFuture.channel().close(); + workerGroup.shutdownGracefully(); + dispatchGroup.shutdownGracefully(); + } + } + } diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/PoolTest.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/PoolTest.java index 048f8ab..7478802 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/PoolTest.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/PoolTest.java @@ -1,7 +1,12 @@ package org.xbib.netty.http.client.test.pool; import io.netty.bootstrap.Bootstrap; +import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.http.HttpVersion; import io.netty.util.AttributeKey; import org.junit.Test; @@ -44,21 +49,42 @@ public class PoolTest { @Parameterized.Parameters public static Collection generateData() { return Arrays.asList(new Object[][] { - {1, 1}, - {10, 1}, {10, 2}, {10, 5}, {10, 10}, - {100, 1}, {100, 2}, {100, 5}, {100, 10}, - {1000, 1}, {1000, 2}, {1000, 5}, {1000, 10} - }); + {1, 1}, + {10, 1}, + {10, 2}, + //{10, 5}, + //{10, 10}, + {100, 1}, + {100, 2}, + //{100, 5}, + //{100, 10}, + //{1000, 1}, + //{1000, 2}, + //{1000, 5}, + //{1000, 10} + }); } - public PoolTest(int concurrencyLevel, int nodeCount) { + public PoolTest(int concurrencyLevel, int nodeCount) throws InterruptedException { + + ServerBootstrap serverBootstrap = new ServerBootstrap() + .group(new NioEventLoopGroup()) + .channel(NioServerSocketChannel.class) + .childHandler(new ChannelInitializer() { + @Override + protected void initChannel(Channel ch) { + } + }); + Channel serverChannel = serverBootstrap.bind("localhost", 8008).sync().channel(); + this.nodeCount = nodeCount; List nodes = new ArrayList<>(); for (int i = 0; i < nodeCount; i ++) { - nodes.add(HttpAddress.http1("localhost" + i)); + nodes.add(HttpAddress.http1("localhost", 8008)); } - try (Pool pool = new BoundedChannelPool<>(new Semaphore(concurrencyLevel), HttpVersion.HTTP_1_1, false, - nodes, new Bootstrap(), null, 0)) { + try (Pool pool = new BoundedChannelPool<>(new Semaphore(concurrencyLevel), HttpVersion.HTTP_1_1, + nodes, new Bootstrap().group(new NioEventLoopGroup()).channel(NioSocketChannel.class), + null, 0, BoundedChannelPool.PoolKeySelectorType.ROUNDROBIN)) { int n = Runtime.getRuntime().availableProcessors(); ExecutorService executorService = Executors.newFixedThreadPool(n); for(int i = 0; i < n; i ++) { @@ -80,7 +106,7 @@ public class PoolTest { channels.add(channel); } for (k = 0; k < j; k ++) { - pool.release(channels.get(k)); + pool.release(channels.get(k), false); } channels.clear(); } @@ -99,6 +125,7 @@ public class PoolTest { } catch (Throwable t) { logger.log(Level.WARNING, t.getMessage(), t); } finally { + serverChannel.close(); long connCountSum = nodeFreq.values().stream().mapToLong(LongAdder::sum).sum(); logger.log(Level.INFO, "concurrency = " + concurrencyLevel + ", nodes = " + nodeCount + " -> rate: " + connCountSum / TEST_STEP_TIME_SECONDS); diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/PooledClientTest.java b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/PooledClientTest.java similarity index 75% rename from netty-http-client/src/test/java/org/xbib/netty/http/client/test/PooledClientTest.java rename to netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/PooledClientTest.java index a659f4f..42563a7 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/PooledClientTest.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/client/test/pool/PooledClientTest.java @@ -1,9 +1,11 @@ -package org.xbib.netty.http.client.test; +package org.xbib.netty.http.client.test.pool; import io.netty.handler.codec.http.HttpVersion; import org.junit.Test; import org.xbib.net.URL; import org.xbib.netty.http.client.Client; +import org.xbib.TestBase; +import org.xbib.netty.http.client.listener.ResponseListener; import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.client.Request; @@ -16,7 +18,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; -public class PooledClientTest extends LoggingBase { +public class PooledClientTest extends TestBase { private static final Logger logger = Logger.getLogger(""); @@ -31,6 +33,11 @@ public class PooledClientTest extends LoggingBase { .setPoolNodeConnectionLimit(threads) .build(); AtomicInteger count = new AtomicInteger(); + ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + //logger.log(Level.INFO, "status = " + fullHttpResponse.status() + " response body = " + response); + count.getAndIncrement(); + }; try { ExecutorService executorService = Executors.newFixedThreadPool(threads); for (int n = 0; n < threads; n++) { @@ -38,17 +45,12 @@ public class PooledClientTest extends LoggingBase { try { logger.log(Level.INFO, "starting " + Thread.currentThread()); for (int i = 0; i < loop; i++) { - Request request = Request.get() + Request request = Request.get().setVersion(httpAddress.getVersion()) .url(url.toString()) - .setVersion(httpAddress.getVersion()) //.setTimeoutInMillis(25000L) .build() - .setResponseListener(fullHttpResponse -> { - String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); - //logger.log(Level.INFO, "status = " + fullHttpResponse.status() + " response body = " + response); - count.getAndIncrement(); - }); - client.pooledExecute(request).get(); + .setResponseListener(responseListener); + client.newTransport().execute(request).get(); } logger.log(Level.INFO, "done " + Thread.currentThread()); } catch (Throwable e) { diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/simple/Http2FramesTest.java b/netty-http-client/src/test/java/org/xbib/netty/http/hacks/Http2FramesTest.java similarity index 98% rename from netty-http-client/src/test/java/org/xbib/netty/http/client/test/simple/Http2FramesTest.java rename to netty-http-client/src/test/java/org/xbib/netty/http/hacks/Http2FramesTest.java index 9ae6ef6..0cba6a2 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/simple/Http2FramesTest.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/hacks/Http2FramesTest.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.client.test.simple; +package org.xbib.netty.http.hacks; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; @@ -28,6 +28,7 @@ import io.netty.handler.ssl.SupportedCipherSuiteFilter; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import org.junit.Ignore; import org.junit.Test; +import org.xbib.TestBase; import javax.net.ssl.SNIHostName; import javax.net.ssl.SNIServerName; @@ -39,7 +40,8 @@ import java.util.concurrent.CompletableFuture; import java.util.logging.Level; import java.util.logging.Logger; -public class Http2FramesTest { +@Ignore +public class Http2FramesTest extends TestBase { private static final Logger logger = Logger.getLogger(Http2FramesTest.class.getName()); diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/simple/SimpleHttp1Test.java b/netty-http-client/src/test/java/org/xbib/netty/http/hacks/SimpleHttp1Test.java similarity index 98% rename from netty-http-client/src/test/java/org/xbib/netty/http/client/test/simple/SimpleHttp1Test.java rename to netty-http-client/src/test/java/org/xbib/netty/http/hacks/SimpleHttp1Test.java index 15d75b5..6368175 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/simple/SimpleHttp1Test.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/hacks/SimpleHttp1Test.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.client.test.simple; +package org.xbib.netty.http.hacks; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; @@ -20,7 +20,9 @@ import io.netty.handler.codec.http.HttpObjectAggregator; import io.netty.handler.codec.http.HttpVersion; import io.netty.util.AttributeKey; import org.junit.After; +import org.junit.Ignore; import org.junit.Test; +import org.xbib.TestBase; import java.io.IOException; import java.net.InetSocketAddress; @@ -38,17 +40,14 @@ import java.util.logging.LogManager; import java.util.logging.Logger; import java.util.logging.SimpleFormatter; -/** - * - */ -public class SimpleHttp1Test { +@Ignore +public class SimpleHttp1Test extends TestBase { private static final Logger logger = Logger.getLogger(SimpleHttp1Test.class.getName()); static { System.setProperty("io.netty.leakDetection.level", "paranoid"); System.setProperty("io.netty.noKeySetOptimization", Boolean.toString(true)); - System.setProperty("java.util.logging.SimpleFormatter.format", "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %2$s %5$s %6$s%n"); LogManager.getLogManager().reset(); diff --git a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/simple/SimpleHttp2Test.java b/netty-http-client/src/test/java/org/xbib/netty/http/hacks/SimpleHttp2Test.java similarity index 94% rename from netty-http-client/src/test/java/org/xbib/netty/http/client/test/simple/SimpleHttp2Test.java rename to netty-http-client/src/test/java/org/xbib/netty/http/hacks/SimpleHttp2Test.java index adac51c..759d067 100644 --- a/netty-http-client/src/test/java/org/xbib/netty/http/client/test/simple/SimpleHttp2Test.java +++ b/netty-http-client/src/test/java/org/xbib/netty/http/hacks/SimpleHttp2Test.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.client.test.simple; +package org.xbib.netty.http.hacks; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; @@ -35,14 +35,14 @@ import io.netty.handler.ssl.SslProvider; import io.netty.handler.ssl.SupportedCipherSuiteFilter; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import io.netty.util.AttributeKey; +import org.junit.Ignore; import org.junit.Test; +import org.xbib.TestBase; import javax.net.ssl.SSLException; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; @@ -53,9 +53,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; -/** - */ -public class SimpleHttp2Test { +@Ignore +public class SimpleHttp2Test extends TestBase { private static final Logger logger = Logger.getLogger(SimpleHttp2Test.class.getName()); @@ -113,8 +112,6 @@ public class SimpleHttp2Test { private final Initializer initializer; - private final List transports; - Client() { eventLoopGroup = new NioEventLoopGroup(); http2SettingsHandler = new Http2SettingsHandler(); @@ -124,7 +121,6 @@ public class SimpleHttp2Test { .group(eventLoopGroup) .channel(NioSocketChannel.class) .handler(initializer); - transports = new ArrayList<>(); } Bootstrap bootstrap() { @@ -137,23 +133,8 @@ public class SimpleHttp2Test { Http2Transport newTransport(String host, int port) { Http2Transport transport = new Http2Transport(this, new InetSocketAddress(host, port)); - transports.add(transport); return transport; } - - List transports() { - return transports; - } - - void close(Http2Transport transport) { - transports.remove(transport); - } - - void close() { - for (Http2Transport transport : transports) { - transport.close(); - } - } } class Http2Transport { @@ -179,10 +160,6 @@ public class SimpleHttp2Test { streamIdCounter = new AtomicInteger(3); } - Client client() { - return client; - } - InetSocketAddress inetSocketAddress() { return inetSocketAddress; } @@ -270,12 +247,6 @@ public class SimpleHttp2Test { } } - void complete() { - for (CompletableFuture promise : streamidPromiseMap.values()) { - promise.complete(true); - } - } - void fail(Throwable throwable) { for (CompletableFuture promise : streamidPromiseMap.values()) { promise.completeExceptionally(throwable); @@ -286,7 +257,6 @@ public class SimpleHttp2Test { if (channel != null) { channel.close(); } - client.close(this); } } diff --git a/netty-http-common/src/main/java/org/xbib/netty/http/common/HttpAddress.java b/netty-http-common/src/main/java/org/xbib/netty/http/common/HttpAddress.java index 99add9a..ca3212f 100644 --- a/netty-http-common/src/main/java/org/xbib/netty/http/common/HttpAddress.java +++ b/netty-http-common/src/main/java/org/xbib/netty/http/common/HttpAddress.java @@ -39,10 +39,18 @@ public class HttpAddress implements PoolKey { } public static HttpAddress http2(String host) { - return new HttpAddress(host, 443, HTTP_2_0, true); + return new HttpAddress(host, 443, HTTP_2_0, false); } public static HttpAddress http2(String host, int port) { + return new HttpAddress(host, port, HTTP_2_0, false); + } + + public static HttpAddress secureHttp2(String host) { + return new HttpAddress(host, 443, HTTP_2_0, true); + } + + public static HttpAddress secureHttp2(String host, int port) { return new HttpAddress(host, port, HTTP_2_0, true); } diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/Server.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/Server.java index 4287023..f506a2a 100644 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/Server.java +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/Server.java @@ -22,9 +22,9 @@ import io.netty.util.DomainNameMapping; import io.netty.util.DomainNameMappingBuilder; import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.server.context.VirtualServer; -import org.xbib.netty.http.server.handler.http1.HttpChannelInitializer; +import org.xbib.netty.http.server.handler.http.HttpChannelInitializer; import org.xbib.netty.http.server.handler.http2.Http2ChannelInitializer; -import org.xbib.netty.http.server.transport.Http1ServerTransport; +import org.xbib.netty.http.server.transport.HttpServerTransport; import org.xbib.netty.http.server.transport.Http2ServerTransport; import org.xbib.netty.http.server.transport.ServerTransport; import org.xbib.netty.http.server.util.NetworkUtils; @@ -49,7 +49,9 @@ public final class Server { static { // extend Java system properties by detected network interfaces - //NetworkUtils.extendSystemProperties(); + if (System.getProperty("xbib.netty.http.client.extendsystemproperties") != null) { + NetworkUtils.extendSystemProperties(); + } // change Netty defaults to safer ones, but still allow override from arg line if (System.getProperty("io.netty.noUnsafe") == null) { System.setProperty("io.netty.noUnsafe", Boolean.toString(true)); @@ -57,12 +59,6 @@ public final class Server { if (System.getProperty("io.netty.noKeySetOptimization") == null) { System.setProperty("io.netty.noKeySetOptimization", Boolean.toString(true)); } - if (System.getProperty("io.netty.recycler.maxCapacity") == null) { - System.setProperty("io.netty.recycler.maxCapacity", Integer.toString(0)); - } - //if (System.getProperty("io.netty.leakDetection.level") == null) { - // System.setProperty("io.netty.leakDetection.level", "paranoid"); - //} } private final ServerConfig serverConfig; @@ -83,6 +79,12 @@ public final class Server { /** * Create a new HTTP server. Use {@link #builder()} to build HTTP client instance. + * @param serverConfig server configuration + * @param byteBufAllocator byte buf allocator + * @param parentEventLoopGroup parent event loop group + * @param childEventLoopGroup child event loop group + * @param socketChannelClass socket channel class + * @throws SSLException if SSL can not be configured */ public Server(ServerConfig serverConfig, ByteBufAllocator byteBufAllocator, @@ -112,7 +114,6 @@ public final class Server { .childOption(ChannelOption.SO_RCVBUF, serverConfig.getTcpReceiveBufferSize()) .childOption(ChannelOption.CONNECT_TIMEOUT_MILLIS, serverConfig.getConnectTimeoutMillis()) .childOption(ChannelOption.WRITE_BUFFER_WATER_MARK, serverConfig.getWriteBufferWaterMark()); - if (serverConfig.isDebug()) { bootstrap.handler(new LoggingHandler("bootstrap-server", serverConfig.getDebugLogLevel())); } @@ -178,6 +179,7 @@ public final class Server { /** * Start accepting incoming connections. + * @return the channel future */ public ChannelFuture accept() { logger.log(Level.INFO, () -> "trying to bind to " + serverConfig.getAddress()); @@ -198,7 +200,7 @@ public final class Server { } public ServerTransport newTransport(HttpVersion httpVersion) { - return httpVersion.majorVersion() == 1 ? new Http1ServerTransport(this) : new Http2ServerTransport(this); + return httpVersion.majorVersion() == 1 ? new HttpServerTransport(this) : new Http2ServerTransport(this); } public synchronized void shutdownGracefully() throws IOException { diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/HttpServerChannelInitializer.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/HttpServerChannelInitializer.java deleted file mode 100644 index 0ba5172..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/HttpServerChannelInitializer.java +++ /dev/null @@ -1,220 +0,0 @@ -package org.xbib.netty.http.server.handler; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.codec.http.HttpContentDecompressor; -import io.netty.handler.codec.http.HttpObjectAggregator; -import io.netty.handler.codec.http.HttpServerCodec; -import io.netty.handler.codec.http.HttpServerUpgradeHandler; -import io.netty.handler.codec.http2.CleartextHttp2ServerUpgradeHandler; -import io.netty.handler.codec.http2.Http2ConnectionHandler; -import io.netty.handler.codec.http2.Http2MultiplexCodec; -import io.netty.handler.codec.http2.Http2MultiplexCodecBuilder; -import io.netty.handler.codec.http2.Http2ServerUpgradeCodec; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.ssl.ApplicationProtocolNames; -import io.netty.handler.ssl.ApplicationProtocolNegotiationHandler; -import io.netty.handler.ssl.SniHandler; -import io.netty.handler.ssl.SslContext; -import io.netty.util.AsciiString; -import io.netty.util.DomainNameMapping; -import org.xbib.netty.http.server.Server; -import org.xbib.netty.http.server.ServerConfig; -import org.xbib.netty.http.server.handler.http1.HttpHandler; -import org.xbib.netty.http.server.handler.http1.IdleTimeoutHandler; -import org.xbib.netty.http.server.handler.http1.TrafficLoggingHandler; -import org.xbib.netty.http.server.handler.http2.UserEventLogger; -import org.xbib.netty.http.server.internal.Http1ObjectEncoder; -import org.xbib.netty.http.server.internal.Http2ObjectEncoder; -import org.xbib.netty.http.server.internal.HttpObjectEncoder; - -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * HTTP server channel initializer. - */ -public class HttpServerChannelInitializer extends ChannelInitializer { - - private static final Logger logger = Logger.getLogger(HttpServerChannelInitializer.class.getName()); - - private final Server server; - - private final ServerConfig serverConfig; - - private final Http2ConnectionHandler http2ConnectionHandler; - - private final DomainNameMapping domainNameMapping; - - public HttpServerChannelInitializer(Server server, ServerConfig serverConfig, - DomainNameMapping domainNameMapping) { - this.server = server; - this.serverConfig = serverConfig; - this.domainNameMapping = domainNameMapping; - this.http2ConnectionHandler = null;//createHttp2ConnectionHandler(serverConfig); - - } - - @Override - public void initChannel(SocketChannel ch) { - if (serverConfig.isDebug()) { - ch.pipeline().addLast(new TrafficLoggingHandler(LogLevel.DEBUG)); - } - if (serverConfig.getAddress().isSecure()) { - configureSecure(ch); - } else { - configureClearText(ch); - } - HttpObjectEncoder encoder = serverConfig.getAddress().getVersion().majorVersion() == 2 ? - new Http2ObjectEncoder(http2ConnectionHandler.encoder()) : - new Http1ObjectEncoder(); - if (serverConfig.isDebug()) { - logger.log(Level.FINE, "server channel initialized: " + ch.pipeline().names()); - } - } - - private void configureClearText(SocketChannel ch) { - ChannelPipeline pipeline = ch.pipeline(); - if (serverConfig.getAddress().getVersion().majorVersion() == 1) { - if (serverConfig.isInstallHttp2Upgrade()) { - installHttp2Upgrade(pipeline); - } else { - pipeline.addFirst(new IdleTimeoutHandler()); - pipeline.addLast(new UserEventLogger()); - pipeline.addLast(createHttp1ConnectionHandler(serverConfig)); - configureHttp1Pipeline(pipeline); - } - } else if (serverConfig.getAddress().getVersion().majorVersion() == 2) { - pipeline.addLast(http2ConnectionHandler); - configureHttp2Pipeline(pipeline); - } - } - - private void installHttp2Upgrade(ChannelPipeline pipeline) { - HttpServerCodec httpServerCodec = new HttpServerCodec(); - HttpServerUpgradeHandler httpServerUpgradeHandler = new HttpServerUpgradeHandler(httpServerCodec, protocol -> { - if (AsciiString.contentEquals("h2c", protocol)) { - return new Http2ServerUpgradeCodec(http2ConnectionHandler); - } else { - return null; - } - }); - pipeline.addLast(new CleartextHttp2ServerUpgradeHandler(httpServerCodec, httpServerUpgradeHandler, - new HttpHandler(server))); - } - - private void configureSecure(SocketChannel ch) { - ChannelPipeline pipeline = ch.pipeline(); - pipeline.addLast(new SniHandler(domainNameMapping)); - pipeline.addLast(new Http2NegotiationHandler(ApplicationProtocolNames.HTTP_1_1)); - } - - private HttpServerCodec createHttp1ConnectionHandler(ServerConfig context) { - return new HttpServerCodec(context.getMaxInitialLineLength(), - context.getMaxHeadersSize(), context.getMaxChunkSize()); - } - - private void configureHttp1Pipeline(ChannelPipeline pipeline) { - if (serverConfig.isEnableGzip()) { - pipeline.addLast(new HttpContentDecompressor()); - } - HttpObjectAggregator httpObjectAggregator = - new HttpObjectAggregator(serverConfig.getMaxContentLength(), false); - httpObjectAggregator.setMaxCumulationBufferComponents(serverConfig.getMaxCompositeBufferComponents()); - pipeline.addLast(httpObjectAggregator); - pipeline.addLast(new HttpHandler(server)); - } - - private void configureHttp2Pipeline(ChannelPipeline pipeline) { - pipeline.addLast(new UserEventLogger()); - pipeline.addLast(new HttpHandler(server)); - } - - /*private static Http2ConnectionHandler createHttp2ConnectionHandler(ServerConfig serverConfig) { - Http2Settings initialSettings = serverConfig.getHttp2Settings(); - Http2Connection http2Connection = new DefaultHttp2Connection(true); - Long maxHeaderListSize = initialSettings.maxHeaderListSize(); - Http2FrameReader frameReader = new DefaultHttp2FrameReader(maxHeaderListSize == null ? - new DefaultHttp2HeadersDecoder(true) : - new DefaultHttp2HeadersDecoder(true, maxHeaderListSize)); - Http2FrameWriter frameWriter = new DefaultHttp2FrameWriter(); - Http2FrameLogger frameLogger = null; - if (serverConfig.isDebug()) { - frameLogger = new Http2FrameLogger(serverConfig.getDebugLogLevel(), "server"); - } - if (frameLogger != null) { - frameWriter = new Http2OutboundFrameLogger(frameWriter, frameLogger); - frameReader = new Http2InboundFrameLogger(frameReader, frameLogger); - } - Http2ConnectionEncoder encoder = new DefaultHttp2ConnectionEncoder(http2Connection, frameWriter); - Http2ConnectionDecoder decoder = new DefaultHttp2ConnectionDecoder(http2Connection, encoder, frameReader); - Http2ConnectionHandler http2ConnectionHandler = new Http2ServerConnectionHandler(decoder, encoder, initialSettings); - Http2Handler http2Handler = new Http2Handler(serverConfig, http2Connection, true); - http2ConnectionHandler.connection().addListener(http2Handler); - http2ConnectionHandler.decoder().frameListener(new DelegatingDecompressorFrameListener(http2Connection, http2Handler)); - if (serverConfig.getIdleTimeoutMillis() > 0) { - http2ConnectionHandler.gracefulShutdownTimeoutMillis(serverConfig.getIdleTimeoutMillis()); - } - return http2ConnectionHandler; - }*/ - - private ChannelHandler createMultiplexInitializer() { - /*HttpObjectAggregator httpObjectAggregator = - new HttpObjectAggregator(serverConfig.getMaxContentLength(), false); - httpObjectAggregator.setMaxCumulationBufferComponents(serverConfig.getMaxCompositeBufferComponents());*/ - return new ChannelInitializer() { - @Override - protected void initChannel(Channel ch) { - ch.pipeline().addLast(http2ConnectionHandler); - configureHttp2Pipeline(ch.pipeline()); - //.addLast(new Http2StreamFrameToHttpObjectCodec(true)) - //.addLast(httpObjectAggregator) - //.addLast(httpHandler); - } - }; - } - - private Http2MultiplexCodec createHttp2MultiplexCodec() { - Http2MultiplexCodecBuilder multiplexCodecBuilder = Http2MultiplexCodecBuilder.forServer(createMultiplexInitializer()); - multiplexCodecBuilder.initialSettings(serverConfig.getHttp2Settings()); - if (serverConfig.getIdleTimeoutMillis() > 0) { - multiplexCodecBuilder.gracefulShutdownTimeoutMillis(serverConfig.getIdleTimeoutMillis()); - } - return multiplexCodecBuilder.build(); - } - - /** - * Negotiates with the browser if HTTP/2 or HTTP is going to be used. Once decided, the - * pipeline is setup with the correct handlers for the selected protocol. - */ - class Http2NegotiationHandler extends ApplicationProtocolNegotiationHandler { - - Http2NegotiationHandler(String fallbackProtocol) { - super(fallbackProtocol); - } - - @Override - protected void configurePipeline(ChannelHandlerContext ctx, String protocol) { - ChannelPipeline pipeline = ctx.pipeline(); - if (ApplicationProtocolNames.HTTP_1_1.equals(protocol)) { - pipeline.addLast(createHttp1ConnectionHandler(serverConfig)); - configureHttp1Pipeline(pipeline); - return; - } - if (ApplicationProtocolNames.HTTP_2.equals(protocol)) { - pipeline.addLast(http2ConnectionHandler); - configureHttp2Pipeline(pipeline); - if (serverConfig.isDebug()) { - logger.log(Level.INFO, "after successful HTTP/2 negotiation: " + pipeline.names()); - } - return; - } - ctx.close(); - throw new IllegalStateException("unknown protocol: " + protocol); - } - } -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/IdleTimeoutHandler.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/IdleTimeoutHandler.java similarity index 94% rename from netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/IdleTimeoutHandler.java rename to netty-http-server/src/main/java/org/xbib/netty/http/server/handler/IdleTimeoutHandler.java index 62f7a09..1f26775 100644 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/IdleTimeoutHandler.java +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/IdleTimeoutHandler.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.server.handler.http1; +package org.xbib.netty.http.server.handler; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/TrafficLoggingHandler.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/TrafficLoggingHandler.java similarity index 95% rename from netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/TrafficLoggingHandler.java rename to netty-http-server/src/main/java/org/xbib/netty/http/server/handler/TrafficLoggingHandler.java index 63a2c19..92a27fe 100644 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/TrafficLoggingHandler.java +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/TrafficLoggingHandler.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.server.handler.http1; +package org.xbib.netty.http.server.handler; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandler; diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/HttpChannelInitializer.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpChannelInitializer.java similarity index 54% rename from netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/HttpChannelInitializer.java rename to netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpChannelInitializer.java index 05a5813..7e18fee 100644 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/HttpChannelInitializer.java +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpChannelInitializer.java @@ -1,11 +1,18 @@ -package org.xbib.netty.http.server.handler.http1; +package org.xbib.netty.http.server.handler.http; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelPipeline; import io.netty.channel.socket.SocketChannel; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.HttpContentDecompressor; import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpServerCodec; +import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.logging.LogLevel; import io.netty.handler.ssl.SniHandler; import io.netty.handler.ssl.SslContext; @@ -13,7 +20,10 @@ import io.netty.util.DomainNameMapping; import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.server.Server; import org.xbib.netty.http.server.ServerConfig; +import org.xbib.netty.http.server.handler.TrafficLoggingHandler; +import org.xbib.netty.http.server.transport.ServerTransport; +import java.nio.charset.StandardCharsets; import java.util.logging.Level; import java.util.logging.Logger; @@ -21,6 +31,8 @@ public class HttpChannelInitializer extends ChannelInitializer { private static final Logger logger = Logger.getLogger(HttpChannelInitializer.class.getName()); + private final Server server; + private final ServerConfig serverConfig; private final HttpAddress httpAddress; @@ -32,6 +44,7 @@ public class HttpChannelInitializer extends ChannelInitializer { public HttpChannelInitializer(Server server, HttpAddress httpAddress, DomainNameMapping domainNameMapping) { + this.server = server; this.serverConfig = server.getServerConfig(); this.httpAddress = httpAddress; this.domainNameMapping = domainNameMapping; @@ -40,6 +53,8 @@ public class HttpChannelInitializer extends ChannelInitializer { @Override public void initChannel(SocketChannel channel) { + ServerTransport serverTransport = server.newTransport(httpAddress.getVersion()); + channel.attr(ServerTransport.TRANSPORT_ATTRIBUTE_KEY).set(serverTransport); if (serverConfig.isDebug()) { channel.pipeline().addLast(new TrafficLoggingHandler(LogLevel.DEBUG)); } @@ -70,6 +85,46 @@ public class HttpChannelInitializer extends ChannelInitializer { false); httpObjectAggregator.setMaxCumulationBufferComponents(serverConfig.getMaxCompositeBufferComponents()); pipeline.addLast(httpObjectAggregator); + pipeline.addLast(new HttpPipeliningHandler(1024)); pipeline.addLast(httpHandler); } + + @Sharable + class HttpHandler extends ChannelInboundHandlerAdapter { + + private final Logger logger = Logger.getLogger(HttpHandler.class.getName()); + + private final Server server; + + public HttpHandler(Server server) { + this.server = server; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (msg instanceof HttpPipelinedRequest) { + HttpPipelinedRequest httpPipelinedRequest = (HttpPipelinedRequest) msg; + if (httpPipelinedRequest.getRequest() instanceof FullHttpRequest) { + FullHttpRequest fullHttpRequest = (FullHttpRequest) httpPipelinedRequest.getRequest(); + ServerTransport serverTransport = server.newTransport(fullHttpRequest.protocolVersion()); + serverTransport.requestReceived(ctx, fullHttpRequest, httpPipelinedRequest.getSequenceId()); + } + } else { + super.channelRead(ctx, msg); + } + } + + @Override + public void channelReadComplete(ChannelHandlerContext ctx) { + ctx.flush(); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + logger.log(Level.WARNING, cause.getMessage(), cause); + ctx.writeAndFlush(new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, + HttpResponseStatus.INTERNAL_SERVER_ERROR, + Unpooled.copiedBuffer(cause.getMessage().getBytes(StandardCharsets.UTF_8)))); + } + } } diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpPipelinedRequest.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpPipelinedRequest.java new file mode 100644 index 0000000..990eccf --- /dev/null +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpPipelinedRequest.java @@ -0,0 +1,23 @@ +package org.xbib.netty.http.server.handler.http; + +import io.netty.handler.codec.http.LastHttpContent; + +public class HttpPipelinedRequest { + + private final LastHttpContent request; + + private final int sequenceId; + + public HttpPipelinedRequest(LastHttpContent request, int sequenceId) { + this.request = request; + this.sequenceId = sequenceId; + } + + public LastHttpContent getRequest() { + return request; + } + + public int getSequenceId() { + return sequenceId; + } +} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpPipelinedResponse.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpPipelinedResponse.java new file mode 100644 index 0000000..575ac05 --- /dev/null +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpPipelinedResponse.java @@ -0,0 +1,34 @@ +package org.xbib.netty.http.server.handler.http; + +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.HttpResponse; + +public class HttpPipelinedResponse implements Comparable { + + private final HttpResponse response; + private final ChannelPromise promise; + private final int sequenceId; + + public HttpPipelinedResponse(HttpResponse response, ChannelPromise promise, int sequenceId) { + this.response = response; + this.promise = promise; + this.sequenceId = sequenceId; + } + + public int getSequenceId() { + return sequenceId; + } + + public HttpResponse getResponse() { + return response; + } + + public ChannelPromise getPromise() { + return promise; + } + + @Override + public int compareTo(HttpPipelinedResponse other) { + return this.sequenceId - other.sequenceId; + } +} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpPipeliningHandler.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpPipeliningHandler.java new file mode 100644 index 0000000..d1b8f5e --- /dev/null +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http/HttpPipeliningHandler.java @@ -0,0 +1,75 @@ +package org.xbib.netty.http.server.handler.http; + +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.LastHttpContent; + +import java.util.PriorityQueue; +import java.util.Queue; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Implements HTTP pipelining ordering, ensuring that responses are completely served in the same order as their + * corresponding requests. + * + * Based on https://github.com/typesafehub/netty-http-pipelining - which uses netty 3 + */ +public class HttpPipeliningHandler extends ChannelDuplexHandler { + + private final int pipelineCapacity; + + private final Queue httpPipelinedResponses; + + private final AtomicInteger requestCounter; + + private final AtomicInteger writtenRequests; + + /** + * @param pipelineCapacity the maximum number of channel events that will be retained prior to aborting the channel + * connection. This is required as events cannot queue up indefinitely; we would run out of + * memory if this was the case. + */ + public HttpPipeliningHandler(int pipelineCapacity) { + this.pipelineCapacity = pipelineCapacity; + this.httpPipelinedResponses = new PriorityQueue<>(3); + this.requestCounter = new AtomicInteger(); + this.writtenRequests = new AtomicInteger(); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (msg instanceof LastHttpContent) { + super.channelRead(ctx, new HttpPipelinedRequest((LastHttpContent) msg, requestCounter.getAndIncrement())); + } + } + + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { + if (msg instanceof HttpPipelinedResponse) { + boolean channelShouldClose = false; + synchronized (httpPipelinedResponses) { + if (httpPipelinedResponses.size() < pipelineCapacity) { + HttpPipelinedResponse currentEvent = (HttpPipelinedResponse) msg; + httpPipelinedResponses.add(currentEvent); + while (!httpPipelinedResponses.isEmpty()) { + HttpPipelinedResponse queuedPipelinedResponse = httpPipelinedResponses.peek(); + if (queuedPipelinedResponse.getSequenceId() != writtenRequests.get()) { + break; + } + httpPipelinedResponses.remove(); + super.write(ctx, queuedPipelinedResponse.getResponse(), queuedPipelinedResponse.getPromise()); + writtenRequests.getAndIncrement(); + } + } else { + channelShouldClose = true; + } + } + if (channelShouldClose) { + ctx.close(); + } + } else { + super.write(ctx, msg, promise); + } + } +} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/HttpHandler.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/HttpHandler.java deleted file mode 100644 index 442a758..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http1/HttpHandler.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.xbib.netty.http.server.handler.http1; - -import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.codec.http.DefaultFullHttpResponse; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpVersion; -import org.xbib.netty.http.server.Server; -import org.xbib.netty.http.server.transport.ServerTransport; - -import java.nio.charset.StandardCharsets; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * HTTP handler. - */ -@ChannelHandler.Sharable -public class HttpHandler extends ChannelInboundHandlerAdapter { - - private static final Logger logger = Logger.getLogger(HttpHandler.class.getName()); - - private final Server server; - - public HttpHandler(Server server) { - this.server = server; - } - - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - if (msg instanceof FullHttpRequest) { - FullHttpRequest fullHttpRequest = (FullHttpRequest) msg; - ServerTransport serverTransport = server.newTransport(fullHttpRequest.protocolVersion()); - serverTransport.requestReceived(ctx, fullHttpRequest); - } else { - super.channelRead(ctx, msg); - } - } - - @Override - public void channelReadComplete(ChannelHandlerContext ctx) { - ctx.flush(); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - logger.log(Level.WARNING, cause.getMessage(), cause); - ctx.writeAndFlush(new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, - HttpResponseStatus.INTERNAL_SERVER_ERROR, - Unpooled.copiedBuffer(cause.getMessage().getBytes(StandardCharsets.UTF_8)))); - } -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/DummyHandler.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/DummyHandler.java deleted file mode 100644 index 221bcaf..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/DummyHandler.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.xbib.netty.http.server.handler.http2; - -import io.netty.channel.ChannelDuplexHandler; -import io.netty.channel.ChannelHandlerContext; - -import java.util.logging.Level; -import java.util.logging.Logger; - -public class DummyHandler extends ChannelDuplexHandler { - - private static final Logger logger = Logger.getLogger(DummyHandler.class.getName()); - - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - logger.log(Level.INFO, "msg = " + msg + " class = " + msg.getClass().getName()); - } -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/FrameListener.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/FrameListener.java deleted file mode 100644 index 0b4094d..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/FrameListener.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.xbib.netty.http.server.handler.http2; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http2.DefaultHttp2DataFrame; -import io.netty.handler.codec.http2.DefaultHttp2HeadersFrame; -import io.netty.handler.codec.http2.Http2DataFrame; -import io.netty.handler.codec.http2.Http2EventAdapter; -import io.netty.handler.codec.http2.Http2Headers; -import io.netty.handler.codec.http2.Http2HeadersFrame; - -import java.util.logging.Level; -import java.util.logging.Logger; - -public class FrameListener extends Http2EventAdapter { - - private static final Logger logger = Logger.getLogger(FrameListener.class.getName()); - - @Override - public void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers headers, int streamDependency, - short weight, boolean exclusive, int padding, boolean endStream) { - logger.log(Level.FINE, "onHeadersRead"); - Http2HeadersFrame frame = new DefaultHttp2HeadersFrame(headers,endStream,padding); - ctx.fireChannelRead(frame); - } - - public int onDataRead(ChannelHandlerContext ctx, int streamId, ByteBuf data, int padding, boolean endOfStream) { - logger.log(Level.FINE, "onDataRead"); - Http2DataFrame frame = new DefaultHttp2DataFrame(data, endOfStream, padding); - ctx.fireChannelRead(frame); - return data.readableBytes() + padding; - } -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/HelloWorldHttp1Handler.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/HelloWorldHttp1Handler.java deleted file mode 100644 index e6870b6..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/HelloWorldHttp1Handler.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.xbib.netty.http.server.handler.http2; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufUtil; -import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.handler.codec.http.DefaultFullHttpResponse; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.FullHttpResponse; -import io.netty.handler.codec.http.HttpHeaderValues; -import io.netty.handler.codec.http.HttpUtil; - -import java.nio.charset.StandardCharsets; - -import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; -import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; -import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE; -import static io.netty.handler.codec.http.HttpResponseStatus.CONTINUE; -import static io.netty.handler.codec.http.HttpResponseStatus.OK; -import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; -import static io.netty.util.internal.ObjectUtil.checkNotNull; - -/** - * HTTP handler that responds with a "Hello World" - */ -public class HelloWorldHttp1Handler extends SimpleChannelInboundHandler { - - static final ByteBuf RESPONSE_BYTES = Unpooled.unreleasableBuffer(Unpooled.copiedBuffer("Hello World", StandardCharsets.UTF_8)); - - private final String establishApproach; - - public HelloWorldHttp1Handler(String establishApproach) { - this.establishApproach = checkNotNull(establishApproach, "establishApproach"); - } - - @Override - public void channelRead0(ChannelHandlerContext ctx, FullHttpRequest req) { - if (HttpUtil.is100ContinueExpected(req)) { - ctx.write(new DefaultFullHttpResponse(HTTP_1_1, CONTINUE)); - } - boolean keepAlive = HttpUtil.isKeepAlive(req); - - ByteBuf content = ctx.alloc().buffer(); - content.writeBytes(RESPONSE_BYTES.duplicate()); - ByteBufUtil.writeAscii(content, " - via " + req.protocolVersion() + " (" + establishApproach + ")"); - - FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, OK, content); - response.headers().set(CONTENT_TYPE, "text/plain; charset=UTF-8"); - response.headers().setInt(CONTENT_LENGTH, response.content().readableBytes()); - - if (!keepAlive) { - ctx.write(response).addListener(ChannelFutureListener.CLOSE); - } else { - response.headers().set(CONNECTION, HttpHeaderValues.KEEP_ALIVE); - ctx.write(response); - } - } - - @Override - public void channelReadComplete(ChannelHandlerContext ctx) { - ctx.flush(); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - cause.printStackTrace(); - ctx.close(); - } -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/HelloWorldHttp2Handler.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/HelloWorldHttp2Handler.java deleted file mode 100644 index e5e490b..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/HelloWorldHttp2Handler.java +++ /dev/null @@ -1,87 +0,0 @@ -package org.xbib.netty.http.server.handler.http2; - -import static io.netty.buffer.Unpooled.copiedBuffer; -import static io.netty.buffer.Unpooled.unreleasableBuffer; -import static io.netty.handler.codec.http.HttpResponseStatus.OK; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufUtil; -import io.netty.channel.ChannelDuplexHandler; -import io.netty.channel.ChannelHandler.Sharable; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http2.DefaultHttp2DataFrame; -import io.netty.handler.codec.http2.DefaultHttp2Headers; -import io.netty.handler.codec.http2.DefaultHttp2HeadersFrame; -import io.netty.handler.codec.http2.Http2DataFrame; -import io.netty.handler.codec.http2.Http2Headers; -import io.netty.handler.codec.http2.Http2HeadersFrame; -import io.netty.util.CharsetUtil; - -/** - * A simple handler that responds with the message "Hello World!". - * - *

This example is making use of the "multiplexing" http2 API, where streams are mapped to child - * Channels. This API is very experimental and incomplete. - */ -@Sharable -public class HelloWorldHttp2Handler extends ChannelDuplexHandler { - - private static final ByteBuf RESPONSE_BYTES = unreleasableBuffer(copiedBuffer("Hello World", CharsetUtil.UTF_8)); - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - super.exceptionCaught(ctx, cause); - cause.printStackTrace(); - ctx.close(); - } - - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - if (msg instanceof Http2HeadersFrame) { - onHeadersRead(ctx, (Http2HeadersFrame) msg); - } else if (msg instanceof Http2DataFrame) { - onDataRead(ctx, (Http2DataFrame) msg); - } else { - super.channelRead(ctx, msg); - } - } - - @Override - public void channelReadComplete(ChannelHandlerContext ctx) throws Exception { - ctx.flush(); - } - - /** - * If receive a frame with end-of-stream set, send a pre-canned response. - */ - private static void onDataRead(ChannelHandlerContext ctx, Http2DataFrame data) throws Exception { - if (data.isEndStream()) { - sendResponse(ctx, data.content()); - } else { - // We do not send back the response to the remote-peer, so we need to release it. - data.release(); - } - } - - /** - * If receive a frame with end-of-stream set, send a pre-canned response. - */ - private static void onHeadersRead(ChannelHandlerContext ctx, Http2HeadersFrame headers) - throws Exception { - if (headers.isEndStream()) { - ByteBuf content = ctx.alloc().buffer(); - content.writeBytes(RESPONSE_BYTES.duplicate()); - ByteBufUtil.writeAscii(content, " - via HTTP/2"); - sendResponse(ctx, content); - } - } - - /** - * Sends a "Hello World" DATA frame to the client. - */ - private static void sendResponse(ChannelHandlerContext ctx, ByteBuf payload) { - // Send a frame for the response status - Http2Headers headers = new DefaultHttp2Headers().status(OK.codeAsText()); - ctx.write(new DefaultHttp2HeadersFrame(headers)); - ctx.write(new DefaultHttp2DataFrame(payload, true)); - } -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2ChannelInitializer.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2ChannelInitializer.java index 41258e4..18662e4 100644 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2ChannelInitializer.java +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2ChannelInitializer.java @@ -2,51 +2,34 @@ package org.xbib.netty.http.server.handler.http2; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelPipeline; import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.codec.http.HttpMessage; +import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.HttpObjectAggregator; import io.netty.handler.codec.http.HttpServerCodec; import io.netty.handler.codec.http.HttpServerUpgradeHandler; -import io.netty.handler.codec.http2.DefaultHttp2Connection; -import io.netty.handler.codec.http2.DefaultHttp2ConnectionDecoder; -import io.netty.handler.codec.http2.DefaultHttp2ConnectionEncoder; -import io.netty.handler.codec.http2.DefaultHttp2FrameReader; -import io.netty.handler.codec.http2.DefaultHttp2FrameWriter; -import io.netty.handler.codec.http2.DefaultHttp2HeadersDecoder; -import io.netty.handler.codec.http2.DelegatingDecompressorFrameListener; +import io.netty.handler.codec.http2.CleartextHttp2ServerUpgradeHandler; +import io.netty.handler.codec.http2.DefaultHttp2SettingsFrame; import io.netty.handler.codec.http2.Http2CodecUtil; -import io.netty.handler.codec.http2.Http2Connection; -import io.netty.handler.codec.http2.Http2ConnectionDecoder; -import io.netty.handler.codec.http2.Http2ConnectionEncoder; -import io.netty.handler.codec.http2.Http2ConnectionHandler; -import io.netty.handler.codec.http2.Http2ConnectionHandlerBuilder; import io.netty.handler.codec.http2.Http2FrameLogger; -import io.netty.handler.codec.http2.Http2FrameReader; -import io.netty.handler.codec.http2.Http2FrameWriter; -import io.netty.handler.codec.http2.Http2InboundFrameLogger; import io.netty.handler.codec.http2.Http2MultiplexCodec; import io.netty.handler.codec.http2.Http2MultiplexCodecBuilder; -import io.netty.handler.codec.http2.Http2OutboundFrameLogger; import io.netty.handler.codec.http2.Http2ServerUpgradeCodec; import io.netty.handler.codec.http2.Http2Settings; -import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandlerBuilder; -import io.netty.handler.codec.http2.InboundHttp2ToHttpAdapter; -import io.netty.handler.codec.http2.InboundHttp2ToHttpAdapterBuilder; import io.netty.handler.logging.LogLevel; import io.netty.handler.ssl.SniHandler; import io.netty.handler.ssl.SslContext; import io.netty.util.AsciiString; import io.netty.util.DomainNameMapping; -import io.netty.util.ReferenceCountUtil; import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.server.Server; import org.xbib.netty.http.server.ServerConfig; -import org.xbib.netty.http.server.handler.http1.HttpHandler; -import org.xbib.netty.http.server.handler.http1.TrafficLoggingHandler; +import org.xbib.netty.http.server.handler.TrafficLoggingHandler; +import org.xbib.netty.http.server.transport.ServerTransport; +import java.io.IOException; import java.util.logging.Level; import java.util.logging.Logger; @@ -71,13 +54,10 @@ public class Http2ChannelInitializer extends ChannelInitializer { this.domainNameMapping = domainNameMapping; } - /** - * The channel initialization for HTTP/2. - * - * @param channel socket channel - */ @Override public void initChannel(Channel channel) { + ServerTransport serverTransport = server.newTransport(httpAddress.getVersion()); + channel.attr(ServerTransport.TRANSPORT_ATTRIBUTE_KEY).set(serverTransport); if (serverConfig.isDebug()) { channel.pipeline().addLast(new TrafficLoggingHandler(LogLevel.DEBUG)); } @@ -97,155 +77,71 @@ public class Http2ChannelInitializer extends ChannelInitializer { } private void configureCleartext(Channel ch) { - Http2SettingsHandler http2SettingsHandler = new Http2SettingsHandler(); - Http2RequestHandler http2RequestHandler = new Http2RequestHandler(); - //HttpHandler httpHandler = new HttpHandler(server); - - ch.pipeline() - //.addLast(newConnectionHandler()) - .addLast(upgradeHandler()); - //.addLast(http2SettingsHandler) - //.addLast(http2RequestHandler); - // .addLast(sourceCodec) - - /*final Http2MultiplexCodec http2Codec = Http2MultiplexCodecBuilder.forServer(new ChannelInitializer() { - @Override - protected void initChannel(Channel ch) { - logger.log(Level.INFO, "initChannel multiplex "); - } - }).build(); - HttpServerUpgradeHandler.UpgradeCodecFactory upgradeCodecFactory = protocol -> - new Http2ServerUpgradeCodec(http2Codec); - final HttpServerCodec serverCodec = new HttpServerCodec(); - ch.pipeline().addLast(serverCodec) - .addLast(new HttpServerUpgradeHandler(serverCodec, upgradeCodecFactory)) - .addLast(new SimpleChannelInboundHandler() { + ChannelPipeline p = ch.pipeline(); + Http2MultiplexCodecBuilder serverMultiplexCodecBuilder = Http2MultiplexCodecBuilder.forServer(new ChannelInitializer() { @Override - protected void channelRead0(ChannelHandlerContext ctx, HttpMessage msg) throws Exception { - // If this handler is hit then no upgrade has been attempted and the client is just talking HTTP. - System.err.println("Directly talking: " + msg.protocolVersion() + " (no upgrade was attempted)"); - ChannelPipeline pipeline = ctx.pipeline(); - ChannelHandlerContext thisCtx = pipeline.context(this); - pipeline.addAfter(thisCtx.name(), null, new HelloWorldHttp1Handler("Direct. No Upgrade Attempted.")); - pipeline.replace(this, null, new HttpObjectAggregator(Integer.MAX_VALUE)); - ctx.fireChannelRead(ReferenceCountUtil.retain(msg)); + protected void initChannel(Channel channel) { + ServerTransport serverTransport = server.newTransport(httpAddress.getVersion()); + channel.attr(ServerTransport.TRANSPORT_ATTRIBUTE_KEY).set(serverTransport); + ChannelPipeline p = channel.pipeline(); + p.addLast("multiplex-server-frame-converter", + new Http2StreamFrameToHttpObjectCodec(true)); + p.addLast("multiplex-server-chunk-aggregator", + new HttpObjectAggregator(serverConfig.getMaxContentLength())); + p.addLast("multiplex-server-request-handler", + new ServerRequestHandler()); } }) - .addLast(new UserEventLogger()) - .addLast(new HttpHandler(server)); -*/ - - /* - Http2FrameCodec http2FrameCodec = Http2FrameCodecBuilder.forServer().build(); - - Http2StreamFrameToHttpObjectCodec http2StreamFrameToHttpObjectCodec = - new Http2StreamFrameToHttpObjectCodec(true, true); - - HttpObjectAggregator httpObjectAggregator = - new HttpObjectAggregator(serverConfig.getMaxContentLength(), false); - httpObjectAggregator.setMaxCumulationBufferComponents(serverConfig.getMaxCompositeBufferComponents()); - HttpHandler httpHandler = new HttpHandler(server); - - Http2ConnectionHandler http2ConnectionHandler = newConnectionHandler(server.getServerConfig()); - Http2Connection http2Connection = http2ConnectionHandler.connection(); - Http2Handler http2Handler = new Http2Handler(serverConfig, http2Connection, true); - http2Connection.addListener(http2Handler); - http2ConnectionHandler.decoder().frameListener(new DelegatingDecompressorFrameListener(http2Connection, http2Handler)); - - channel.pipeline().addLast(http2ConnectionHandler) - .addLast(new UserEventLogger()) - .addLast(new HttpHandler(server)); - - //.addLast(new Http2StreamFrameToHttpObjectCodec(true)) - //.addLast(httpObjectAggregator) - //.addLast(httpHandler); - */ - } - - private Http2ConnectionHandler newStandardConnectionHandler() { - Http2Connection http2Connection = new DefaultHttp2Connection(true); - InboundHttp2ToHttpAdapter inboundHttp2ToHttpAdapter = - new InboundHttp2ToHttpAdapterBuilder(http2Connection) - .maxContentLength(serverConfig.getMaxContentLength()) - .propagateSettings(true) - .validateHttpHeaders(true) - .build(); - Http2ConnectionHandlerBuilder builder = new Http2ConnectionHandlerBuilder() - .connection(http2Connection) - .initialSettings(serverConfig.getHttp2Settings()) - .frameListener(new DelegatingDecompressorFrameListener(http2Connection, inboundHttp2ToHttpAdapter)); + .initialSettings(Http2Settings.defaultSettings()); if (serverConfig.isDebug()) { - builder.frameLogger(new Http2FrameLogger(serverConfig.getDebugLogLevel(), "server")); + serverMultiplexCodecBuilder.frameLogger(new Http2FrameLogger(LogLevel.DEBUG, "server")); } - return builder.build(); - } - - private Http2ConnectionHandler newConnectionHandler() { - Http2Settings initialSettings = serverConfig.getHttp2Settings(); - Http2Connection http2Connection = new DefaultHttp2Connection(true); - Long maxHeaderListSize = initialSettings.maxHeaderListSize(); - Http2FrameReader frameReader = new DefaultHttp2FrameReader(maxHeaderListSize == null ? - new DefaultHttp2HeadersDecoder(true) : - new DefaultHttp2HeadersDecoder(true, maxHeaderListSize)); - Http2FrameWriter frameWriter = new DefaultHttp2FrameWriter(); - Http2FrameLogger frameLogger = null; - if (serverConfig.isDebug()) { - frameLogger = new Http2FrameLogger(serverConfig.getDebugLogLevel(), "server"); - } - if (frameLogger != null) { - frameWriter = new Http2OutboundFrameLogger(frameWriter, frameLogger); - frameReader = new Http2InboundFrameLogger(frameReader, frameLogger); - } - Http2ConnectionEncoder encoder = new DefaultHttp2ConnectionEncoder(http2Connection, frameWriter); - Http2ConnectionDecoder decoder = new DefaultHttp2ConnectionDecoder(http2Connection, encoder, frameReader); - - Http2ConnectionHandlerBuilder builder = new Http2ConnectionHandlerBuilder() - .connection(http2Connection) - //.codec(decoder, encoder) - //.initialSettings(initialSettings) - .frameListener(new FrameListener()) - .frameLogger(new Http2FrameLogger(serverConfig.getDebugLogLevel(), "server")); - if (serverConfig.getIdleTimeoutMillis() > 0) { - builder.gracefulShutdownTimeoutMillis(serverConfig.getIdleTimeoutMillis()); - } - return builder.build(); - //Http2Handler http2Handler = new Http2Handler(server, http2Connection, true); - //http2ConnectionHandler.connection().addListener(http2Handler); - //http2ConnectionHandler.decoder().frameListener(); - //return http2ConnectionHandler; - } - - static class Http2ServerConnectionHandler extends Http2ConnectionHandler { - - Http2ServerConnectionHandler(Http2ConnectionDecoder decoder, Http2ConnectionEncoder encoder, - Http2Settings initialSettings) { - super(decoder, encoder, initialSettings); - } - } - - private final HttpServerUpgradeHandler.UpgradeCodecFactory upgradeCodecFactory = protocol -> { - if (AsciiString.contentEquals(Http2CodecUtil.HTTP_UPGRADE_PROTOCOL_NAME, protocol)) { - return upgradeCodec(); - } else { - return null; - } - }; - - private Http2ServerUpgradeCodec upgradeCodec() { - return new Http2ServerUpgradeCodec(Http2MultiplexCodecBuilder.forServer(http2MultiplexCodec()).build()); - } - - private HttpServerUpgradeHandler upgradeHandler() { + Http2MultiplexCodec serverMultiplexCodec = serverMultiplexCodecBuilder.build(); + HttpServerUpgradeHandler.UpgradeCodecFactory upgradeCodecFactory = protocol -> { + if (AsciiString.contentEquals(Http2CodecUtil.HTTP_UPGRADE_PROTOCOL_NAME, protocol)) { + return new Http2ServerUpgradeCodec("server-codec", serverMultiplexCodec); + } else { + return null; + } + }; HttpServerCodec sourceCodec = new HttpServerCodec(); - return new HttpServerUpgradeHandler(sourceCodec, upgradeCodecFactory); + HttpServerUpgradeHandler upgradeHandler = new HttpServerUpgradeHandler(sourceCodec, upgradeCodecFactory); + CleartextHttp2ServerUpgradeHandler cleartextHttp2ServerUpgradeHandler = + new CleartextHttp2ServerUpgradeHandler(sourceCodec, upgradeHandler, serverMultiplexCodec); + p.addLast("server-upgrade", cleartextHttp2ServerUpgradeHandler); + p.addLast("server-messages", new ServerMessages()); } - private Http2MultiplexCodec http2MultiplexCodec() { - Http2FrameLogger frameLogger = new Http2FrameLogger(serverConfig.getDebugLogLevel(), "server"); - return Http2MultiplexCodecBuilder.forServer(new DummyHandler()) - .frameLogger(frameLogger) - .initialSettings(serverConfig.getHttp2Settings()) - .build(); + class ServerRequestHandler extends SimpleChannelInboundHandler { + + @Override + protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest fullHttpRequest) throws IOException { + ServerTransport serverTransport = ctx.channel().attr(ServerTransport.TRANSPORT_ATTRIBUTE_KEY).get(); + serverTransport.requestReceived(ctx, fullHttpRequest); + } } + class ServerMessages extends ChannelInboundHandlerAdapter { + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + ServerTransport serverTransport = ctx.channel().attr(ServerTransport.TRANSPORT_ATTRIBUTE_KEY).get(); + if (msg instanceof DefaultHttp2SettingsFrame) { + DefaultHttp2SettingsFrame http2SettingsFrame = (DefaultHttp2SettingsFrame) msg; + serverTransport.settingsReceived(ctx, http2SettingsFrame.settings()); + } + } + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) { + ServerTransport serverTransport = ctx.channel().attr(ServerTransport.TRANSPORT_ATTRIBUTE_KEY).get(); + ctx.fireUserEventTriggered(evt); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws IOException { + ServerTransport serverTransport = ctx.channel().attr(ServerTransport.TRANSPORT_ATTRIBUTE_KEY).get(); + serverTransport.exceptionReceived(ctx, cause); + } + } } diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2Handler.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2Handler.java deleted file mode 100644 index 4e22d61..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2Handler.java +++ /dev/null @@ -1,483 +0,0 @@ -package org.xbib.netty.http.server.handler.http2; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.DefaultFullHttpRequest; -import io.netty.handler.codec.http.FullHttpMessage; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpUtil; -import io.netty.handler.codec.http.HttpVersion; -import io.netty.handler.codec.http2.CharSequenceMap; -import io.netty.handler.codec.http2.Http2CodecUtil; -import io.netty.handler.codec.http2.Http2Connection; -import io.netty.handler.codec.http2.Http2Error; -import io.netty.handler.codec.http2.Http2EventAdapter; -import io.netty.handler.codec.http2.Http2Exception; -import io.netty.handler.codec.http2.Http2Headers; -import io.netty.handler.codec.http2.Http2LocalFlowController; -import io.netty.handler.codec.http2.Http2Settings; -import io.netty.handler.codec.http2.Http2Stream; -import io.netty.handler.codec.http2.HttpConversionUtil; -import io.netty.util.AsciiString; -import io.netty.util.internal.ObjectUtil; -import org.xbib.netty.http.server.Server; -import org.xbib.netty.http.server.ServerConfig; -import org.xbib.netty.http.server.transport.ServerTransport; - -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * A HTTP/2 event adapter for a server. - * - * This event adapter expects {@link Http2Settings} are sent from the server before the - * {@link HttpRequest} is submitted by sending a header frame, and, if a body exists, a - * data frame. - */ -@ChannelHandler.Sharable -public class Http2Handler extends Http2EventAdapter { - - private static final Logger logger = Logger.getLogger(Http2Handler.class.getName()); - - private static final HttpVersion HTTP_2_0 = HttpVersion.valueOf("HTTP/2.0"); - - private final Server server; - - private final ServerConfig serverConfig; - - private final ServerTransport serverTransport; - - private final Http2Connection connection; - - private final Http2Connection.PropertyKey messageKey; - - private final boolean validateHttpHeaders; - - /** - * Constructor for {@link Http2Handler}. - * @param server the server - * @param connection the HTTP/2 connection - * @param validateHeaders true if headers should be validated - */ - public Http2Handler(Server server, Http2Connection connection, boolean validateHeaders) { - this.server = server; - this.serverConfig = server.getServerConfig(); - this.connection = connection; - this.validateHttpHeaders = validateHeaders; - this.messageKey = connection.newKey(); - this.serverTransport = server.newTransport(HTTP_2_0); - } - - /** - * Handles an inbound {@code SETTINGS} frame. - * After frame is received, the request is sent. - * - * @param ctx the context from the handler where the frame was read. - * @param settings the settings received from the remote endpoint. - */ - @Override - public void onSettingsRead(ChannelHandlerContext ctx, Http2Settings settings) { - if (serverConfig.isDebug()) { - logger.log(Level.FINE, () -> "settings received " + settings); - } - try { - serverTransport.settingsReceived(ctx, settings); - } catch (Exception e) { - logger.log(Level.WARNING, e.getMessage(), e); - } - } - - /** - * Handles an inbound {@code HEADERS} frame. - *

- * Only one of the following methods will be called for each {@code HEADERS} frame sequence. - * One will be called when the {@code END_HEADERS} flag has been received. - *

    - *
  • {@link #onHeadersRead(ChannelHandlerContext, int, Http2Headers, int, boolean)}
    • - *
  • {@link #onHeadersRead(ChannelHandlerContext, int, Http2Headers, int, short, boolean, int, boolean)}
    • - *
  • {@link #onPushPromiseRead(ChannelHandlerContext, int, int, Http2Headers, int)}
    • - *
- *

- * To say it another way; the {@link Http2Headers} will contain all of the headers - * for the current message exchange step (additional queuing is not necessary). - * - * @param ctx the context from the handler where the frame was read. - * @param streamId the subject stream for the frame. - * @param headers the received headers. - * @param padding additional bytes that should be added to obscure the true content size. Must be between 0 and - * 256 (inclusive). - * @param endOfStream Indicates whether this is the last frame to be sent from the remote endpoint - * for this stream. - */ - @Override - public void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers headers, int padding, - boolean endOfStream) throws Http2Exception { - if (serverConfig.isDebug()) { - logger.log(Level.FINE, () -> "headers received " + headers + " endOfStream " + endOfStream); - } - Http2Stream stream = connection.stream(streamId); - FullHttpMessage msg = beginHeader(ctx, stream, headers); - endHeader(ctx, stream, msg, endOfStream); - } - - /** - * Handles an inbound {@code HEADERS} frame with priority information specified. - * Only called if {@code END_HEADERS} encountered. - *

- * Only one of the following methods will be called for each {@code HEADERS} frame sequence. - * One will be called when the {@code END_HEADERS} flag has been received. - *

    - *
  • {@link #onHeadersRead(ChannelHandlerContext, int, Http2Headers, int, boolean)}
    • - *
  • {@link #onHeadersRead(ChannelHandlerContext, int, Http2Headers, int, short, boolean, int, boolean)}
    • - *
  • {@link #onPushPromiseRead(ChannelHandlerContext, int, int, Http2Headers, int)}
    • - *
- *

- * To say it another way; the {@link Http2Headers} will contain all of the headers - * for the current message exchange step (additional queuing is not necessary). - * - * @param ctx the context from the handler where the frame was read. - * @param streamId the subject stream for the frame. - * @param headers the received headers. - * @param streamDependency the stream on which this stream depends, or 0 if dependent on the - * connection. - * @param weight the new weight for the stream. - * @param exclusive whether or not the stream should be the exclusive dependent of its parent. - * @param padding additional bytes that should be added to obscure the true content size. Must be between 0 and - * 256 (inclusive). - * @param endOfStream Indicates whether this is the last frame to be sent from the remote endpoint - * for this stream. - */ - @Override - public void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers headers, int streamDependency, - short weight, boolean exclusive, int padding, boolean endOfStream) throws Http2Exception { - if (serverConfig.isDebug()) { - logger.log(Level.FINE, () -> "headers received (weighted) " + headers + " endOfStream " + endOfStream); - } - Http2Stream stream = connection.stream(streamId); - FullHttpMessage msg = beginHeader(ctx, stream, headers); - if (streamDependency != Http2CodecUtil.CONNECTION_STREAM_ID) { - msg.headers().setInt(HttpConversionUtil.ExtensionHeaderNames.STREAM_DEPENDENCY_ID.text(), - streamDependency); - } - msg.headers().setShort(HttpConversionUtil.ExtensionHeaderNames.STREAM_WEIGHT.text(), weight); - endHeader(ctx, stream, msg, endOfStream); - } - - /** - * Handles an inbound {@code DATA} frame. - * - * @param ctx the context from the handler where the frame was read. - * @param streamId the subject stream for the frame. - * @param data payload buffer for the frame. This buffer will be released by the codec. - * @param padding additional bytes that should be added to obscure the true content size. Must be between 0 and - * 256 (inclusive). - * @param endOfStream Indicates whether this is the last frame to be sent from the remote endpoint for this stream. - * @return the number of bytes that have been processed by the application. The returned bytes are used by the - * inbound flow controller to determine the appropriate time to expand the inbound flow control window (i.e. send - * {@code WINDOW_UPDATE}). Returning a value equal to the length of {@code data} + {@code padding} will effectively - * opt-out of application-level flow control for this frame. Returning a value less than the length of {@code data} - * + {@code padding} will defer the returning of the processed bytes, which the application must later return via - * {@link Http2LocalFlowController#consumeBytes(Http2Stream, int)}. The returned value must - * be >= {@code 0} and <= {@code data.readableBytes()} + {@code padding}. - */ - @Override - public int onDataRead(ChannelHandlerContext ctx, int streamId, ByteBuf data, int padding, boolean endOfStream) - throws Http2Exception { - if (serverConfig.isDebug()) { - logger.log(Level.FINE, () -> "data received " + data); - } - Http2Stream stream = connection.stream(streamId); - FullHttpMessage msg = getMessage(stream); - if (msg == null) { - throw Http2Exception.connectionError(Http2Error.PROTOCOL_ERROR, - "data frame received for unknown stream id %d", streamId); - } - ByteBuf content = msg.content(); - final int dataReadableBytes = data.readableBytes(); - if (content.readableBytes() > serverConfig.getMaxContentLength() - dataReadableBytes) { - throw Http2Exception.connectionError(Http2Error.INTERNAL_ERROR, - "content length exceeded maximum of %d for stream id %d", - serverConfig.getMaxContentLength(), streamId); - } - content.writeBytes(data, data.readerIndex(), dataReadableBytes); - if (endOfStream) { - fireChannelRead(ctx, msg, false, stream); - } - return dataReadableBytes + padding; - } - - /** - * Handles an inbound {@code RST_STREAM} frame. Deletes push stream id if present. - * - * @param ctx the context from the handler where the frame was read. - * @param streamId the stream that is terminating. - * @param errorCode the error code identifying the type of failure. - */ - @Override - public void onRstStreamRead(ChannelHandlerContext ctx, int streamId, long errorCode) { - if (serverConfig.isDebug()) { - logger.log(Level.FINE, () -> "rst stream received: error code = " + errorCode); - } - Http2Stream stream = connection.stream(streamId); - FullHttpMessage msg = getMessage(stream); - if (msg != null) { - removeMessage(stream, true); - } - } - - /** - * Handles an inbound {@code PUSH_PROMISE} frame. Only called if {@code END_HEADERS} encountered. - *

- * Promised requests MUST be authoritative, cacheable, and safe. - * See [RFC http2], Section 8.2. - *

- * Only one of the following methods will be called for each {@code HEADERS} frame sequence. - * One will be called when the {@code END_HEADERS} flag has been received. - *

    - *
  • {@link #onHeadersRead(ChannelHandlerContext, int, Http2Headers, int, boolean)}
    • - *
  • {@link #onHeadersRead(ChannelHandlerContext, int, Http2Headers, int, short, boolean, int, boolean)}
    • - *
  • {@link #onPushPromiseRead(ChannelHandlerContext, int, int, Http2Headers, int)}
    • - *
- *

- * To say it another way; the {@link Http2Headers} will contain all of the headers - * for the current message exchange step (additional queuing is not necessary). - * - * @param ctx the context from the handler where the frame was read. - * @param streamId the stream the frame was sent on. - * @param promisedStreamId the ID of the promised stream. - * @param headers the received headers. - * @param padding additional bytes that should be added to obscure the true content size. Must be between 0 and - * 256 (inclusive). - */ - @Override - public void onPushPromiseRead(ChannelHandlerContext ctx, int streamId, int promisedStreamId, - Http2Headers headers, int padding) { - if (serverConfig.isDebug()) { - logger.log(Level.FINE, () -> "push promise received: streamId " + streamId + - " promised stream ID = " + promisedStreamId + " headers =" + headers); - } - throw new IllegalStateException("server is not allowd to receive push promise"); - } - - /** - * Notifies the listener that the given stream has now been removed from the connection and - * will no longer be returned via {@link Http2Connection#stream(int)}. The connection may - * maintain inactive streams for some time before removing them. - *

- * If a {@link RuntimeException} is thrown it will be logged and not propagated. - * Throwing from this method is not supported and is considered a programming error. - */ - @Override - public void onStreamRemoved(Http2Stream stream) { - if (serverConfig.isDebug()) { - logger.log(Level.FINE, () -> "stream removed " + stream); - } - removeMessage(stream, true); - } - - - /** - * Get the {@link FullHttpMessage} associated with {@code stream}. - * @param stream The stream to get the associated state from - * @return The {@link FullHttpMessage} associated with {@code stream}. - */ - private FullHttpMessage getMessage(Http2Stream stream) { - return (FullHttpMessage) stream.getProperty(messageKey); - } - - /** - * Make {@code message} be the state associated with {@code stream}. - * @param stream The stream which {@code message} is associated with. - * @param message The message which contains the HTTP semantics. - */ - private void putMessage(Http2Stream stream, FullHttpMessage message) { - FullHttpMessage previous = stream.setProperty(messageKey, message); - if (previous != message && previous != null) { - previous.release(); - } - } - /** - * The stream is out of scope for the HTTP message flow and will no longer be tracked. - * @param stream The stream to remove associated state with - * @param release {@code true} to call release on the value if it is present. {@code false} to not call release. - */ - private void removeMessage(Http2Stream stream, boolean release) { - FullHttpMessage msg = stream.removeProperty(messageKey); - if (release && msg != null) { - msg.release(); - } - } - - private FullHttpMessage beginHeader(ChannelHandlerContext ctx, Http2Stream stream, Http2Headers headers) throws Http2Exception { - FullHttpMessage msg = getMessage(stream); - if (msg == null) { - msg = newMessage(stream, headers, validateHttpHeaders, ctx.alloc()); - } else { - addHttp2ToHttpHeaders(stream.id(), headers, msg.headers(), msg.protocolVersion(), - true, msg instanceof HttpRequest); - } - return msg; - } - - private void endHeader(ChannelHandlerContext ctx, Http2Stream stream, FullHttpMessage msg, boolean endOfStream) { - if (endOfStream) { - fireChannelRead(ctx, msg, getMessage(stream) != msg, stream); - } else { - putMessage(stream, msg); - } - } - - /** - * Set final headers and fire a channel read event. - * - * @param ctx The context to fire the event on - * @param msg The message to send - * @param release {@code true} to call release on the value if it is present. {@code false} to not call release. - * @param stream the stream of the message which is being fired - */ - private void fireChannelRead(ChannelHandlerContext ctx, FullHttpMessage msg, boolean release, - Http2Stream stream) { - removeMessage(stream, release); - HttpUtil.setContentLength(msg, msg.content().readableBytes()); - ctx.fireChannelRead(msg); - } - - /** - * Create a new {@link FullHttpMessage} based upon the current connection parameters. - * - * @param stream The stream to create a message for - * @param headers The headers associated with {@code stream} - * @param validateHttpHeaders - *

    - *
  • {@code true} to validate HTTP headers in the http-codec
    • - *
  • {@code false} not to validate HTTP headers in the http-codec
    • - *
- * @param alloc The {@link ByteBufAllocator} to use to generate the content of the message - * @throws Http2Exception if message can not be created - */ - private FullHttpMessage newMessage(Http2Stream stream, Http2Headers headers, boolean validateHttpHeaders, - ByteBufAllocator alloc) throws Http2Exception { - FullHttpMessage fullHttpMessage = toFullHttpRequest(stream.id(), headers, alloc, validateHttpHeaders); - if (serverConfig.isDebug()) { - logger.log(Level.FINE, headers.toString()); - logger.log(Level.FINE, fullHttpMessage::toString); - } - return fullHttpMessage; - } - - /** - * Create a new object to contain the request data - * - * @param streamId The stream associated with the request - * @param http2Headers The initial set of HTTP/2 headers to create the request with - * @param alloc The {@link ByteBufAllocator} to use to generate the content of the message - * @param validateHttpHeaders
    - *
  • {@code true} to validate HTTP headers in the http-codec
    • - *
  • {@code false} not to validate HTTP headers in the http-codec
    • - *
- * @return A new request object which represents headers/data - * @throws Http2Exception If not all HTTP/2 headers can be translated to HTTP/1.x. - */ - public static FullHttpRequest toFullHttpRequest(int streamId, Http2Headers http2Headers, - ByteBufAllocator alloc, - boolean validateHttpHeaders) - throws Http2Exception { - final CharSequence method = ObjectUtil.checkNotNull(http2Headers.method(),"method header cannot be null"); - final CharSequence path = ObjectUtil.checkNotNull(http2Headers.path(),"path header cannot be null "); - ByteBuf byteBuf = alloc.buffer(); - FullHttpRequest msg = new DefaultFullHttpRequest(HTTP_2_0, HttpMethod.valueOf(method.toString()), - path.toString(), byteBuf, validateHttpHeaders); - try { - addHttp2ToHttpHeaders(streamId, http2Headers, msg.headers(), msg.protocolVersion(), false, true); - } catch (Http2Exception e) { - msg.release(); - throw e; - } catch (Throwable t) { - msg.release(); - throw Http2Exception.streamError(streamId, Http2Error.PROTOCOL_ERROR, t, "HTTP/2 full request conversion error"); - } - return msg; - } - - /** - * Translate and add HTTP/2 headers to HTTP/1.x headers. - * - * @param streamId The stream associated with {@code sourceHeaders}. - * @param inputHeaders The HTTP/2 headers to convert. - * @param outputHeaders The object which will contain the resulting HTTP/1.x headers.. - * @param httpVersion What HTTP/1.x version {@code outputHeaders} should be treated as when doing the conversion. - * @param isTrailer {@code true} if {@code outputHeaders} should be treated as trailing headers. - * {@code false} otherwise. - * @param isRequest {@code true} if the {@code outputHeaders} will be used in a request message. - * {@code false} for response message. - * @throws Http2Exception If not all HTTP/2 headers can be translated to HTTP/1.x. - */ - public static void addHttp2ToHttpHeaders(int streamId, Http2Headers inputHeaders, - HttpHeaders outputHeaders, - HttpVersion httpVersion, - boolean isTrailer, - boolean isRequest) throws Http2Exception { - - final CharSequenceMap translations = isRequest ? REQUEST_HEADER_TRANSLATIONS : RESPONSE_HEADER_TRANSLATIONS; - try { - for (Map.Entry entry : inputHeaders) { - final CharSequence name = entry.getKey(); - final CharSequence value = entry.getValue(); - AsciiString translatedName = translations.get(name); - if (translatedName != null) { - outputHeaders.add(translatedName, AsciiString.of(value)); - } else if (!Http2Headers.PseudoHeaderName.isPseudoHeader(name)) { - // https://tools.ietf.org/html/rfc7540#section-8.1.2.3 - // All headers that start with ':' are only valid in HTTP/2 context - if (name.length() == 0 || name.charAt(0) == ':') { - throw Http2Exception.streamError(streamId, Http2Error.PROTOCOL_ERROR, - "Invalid HTTP/2 header '%s' encountered in translation to HTTP/1.x", name); - } - if (HttpHeaderNames.COOKIE.equals(name)) { - // combine the cookie values into 1 header entry. - // https://tools.ietf.org/html/rfc7540#section-8.1.2.5 - String existingCookie = outputHeaders.get(HttpHeaderNames.COOKIE); - outputHeaders.set(HttpHeaderNames.COOKIE, - (existingCookie != null) ? (existingCookie + "; " + value) : value); - } else { - outputHeaders.add(name, value); - } - } - } - } catch (Http2Exception ex) { - throw ex; - } catch (Throwable t) { - throw Http2Exception.streamError(streamId, Http2Error.PROTOCOL_ERROR, t, "HTTP/2 headers conversion error"); - } - outputHeaders.remove(HttpHeaderNames.TRANSFER_ENCODING); - outputHeaders.remove(HttpHeaderNames.TRAILER); - if (!isTrailer) { - outputHeaders.setInt(HttpConversionUtil.ExtensionHeaderNames.STREAM_ID.text(), streamId); - HttpUtil.setKeepAlive(outputHeaders, httpVersion, true); - } - } - - /** - * Translations from HTTP/2 header name to the HTTP/1.x equivalent. - */ - private static final CharSequenceMap - REQUEST_HEADER_TRANSLATIONS = new CharSequenceMap(); - private static final CharSequenceMap - RESPONSE_HEADER_TRANSLATIONS = new CharSequenceMap(); - static { - RESPONSE_HEADER_TRANSLATIONS.add(Http2Headers.PseudoHeaderName.AUTHORITY.value(), - HttpHeaderNames.HOST); - RESPONSE_HEADER_TRANSLATIONS.add(Http2Headers.PseudoHeaderName.SCHEME.value(), - HttpConversionUtil.ExtensionHeaderNames.SCHEME.text()); - REQUEST_HEADER_TRANSLATIONS.add(RESPONSE_HEADER_TRANSLATIONS); - RESPONSE_HEADER_TRANSLATIONS.add(Http2Headers.PseudoHeaderName.PATH.value(), - HttpConversionUtil.ExtensionHeaderNames.PATH.text()); - } -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2RequestHandler.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2RequestHandler.java deleted file mode 100644 index 78734ed..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2RequestHandler.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.xbib.netty.http.server.handler.http2; - -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.handler.codec.http.FullHttpRequest; -import org.xbib.netty.http.server.transport.ServerTransport; - -import java.io.IOException; - -@ChannelHandler.Sharable -public class Http2RequestHandler extends SimpleChannelInboundHandler { - - @Override - protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest httpRequest) throws IOException { - ServerTransport transport = ctx.channel().attr(ServerTransport.TRANSPORT_ATTRIBUTE_KEY).get(); - transport.requestReceived(ctx, httpRequest); - } - - @Override - public void channelInactive(ChannelHandlerContext ctx) { - ctx.fireChannelInactive(); - ServerTransport transport = ctx.channel().attr(ServerTransport.TRANSPORT_ATTRIBUTE_KEY).get(); - //transport.fail(new IOException("channel closed")); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - ServerTransport transport = ctx.channel().attr(ServerTransport.TRANSPORT_ATTRIBUTE_KEY).get(); - //transport.fail(cause); - ctx.channel().close(); - } -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2SettingsHandler.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2SettingsHandler.java deleted file mode 100644 index 17f7fb6..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2SettingsHandler.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.xbib.netty.http.server.handler.http2; - -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.handler.codec.http2.Http2Settings; -import org.xbib.netty.http.server.transport.ServerTransport; - -@ChannelHandler.Sharable -public class Http2SettingsHandler extends SimpleChannelInboundHandler { - - @Override - protected void channelRead0(ChannelHandlerContext ctx, Http2Settings http2Settings) throws Exception { - ServerTransport transport = ctx.channel().attr(ServerTransport.TRANSPORT_ATTRIBUTE_KEY).get(); - transport.settingsReceived(ctx, http2Settings); - ctx.pipeline().remove(this); - } -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2StreamFrameToHttpObjectCodec.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2StreamFrameToHttpObjectCodec.java new file mode 100644 index 0000000..d4122be --- /dev/null +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/Http2StreamFrameToHttpObjectCodec.java @@ -0,0 +1,228 @@ +package org.xbib.netty.http.server.handler.http2; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandler.Sharable; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.EncoderException; +import io.netty.handler.codec.MessageToMessageCodec; +import io.netty.handler.codec.http.DefaultHttpContent; +import io.netty.handler.codec.http.DefaultLastHttpContent; +import io.netty.handler.codec.http.FullHttpMessage; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaderValues; +import io.netty.handler.codec.http.HttpMessage; +import io.netty.handler.codec.http.HttpObject; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpScheme; +import io.netty.handler.codec.http.HttpUtil; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.codec.http2.DefaultHttp2DataFrame; +import io.netty.handler.codec.http2.DefaultHttp2HeadersFrame; +import io.netty.handler.codec.http2.Http2DataFrame; +import io.netty.handler.codec.http2.Http2Exception; +import io.netty.handler.codec.http2.Http2Headers; +import io.netty.handler.codec.http2.Http2HeadersFrame; +import io.netty.handler.codec.http2.Http2MultiplexCodec; +import io.netty.handler.codec.http2.Http2StreamChannel; +import io.netty.handler.codec.http2.Http2StreamFrame; +import io.netty.handler.codec.http2.HttpConversionUtil; +import io.netty.handler.ssl.SslHandler; +import io.netty.util.internal.UnstableApi; + +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * This handler converts from {@link Http2StreamFrame} to {@link HttpObject}, + * and back. It can be used as an adapter in conjunction with {@link + * Http2MultiplexCodec} to make http/2 connections backward-compatible with + * {@link ChannelHandler}s expecting {@link HttpObject}. + * + * For simplicity, it converts to chunked encoding unless the entire stream + * is a single header. + */ +@UnstableApi +@Sharable +public class Http2StreamFrameToHttpObjectCodec extends MessageToMessageCodec { + + private static final Logger logger = Logger.getLogger(Http2StreamFrameToHttpObjectCodec.class.getName()); + + private final boolean isServer; + + private final boolean validateHeaders; + + private HttpScheme scheme; + + public Http2StreamFrameToHttpObjectCodec(final boolean isServer, + final boolean validateHeaders) { + this.isServer = isServer; + this.validateHeaders = validateHeaders; + scheme = HttpScheme.HTTP; + } + + public Http2StreamFrameToHttpObjectCodec(final boolean isServer) { + this(isServer, true); + } + + @Override + public boolean acceptInboundMessage(Object msg) throws Exception { + return (msg instanceof Http2HeadersFrame) || (msg instanceof Http2DataFrame); + } + + @Override + protected void decode(ChannelHandlerContext ctx, Http2StreamFrame frame, List out) throws Exception { + if (frame instanceof Http2HeadersFrame) { + int id = frame.stream() != null ? frame.stream().id() : -1; + Http2HeadersFrame headersFrame = (Http2HeadersFrame) frame; + Http2Headers headers = headersFrame.headers(); + + final CharSequence status = headers.status(); + + // 100-continue response is a special case where Http2HeadersFrame#isEndStream=false + // but we need to decode it as a FullHttpResponse to play nice with HttpObjectAggregator. + if (null != status && HttpResponseStatus.CONTINUE.codeAsText().contentEquals(status)) { + final FullHttpMessage fullMsg = newFullMessage(id, headers, ctx.alloc()); + out.add(fullMsg); + return; + } + + if (headersFrame.isEndStream()) { + if (headers.method() == null && status == null) { + LastHttpContent last = new DefaultLastHttpContent(Unpooled.EMPTY_BUFFER, validateHeaders); + HttpConversionUtil.addHttp2ToHttpHeaders(id, headers, last.trailingHeaders(), + HttpVersion.HTTP_1_1, true, true); + out.add(last); + } else { + FullHttpMessage full = newFullMessage(id, headers, ctx.alloc()); + out.add(full); + } + } else { + HttpMessage req = newMessage(id, headers); + if (!HttpUtil.isContentLengthSet(req)) { + req.headers().add(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED); + } + out.add(req); + } + } else if (frame instanceof Http2DataFrame) { + Http2DataFrame dataFrame = (Http2DataFrame) frame; + if (dataFrame.isEndStream()) { + out.add(new DefaultLastHttpContent(dataFrame.content().retain(), validateHeaders)); + } else { + out.add(new DefaultHttpContent(dataFrame.content().retain())); + } + } + } + + private void encodeLastContent(LastHttpContent last, List out) { + boolean needFiller = !(last instanceof FullHttpMessage) && last.trailingHeaders().isEmpty(); + if (last.content().isReadable() || needFiller) { + out.add(new DefaultHttp2DataFrame(last.content().retain(), last.trailingHeaders().isEmpty())); + } + if (!last.trailingHeaders().isEmpty()) { + Http2Headers headers = HttpConversionUtil.toHttp2Headers(last.trailingHeaders(), validateHeaders); + out.add(new DefaultHttp2HeadersFrame(headers, true)); + } + } + + /** + * Encode from an {@link HttpObject} to an {@link Http2StreamFrame}. This method will + * be called for each written message that can be handled by this encoder. + * + * NOTE: 100-Continue responses that are NOT {@link FullHttpResponse} will be rejected. + * + * @param ctx the {@link ChannelHandlerContext} which this handler belongs to + * @param obj the {@link HttpObject} message to encode + * @param out the {@link List} into which the encoded msg should be added + * needs to do some kind of aggregation + * @throws Exception is thrown if an error occurs + */ + @Override + protected void encode(ChannelHandlerContext ctx, HttpObject obj, List out) throws Exception { + // 100-continue is typically a FullHttpResponse, but the decoded + // Http2HeadersFrame should not be marked as endStream=true + if (obj instanceof HttpResponse) { + final HttpResponse res = (HttpResponse) obj; + if (res.status().equals(HttpResponseStatus.CONTINUE)) { + if (res instanceof FullHttpResponse) { + final Http2Headers headers = toHttp2Headers(res); + out.add(new DefaultHttp2HeadersFrame(headers, false)); + return; + } else { + throw new EncoderException( + HttpResponseStatus.CONTINUE.toString() + " must be a FullHttpResponse"); + } + } + } + + if (obj instanceof HttpMessage) { + Http2Headers headers = toHttp2Headers((HttpMessage) obj); + boolean noMoreFrames = false; + if (obj instanceof FullHttpMessage) { + FullHttpMessage full = (FullHttpMessage) obj; + noMoreFrames = !full.content().isReadable() && full.trailingHeaders().isEmpty(); + } + + out.add(new DefaultHttp2HeadersFrame(headers, noMoreFrames)); + } + + if (obj instanceof LastHttpContent) { + LastHttpContent last = (LastHttpContent) obj; + encodeLastContent(last, out); + } else if (obj instanceof HttpContent) { + HttpContent cont = (HttpContent) obj; + out.add(new DefaultHttp2DataFrame(cont.content().retain(), false)); + } + } + + private Http2Headers toHttp2Headers(final HttpMessage msg) { + if (msg instanceof HttpRequest) { + msg.headers().set( + HttpConversionUtil.ExtensionHeaderNames.SCHEME.text(), + scheme.name()); + } + + return HttpConversionUtil.toHttp2Headers(msg, validateHeaders); + } + + private HttpMessage newMessage(final int id, + final Http2Headers headers) throws Http2Exception { + return isServer ? + HttpConversionUtil.toHttpRequest(id, headers, validateHeaders) : + HttpConversionUtil.toHttpResponse(id, headers, validateHeaders); + } + + private FullHttpMessage newFullMessage(final int id, + final Http2Headers headers, + final ByteBufAllocator alloc) throws Http2Exception { + return isServer ? + HttpConversionUtil.toFullHttpRequest(id, headers, alloc, validateHeaders) : + HttpConversionUtil.toFullHttpResponse(id, headers, alloc, validateHeaders); + } + + @Override + public void handlerAdded(final ChannelHandlerContext ctx) throws Exception { + super.handlerAdded(ctx); + + // this handler is typically used on an Http2StreamChannel. at this + // stage, ssl handshake should've been established. checking for the + // presence of SslHandler in the parent's channel pipeline to + // determine the HTTP scheme should suffice, even for the case where + // SniHandler is used. + scheme = isSsl(ctx) ? HttpScheme.HTTPS : HttpScheme.HTTP; + } + + protected boolean isSsl(final ChannelHandlerContext ctx) { + final Channel ch = ctx.channel(); + final Channel connChannel = (ch instanceof Http2StreamChannel) ? ch.parent() : ch; + return null != connChannel.pipeline().get(SslHandler.class); + } +} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/UserEventLogger.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/UserEventLogger.java deleted file mode 100644 index b0341c7..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/handler/http2/UserEventLogger.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.xbib.netty.http.server.handler.http2; - -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; - -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * A Netty handler that logs user events. - */ -@ChannelHandler.Sharable -public class UserEventLogger extends ChannelInboundHandlerAdapter { - - private static final Logger logger = Logger.getLogger(UserEventLogger.class.getName()); - - @Override - public void userEventTriggered(ChannelHandlerContext ctx, Object evt) { - logger.log(Level.FINE, () -> "got user event " + evt); - ctx.fireUserEventTriggered(evt); - } -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/ClosedSessionException.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/ClosedSessionException.java deleted file mode 100644 index 9048ce1..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/ClosedSessionException.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.xbib.netty.http.server.internal; - -/** - * A {@link RuntimeException} raised when the connection to the remote peer has been closed unexpectedly. - */ -public final class ClosedSessionException extends RuntimeException { -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/Http1ObjectEncoder.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/Http1ObjectEncoder.java deleted file mode 100644 index 4f42a46..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/Http1ObjectEncoder.java +++ /dev/null @@ -1,192 +0,0 @@ -package org.xbib.netty.http.server.internal; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelPromise; -import io.netty.handler.codec.http.DefaultHttpContent; -import io.netty.handler.codec.http.DefaultHttpResponse; -import io.netty.handler.codec.http.DefaultLastHttpContent; -import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpObject; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpVersion; -import io.netty.handler.codec.http2.Http2Error; -import io.netty.util.collection.IntObjectHashMap; -import io.netty.util.collection.IntObjectMap; - -import java.util.AbstractMap.SimpleImmutableEntry; -import java.util.ArrayDeque; -import java.util.Map.Entry; -import java.util.Queue; -import java.util.logging.Logger; - -/** - * HTTP 1 object encoder. - */ -public final class Http1ObjectEncoder extends HttpObjectEncoder { - - private static final Logger logger = Logger.getLogger(Http1ObjectEncoder.class.getName()); - - /** - * The map which maps a request ID to its related pending response. - */ - private final IntObjectMap pendingWrites = new IntObjectHashMap<>(); - /** - * The ID of the request which is at its turn to send a response. - */ - private int currentId = 1; - /** - * The minimum ID of the request whose stream has been closed/reset. - */ - private int minClosedId = Integer.MAX_VALUE; - /** - * The maximum known ID with pending writes. - */ - private int maxIdWithPendingWrites = Integer.MIN_VALUE; - - @Override - protected ChannelFuture doWriteHeaders(ChannelHandlerContext ctx, int id, int streamId, - HttpHeaders headers, HttpResponseStatus status, boolean endStream) { - if (id >= minClosedId) { - return ctx.newFailedFuture(new ClosedSessionException()); - } - try { - return write(ctx, id, new DefaultHttpResponse(HttpVersion.HTTP_1_1, status, headers), endStream); - } catch (Throwable t) { - return ctx.newFailedFuture(t); - } - } - - @Override - protected ChannelFuture doWriteData(ChannelHandlerContext ctx, int id, int streamId, ByteBuf buf, boolean endStream) { - if (id >= minClosedId) { - return ctx.newFailedFuture(new ClosedSessionException()); - } - try { - final HttpContent content; - if (endStream) { - content = new DefaultLastHttpContent(buf); - } else { - content = new DefaultHttpContent(buf); - } - return write(ctx, id, content, endStream); - } catch (Throwable t) { - return ctx.newFailedFuture(t); - } - } - - private ChannelFuture write(ChannelHandlerContext ctx, int id, HttpObject obj, boolean endStream) { - if (id < currentId) { - return ctx.newFailedFuture(new ClosedSessionException()); - } - final PendingWrites currentPendingWrites = pendingWrites.get(id); - if (id == currentId) { - if (currentPendingWrites != null) { - pendingWrites.remove(id); - flushPendingWrites(ctx, currentPendingWrites); - } - final ChannelFuture future = ctx.write(obj); - if (endStream) { - currentId++; - for (;;) { - final PendingWrites nextPendingWrites = pendingWrites.get(currentId); - if (nextPendingWrites == null) { - break; - } - flushPendingWrites(ctx, nextPendingWrites); - if (!nextPendingWrites.isEndOfStream()) { - break; - } - pendingWrites.remove(currentId); - currentId++; - } - } - ctx.flush(); - return future; - } else { - final ChannelPromise promise = ctx.newPromise(); - final Entry entry = new SimpleImmutableEntry<>(obj, promise); - if (currentPendingWrites == null) { - final PendingWrites newPendingWrites = new PendingWrites(); - maxIdWithPendingWrites = Math.max(maxIdWithPendingWrites, id); - newPendingWrites.add(entry); - pendingWrites.put(id, newPendingWrites); - } else { - currentPendingWrites.add(entry); - if (endStream) { - currentPendingWrites.setEndOfStream(); - } - } - return promise; - } - } - - private static void flushPendingWrites(ChannelHandlerContext ctx, PendingWrites pendingWrites) { - while (true) { - final Entry e = pendingWrites.poll(); - if (e == null) { - break; - } - ctx.write(e.getKey(), e.getValue()); - } - } - - @Override - protected ChannelFuture doWriteReset(ChannelHandlerContext ctx, int id, int streamId, Http2Error error) { - minClosedId = Math.min(minClosedId, id); - for (int i = minClosedId; i <= maxIdWithPendingWrites; i++) { - final PendingWrites pendingWrites = this.pendingWrites.remove(i); - while (true) { - final Entry e = pendingWrites.poll(); - if (e == null) { - break; - } - e.getValue().tryFailure(new ClosedSessionException()); - } - } - final ChannelFuture f = ctx.write(Unpooled.EMPTY_BUFFER); - if (currentId >= minClosedId) { - f.addListener(ChannelFutureListener.CLOSE); - } - return f; - } - - @Override - protected void doClose() { - if (pendingWrites.isEmpty()) { - return; - } - ClosedSessionException cause = new ClosedSessionException(); - for (Queue> queue : pendingWrites.values()) { - while (true) { - final Entry e = queue.poll(); - if (e == null) { - break; - } - e.getValue().tryFailure(cause); - } - } - pendingWrites.clear(); - } - - private static final class PendingWrites extends ArrayDeque> { - - private boolean endOfStream; - - PendingWrites() { - super(4); - } - - boolean isEndOfStream() { - return endOfStream; - } - - void setEndOfStream() { - endOfStream = true; - } - } -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/Http2ObjectEncoder.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/Http2ObjectEncoder.java deleted file mode 100644 index dd299a9..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/Http2ObjectEncoder.java +++ /dev/null @@ -1,167 +0,0 @@ -package org.xbib.netty.http.server.internal; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaderValues; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http2.CharSequenceMap; -import io.netty.handler.codec.http2.DefaultHttp2Headers; -import io.netty.handler.codec.http2.Http2ConnectionEncoder; -import io.netty.handler.codec.http2.Http2Error; -import io.netty.handler.codec.http2.Http2Headers; -import io.netty.handler.codec.http2.Http2Stream; -import io.netty.handler.codec.http2.HttpConversionUtil; -import io.netty.util.AsciiString; - -import java.util.Iterator; -import java.util.Map; -import java.util.Objects; - -import static io.netty.util.AsciiString.EMPTY_STRING; -import static io.netty.util.ByteProcessor.FIND_SEMI_COLON; - -/** - * - */ -public final class Http2ObjectEncoder extends HttpObjectEncoder { - - /** - * The set of headers that should not be directly copied when converting headers from HTTP to HTTP/2. - */ - private static final CharSequenceMap HTTP_TO_HTTP2_HEADER_BLACKLIST = - new CharSequenceMap(); - static { - HTTP_TO_HTTP2_HEADER_BLACKLIST.add(HttpHeaderNames.CONNECTION, EMPTY_STRING); - @SuppressWarnings("deprecation") - AsciiString keepAlive = HttpHeaderNames.KEEP_ALIVE; - HTTP_TO_HTTP2_HEADER_BLACKLIST.add(keepAlive, EMPTY_STRING); - @SuppressWarnings("deprecation") - AsciiString proxyConnection = HttpHeaderNames.PROXY_CONNECTION; - HTTP_TO_HTTP2_HEADER_BLACKLIST.add(proxyConnection, EMPTY_STRING); - HTTP_TO_HTTP2_HEADER_BLACKLIST.add(HttpHeaderNames.TRANSFER_ENCODING, EMPTY_STRING); - HTTP_TO_HTTP2_HEADER_BLACKLIST.add(HttpHeaderNames.HOST, EMPTY_STRING); - HTTP_TO_HTTP2_HEADER_BLACKLIST.add(HttpHeaderNames.UPGRADE, EMPTY_STRING); - HTTP_TO_HTTP2_HEADER_BLACKLIST.add(HttpConversionUtil.ExtensionHeaderNames.STREAM_ID.text(), EMPTY_STRING); - HTTP_TO_HTTP2_HEADER_BLACKLIST.add(HttpConversionUtil.ExtensionHeaderNames.SCHEME.text(), EMPTY_STRING); - HTTP_TO_HTTP2_HEADER_BLACKLIST.add(HttpConversionUtil.ExtensionHeaderNames.PATH.text(), EMPTY_STRING); - } - - private final Http2ConnectionEncoder encoder; - - public Http2ObjectEncoder(Http2ConnectionEncoder encoder) { - super(); - this.encoder = Objects.requireNonNull(encoder, "encoder"); - } - - @Override - protected ChannelFuture doWriteHeaders(ChannelHandlerContext ctx, int id, int streamId, - HttpHeaders headers, HttpResponseStatus status, - boolean endStream) { - final ChannelFuture future = validateStream(ctx, streamId); - if (future != null) { - return future; - } - Http2Headers http2Headers = toHttp2Headers(headers, status, false); - return encoder.writeHeaders(ctx, streamId, http2Headers, 0, endStream, ctx.newPromise()); - } - - @Override - protected ChannelFuture doWriteData(ChannelHandlerContext ctx, int id, int streamId, ByteBuf data, - boolean endStream) { - final ChannelFuture future = validateStream(ctx, streamId); - if (future != null) { - return future; - } - return encoder.writeData(ctx, streamId, data, 0, endStream, ctx.newPromise()); - } - - @Override - protected ChannelFuture doWriteReset(ChannelHandlerContext ctx, int id, int streamId, Http2Error error) { - final ChannelFuture future = validateStream(ctx, streamId); - if (future != null) { - return future; - } - return encoder.writeRstStream(ctx, streamId, error.code(), ctx.newPromise()); - } - - - @Override - protected void doClose() { - } - - private ChannelFuture validateStream(ChannelHandlerContext ctx, int streamId) { - final Http2Stream stream = encoder.connection().stream(streamId); - if (stream != null) { - switch (stream.state()) { - case RESERVED_LOCAL: - case OPEN: - case HALF_CLOSED_REMOTE: - break; - default: - return ctx.newFailedFuture(new IllegalStateException("stream state = " + stream.state().name())); - } - } else if (encoder.connection().streamMayHaveExisted(streamId)) { - return ctx.newFailedFuture(new IllegalStateException("stream may have existed")); - } - return null; - } - - public static Http2Headers toHttp2Headers(HttpHeaders inHeaders, - HttpResponseStatus status, - boolean validateHeaders) { - final Http2Headers out = new DefaultHttp2Headers(validateHeaders, inHeaders.size()); - out.status(status.codeAsText()); - toHttp2Headers(inHeaders, out); - return out; - } - - public static void toHttp2Headers(HttpHeaders inHeaders, Http2Headers outHeaders) { - Iterator> iter = inHeaders.iteratorCharSequence(); - while (iter.hasNext()) { - Map.Entry entry = iter.next(); - final AsciiString aName = AsciiString.of(entry.getKey()).toLowerCase(); - if (!HTTP_TO_HTTP2_HEADER_BLACKLIST.contains(aName)) { - // https://tools.ietf.org/html/rfc7540#section-8.1.2.2 makes a special exception for TE - if (aName.contentEqualsIgnoreCase(HttpHeaderNames.TE) && - !AsciiString.contentEqualsIgnoreCase(entry.getValue(), HttpHeaderValues.TRAILERS)) { - throw new IllegalArgumentException("Invalid value for " + HttpHeaderNames.TE + ": " + - entry.getValue()); - } - if (aName.contentEqualsIgnoreCase(HttpHeaderNames.COOKIE)) { - AsciiString value = AsciiString.of(entry.getValue()); - // split up cookies to allow for better compression - // https://tools.ietf.org/html/rfc7540#section-8.1.2.5 - try { - int index = value.forEachByte(FIND_SEMI_COLON); - if (index != -1) { - int start = 0; - do { - outHeaders.add(HttpHeaderNames.COOKIE, value.subSequence(start, index, false)); - // skip 2 characters "; " (see https://tools.ietf.org/html/rfc6265#section-4.2.1) - start = index + 2; - } while (start < value.length() && - (index = value.forEachByte(start, value.length() - start, FIND_SEMI_COLON)) != -1); - if (start >= value.length()) { - throw new IllegalArgumentException("cookie value is of unexpected format: " + value); - } - outHeaders.add(HttpHeaderNames.COOKIE, value.subSequence(start, value.length(), false)); - } else { - outHeaders.add(HttpHeaderNames.COOKIE, value); - } - } catch (Exception e) { - // This is not expect to happen because FIND_SEMI_COLON never throws but must be caught - // because of the ByteProcessor interface. - throw new IllegalStateException(e); - } - } else { - outHeaders.add(aName, entry.getValue()); - } - } - } - } -} - - diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/HttpObjectEncoder.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/HttpObjectEncoder.java deleted file mode 100644 index d60fa23..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/HttpObjectEncoder.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.xbib.netty.http.server.internal; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http2.Http2Error; - -/** - * HTTP object encoder. - */ -public abstract class HttpObjectEncoder { - - private volatile boolean closed; - - /** - * Writes an {@link HttpHeaders}. - */ - public final ChannelFuture writeHeaders(ChannelHandlerContext ctx, int id, int streamId, HttpHeaders headers, - HttpResponseStatus status, boolean endStream) { - if (!ctx.channel().eventLoop().inEventLoop()) { - throw new IllegalStateException(); - } - if (closed) { - return newFailedFuture(ctx); - } - return doWriteHeaders(ctx, id, streamId, headers, status, endStream); - } - - protected abstract ChannelFuture doWriteHeaders(ChannelHandlerContext ctx, int id, int streamId, - HttpHeaders headers, HttpResponseStatus status, boolean endStream); - - public final ChannelFuture writeData(ChannelHandlerContext ctx, int id, int streamId, ByteBuf data, boolean endStream) { - if (!ctx.channel().eventLoop().inEventLoop()) { - throw new IllegalStateException(); - } - if (closed) { - return newFailedFuture(ctx); - } - return doWriteData(ctx, id, streamId, data, endStream); - } - - protected abstract ChannelFuture doWriteData(ChannelHandlerContext ctx, int id, int streamId, ByteBuf data, - boolean endStream); - - /** - * Resets the specified stream. If the session protocol doesn't support multiplexing or the connection - * is in unrecoverable state, the connection will be closed. For example, in an HTTP/1 connection, this - * will lead the connection to be closed immediately or after the previous requests that are not reset. - */ - public final ChannelFuture writeReset(ChannelHandlerContext ctx, int id, int streamId, Http2Error error) { - if (closed) { - return newFailedFuture(ctx); - } - return doWriteReset(ctx, id, streamId, error); - } - - protected abstract ChannelFuture doWriteReset(ChannelHandlerContext ctx, int id, int streamId, Http2Error error); - - /** - * Releases the resources related with this encoder and fails any unfinished writes. - */ - public void close() { - if (closed) { - return; - } - closed = true; - doClose(); - } - - protected abstract void doClose(); - - private static ChannelFuture newFailedFuture(ChannelHandlerContext ctx) { - return ctx.newFailedFuture(new ClosedSessionException()); - } -} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/package-info.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/package-info.java deleted file mode 100644 index cd28b37..0000000 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/internal/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Internal classes for Netty HTTP server. - */ -package org.xbib.netty.http.server.internal; diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/BaseServerTransport.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/BaseServerTransport.java index 780eaab..251e4a3 100644 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/BaseServerTransport.java +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/BaseServerTransport.java @@ -1,5 +1,6 @@ package org.xbib.netty.http.server.transport; +import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpVersion; @@ -14,9 +15,13 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; abstract class BaseServerTransport implements ServerTransport { + private static final Logger logger = Logger.getLogger(BaseServerTransport.class.getName()); + protected static final AtomicInteger requestCounter = new AtomicInteger(); private static final List METHODS = Arrays.asList("GET", "HEAD", "OPTIONS"); @@ -27,6 +32,11 @@ abstract class BaseServerTransport implements ServerTransport { this.server = server; } + @Override + public void exceptionReceived(ChannelHandlerContext ctx, Throwable throwable) throws IOException { + logger.log(Level.WARNING, throwable.getMessage(), throwable); + } + /** * Accepts a request, performing various validation checks * and required special header handling, possibly returning an @@ -92,9 +102,9 @@ abstract class BaseServerTransport implements ServerTransport { // "*" is a special server-wide (no-context) request supported by OPTIONS boolean isServerOptions = path.equals("*") && method.equals("OPTIONS"); methods.addAll(isServerOptions ? virtualServer.getMethods() : handlers.keySet()); - serverResponse.getHeaders().add(HttpHeaderNames.ALLOW, String.join(", ", methods)); + serverResponse.setHeader(HttpHeaderNames.ALLOW, String.join(", ", methods)); if (method.equals("OPTIONS")) { // default OPTIONS handler - serverResponse.getHeaders().add(HttpHeaderNames.CONTENT_LENGTH, "0"); // RFC2616#9.2 + serverResponse.setHeader(HttpHeaderNames.CONTENT_LENGTH, "0"); // RFC2616#9.2 serverResponse.write(200); } else if (virtualServer.getMethods().contains(method)) { serverResponse.write(405); // supported by server, but not this context (nor built-in) diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http2ServerResponse.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http2ServerResponse.java new file mode 100644 index 0000000..e495544 --- /dev/null +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http2ServerResponse.java @@ -0,0 +1,175 @@ +package org.xbib.netty.http.server.transport; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufUtil; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaderValues; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http2.DefaultHttp2DataFrame; +import io.netty.handler.codec.http2.DefaultHttp2Headers; +import io.netty.handler.codec.http2.DefaultHttp2HeadersFrame; +import io.netty.handler.codec.http2.Http2Headers; +import io.netty.handler.codec.http2.HttpConversionUtil; +import io.netty.util.AsciiString; +import org.xbib.netty.http.server.ServerName; + +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +public class Http2ServerResponse implements ServerResponse { + + private final ServerRequest serverRequest; + + private final ChannelHandlerContext ctx; + + private Http2Headers headers; + + public Http2ServerResponse(ServerRequest serverRequest, ChannelHandlerContext ctx) { + this.serverRequest = serverRequest; + this.ctx = ctx; + this.headers = new DefaultHttp2Headers(); + } + + @Override + public void setHeader(AsciiString name, String value) { + headers.set(name, value); + } + + @Override + public void write(String text) { + write(200, "text/plain; charset=utf-8", text); + } + + /** + * Sends an error response with the given status and default body. + * + * @param status the response status + */ + @Override + public void writeError(int status) { + writeError(status, status < 400 ? ":)" : "sorry it didn't work out :("); + } + + /** + * Sends an error response with the given status and detailed message. + * An HTML body is created containing the status and its description, + * as well as the message, which is escaped using the + * {@link #escapeHTML escape} method. + * + * @param status the response status + * @param text the text body (sent as text/html) + */ + @Override + public void writeError(int status, String text) { + write(status, "text/html; charset=utf-8", + String.format("%n%n%d %s%n" + + "

%d %s

%n

%s

%n", + status, HttpResponseStatus.valueOf(status).reasonPhrase(), + status, HttpResponseStatus.valueOf(status).reasonPhrase(), + escapeHTML(text))); + } + + @Override + public void write(int status) { + write(status, null, (ByteBuf) null); + } + + @Override + public void write(int status, String contentType, String text) { + write(status, contentType, ByteBufUtil.writeUtf8(ctx.alloc(), text)); + } + + @Override + public void write(int status, String contentType, String text, Charset charset) { + write(status, contentType, ByteBufUtil.encodeString(ctx.alloc(), CharBuffer.allocate(text.length()).append(text), charset)); + } + + @Override + public void write(int status, String contentType, ByteBuf byteBuf) { + if (byteBuf != null) { + CharSequence s = headers.get(HttpHeaderNames.CONTENT_TYPE); + if (s == null) { + s = contentType != null ? contentType : HttpHeaderValues.APPLICATION_OCTET_STREAM; + headers.add(HttpHeaderNames.CONTENT_TYPE, s); + } + if (!headers.contains(HttpHeaderNames.CONTENT_LENGTH) && !headers.contains(HttpHeaderNames.TRANSFER_ENCODING)) { + int length = byteBuf.readableBytes(); + if (length < 0) { + headers.add(HttpHeaderNames.TRANSFER_ENCODING, "chunked"); + } else { + headers.add(HttpHeaderNames.CONTENT_LENGTH, Long.toString(length)); + } + } + if (serverRequest != null && "close".equalsIgnoreCase(serverRequest.getRequest().headers().get(HttpHeaderNames.CONNECTION)) && + !headers.contains(HttpHeaderNames.CONNECTION)) { + headers.add(HttpHeaderNames.CONNECTION, "close"); + } + if (!headers.contains(HttpHeaderNames.DATE)) { + headers.add(HttpHeaderNames.DATE, DateTimeFormatter.RFC_1123_DATE_TIME.format(ZonedDateTime.now(ZoneOffset.UTC))); + } + headers.add(HttpHeaderNames.SERVER, ServerName.getServerName()); + } + if (serverRequest != null) { + Integer streamId = serverRequest.streamId(); + if (streamId != null) { + headers.setInt(HttpConversionUtil.ExtensionHeaderNames.STREAM_ID.text(), streamId); + } + } + Http2Headers http2Headers = new DefaultHttp2Headers() + .status(HttpResponseStatus.valueOf(status).codeAsText()) + .add(headers); + ctx.channel().write(new DefaultHttp2HeadersFrame(http2Headers,byteBuf == null)); + if (byteBuf != null) { + ctx.channel().write(new DefaultHttp2DataFrame(byteBuf, true)); + } + ctx.channel().flush(); + } + + /** + * Returns an HTML-escaped version of the given string for safe display + * within a web page. The characters '&', '>' and '<' must always + * be escaped, and single and double quotes must be escaped within + * attribute values; this method escapes them always. This method can + * be used for generating both HTML and XHTML valid content. + * + * @param s the string to escape + * @return the escaped string + * @see The W3C FAQ + */ + private static String escapeHTML(String s) { + int len = s.length(); + StringBuilder es = new StringBuilder(len + 30); + int start = 0; + for (int i = 0; i < len; i++) { + String ref = null; + switch (s.charAt(i)) { + case '&': + ref = "&"; + break; + case '>': + ref = ">"; + break; + case '<': + ref = "<"; + break; + case '"': + ref = """; + break; + case '\'': + ref = "'"; + break; + default: + break; + } + if (ref != null) { + es.append(s.substring(start, i)).append(ref); + start = i + 1; + } + } + return start == 0 ? s : es.append(s.substring(start)).toString(); + } +} diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http2ServerTransport.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http2ServerTransport.java index ea5aefe..69e29b6 100644 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http2ServerTransport.java +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http2ServerTransport.java @@ -4,41 +4,42 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http2.Http2Settings; +import io.netty.handler.codec.http2.HttpConversionUtil; import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.server.Server; import org.xbib.netty.http.server.context.VirtualServer; import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; public class Http2ServerTransport extends BaseServerTransport { - private static final Logger logger = Logger.getLogger(Http2ServerTransport.class.getName()); - public Http2ServerTransport(Server server) { super(server); } @Override public void requestReceived(ChannelHandlerContext ctx, FullHttpRequest fullHttpRequest) throws IOException { - logger.log(Level.INFO, "requestReceived"); + requestReceived(ctx, fullHttpRequest, null); + } + + @Override + public void requestReceived(ChannelHandlerContext ctx, FullHttpRequest fullHttpRequest, Integer sequenceId) throws IOException { int requestId = requestCounter.incrementAndGet(); VirtualServer virtualServer = server.getVirtualServer(fullHttpRequest.headers().get(HttpHeaderNames.HOST)); if (virtualServer == null) { virtualServer = server.getDefaultVirtualServer(); } HttpAddress httpAddress = server.getServerConfig().getAddress(); + Integer streamId = fullHttpRequest.headers().getInt(HttpConversionUtil.ExtensionHeaderNames.STREAM_ID.text()); ServerRequest serverRequest = new ServerRequest(virtualServer, httpAddress, fullHttpRequest, - null, requestId); - ServerResponse serverResponse = new Http1ServerResponse(httpAddress.getVersion(), serverRequest, ctx); + sequenceId, streamId, requestId); + ServerResponse serverResponse = new Http2ServerResponse(serverRequest, ctx); if (acceptRequest(serverRequest, serverResponse)) { handle(serverRequest, serverResponse); } } @Override - public void settingsReceived(ChannelHandlerContext ctx, Http2Settings http2Settings) throws Exception { - logger.log(Level.INFO, "settings received"); + public void settingsReceived(ChannelHandlerContext ctx, Http2Settings http2Settings) { } } diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http1ServerResponse.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/HttpServerResponse.java similarity index 84% rename from netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http1ServerResponse.java rename to netty-http-server/src/main/java/org/xbib/netty/http/server/transport/HttpServerResponse.java index 6452e16..e381b69 100644 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http1ServerResponse.java +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/HttpServerResponse.java @@ -12,17 +12,20 @@ import io.netty.handler.codec.http.HttpHeaderValues; import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpVersion; +import io.netty.util.AsciiString; import org.xbib.netty.http.server.ServerName; +import org.xbib.netty.http.server.handler.http.HttpPipelinedResponse; import java.nio.CharBuffer; import java.nio.charset.Charset; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.logging.Logger; -public class Http1ServerResponse implements ServerResponse { +public class HttpServerResponse implements ServerResponse { - private final HttpVersion httpVersion; + private static final Logger logger = Logger.getLogger(HttpServerResponse.class.getName()); private final ServerRequest serverRequest; @@ -32,8 +35,7 @@ public class Http1ServerResponse implements ServerResponse { private HttpHeaders trailingHeaders; - public Http1ServerResponse(HttpVersion httpVersion, ServerRequest serverRequest, ChannelHandlerContext ctx) { - this.httpVersion = httpVersion; + public HttpServerResponse(ServerRequest serverRequest, ChannelHandlerContext ctx) { this.serverRequest = serverRequest; this.ctx = ctx; this.headers = new DefaultHttpHeaders(); @@ -41,8 +43,8 @@ public class Http1ServerResponse implements ServerResponse { } @Override - public HttpHeaders getHeaders() { - return headers; + public void setHeader(AsciiString name, String value) { + headers.set(name, value); } @Override @@ -120,12 +122,20 @@ public class Http1ServerResponse implements ServerResponse { headers.add(HttpHeaderNames.SERVER, ServerName.getServerName()); } FullHttpResponse fullHttpResponse = byteBuf != null ? - new DefaultFullHttpResponse(httpVersion, + new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.valueOf(status), byteBuf, headers, trailingHeaders) : - new DefaultFullHttpResponse(httpVersion, + new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.valueOf(status), Unpooled.EMPTY_BUFFER, headers, trailingHeaders); - if (ctx.channel().isWritable()) { - ctx.channel().writeAndFlush(fullHttpResponse); + if (serverRequest != null && serverRequest.getSequenceId() != null) { + HttpPipelinedResponse httpPipelinedResponse = new HttpPipelinedResponse(fullHttpResponse, + ctx.channel().newPromise(), serverRequest.getSequenceId()); + if (ctx.channel().isWritable()) { + ctx.channel().writeAndFlush(httpPipelinedResponse); + } + } else { + if (ctx.channel().isWritable()) { + ctx.channel().writeAndFlush(fullHttpResponse); + } } } diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http1ServerTransport.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/HttpServerTransport.java similarity index 70% rename from netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http1ServerTransport.java rename to netty-http-server/src/main/java/org/xbib/netty/http/server/transport/HttpServerTransport.java index 4d8af49..c6962dc 100644 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/Http1ServerTransport.java +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/HttpServerTransport.java @@ -10,14 +10,20 @@ import org.xbib.netty.http.server.context.VirtualServer; import java.io.IOException; -public class Http1ServerTransport extends BaseServerTransport { +public class HttpServerTransport extends BaseServerTransport { - public Http1ServerTransport(Server server) { + public HttpServerTransport(Server server) { super(server); } @Override public void requestReceived(ChannelHandlerContext ctx, FullHttpRequest fullHttpRequest) throws IOException { + requestReceived(ctx, fullHttpRequest, 0); + } + + @Override + public void requestReceived(ChannelHandlerContext ctx, FullHttpRequest fullHttpRequest, Integer sequenceId) + throws IOException { int requestId = requestCounter.incrementAndGet(); VirtualServer virtualServer = server.getVirtualServer(fullHttpRequest.headers().get(HttpHeaderNames.HOST)); if (virtualServer == null) { @@ -25,15 +31,15 @@ public class Http1ServerTransport extends BaseServerTransport { } HttpAddress httpAddress = server.getServerConfig().getAddress(); ServerRequest serverRequest = new ServerRequest(virtualServer, httpAddress, fullHttpRequest, - null, requestId); - ServerResponse serverResponse = new Http1ServerResponse(httpAddress.getVersion(), serverRequest, ctx); + sequenceId, null, requestId); + ServerResponse serverResponse = new HttpServerResponse(serverRequest, ctx); if (acceptRequest(serverRequest, serverResponse)) { handle(serverRequest, serverResponse); } } @Override - public void settingsReceived(ChannelHandlerContext ctx, Http2Settings http2Settings) throws Exception { - + public void settingsReceived(ChannelHandlerContext ctx, Http2Settings http2Settings) { + // there are no settings in HTTP 1 } } diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/ServerRequest.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/ServerRequest.java index eaf506f..3bd4559 100644 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/ServerRequest.java +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/ServerRequest.java @@ -15,15 +15,18 @@ public class ServerRequest { private final FullHttpRequest httpRequest; + private final Integer sequenceId; + private final Integer streamId; private final Integer requestId; - public ServerRequest(VirtualServer virtualServer, HttpAddress httpAddress, FullHttpRequest httpRequest, - Integer streamId, Integer requestId) { + public ServerRequest(VirtualServer virtualServer, HttpAddress httpAddress, + FullHttpRequest httpRequest, Integer sequenceId, Integer streamId, Integer requestId) { this.virtualServer = virtualServer; this.httpAddress = httpAddress; this.httpRequest = httpRequest; + this.sequenceId = sequenceId; this.streamId = streamId; this.requestId = requestId; } @@ -40,6 +43,10 @@ public class ServerRequest { return httpRequest; } + public Integer getSequenceId() { + return sequenceId; + } + public Integer streamId() { return streamId; } diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/ServerResponse.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/ServerResponse.java index 2643be6..297e813 100644 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/ServerResponse.java +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/ServerResponse.java @@ -1,7 +1,7 @@ package org.xbib.netty.http.server.transport; import io.netty.buffer.ByteBuf; -import io.netty.handler.codec.http.HttpHeaders; +import io.netty.util.AsciiString; import java.nio.charset.Charset; @@ -10,7 +10,7 @@ import java.nio.charset.Charset; */ public interface ServerResponse { - HttpHeaders getHeaders(); + void setHeader(AsciiString name, String value); void write(String text); diff --git a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/ServerTransport.java b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/ServerTransport.java index a62fd0e..14ef451 100644 --- a/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/ServerTransport.java +++ b/netty-http-server/src/main/java/org/xbib/netty/http/server/transport/ServerTransport.java @@ -13,6 +13,9 @@ public interface ServerTransport { void requestReceived(ChannelHandlerContext ctx, FullHttpRequest fullHttpRequest) throws IOException; + void requestReceived(ChannelHandlerContext ctx, FullHttpRequest fullHttpRequest, Integer sequenceId) throws IOException; + void settingsReceived(ChannelHandlerContext ctx, Http2Settings http2Settings) throws Exception; + void exceptionReceived(ChannelHandlerContext ctx, Throwable throwable) throws IOException; } diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/LoggingBase.java b/netty-http-server/src/test/java/org/xbib/TestBase.java similarity index 62% rename from netty-http-server/src/test/java/org/xbib/netty/http/server/test/LoggingBase.java rename to netty-http-server/src/test/java/org/xbib/TestBase.java index 18f49d7..7b46847 100644 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/LoggingBase.java +++ b/netty-http-server/src/test/java/org/xbib/TestBase.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.server.test; +package org.xbib; import java.util.logging.ConsoleHandler; import java.util.logging.Handler; @@ -7,20 +7,26 @@ import java.util.logging.LogManager; import java.util.logging.Logger; import java.util.logging.SimpleFormatter; -public class LoggingBase { +public class TestBase { static { + + System.setProperty("io.netty.noUnsafe", Boolean.toString(true)); + System.setProperty("io.netty.noKeySetOptimization", Boolean.toString(true)); + //System.setProperty("io.netty.recycler.maxCapacity", Integer.toString(0)); + //System.setProperty("io.netty.leakDetection.level", "paranoid"); + System.setProperty("java.util.logging.SimpleFormatter.format", - "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %2$s %5$s %6$s%n"); + "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %5$s %6$s%n"); LogManager.getLogManager().reset(); Logger rootLogger = LogManager.getLogManager().getLogger(""); Handler handler = new ConsoleHandler(); handler.setFormatter(new SimpleFormatter()); rootLogger.addHandler(handler); - rootLogger.setLevel(Level.ALL); + rootLogger.setLevel(Level.FINE); for (Handler h : rootLogger.getHandlers()) { handler.setFormatter(new SimpleFormatter()); - h.setLevel(Level.ALL); + h.setLevel(Level.FINE); } } } diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/simple/CleartextHttp2Test.java b/netty-http-server/src/test/java/org/xbib/netty/http/hacks/CleartextHttp2Test.java similarity index 86% rename from netty-http-server/src/test/java/org/xbib/netty/http/server/test/simple/CleartextHttp2Test.java rename to netty-http-server/src/test/java/org/xbib/netty/http/hacks/CleartextHttp2Test.java index 3fa08ba..a4ffe5d 100644 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/simple/CleartextHttp2Test.java +++ b/netty-http-server/src/test/java/org/xbib/netty/http/hacks/CleartextHttp2Test.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.server.test.simple; +package org.xbib.netty.http.hacks; import io.netty.bootstrap.Bootstrap; import io.netty.bootstrap.ServerBootstrap; @@ -29,19 +29,18 @@ import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandlerBuilder; import io.netty.handler.codec.http2.InboundHttp2ToHttpAdapterBuilder; import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; +import org.junit.Ignore; import org.junit.Test; +import org.xbib.TestBase; import java.net.InetSocketAddress; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.logging.ConsoleHandler; -import java.util.logging.Handler; import java.util.logging.Level; -import java.util.logging.LogManager; import java.util.logging.Logger; -import java.util.logging.SimpleFormatter; -public class CleartextHttp2Test { +@Ignore +public class CleartextHttp2Test extends TestBase { private static final Logger clientLogger = Logger.getLogger("client"); private static final Logger serverLogger = Logger.getLogger("server"); @@ -52,36 +51,19 @@ public class CleartextHttp2Test { private static final Http2FrameLogger serverFrameLogger = new Http2FrameLogger(logLevel, "server"); private static final Http2FrameLogger clientFrameLogger = new Http2FrameLogger(logLevel, "client"); - static { - System.setProperty("io.netty.noUnsafe", Boolean.toString(true)); - System.setProperty("io.netty.noKeySetOptimization", Boolean.toString(true)); - //System.setProperty("io.netty.recycler.maxCapacity", Integer.toString(0)); - //System.setProperty("io.netty.leakDetection.level", "paranoid"); + private CompletableFuture settingsPrefaceFuture; - // expand Java logging to full level - System.setProperty("java.util.logging.SimpleFormatter.format", - "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %2$s %5$s %6$s%n"); - LogManager.getLogManager().reset(); - Logger rootLogger = LogManager.getLogManager().getLogger(""); - Handler handler = new ConsoleHandler(); - handler.setFormatter(new SimpleFormatter()); - rootLogger.addHandler(handler); - rootLogger.setLevel(Level.ALL); - for (Handler h : rootLogger.getHandlers()) { - handler.setFormatter(new SimpleFormatter()); - h.setLevel(Level.ALL); - } - } - - private final InetSocketAddress inetSocketAddress = new InetSocketAddress("localhost", 8008); - - private final CompletableFuture settingsPrefaceFuture = new CompletableFuture<>(); - - private final CompletableFuture completableFuture = new CompletableFuture<>(); + private CompletableFuture completableFuture; @Test public void testHttp2() throws Exception { + final InetSocketAddress inetSocketAddress = new InetSocketAddress("localhost", 8008); + + settingsPrefaceFuture = new CompletableFuture<>(); + + completableFuture = new CompletableFuture<>(); + EventLoopGroup serverEventLoopGroup = new NioEventLoopGroup(); EventLoopGroup clientEventLoopGroup = new NioEventLoopGroup(); diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/hacks/HttpPipeliningHandlerTest.java b/netty-http-server/src/test/java/org/xbib/netty/http/hacks/HttpPipeliningHandlerTest.java new file mode 100644 index 0000000..5f82f00 --- /dev/null +++ b/netty-http-server/src/test/java/org/xbib/netty/http/hacks/HttpPipeliningHandlerTest.java @@ -0,0 +1,207 @@ +package org.xbib.netty.http.hacks; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufUtil; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.DefaultHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.codec.http.QueryStringDecoder; +import org.junit.After; +import org.junit.Ignore; +import org.junit.Test; +import org.xbib.netty.http.server.handler.http.HttpPipelinedRequest; +import org.xbib.netty.http.server.handler.http.HttpPipelinedResponse; +import org.xbib.netty.http.server.handler.http.HttpPipeliningHandler; +import org.xbib.TestBase; + +import java.nio.channels.ClosedChannelException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertNotNull; + +@Ignore +public class HttpPipeliningHandlerTest extends TestBase { + + private static final Logger logger = Logger.getLogger(HttpPipeliningHandlerTest.class.getName()); + + private static Map waitingRequests = new ConcurrentHashMap<>(); + + @After + public void closeResources() { + for (String url : waitingRequests.keySet()) { + finishRequest(url); + } + } + + @Test + public void testThatPipeliningWorksWithFastSerializedRequests() { + WorkEmulatorHandler handler = new WorkEmulatorHandler(); + EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(10000), + handler); + for (int i = 0; i < 5; i++) { + embeddedChannel.writeInbound(createHttpRequest("/" + String.valueOf(i))); + } + for (String url : waitingRequests.keySet()) { + finishRequest(url); + } + handler.shutdownExecutorService(); + for (int i = 0; i < 5; i++) { + assertReadHttpMessageHasContent(embeddedChannel, String.valueOf(i)); + } + assertThat(embeddedChannel.isOpen(), is(true)); + } + + @Test + public void testThatPipeliningWorksWhenSlowRequestsInDifferentOrder() { + WorkEmulatorHandler handler = new WorkEmulatorHandler(); + EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(10000), + handler); + for (int i = 0; i < 5; i++) { + embeddedChannel.writeInbound(createHttpRequest("/" + String.valueOf(i))); + } + List urls = new ArrayList<>(waitingRequests.keySet()); + Collections.shuffle(urls); + for (String url : urls) { + finishRequest(url); + } + handler.shutdownExecutorService(); + for (int i = 0; i < 5; i++) { + assertReadHttpMessageHasContent(embeddedChannel, String.valueOf(i)); + } + assertThat(embeddedChannel.isOpen(), is(true)); + } + + @Test + public void testThatPipeliningWorksWithChunkedRequests() { + WorkEmulatorHandler handler = new WorkEmulatorHandler(); + EmbeddedChannel embeddedChannel = new EmbeddedChannel(new AggregateUrisAndHeadersHandler(), + new HttpPipeliningHandler(10000), handler); + DefaultHttpRequest httpRequest = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/0"); + embeddedChannel.writeInbound(httpRequest); + embeddedChannel.writeInbound(LastHttpContent.EMPTY_LAST_CONTENT); + httpRequest = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/1"); + embeddedChannel.writeInbound(httpRequest); + embeddedChannel.writeInbound(LastHttpContent.EMPTY_LAST_CONTENT); + finishRequest("1"); + finishRequest("0"); + handler.shutdownExecutorService(); + for (int i = 0; i < 2; i++) { + assertReadHttpMessageHasContent(embeddedChannel, String.valueOf(i)); + } + assertThat(embeddedChannel.isOpen(), is(true)); + } + + @Test(expected = ClosedChannelException.class) + public void testThatPipeliningClosesConnectionWithTooManyEvents() { + WorkEmulatorHandler handler = new WorkEmulatorHandler(); + EmbeddedChannel embeddedChannel = new EmbeddedChannel(new HttpPipeliningHandler(2), + handler); + embeddedChannel.writeInbound(createHttpRequest("/0")); + embeddedChannel.writeInbound(createHttpRequest("/1")); + embeddedChannel.writeInbound(createHttpRequest("/2")); + embeddedChannel.writeInbound(createHttpRequest("/3")); + finishRequest("1"); + finishRequest("2"); + finishRequest("3"); + finishRequest("0"); + handler.shutdownExecutorService(); + embeddedChannel.writeInbound(createHttpRequest("/")); + } + + private void assertReadHttpMessageHasContent(EmbeddedChannel embeddedChannel, String expectedContent) { + FullHttpResponse response = (FullHttpResponse) embeddedChannel.outboundMessages().poll(); + assertNotNull("Expected response to exist, maybe you did not wait long enough?", response); + assertNotNull("Expected response to have content " + expectedContent, response.content()); + String data = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8); + assertThat(data, is(expectedContent)); + } + + private void finishRequest(String url) { + waitingRequests.get(url).countDown(); + } + + private FullHttpRequest createHttpRequest(String uri) { + return new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri); + } + + private static class AggregateUrisAndHeadersHandler extends SimpleChannelInboundHandler { + + static final Queue STRINGS = new LinkedTransferQueue<>(); + + @Override + protected void channelRead0(ChannelHandlerContext ctx, HttpRequest request) { + STRINGS.add(request.uri()); + } + } + + private class WorkEmulatorHandler extends SimpleChannelInboundHandler { + + private final ExecutorService executorService = Executors.newFixedThreadPool(5); + + @Override + protected void channelRead0(ChannelHandlerContext ctx, HttpPipelinedRequest pipelinedRequest) { + QueryStringDecoder decoder; + if (pipelinedRequest.getRequest() instanceof FullHttpRequest) { + FullHttpRequest fullHttpRequest = (FullHttpRequest) pipelinedRequest.getRequest(); + decoder = new QueryStringDecoder(fullHttpRequest.uri()); + } else { + decoder = new QueryStringDecoder(AggregateUrisAndHeadersHandler.STRINGS.poll()); + } + String uri = decoder.path().replace("/", ""); + ByteBuf content = Unpooled.copiedBuffer(uri, StandardCharsets.UTF_8); + DefaultFullHttpResponse httpResponse = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, + HttpResponseStatus.OK, content); + httpResponse.headers().add(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes()); + CountDownLatch latch = new CountDownLatch(1); + waitingRequests.put(uri, latch); + executorService.submit(() -> { + try { + latch.await(2, TimeUnit.SECONDS); + HttpPipelinedResponse httpPipelinedResponse = new HttpPipelinedResponse(httpResponse, + ctx.channel().newPromise(), pipelinedRequest.getSequenceId()); + ctx.writeAndFlush(httpPipelinedResponse); + } catch (InterruptedException e) { + logger.log(Level.WARNING, e.getMessage(), e); + } + }); + } + + void shutdownExecutorService() { + if (!executorService.isShutdown()) { + executorService.shutdown(); + try { + executorService.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + logger.log(Level.WARNING, e.getMessage(), e); + } + } + } + } +} diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/simple/MultiplexCodecCleartextHttp2Test.java b/netty-http-server/src/test/java/org/xbib/netty/http/hacks/MultiplexCodecCleartextHttp2Test.java similarity index 90% rename from netty-http-server/src/test/java/org/xbib/netty/http/server/test/simple/MultiplexCodecCleartextHttp2Test.java rename to netty-http-server/src/test/java/org/xbib/netty/http/hacks/MultiplexCodecCleartextHttp2Test.java index 7a9b666..19eaee5 100644 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/simple/MultiplexCodecCleartextHttp2Test.java +++ b/netty-http-server/src/test/java/org/xbib/netty/http/hacks/MultiplexCodecCleartextHttp2Test.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.server.test.simple; +package org.xbib.netty.http.hacks; import io.netty.bootstrap.Bootstrap; import io.netty.bootstrap.ServerBootstrap; @@ -40,17 +40,15 @@ import io.netty.handler.codec.http2.Http2StreamFrameToHttpObjectCodec; import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; import io.netty.util.AsciiString; +import org.junit.Ignore; import org.junit.Test; +import org.xbib.TestBase; import java.net.InetSocketAddress; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.logging.ConsoleHandler; -import java.util.logging.Handler; import java.util.logging.Level; -import java.util.logging.LogManager; import java.util.logging.Logger; -import java.util.logging.SimpleFormatter; /** * @@ -70,32 +68,12 @@ import java.util.logging.SimpleFormatter; * * */ -public class MultiplexCodecCleartextHttp2Test { +@Ignore +public class MultiplexCodecCleartextHttp2Test extends TestBase { private static final Logger clientLogger = Logger.getLogger("client"); private static final Logger serverLogger = Logger.getLogger("server"); - static { - System.setProperty("io.netty.noUnsafe", Boolean.toString(true)); - System.setProperty("io.netty.noKeySetOptimization", Boolean.toString(true)); - System.setProperty("io.netty.recycler.maxCapacity", Integer.toString(0)); - System.setProperty("io.netty.leakDetection.level", "paranoid"); - - // expand Java logging to full level - System.setProperty("java.util.logging.SimpleFormatter.format", - "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %2$s %5$s %6$s%n"); - LogManager.getLogManager().reset(); - Logger rootLogger = LogManager.getLogManager().getLogger(""); - Handler handler = new ConsoleHandler(); - handler.setFormatter(new SimpleFormatter()); - rootLogger.addHandler(handler); - rootLogger.setLevel(Level.ALL); - for (Handler h : rootLogger.getHandlers()) { - handler.setFormatter(new SimpleFormatter()); - h.setLevel(Level.ALL); - } - } - private final InetSocketAddress inetSocketAddress = new InetSocketAddress("localhost", 8443); private final CompletableFuture settingsPrefaceFuture = new CompletableFuture<>(); @@ -195,10 +173,11 @@ public class MultiplexCodecCleartextHttp2Test { p.addLast("child-client-response-handler", new ClientResponseHandler()); } }).open().syncUninterruptibly().getNow(); - Http2Headers request = new DefaultHttp2Headers().method(HttpMethod.GET.asciiName()) + Http2Headers request = new DefaultHttp2Headers() + .method(HttpMethod.GET.asciiName()) .path("/foobar/0/0") .scheme("http") - .authority(inetSocketAddress.getHostName()); + .authority(inetSocketAddress.getHostName() + ":" + inetSocketAddress.getPort()); childChannel.writeAndFlush(new DefaultHttp2HeadersFrame(request, true)); clientLogger.log(Level.INFO, "waiting max. 10 seconds"); responseFuture.get(10, TimeUnit.SECONDS); diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/multithread/MultithreadedCleartextHttp2Test.java b/netty-http-server/src/test/java/org/xbib/netty/http/hacks/MultithreadedCleartextHttp2Test.java similarity index 84% rename from netty-http-server/src/test/java/org/xbib/netty/http/server/test/multithread/MultithreadedCleartextHttp2Test.java rename to netty-http-server/src/test/java/org/xbib/netty/http/hacks/MultithreadedCleartextHttp2Test.java index e681b69..47a45fe 100644 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/multithread/MultithreadedCleartextHttp2Test.java +++ b/netty-http-server/src/test/java/org/xbib/netty/http/hacks/MultithreadedCleartextHttp2Test.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.server.test.multithread; +package org.xbib.netty.http.hacks; import io.netty.bootstrap.Bootstrap; import io.netty.bootstrap.ServerBootstrap; @@ -27,7 +27,9 @@ import io.netty.handler.codec.http2.Http2ConnectionPrefaceAndSettingsFrameWritte import io.netty.handler.codec.http2.Http2Settings; import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandlerBuilder; import io.netty.handler.codec.http2.InboundHttp2ToHttpAdapterBuilder; +import org.junit.Ignore; import org.junit.Test; +import org.xbib.TestBase; import java.net.InetSocketAddress; import java.util.concurrent.CompletableFuture; @@ -35,56 +37,36 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.ConsoleHandler; -import java.util.logging.Handler; import java.util.logging.Level; -import java.util.logging.LogManager; import java.util.logging.Logger; -import java.util.logging.SimpleFormatter; -public class MultithreadedCleartextHttp2Test { +@Ignore +public class MultithreadedCleartextHttp2Test extends TestBase { private static final Logger clientLogger = Logger.getLogger("client"); private static final Logger serverLogger = Logger.getLogger("server"); private static final Level level = Level.FINE; - static { - System.setProperty("io.netty.noUnsafe", Boolean.toString(true)); - System.setProperty("io.netty.noKeySetOptimization", Boolean.toString(true)); - //System.setProperty("io.netty.recycler.maxCapacity", Integer.toString(0)); - //System.setProperty("io.netty.leakDetection.level", "paranoid"); + private InetSocketAddress inetSocketAddress; - // expand Java logging to full level - System.setProperty("java.util.logging.SimpleFormatter.format", - "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %2$s %5$s %6$s%n"); - LogManager.getLogManager().reset(); - Logger rootLogger = LogManager.getLogManager().getLogger(""); - Handler handler = new ConsoleHandler(); - handler.setFormatter(new SimpleFormatter()); - rootLogger.addHandler(handler); - rootLogger.setLevel(Level.OFF); - for (Handler h : rootLogger.getHandlers()) { - handler.setFormatter(new SimpleFormatter()); - h.setLevel(Level.ALL); - } - } + private CompletableFuture settingsPrefaceFuture; - private final InetSocketAddress inetSocketAddress = new InetSocketAddress("localhost", 8008); + private CompletableFuture responseFuture; - private final CompletableFuture settingsPrefaceFuture = new CompletableFuture<>(); + private final int threads = 4; - private final CompletableFuture responseFuture = new CompletableFuture<>(); - - private final int threads = 10; - - private final int requestsPerThread = 100000; + private final int requestsPerThread = 500; private final AtomicInteger responseCounter = new AtomicInteger(); @Test public void testMultiThreadedHttp2() throws Exception { + inetSocketAddress = new InetSocketAddress("localhost", 8008); + settingsPrefaceFuture = new CompletableFuture<>(); + responseFuture = new CompletableFuture<>(); + EventLoopGroup serverEventLoopGroup = new NioEventLoopGroup(); EventLoopGroup clientEventLoopGroup = new NioEventLoopGroup(); @@ -180,7 +162,7 @@ public class MultithreadedCleartextHttp2Test { clientLogger.log(level, "waiting"); responseFuture.get(60, TimeUnit.SECONDS); if (responseFuture.isDone()) { - clientLogger.log(Level.INFO, "done"); + clientLogger.log(Level.INFO, "stop"); } } finally { @@ -229,7 +211,7 @@ public class MultithreadedCleartextHttp2Test { if (msg instanceof FullHttpRequest) { FullHttpResponse response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); - ctx.writeAndFlush(response); + ctx.write(response); } } diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/multithread/MultithreadedMultiplexCodecCleartextHttp2Test.java b/netty-http-server/src/test/java/org/xbib/netty/http/hacks/MultithreadedMultiplexCodecCleartextHttp2Test.java similarity index 72% rename from netty-http-server/src/test/java/org/xbib/netty/http/server/test/multithread/MultithreadedMultiplexCodecCleartextHttp2Test.java rename to netty-http-server/src/test/java/org/xbib/netty/http/hacks/MultithreadedMultiplexCodecCleartextHttp2Test.java index 268d40a..373be89 100644 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/multithread/MultithreadedMultiplexCodecCleartextHttp2Test.java +++ b/netty-http-server/src/test/java/org/xbib/netty/http/hacks/MultithreadedMultiplexCodecCleartextHttp2Test.java @@ -1,4 +1,4 @@ -package org.xbib.netty.http.server.test.multithread; +package org.xbib.netty.http.hacks; import io.netty.bootstrap.Bootstrap; import io.netty.bootstrap.ServerBootstrap; @@ -35,7 +35,9 @@ import io.netty.handler.codec.http2.Http2StreamChannel; import io.netty.handler.codec.http2.Http2StreamChannelBootstrap; import io.netty.handler.codec.http2.Http2StreamFrameToHttpObjectCodec; import io.netty.util.AsciiString; +import org.junit.Ignore; import org.junit.Test; +import org.xbib.TestBase; import java.net.InetSocketAddress; import java.util.concurrent.CompletableFuture; @@ -43,61 +45,41 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.ConsoleHandler; -import java.util.logging.Handler; import java.util.logging.Level; -import java.util.logging.LogManager; import java.util.logging.Logger; -import java.util.logging.SimpleFormatter; /** * * Multithreaded Http2MultiplexCodec demo for cleartext HTTP/2 between a server and a client. * */ -public class MultithreadedMultiplexCodecCleartextHttp2Test { +@Ignore +public class MultithreadedMultiplexCodecCleartextHttp2Test extends TestBase { private static final Logger clientLogger = Logger.getLogger("client"); private static final Logger serverLogger = Logger.getLogger("server"); private Level level = Level.FINE; - static { - System.setProperty("io.netty.noUnsafe", Boolean.toString(true)); - System.setProperty("io.netty.noKeySetOptimization", Boolean.toString(true)); - //System.setProperty("io.netty.recycler.maxCapacity", Integer.toString(0)); - //System.setProperty("io.netty.leakDetection.level", "paranoid"); + private InetSocketAddress inetSocketAddress; - // expand Java logging to full level - System.setProperty("java.util.logging.SimpleFormatter.format", - "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %2$s %5$s %6$s%n"); - LogManager.getLogManager().reset(); - Logger rootLogger = LogManager.getLogManager().getLogger(""); - Handler handler = new ConsoleHandler(); - handler.setFormatter(new SimpleFormatter()); - rootLogger.addHandler(handler); - rootLogger.setLevel(Level.INFO); - for (Handler h : rootLogger.getHandlers()) { - handler.setFormatter(new SimpleFormatter()); - h.setLevel(Level.ALL); - } - } + private CompletableFuture settingsPrefaceFuture; - private final InetSocketAddress inetSocketAddress = new InetSocketAddress("localhost", 8008); + private CompletableFuture responseFuture; - private final CompletableFuture settingsPrefaceFuture = new CompletableFuture<>(); + private final int threads = 4; - private final CompletableFuture responseFuture = new CompletableFuture<>(); - - private final int threads = 10; - - private final int requestsPerThread = 100000; + private final int requestsPerThread = 500; private final AtomicInteger responseCounter = new AtomicInteger(); @Test public void testMultithreadedMultiplexHttp2() throws Exception { + inetSocketAddress = new InetSocketAddress("localhost", 8008); + settingsPrefaceFuture = new CompletableFuture<>(); + responseFuture = new CompletableFuture<>(); + EventLoopGroup serverEventLoopGroup = new NioEventLoopGroup(); EventLoopGroup clientEventLoopGroup = new NioEventLoopGroup(); @@ -108,31 +90,31 @@ public class MultithreadedMultiplexCodecCleartextHttp2Test { .childHandler(new ChannelInitializer() { @Override protected void initChannel(Channel ch) { - ChannelPipeline p = ch.pipeline(); - Http2MultiplexCodec serverMultiplexCodec = Http2MultiplexCodecBuilder.forServer(new ChannelInitializer() { - @Override - protected void initChannel(Channel channel) { - ChannelPipeline p = channel.pipeline(); - p.addLast("multiplex-server-frame-converter", new Http2StreamFrameToHttpObjectCodec(true)); - p.addLast("multiplex-server-chunk-aggregator", new HttpObjectAggregator(1048576)); - p.addLast("multiplex-server-request-handler", new ServerRequestHandler()); + ChannelPipeline p = ch.pipeline(); + Http2MultiplexCodec serverMultiplexCodec = Http2MultiplexCodecBuilder.forServer(new ChannelInitializer() { + @Override + protected void initChannel(Channel channel) { + ChannelPipeline p = channel.pipeline(); + p.addLast("multiplex-server-frame-converter", new Http2StreamFrameToHttpObjectCodec(true)); + p.addLast("multiplex-server-chunk-aggregator", new HttpObjectAggregator(1048576)); + p.addLast("multiplex-server-request-handler", new ServerRequestHandler()); + } + }) + .initialSettings(Http2Settings.defaultSettings()) + .build(); + HttpServerUpgradeHandler.UpgradeCodecFactory upgradeCodecFactory = protocol -> { + if (AsciiString.contentEquals(Http2CodecUtil.HTTP_UPGRADE_PROTOCOL_NAME, protocol)) { + return new Http2ServerUpgradeCodec("server-codec", serverMultiplexCodec); + } else { + return null; } - }) - .initialSettings(Http2Settings.defaultSettings()) - .build(); - HttpServerUpgradeHandler.UpgradeCodecFactory upgradeCodecFactory = protocol -> { - if (AsciiString.contentEquals(Http2CodecUtil.HTTP_UPGRADE_PROTOCOL_NAME, protocol)) { - return new Http2ServerUpgradeCodec("server-codec", serverMultiplexCodec); - } else { - return null; - } - }; - HttpServerCodec sourceCodec = new HttpServerCodec(); - HttpServerUpgradeHandler upgradeHandler = new HttpServerUpgradeHandler(sourceCodec, upgradeCodecFactory); - CleartextHttp2ServerUpgradeHandler cleartextHttp2ServerUpgradeHandler = - new CleartextHttp2ServerUpgradeHandler(sourceCodec, upgradeHandler, serverMultiplexCodec); - p.addLast("server-upgrade", cleartextHttp2ServerUpgradeHandler); - p.addLast("server-messages", new ServerMessages()); + }; + HttpServerCodec sourceCodec = new HttpServerCodec(); + HttpServerUpgradeHandler upgradeHandler = new HttpServerUpgradeHandler(sourceCodec, upgradeCodecFactory); + CleartextHttp2ServerUpgradeHandler cleartextHttp2ServerUpgradeHandler = + new CleartextHttp2ServerUpgradeHandler(sourceCodec, upgradeHandler, serverMultiplexCodec); + p.addLast("server-upgrade", cleartextHttp2ServerUpgradeHandler); + p.addLast("server-messages", new ServerMessages()); } }); Channel serverChannel = serverBootstrap.bind(inetSocketAddress).sync().channel(); @@ -193,8 +175,8 @@ public class MultithreadedMultiplexCodecCleartextHttp2Test { childChannel.write(new DefaultHttp2HeadersFrame(request, true)); //do not close child channel after write, a response is expected } + clientChannel.flush(); }); - clientChannel.flush(); } executorService.shutdown(); executorService.awaitTermination(60, TimeUnit.SECONDS); @@ -252,7 +234,7 @@ public class MultithreadedMultiplexCodecCleartextHttp2Test { protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest msg) { DefaultFullHttpResponse response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); - ctx.writeAndFlush(response); + ctx.write(response); } } diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/hacks/package-info.java b/netty-http-server/src/test/java/org/xbib/netty/http/hacks/package-info.java new file mode 100644 index 0000000..61b2924 --- /dev/null +++ b/netty-http-server/src/test/java/org/xbib/netty/http/hacks/package-info.java @@ -0,0 +1,4 @@ +/** + * Hacking Netty for showing server functions. + */ +package org.xbib.netty.http.hacks; diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/CleartextHttp1Test.java b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/CleartextHttp1Test.java index 6e2cf62..9381ef6 100644 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/CleartextHttp1Test.java +++ b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/CleartextHttp1Test.java @@ -1,61 +1,165 @@ package org.xbib.netty.http.server.test; +import io.netty.handler.codec.http.HttpVersion; import org.junit.Test; +import org.xbib.TestBase; import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Request; +import org.xbib.netty.http.client.listener.ResponseListener; import org.xbib.netty.http.client.transport.Transport; import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.server.Server; import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; -public class CleartextHttp1Test extends LoggingBase { +import static org.junit.Assert.assertEquals; - private static final Logger logger = Logger.getLogger(""); +public class CleartextHttp1Test extends TestBase { + + private static final Logger logger = Logger.getLogger(CleartextHttp1Test.class.getName()); @Test - public void testClearTextHttp1() throws Exception { - int loop = 1024; + public void testSimpleClearTextHttp1() throws Exception { + HttpAddress httpAddress = HttpAddress.http1("localhost", 8008); + Server server = Server.builder() + .bind(httpAddress).build(); + server.getDefaultVirtualServer().addContext("/", (request, response) -> + response.write(200, "text/plain", request.getRequest().content().retain())); + server.accept(); + Client client = Client.builder() + .build(); + AtomicInteger counter = new AtomicInteger(); + final ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + //logger.log(Level.INFO, "status = " + fullHttpResponse.status() + " response body = " + response); + counter.incrementAndGet(); + }; + try { + Request request = Request.get().setVersion(HttpVersion.HTTP_1_1) + .url(server.getServerConfig().getAddress().base()) + .content("Hello world", "text/plain") + .build() + .setResponseListener(responseListener); + client.execute(request).get(); + } finally { + client.shutdownGracefully(); + server.shutdownGracefully(); + } + logger.log(Level.INFO, "exepecting=1 counter=" + counter.get()); + assertEquals(1, counter.get()); + } + + @Test + public void testPooledClearTextHttp1() throws Exception { + int loop = 4096; HttpAddress httpAddress = HttpAddress.http1("localhost", 8008); Server server = Server.builder() //.enableDebug() .bind(httpAddress).build(); server.getDefaultVirtualServer().addContext("/", (request, response) -> { - response.write("Hello World " + request.getRequest().content().toString(StandardCharsets.UTF_8)); + response.write(200, "text/plain", request.getRequest().content().retain()); }); server.accept(); - Client httpClient = Client.builder() + Client client = Client.builder() //.enableDebug() + .addPoolNode(httpAddress) + .setPoolNodeConnectionLimit(2) .build(); AtomicInteger counter = new AtomicInteger(); + final ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + //logger.log(Level.INFO, "status = " + fullHttpResponse.status() + " response body = " + response); + counter.incrementAndGet(); + }; try { - // will not work for several thousands channels - "java.net.SocketException: Too many open files in system" for (int i = 0; i < loop; i++) { Request request = Request.get().setVersion("HTTP/1.1") .url(server.getServerConfig().getAddress().base()) - .addParameter("test", Integer.toString(i)) .content(Integer.toString(i), "text/plain") .build() - .setResponseListener(fullHttpResponse -> { - String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); - //logger.log(Level.INFO, "status = " + fullHttpResponse.status() + " response body = " + response); - counter.incrementAndGet(); - }); - Transport transport = httpClient.execute(request); + .setResponseListener(responseListener); + Transport transport = client.newTransport(); + transport.execute(request); if (transport.isFailed()) { logger.log(Level.WARNING, transport.getFailure().getMessage(), transport.getFailure()); break; } - // each execution needs to be synchronized transport.get(); } } finally { - httpClient.shutdownGracefully(); + client.shutdownGracefully(); server.shutdownGracefully(); } - logger.log(Level.INFO, "counter=" + counter.get()); + logger.log(Level.INFO, "expecting=" + loop + " counter=" + counter.get()); + assertEquals(loop, counter.get()); + } + + @Test + public void testMultithreadedPooledClearTextHttp1() throws Exception { + int threads = 4; + int loop = 4 * 1024; + HttpAddress httpAddress = HttpAddress.http1("localhost", 8008); + Server server = Server.builder() + //.enableDebug() + .bind(httpAddress).build(); + server.getDefaultVirtualServer().addContext("/", (request, response) -> { + response.write(200, "text/plain", request.getRequest().content().retain()); + }); + server.accept(); + Client client = Client.builder() + .addPoolNode(httpAddress) + .setPoolNodeConnectionLimit(threads) + .build(); + AtomicInteger counter = new AtomicInteger(); + final ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + //logger.log(Level.INFO, "status = " + fullHttpResponse.status() + + // " response=" + response + " payload=" + payload); + counter.incrementAndGet(); + }; + try { + ExecutorService executorService = Executors.newFixedThreadPool(threads); + for (int n = 0; n < threads; n++) { + final int t = n; + executorService.submit(() -> { + try { + for (int i = 0; i < loop; i++) { + String payload = Integer.toString(t) + "/" + Integer.toString(i); + Request request = Request.get().setVersion(HttpVersion.HTTP_1_1) + .url(server.getServerConfig().getAddress().base()) + .content(payload, "text/plain") + .build() + .setResponseListener(responseListener); + // note: a new transport is created per execution + Transport transport = client.newTransport(); + transport.execute(request); + if (transport.isFailed()) { + logger.log(Level.WARNING, "transport failed: " + transport.getFailure().getMessage(), transport.getFailure()); + break; + } + transport.get(); + } + } catch (Exception e) { + logger.log(Level.SEVERE, e.getMessage(), e); + } + }); + } + executorService.shutdown(); + boolean terminated = executorService.awaitTermination(30, TimeUnit.SECONDS); + logger.log(Level.INFO, "terminated = " + terminated + ", now waiting for transport to complete"); + } catch (Exception e) { + logger.log(Level.SEVERE, e.getMessage(), e); + } finally { + client.shutdownGracefully(); + server.shutdownGracefully(); + } + logger.log(Level.INFO, "expecting=" + (threads * loop) + " counter=" + counter.get()); + assertEquals(threads * loop, counter.get()); } } diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/CleartextHttp2Test.java b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/CleartextHttp2Test.java index 243e23a..36d82e8 100644 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/CleartextHttp2Test.java +++ b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/CleartextHttp2Test.java @@ -1,73 +1,255 @@ package org.xbib.netty.http.server.test; -import io.netty.channel.WriteBufferWaterMark; -import io.netty.handler.codec.http.HttpVersion; import org.junit.Test; -import org.xbib.net.URL; +import org.xbib.TestBase; import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Request; +import org.xbib.netty.http.client.listener.ResponseListener; import org.xbib.netty.http.client.transport.Transport; import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.server.Server; +import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; -public class CleartextHttp2Test extends LoggingBase { +import static org.junit.Assert.assertEquals; + +public class CleartextHttp2Test extends TestBase { private static final Logger logger = Logger.getLogger(CleartextHttp2Test.class.getName()); @Test - public void testCleartextHttp2() throws Exception { - int loop = 1; - // we assume slow server and reserve a large write buffer for the client, 32-64 bytes for each request, - // to avoid channel.isWritable() drop-outs - int low = 32 * loop; - int high = 64 * loop; - HttpAddress httpAddress = HttpAddress.of("localhost", 8008, HttpVersion.valueOf("HTTP/2.0"), false); + public void testSimpleCleartextHttp2() throws Exception { + HttpAddress httpAddress = HttpAddress.http2("localhost", 8008); Server server = Server.builder() - .enableDebug() .bind(httpAddress) .build(); - //server.logDiagnostics(Level.INFO); server.getDefaultVirtualServer().addContext("/", (request, response) -> - response.write("Hello World " + request.getRequest().content().toString(StandardCharsets.UTF_8))); + response.write(200, "text/plain", request.getRequest().content().retain())); server.accept(); - Client httpClient = Client.builder() - .enableDebug() - .setWriteBufferWaterMark(new WriteBufferWaterMark(low, high)) + Client client = Client.builder() .build(); AtomicInteger counter = new AtomicInteger(); + // a single instance of HTTP/2 response listener, always receives responses out-of-order + ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + logger.log(Level.INFO, "response listener: headers = " + fullHttpResponse.headers().entries() + + " response body = " + response); + counter.incrementAndGet(); + }; try { - URL serverURL = server.getServerConfig().getAddress().base(); - HttpVersion serverVersion = server.getServerConfig().getAddress().getVersion(); - // yes, HTTP/2 uses a single transport, and we can send many thousand requests per second asynchronously - Transport transport = httpClient.newTransport(serverURL, serverVersion); + String payload = Integer.toString(0) + "/" + Integer.toString(0); + Request request = Request.get().setVersion("HTTP/2.0") + .url(server.getServerConfig().getAddress().base()) + .content(payload, "text/plain") + .build() + .setResponseListener(responseListener); + Transport transport = client.newTransport(httpAddress); + transport.execute(request); + if (transport.isFailed()) { + logger.log(Level.WARNING, transport.getFailure().getMessage(), transport.getFailure()); + } + transport.get(); + } finally { + client.shutdownGracefully(); + server.shutdownGracefully(); + } + logger.log(Level.INFO, "counter = " + counter.get()); + assertEquals(1, counter.get()); + } + + @Test + public void testPooledClearTextHttp2() throws Exception { + int loop = 4096; + HttpAddress httpAddress = HttpAddress.http2("localhost", 8008); + Server server = Server.builder() + .bind(httpAddress).build(); + server.getDefaultVirtualServer().addContext("/", (request, response) -> + response.write(200, "text/plain", request.getRequest().content().retain())); + //server.getDefaultVirtualServer().addContext("/", (request, response) -> + // response.write(request.getRequest().content().toString(StandardCharsets.UTF_8))); + server.accept(); + Client client = Client.builder() + //.enableDebug() + .addPoolNode(httpAddress) + .setPoolNodeConnectionLimit(2) + .build(); + AtomicInteger counter = new AtomicInteger(); + // a single instance of HTTP/2 response listener, always receives responses out-of-order + final ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + //logger.log(Level.INFO, "response listener: headers = " + fullHttpResponse.headers().entries() + + // " response body = " + response); + counter.incrementAndGet(); + }; + try { + // single transport, single thread + Transport transport = client.newTransport(); for (int i = 0; i < loop; i++) { + String payload = Integer.toString(0) + "/" + Integer.toString(i); Request request = Request.get().setVersion("HTTP/2.0") .url(server.getServerConfig().getAddress().base()) - .content(Integer.toString(i), "text/plain") + .content(payload, "text/plain") .build() - .setResponseListener(fullHttpResponse -> { - String content = fullHttpResponse.content().toString(StandardCharsets.UTF_8); - logger.log(Level.INFO, fullHttpResponse.toString() + " content=" + content); - counter.incrementAndGet(); - }); - // submit request + .setResponseListener(responseListener); transport.execute(request); if (transport.isFailed()) { logger.log(Level.WARNING, transport.getFailure().getMessage(), transport.getFailure()); break; } } - // wait for transport to complete transport.get(); } finally { - httpClient.shutdownGracefully(); + client.shutdownGracefully(); server.shutdownGracefully(); } - logger.log(Level.INFO, "counter = " + counter.get()); + logger.log(Level.INFO, "expecting=" + loop + " counter=" + counter.get()); + assertEquals(loop, counter.get()); + } + + @Test + public void testMultithreadPooledClearTextHttp2() throws Exception { + int threads = 2; + int loop = 4 * 1024; + HttpAddress httpAddress = HttpAddress.http2("localhost", 8008); + Server server = Server.builder() + .bind(httpAddress) + .build(); + server.getDefaultVirtualServer().addContext("/", (request, response) -> + response.write(request.getRequest().content().toString(StandardCharsets.UTF_8)) + ); + server.accept(); + Client client = Client.builder() + .addPoolNode(httpAddress) + .setPoolNodeConnectionLimit(threads) + .build(); + AtomicInteger counter = new AtomicInteger(); + // a HTTP/2 listener always receives responses out-of-order + final ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + //logger.log(Level.INFO, "response listener: headers = " + fullHttpResponse.headers().entries() + + // " response body = " + response); + counter.incrementAndGet(); + }; + try { + // note: for HTTP/2 only, we can use a single shared transport + final Transport transport = client.newTransport(); + ExecutorService executorService = Executors.newFixedThreadPool(threads); + for (int n = 0; n < threads; n++) { + final int t = n; + executorService.submit(() -> { + try { + for (int i = 0; i < loop; i++) { + String payload = Integer.toString(t) + "/" + Integer.toString(i); + Request request = Request.get().setVersion("HTTP/2.0") + .url(server.getServerConfig().getAddress().base()) + .content(payload, "text/plain") + .build() + .setResponseListener(responseListener); + transport.execute(request); + if (transport.isFailed()) { + logger.log(Level.WARNING, transport.getFailure().getMessage(), transport.getFailure()); + break; + } + } + } catch (IOException e) { + logger.log(Level.WARNING, e.getMessage(), e); + } + }); + } + executorService.shutdown(); + boolean terminated = executorService.awaitTermination(30, TimeUnit.SECONDS); + logger.log(Level.INFO, "terminated = " + terminated + ", now waiting for transport to complete"); + transport.get(30, TimeUnit.SECONDS); + } finally { + client.shutdownGracefully(); + server.shutdownGracefully(); + } + logger.log(Level.INFO, "expected=" + (threads * loop) + " counter=" + counter.get()); + assertEquals(threads * loop , counter.get()); + } + + @Test + public void testTwoPooledClearTextHttp2() throws Exception { + int threads = 2; + int loop = 4 * 1024; + + HttpAddress httpAddress1 = HttpAddress.http2("localhost", 8008); + AtomicInteger counter1 = new AtomicInteger(); + Server server1 = Server.builder() + .bind(httpAddress1).build(); + server1.getDefaultVirtualServer().addContext("/", (request, response) -> { + response.write(request.getRequest().content().toString(StandardCharsets.UTF_8)); + counter1.incrementAndGet(); + }); + server1.accept(); + + HttpAddress httpAddress2 = HttpAddress.http2("localhost", 8009); + AtomicInteger counter2 = new AtomicInteger(); + Server server2 = Server.builder() + .bind(httpAddress2).build(); + server2.getDefaultVirtualServer().addContext("/", (request, response) -> { + response.write(request.getRequest().content().toString(StandardCharsets.UTF_8)); + counter2.incrementAndGet(); + }); + server2.accept(); + + Client client = Client.builder() + .addPoolNode(httpAddress1) + .addPoolNode(httpAddress2) + .setPoolNodeConnectionLimit(threads) + .build(); + AtomicInteger counter = new AtomicInteger(); + // a single instance of HTTP/2 response listener, always receives responses out-of-order + final ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + //logger.log(Level.INFO, "response listener: headers = " + fullHttpResponse.headers().entries() + + // " response body = " + response); + counter.incrementAndGet(); + }; + try { + // note: for HTTP/2 only, we can use a single shared transport + final Transport transport = client.newTransport(); + ExecutorService executorService = Executors.newFixedThreadPool(threads); + for (int n = 0; n < threads; n++) { + final int t = n; + executorService.submit(() -> { + try { + for (int i = 0; i < loop; i++) { + String payload = Integer.toString(t) + "/" + Integer.toString(i); + Request request = Request.get().setVersion("HTTP/2.0") + .uri("/") + .content(payload, "text/plain") + .build() + .setResponseListener(responseListener); + transport.execute(request); + if (transport.isFailed()) { + logger.log(Level.WARNING, transport.getFailure().getMessage(), transport.getFailure()); + break; + } + } + } catch (IOException e) { + logger.log(Level.WARNING, e.getMessage(), e); + } + }); + } + executorService.shutdown(); + boolean terminated = executorService.awaitTermination(30, TimeUnit.SECONDS); + logger.log(Level.INFO, "terminated = " + terminated + ", now waiting for transport to complete"); + transport.get(30, TimeUnit.SECONDS); + } finally { + client.shutdownGracefully(); + server1.shutdownGracefully(); + server2.shutdownGracefully(); + } + logger.log(Level.INFO, "counter1=" + counter1.get() + " counter2=" + counter2.get()); + logger.log(Level.INFO, "expecting=" + threads * loop + " counter=" + counter.get()); + assertEquals(threads * loop, counter.get()); } } diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/MultithreadedCleartextHttp2Test.java b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/MultithreadedCleartextHttp2Test.java deleted file mode 100644 index dab5ad2..0000000 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/MultithreadedCleartextHttp2Test.java +++ /dev/null @@ -1,92 +0,0 @@ -package org.xbib.netty.http.server.test; - -import io.netty.channel.WriteBufferWaterMark; -import io.netty.handler.codec.http.HttpVersion; -import org.junit.Test; -import org.xbib.net.URL; -import org.xbib.netty.http.client.Client; -import org.xbib.netty.http.client.Request; -import org.xbib.netty.http.client.transport.Transport; -import org.xbib.netty.http.common.HttpAddress; -import org.xbib.netty.http.server.Server; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class MultithreadedCleartextHttp2Test extends LoggingBase { - - private static final Logger logger = Logger.getLogger(""); - - /** - * 2018-03-09 18:27:08.975 WARNUNG [io.netty.channel.ChannelInitializer] - * io.netty.channel.ChannelInitializer exceptionCaught Failed to initialize a channel. - * Closing: [id: 0x4af3e71a, L:/127.0.0.1:8008 - R:/127.0.0.1:59996] - * io.netty.channel.ChannelPipelineException: org.xbib.netty.http.server.handler.Http2ServerConnectionHandler - * is not a @Sharable handler, so can't be added or removed multiple times. - * @throws Exception if test fails - */ - @Test - public void testmultithreadedCleartextHttp2() throws Exception { - int loop = 1000; - int threads = 4; - // we assume slow server and reserve a large write buffer for the client, 32-64 bytes for each request, - // to avoid channel.isWritable() drop-outs - int low = 32 * loop; - int high = 64 * loop; - HttpAddress httpAddress = HttpAddress.of("localhost", 8008, HttpVersion.valueOf("HTTP/2.0"), false); - Server server = Server.builder() - .bind(httpAddress) - .build(); - server.getDefaultVirtualServer().addContext("/", (request, response) -> - response.write("Hello World " + request.getRequest().content().toString(StandardCharsets.UTF_8))); - server.accept(); - Client httpClient = Client.builder() - .setWriteBufferWaterMark(new WriteBufferWaterMark(low, high)) - .build(); - AtomicInteger counter = new AtomicInteger(); - try { - URL serverURL = server.getServerConfig().getAddress().base(); - HttpVersion serverVersion = server.getServerConfig().getAddress().getVersion(); - ExecutorService executorService = Executors.newFixedThreadPool(threads); - for (int n = 0; n < threads; n++) { - executorService.submit(() -> { - try { - Transport transport = httpClient.newTransport(serverURL, serverVersion); - for (int i = 0; i < loop; i++) { - Request request = Request.get().setVersion("HTTP/2.0") - .content(Integer.toString(i), "text/plain") - .build() - .setResponseListener(fullHttpResponse -> { - String content = fullHttpResponse.content().toString(StandardCharsets.UTF_8); - //logger.log(Level.INFO, fullHttpResponse.toString() + " content=" + content); - counter.incrementAndGet(); - }); - // submit request - transport.execute(request); - if (transport.isFailed()) { - logger.log(Level.WARNING, transport.getFailure().getMessage(), transport.getFailure()); - break; - } - } - // wait for transport to complete - transport.get(); - } catch (IOException e) { - logger.log(Level.WARNING, e.getMessage(), e); - } - }); - } - executorService.shutdown(); - executorService.awaitTermination(60, TimeUnit.SECONDS); - } finally { - httpClient.shutdownGracefully(); - server.shutdownGracefully(); - } - logger.log(Level.INFO, "counter = " + counter.get()); - } -} diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/PooledCleartextHttp1Test.java b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/PooledCleartextHttp1Test.java deleted file mode 100644 index 4fe6ada..0000000 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/PooledCleartextHttp1Test.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.xbib.netty.http.server.test; - -import org.junit.Test; -import org.xbib.netty.http.client.Client; -import org.xbib.netty.http.client.Request; -import org.xbib.netty.http.client.transport.Transport; -import org.xbib.netty.http.common.HttpAddress; -import org.xbib.netty.http.server.Server; - -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class PooledCleartextHttp1Test extends LoggingBase { - - private static final Logger logger = Logger.getLogger(""); - - @Test - public void testClearTextHttp1() throws Exception { - int loop = 10000; - HttpAddress httpAddress = HttpAddress.http1("localhost", 8008); - Server server = Server.builder() - //.enableDebug() - .bind(httpAddress).build(); - server.getDefaultVirtualServer().addContext("/", (request, response) -> { - response.write("Hello World " + request.getRequest().content().toString(StandardCharsets.UTF_8)); - }); - server.accept(); - org.xbib.netty.http.common.HttpAddress poolNode = org.xbib.netty.http.common.HttpAddress.http1("localhost", 8008); - Client httpClient = Client.builder() - //.enableDebug() - .addPoolNode(poolNode) - .setPoolNodeConnectionLimit(8) - .build(); - AtomicInteger counter = new AtomicInteger(); - try { - for (int i = 0; i < loop; i++) { - Request request = Request.get().setVersion("HTTP/1.1") - .url(server.getServerConfig().getAddress().base()) - .addParameter("test", Integer.toString(i)) - .content(Integer.toString(i), "text/plain") - .build() - .setResponseListener(fullHttpResponse -> { - String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); - //logger.log(Level.INFO, "status = " + fullHttpResponse.status() + " response body = " + response); - counter.incrementAndGet(); - }); - Transport transport = httpClient.pooledExecute(request); - if (transport.isFailed()) { - logger.log(Level.WARNING, transport.getFailure().getMessage(), transport.getFailure()); - break; - } - // each execution needs to be synchronized - transport.get(); - } - } finally { - httpClient.shutdownGracefully(); - server.shutdownGracefully(); - } - logger.log(Level.INFO, "counter=" + counter.get()); - } -} diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/SecureHttp1Test.java b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/SecureHttp1Test.java index 231afd9..cd3400b 100644 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/SecureHttp1Test.java +++ b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/SecureHttp1Test.java @@ -1,53 +1,181 @@ package org.xbib.netty.http.server.test; +import io.netty.handler.codec.http.HttpVersion; import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.junit.Ignore; import org.junit.Test; +import org.xbib.TestBase; import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Request; +import org.xbib.netty.http.client.listener.ResponseListener; +import org.xbib.netty.http.client.transport.Transport; import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.server.Server; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.security.Security; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; -public class SecureHttp1Test extends LoggingBase { +import static org.junit.Assert.assertEquals; - private static final Logger logger = Logger.getLogger(""); +public class SecureHttp1Test extends TestBase { - @Test - public void testSecureHttp1() throws Exception { + private static final Logger logger = Logger.getLogger(SecureHttp1Test.class.getName()); + + static { if (Security.getProvider("BC") == null) { Security.addProvider(new BouncyCastleProvider()); } - Server server = Server.builder().bind(HttpAddress.secureHttp1("localhost", 8143)) + } + + @Test + public void testSimpleSecureHttp1() throws Exception { + Server server = Server.builder() + .setJdkSslProvider() .setSelfCert() + .bind(HttpAddress.secureHttp1("localhost", 8143)) .build(); - Client httpClient = Client.builder() + Client client = Client.builder() + .setJdkSslProvider() .trustInsecure() .build(); + AtomicInteger counter = new AtomicInteger(); + final ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + //logger.log(Level.INFO, "status = " + fullHttpResponse.status() + " response body = " + response); + counter.getAndIncrement(); + }; try { server.getDefaultVirtualServer().addContext("/", (request, response) -> - response.write("Hello World")); + response.write(200, "text/plain", request.getRequest().content().retain())); server.accept(); - httpClient.execute(Request.get().setVersion("HTTP/1.1") + Request request = Request.get().setVersion(HttpVersion.HTTP_1_1) .url(server.getServerConfig().getAddress().base()) .build() - .setResponseListener(fullHttpResponse -> { - String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); - logger.log(Level.INFO, "status = " + fullHttpResponse.status() + " response body = " + response); - })).get(); - httpClient.execute(Request.get().setVersion("HTTP/1.1") - .url(server.getServerConfig().getAddress().base()) - .build() - .setResponseListener(fullHttpResponse -> { - String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); - logger.log(Level.INFO, "status = " + fullHttpResponse.status() + " response body = " + response); - })).get(); + .setResponseListener(responseListener); + client.execute(request).get(); } finally { - httpClient.shutdownGracefully(); + client.shutdownGracefully(); server.shutdownGracefully(); } + logger.log(Level.INFO, "counter=" + counter.get()); + assertEquals(1, counter.get()); + } + + @Test + public void testPooledSecureHttp1() throws Exception { + int loop = 4096; + HttpAddress httpAddress = HttpAddress.secureHttp1("localhost", 8143); + Server server = Server.builder() + .setJdkSslProvider() + .setSelfCert() + .bind(httpAddress).build(); + server.getDefaultVirtualServer().addContext("/", (request, response) -> + response.write(200, "text/plain", request.getRequest().content().retain())); + server.accept(); + Client client = Client.builder() + .setJdkSslProvider() + .trustInsecure() + .addPoolNode(httpAddress) + .setPoolNodeConnectionLimit(2) + .build(); + AtomicInteger counter = new AtomicInteger(); + final ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + //logger.log(Level.INFO, "status = " + fullHttpResponse.status() + " response body = " + response); + counter.incrementAndGet(); + }; + try { + for (int i = 0; i < loop; i++) { + Request request = Request.get().setVersion(HttpVersion.HTTP_1_1) + .url(server.getServerConfig().getAddress().base()) + .content(Integer.toString(i), "text/plain") + .build() + .setResponseListener(responseListener); + Transport transport = client.newTransport(); + transport.execute(request); + if (transport.isFailed()) { + logger.log(Level.WARNING, transport.getFailure().getMessage(), transport.getFailure()); + break; + } + transport.get(); + } + } finally { + client.shutdownGracefully(); + server.shutdownGracefully(); + } + logger.log(Level.INFO, "expecting=" + loop + " counter=" + counter.get()); + assertEquals(loop, counter.get()); + } + + @Test + public void testMultithreadPooledSecureHttp1() throws Exception { + int threads = 4; + int loop = 4 * 1024; + HttpAddress httpAddress = HttpAddress.secureHttp1("localhost", 8143); + Server server = Server.builder() + .setJdkSslProvider() + .setSelfCert() + .bind(httpAddress) + .build(); + server.getDefaultVirtualServer().addContext("/", (request, response) -> + response.write(200, "text/plain", request.getRequest().content().retain()) + ); + server.accept(); + Client client = Client.builder() + .setJdkSslProvider() + .trustInsecure() + .addPoolNode(httpAddress) + .setPoolNodeConnectionLimit(threads) + .build(); + AtomicInteger counter = new AtomicInteger(); + final ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + //logger.log(Level.INFO, "response listener: headers = " + fullHttpResponse.headers().entries() + + // " response body = " + response); + counter.incrementAndGet(); + }; + try { + ExecutorService executorService = Executors.newFixedThreadPool(threads); + for (int n = 0; n < threads; n++) { + final int t = n; + executorService.submit(() -> { + try { + for (int i = 0; i < loop; i++) { + String payload = Integer.toString(t) + "/" + Integer.toString(i); + Request request = Request.get().setVersion(HttpVersion.HTTP_1_1) + .url(server.getServerConfig().getAddress().base()) + .content(payload, "text/plain") + .build() + .setResponseListener(responseListener); + // note: a new transport is created per execution + final Transport transport = client.newTransport(); + transport.execute(request); + if (transport.isFailed()) { + logger.log(Level.WARNING, transport.getFailure().getMessage(), transport.getFailure()); + break; + } + transport.get(); + } + } catch (IOException e) { + logger.log(Level.WARNING, e.getMessage(), e); + } + }); + } + executorService.shutdown(); + boolean terminated = executorService.awaitTermination(30, TimeUnit.SECONDS); + logger.log(Level.INFO, "terminated = " + terminated); + } finally { + client.shutdownGracefully(); + server.shutdownGracefully(); + } + logger.log(Level.INFO, "expecting=" + (threads * loop) + " counter=" + counter.get()); + assertEquals(threads * loop , counter.get()); } } diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/SecureHttp2Test.java b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/SecureHttp2Test.java index 22c381a..60a26a3 100644 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/SecureHttp2Test.java +++ b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/SecureHttp2Test.java @@ -1,65 +1,117 @@ package org.xbib.netty.http.server.test; -import io.netty.channel.WriteBufferWaterMark; -import io.netty.handler.codec.http.HttpVersion; import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.junit.Ignore; import org.junit.Test; -import org.xbib.net.URL; +import org.xbib.TestBase; import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Request; +import org.xbib.netty.http.client.listener.ResponseListener; import org.xbib.netty.http.client.transport.Transport; import org.xbib.netty.http.common.HttpAddress; import org.xbib.netty.http.server.Server; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.security.Security; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; -public class SecureHttp2Test extends LoggingBase { +import static org.junit.Assert.assertEquals; - private static final Logger logger = Logger.getLogger(""); +public class SecureHttp2Test extends TestBase { - @Test - public void testSecureHttp2() throws Exception { - // for self-signed certificate, we need Bouncycastle + private static final Logger logger = Logger.getLogger(SecureHttp2Test.class.getName()); + + static { if (Security.getProvider("BC") == null) { Security.addProvider(new BouncyCastleProvider()); } - int threads = 4; - int loop = 100000; - // we assume slow server and reserve a large write buffer for the client, 32-64 bytes for each request, - // to avoid channel.isWritable() drop-outs - int low = 32 * loop; - int high = 64 * loop; + } - Server server = Server.builder().bind(HttpAddress.http2("localhost", 8143)) + @Test + public void testSimpleSecureHttp2() throws Exception { + HttpAddress httpAddress = HttpAddress.secureHttp2("localhost", 8143); + Server server = Server.builder() + .setJdkSslProvider() .setSelfCert() + .bind(httpAddress) .build(); - //server.logDiagnostics(Level.INFO); server.getDefaultVirtualServer().addContext("/", (request, response) -> - response.write("Hello World " + request.getRequest().content().toString(StandardCharsets.UTF_8))); + response.write(200, "text/plain", request.getRequest().content().retain())); server.accept(); - Client httpClient = Client.builder() + Client client = Client.builder() + .setJdkSslProvider() .trustInsecure() - .setWriteBufferWaterMark(new WriteBufferWaterMark(low, high)) .build(); AtomicInteger counter = new AtomicInteger(); + // a single instance of HTTP/2 response listener, always receives responses out-of-order + final ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + logger.log(Level.INFO, "response listener: headers = " + fullHttpResponse.headers().entries() + + " response body = " + response); + counter.incrementAndGet(); + }; try { - URL serverURL = server.getServerConfig().getAddress().base(); - HttpVersion serverVersion = server.getServerConfig().getAddress().getVersion(); - Transport transport = httpClient.newTransport(serverURL, serverVersion); - for (int i = 0; i < loop; i++) { - Request request = Request.get().setVersion("HTTP/2.0") - .url(server.getServerConfig().getAddress().base()) - .content(Integer.toString(i), "text/plain") + Transport transport = client.newTransport(httpAddress); + String payload = Integer.toString(0) + "/" + Integer.toString(0); + Request request = Request.get() + .setVersion("HTTP/2.0") + .uri("/") + //.url(server.getServerConfig().getAddress().base()) + .content(payload, "text/plain") .build() - .setResponseListener(fullHttpResponse -> { - String content = fullHttpResponse.content().toString(StandardCharsets.UTF_8); - //logger.log(Level.INFO, fullHttpResponse.toString() + " content=" + content); - counter.incrementAndGet(); - }); + .setResponseListener(responseListener); + transport.execute(request); + transport.get(); + } finally { + client.shutdownGracefully(); + server.shutdownGracefully(); + } + logger.log(Level.INFO, "counter = " + counter.get()); + assertEquals(1, counter.get()); + } + + @Test + public void testPooledSecureHttp2() throws Exception { + int loop = 4096; + HttpAddress httpAddress = HttpAddress.secureHttp2("localhost", 8143); + Server server = Server.builder() + .setJdkSslProvider() + .setSelfCert() + .bind(httpAddress) + .build(); + server.getDefaultVirtualServer().addContext("/", (request, response) -> + response.write(200, "text/plain", request.getRequest().content().retain())); + server.accept(); + Client client = Client.builder() + .setJdkSslProvider() + .trustInsecure() + .addPoolNode(httpAddress) + .setPoolNodeConnectionLimit(2) + .build(); + AtomicInteger counter = new AtomicInteger(); + // a single instance of HTTP/2 response listener, always receives responses out-of-order + final ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + //logger.log(Level.INFO, "response listener: headers = " + fullHttpResponse.headers().entries() + + // " response body = " + response); + counter.incrementAndGet(); + }; + try { + // single transport, single thread + Transport transport = client.newTransport(); + for (int i = 0; i < loop; i++) { + String payload = Integer.toString(0) + "/" + Integer.toString(i); + Request request = Request.get().setVersion("HTTP/2.0") + .url(server.getServerConfig().getAddress().base()) + .content(payload, "text/plain") + .build() + .setResponseListener(responseListener); transport.execute(request); if (transport.isFailed()) { logger.log(Level.WARNING, transport.getFailure().getMessage(), transport.getFailure()); @@ -68,9 +120,76 @@ public class SecureHttp2Test extends LoggingBase { } transport.get(); } finally { - httpClient.shutdownGracefully(); + client.shutdownGracefully(); server.shutdownGracefully(); } - logger.log(Level.INFO, "counter = " + counter.get()); + logger.log(Level.INFO, "counter=" + counter.get()); + assertEquals(loop, counter.get()); + } + + @Test + public void testMultithreadPooledSecureHttp2() throws Exception { + int threads = 4; + int loop = 4 * 1024; + HttpAddress httpAddress = HttpAddress.secureHttp2("localhost", 8143); + Server server = Server.builder() + .setJdkSslProvider() + .setSelfCert() + .bind(httpAddress) + .build(); + server.getDefaultVirtualServer().addContext("/", (request, response) -> + response.write(200, "text/plain", request.getRequest().content().retain()) + ); + server.accept(); + Client client = Client.builder() + .setJdkSslProvider() + .trustInsecure() + .addPoolNode(httpAddress) + .setPoolNodeConnectionLimit(threads) + .build(); + AtomicInteger counter = new AtomicInteger(); + // a HTTP/2 listener always receives responses out-of-order + final ResponseListener responseListener = fullHttpResponse -> { + String response = fullHttpResponse.content().toString(StandardCharsets.UTF_8); + //logger.log(Level.INFO, "response listener: headers = " + fullHttpResponse.headers().entries() + + // " response body = " + response); + counter.incrementAndGet(); + }; + try { + // note: for HTTP/2 only, we can use a single shared transport + final Transport transport = client.newTransport(); + ExecutorService executorService = Executors.newFixedThreadPool(threads); + for (int n = 0; n < threads; n++) { + final int t = n; + executorService.submit(() -> { + try { + for (int i = 0; i < loop; i++) { + String payload = Integer.toString(t) + "/" + Integer.toString(i); + Request request = Request.get().setVersion("HTTP/2.0") + .url(server.getServerConfig().getAddress().base()) + .content(payload, "text/plain") + .build() + .setResponseListener(responseListener); + transport.execute(request); + if (transport.isFailed()) { + logger.log(Level.WARNING, transport.getFailure().getMessage(), transport.getFailure()); + break; + } + } + } catch (IOException e) { + logger.log(Level.WARNING, e.getMessage(), e); + } + }); + } + executorService.shutdown(); + boolean terminated = executorService.awaitTermination(30, TimeUnit.SECONDS); + logger.log(Level.INFO, "terminated = " + terminated + ", now waiting for transport to complete"); + transport.get(30, TimeUnit.SECONDS); + } finally { + client.shutdownGracefully(); + server.shutdownGracefully(); + } + logger.log(Level.INFO, "expected=" + (threads * loop) + " counter=" + counter.get()); + assertEquals(threads * loop , counter.get()); } } diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/SelfSignedCertificateTest.java b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/SelfSignedCertificateTest.java index 7c30472..ecea3b2 100644 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/SelfSignedCertificateTest.java +++ b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/SelfSignedCertificateTest.java @@ -7,8 +7,6 @@ import org.xbib.netty.http.server.security.tls.SelfSignedCertificate; import java.security.Security; import java.util.logging.Logger; -/** - */ public class SelfSignedCertificateTest { @Test diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/ServerTest.java b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/ServerTest.java index 0610458..cc8db3c 100644 --- a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/ServerTest.java +++ b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/ServerTest.java @@ -1,8 +1,10 @@ package org.xbib.netty.http.server.test; +import org.junit.Ignore; import org.junit.Test; import org.xbib.netty.http.server.Server; +@Ignore public class ServerTest { @Test diff --git a/netty-http-server/src/test/java/org/xbib/netty/http/server/test/ThreadLeakTest.java b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/ThreadLeakTest.java new file mode 100644 index 0000000..bbe63be --- /dev/null +++ b/netty-http-server/src/test/java/org/xbib/netty/http/server/test/ThreadLeakTest.java @@ -0,0 +1,41 @@ +package org.xbib.netty.http.server.test; + +import io.netty.buffer.UnpooledByteBufAllocator; +import org.junit.After; +import org.junit.Test; +import org.xbib.TestBase; +import org.xbib.netty.http.server.Server; + +import java.io.IOException; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class ThreadLeakTest extends TestBase { + + private static final Logger logger = Logger.getLogger(ThreadLeakTest.class.getName()); + + @Test + public void testForLeaks() throws IOException { + Server server = Server.builder() + .setByteBufAllocator(UnpooledByteBufAllocator.DEFAULT) + .build(); + server.getDefaultVirtualServer().addContext("/", (request, response) -> + response.write("Hello World")); + try { + server.accept(); + } finally { + server.shutdownGracefully(); + } + } + + @After + public void checkThreads() throws Exception { + Thread.sleep(1000L); + System.gc(); + Thread.sleep(3000L); + Set threadSet = Thread.getAllStackTraces().keySet(); + logger.log(Level.INFO, "threads = " + threadSet.size() ); + threadSet.forEach( thread -> logger.log(Level.INFO, thread.toString())); + } +}