update to netty 4.1.44

This commit is contained in:
Jörg Prante 2020-01-13 15:26:54 +01:00
parent 8efae7e191
commit 3aa1311188
9 changed files with 59 additions and 67 deletions

View file

@ -1,6 +1,6 @@
plugins { plugins {
id "com.github.spotbugs" version "2.0.0" id "com.github.spotbugs" version "2.0.0"
id "org.sonarqube" version "2.6.1" id "org.sonarqube" version "2.8"
id "io.codearte.nexus-staging" version "0.11.0" id "io.codearte.nexus-staging" version "0.11.0"
id "org.xbib.gradle.plugin.asciidoctor" version "1.6.0.1" id "org.xbib.gradle.plugin.asciidoctor" version "1.6.0.1"
} }
@ -24,7 +24,6 @@ subprojects {
testCompile "org.junit.jupiter:junit-jupiter-api:${project.property('junit.version')}" testCompile "org.junit.jupiter:junit-jupiter-api:${project.property('junit.version')}"
testCompile "org.junit.jupiter:junit-jupiter-params:${project.property('junit.version')}" testCompile "org.junit.jupiter:junit-jupiter-params:${project.property('junit.version')}"
testCompile "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}" testCompile "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}"
//testCompile "org.junit.vintage:junit-vintage-engine:${project.property('junit.version')}"
testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}" testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
} }

View file

@ -1,15 +1,12 @@
group = org.xbib group = org.xbib
name = oai name = oai
version = 2.0.2 version = 2.1.0
xbib-content.version = 2.0.1 xbib-content.version = 2.0.5
xbib-netty-http.version = 4.1.41.0 xbib-netty-http.version = 4.1.44.2
tcnative.version = 2.0.25.Final tcnative.version = 2.0.28.Final
# test # test only
junit.version = 5.5.1 junit.version = 5.5.1
xbib-bibliographic-character-sets.version = 1.0.0 xbib-bibliographic-character-sets.version = 1.0.0
xbib-marc.version = 2.0.1 xbib-marc.version = 2.1.0
# doc
asciidoclet.version = 1.5.4

View file

@ -1,5 +1,5 @@
#Thu Aug 29 18:18:25 CEST 2019 #Mon Jan 13 14:23:25 CET 2020
distributionUrl=https\://services.gradle.org/distributions/gradle-5.6-all.zip distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-all.zip
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
zipStorePath=wrapper/dists zipStorePath=wrapper/dists

View file

@ -2,7 +2,7 @@
dependencies { dependencies {
compile project(':oai-common') compile project(':oai-common')
compile "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}" compile "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}"
testCompile "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}" compile "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}"
testCompile "org.xbib:marc:${project.property('xbib-marc.version')}" testCompile "org.xbib:marc:${project.property('xbib-marc.version')}"
testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}" testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
} }

View file

@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.xbib.net.URL; import org.xbib.net.URL;
import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Client;
import org.xbib.netty.http.client.Request; import org.xbib.netty.http.client.api.Request;
import org.xbib.oai.client.identify.IdentifyRequest; import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.identify.IdentifyResponse; import org.xbib.oai.client.identify.IdentifyResponse;
import org.xbib.oai.client.listrecords.ListRecordsRequest; import org.xbib.oai.client.listrecords.ListRecordsRequest;
@ -34,23 +34,23 @@ class ArxivClientTest {
@Test @Test
void testListRecordsArxiv() { void testListRecordsArxiv() {
final URL url = URL.create("http://export.arxiv.org/oai2/"); final URL url = URL.create("http://export.arxiv.org/oai2/");
try (OAIClient client = new OAIClient(url)) { try (Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
OAIClient client = new OAIClient(url)) {
IdentifyRequest identifyRequest = client.newIdentifyRequest(); IdentifyRequest identifyRequest = client.newIdentifyRequest();
Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
IdentifyResponse identifyResponse = new IdentifyResponse(); IdentifyResponse identifyResponse = new IdentifyResponse();
Request request = Request.get() Request request = Request.get()
.url(identifyRequest.getURL()) .url(identifyRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.build()
.setResponseListener(resp -> { .setResponseListener(resp -> {
logger.log(Level.INFO, logger.log(Level.INFO,
" body = " + resp.getBodyAsString(StandardCharsets.UTF_8)); " body = " + resp.getBodyAsString(StandardCharsets.UTF_8));
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
identifyResponse.receivedResponse(resp, sw); identifyResponse.receivedResponse(resp, sw);
}); })
.build();
httpClient.execute(request).get(); httpClient.execute(request).get();
String granularity = identifyResponse.getGranularity(); String granularity = identifyResponse.getGranularity();
logger.log(Level.INFO, "granularity = " + granularity); logger.log(Level.INFO, "granularity = " + granularity);
@ -76,12 +76,12 @@ class ArxivClientTest {
request = Request.get() request = Request.get()
.url(listRecordsRequest.getURL()) .url(listRecordsRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.build()
.setResponseListener(resp -> { .setResponseListener(resp -> {
listRecordsResponse.receivedResponse(resp, fileWriter); listRecordsResponse.receivedResponse(resp, fileWriter);
logger.log(Level.FINE, "response headers = " + resp.getHeaders() + logger.log(Level.FINE, "response headers = " + resp.getHeaders() +
" resumption-token = " + listRecordsResponse.getResumptionToken()); " resumption-token = " + listRecordsResponse.getResumptionToken());
}); })
.build();
httpClient.execute(request).get(); httpClient.execute(request).get();
listRecordsRequest = client.resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); listRecordsRequest = client.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} catch (IOException e) { } catch (IOException e) {
@ -90,7 +90,6 @@ class ArxivClientTest {
} }
} }
fileWriter.close(); fileWriter.close();
httpClient.shutdownGracefully();
logger.log(Level.INFO, "count = " + handler.count()); logger.log(Level.INFO, "count = " + handler.count());
assertTrue(handler.count() > 0L); assertTrue(handler.count() > 0L);
} catch (Exception e) { } catch (Exception e) {

View file

@ -8,7 +8,7 @@ import org.xbib.marc.json.MarcJsonWriter;
import org.xbib.marc.xml.MarcContentHandler; import org.xbib.marc.xml.MarcContentHandler;
import org.xbib.net.URL; import org.xbib.net.URL;
import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Client;
import org.xbib.netty.http.client.Request; import org.xbib.netty.http.client.api.Request;
import org.xbib.oai.client.identify.IdentifyRequest; import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.identify.IdentifyResponse; import org.xbib.oai.client.identify.IdentifyResponse;
import org.xbib.oai.client.listrecords.ListRecordsRequest; import org.xbib.oai.client.listrecords.ListRecordsRequest;
@ -21,13 +21,14 @@ import java.net.ConnectException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.time.ZoneId; import java.time.ZoneId;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.EnumSet;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
/** /**
* *
*/ */
@Disabled @Disabled("takes very long time")
class BundeskunsthalleTest { class BundeskunsthalleTest {
private static final Logger logger = Logger.getLogger(BundeskunsthalleTest.class.getName()); private static final Logger logger = Logger.getLogger(BundeskunsthalleTest.class.getName());
@ -35,25 +36,25 @@ class BundeskunsthalleTest {
@Test @Test
void testListRecords() { void testListRecords() {
URL url = URL.create("https://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh"); URL url = URL.create("https://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh");
try (OAIClient oaiClient = new OAIClient(url)) { try (Client httpClient = Client.builder()
Client httpClient = Client.builder() .setConnectTimeoutMillis(60 * 1000)
.setConnectTimeoutMillis(60 * 1000) .setReadTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000) .build();
.build(); OAIClient oaiClient = new OAIClient(url)) {
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest(); IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
IdentifyResponse identifyResponse = new IdentifyResponse(); IdentifyResponse identifyResponse = new IdentifyResponse();
Request request = Request.get() Request request = Request.get()
.url(identifyRequest.getURL()) .url(identifyRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.setFollowRedirect(true) .setFollowRedirect(true)
.build()
.setResponseListener(resp -> { .setResponseListener(resp -> {
logger.log(Level.INFO, logger.log(Level.INFO,
"status = " + resp.getStatus() + "status = " + resp.getStatus() +
" body = " + resp.getBodyAsString(StandardCharsets.UTF_8)); " body = " + resp.getBodyAsString(StandardCharsets.UTF_8));
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
identifyResponse.receivedResponse(resp, sw); identifyResponse.receivedResponse(resp, sw);
}); })
.build();
httpClient.execute(request).get(); httpClient.execute(request).get();
String granularity = identifyResponse.getGranularity(); String granularity = identifyResponse.getGranularity();
logger.log(Level.INFO, "granularity = " + granularity); logger.log(Level.INFO, "granularity = " + granularity);
@ -63,7 +64,7 @@ class BundeskunsthalleTest {
listRecordsRequest.setDateTimeFormatter(dateTimeFormatter); listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
listRecordsRequest.setMetadataPrefix("marcxml"); listRecordsRequest.setMetadataPrefix("marcxml");
try (MarcJsonWriter writer = new MarcJsonWriter("bk-bulk%d.jsonl", 1000, try (MarcJsonWriter writer = new MarcJsonWriter("bk-bulk%d.jsonl", 1000,
MarcJsonWriter.Style.ELASTICSEARCH_BULK, 65536, false) EnumSet.of(MarcJsonWriter.Style.ELASTICSEARCH_BULK), 65536, false)
.setIndex("testindex", "testtype")) { .setIndex("testindex", "testtype")) {
writer.startDocument(); writer.startDocument();
writer.beginCollection(); writer.beginCollection();
@ -71,14 +72,18 @@ class BundeskunsthalleTest {
try { try {
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest); ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
logger.log(Level.INFO, "sending " + listRecordsRequest.getURL()); logger.log(Level.INFO, "sending " + listRecordsRequest.getURL());
StringWriter sw = new StringWriter();
request = Request.get() request = Request.get()
.url(listRecordsRequest.getURL()) .url(listRecordsRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.setFollowRedirect(true) .setFollowRedirect(true)
.setTimeoutInMillis(60 * 1000) .setTimeoutInMillis(60 * 1000)
.build()
.setResponseListener(resp -> { .setResponseListener(resp -> {
logger.log(Level.FINE,
"status = " + resp.getStatus() +
" headers = " + resp.getHeaders() +
" resumptiontoken = " + listRecordsResponse.getResumptionToken());
StringWriter sw = new StringWriter();
listRecordsResponse.receivedResponse(resp, sw);
try { try {
Marc.builder() Marc.builder()
.setInputStream(resp.getBodyAsStream()) .setInputStream(resp.getBodyAsStream())
@ -93,12 +98,8 @@ class BundeskunsthalleTest {
} catch (IOException e) { } catch (IOException e) {
throw new OAIException("MARC parser exception: " + e.getMessage(), e); throw new OAIException("MARC parser exception: " + e.getMessage(), e);
} }
listRecordsResponse.receivedResponse(resp, sw); })
logger.log(Level.FINE, .build();
"status = " + resp.getStatus() +
" headers = " + resp.getHeaders() +
" resumptiontoken = " + listRecordsResponse.getResumptionToken());
});
httpClient.execute(request).get(); httpClient.execute(request).get();
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} catch (ConnectException e) { } catch (ConnectException e) {

View file

@ -4,7 +4,7 @@ import io.netty.handler.codec.http.HttpHeaderNames;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.xbib.net.URL; import org.xbib.net.URL;
import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Client;
import org.xbib.netty.http.client.Request; import org.xbib.netty.http.client.api.Request;
import org.xbib.oai.client.identify.IdentifyRequest; import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.identify.IdentifyResponse; import org.xbib.oai.client.identify.IdentifyResponse;
import org.xbib.oai.client.listrecords.ListRecordsRequest; import org.xbib.oai.client.listrecords.ListRecordsRequest;
@ -34,21 +34,21 @@ class DNBClientTest {
@Test @Test
void testBibdat() { void testBibdat() {
URL url = URL.create("http://services.dnb.de/oai/repository"); URL url = URL.create("http://services.dnb.de/oai/repository");
try (OAIClient oaiClient = new OAIClient(url)) { try (Client httpClient = Client.builder()
Client httpClient = Client.builder() .setConnectTimeoutMillis(60 * 1000)
.setConnectTimeoutMillis(60 * 1000) .setReadTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000) .build();
.build(); OAIClient oaiClient = new OAIClient(url)) {
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest(); IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
IdentifyResponse identifyResponse = new IdentifyResponse(); IdentifyResponse identifyResponse = new IdentifyResponse();
Request request = Request.get() Request request = Request.get()
.url(identifyRequest.getURL()) .url(identifyRequest.getURL())
.build()
.setResponseListener(resp -> { .setResponseListener(resp -> {
logger.log(Level.INFO, resp.getBodyAsString(StandardCharsets.UTF_8)); logger.log(Level.INFO, resp.getBodyAsString(StandardCharsets.UTF_8));
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
identifyResponse.receivedResponse(resp, sw); identifyResponse.receivedResponse(resp, sw);
}); })
.build();
httpClient.execute(request).get(); httpClient.execute(request).get();
String granularity = identifyResponse.getGranularity(); String granularity = identifyResponse.getGranularity();
logger.log(Level.INFO, "granularity = " + granularity); logger.log(Level.INFO, "granularity = " + granularity);
@ -70,8 +70,8 @@ class DNBClientTest {
request = Request.get() request = Request.get()
.url(listRecordsRequest.getURL()) .url(listRecordsRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.build() .setResponseListener(resp -> listRecordsResponse.receivedResponse(resp, fileWriter))
.setResponseListener(resp -> listRecordsResponse.receivedResponse(resp, fileWriter)); .build();
httpClient.execute(request).get(); httpClient.execute(request).get();
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} catch (ConnectException e) { } catch (ConnectException e) {
@ -82,7 +82,6 @@ class DNBClientTest {
} }
} }
fileWriter.close(); fileWriter.close();
httpClient.shutdownGracefully();
logger.log(Level.INFO, "count=" + handler.count()); logger.log(Level.INFO, "count=" + handler.count());
} catch (Exception e) { } catch (Exception e) {
logger.log(Level.SEVERE, "skipped, HTTP exception"); logger.log(Level.SEVERE, "skipped, HTTP exception");

View file

@ -5,7 +5,7 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.xbib.net.URL; import org.xbib.net.URL;
import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Client;
import org.xbib.netty.http.client.Request; import org.xbib.netty.http.client.api.Request;
import org.xbib.oai.client.identify.IdentifyRequest; import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.identify.IdentifyResponse; import org.xbib.oai.client.identify.IdentifyResponse;
import org.xbib.oai.client.listrecords.ListRecordsRequest; import org.xbib.oai.client.listrecords.ListRecordsRequest;
@ -33,21 +33,21 @@ class DOAJClientTest {
@Disabled // takes too long time @Disabled // takes too long time
void testListRecordsDOAJ() { void testListRecordsDOAJ() {
URL url = URL.create("https://doaj.org/oai"); URL url = URL.create("https://doaj.org/oai");
try (OAIClient oaiClient = new OAIClient(url)) { try (Client httpClient = Client.builder()
Client httpClient = Client.builder() .setConnectTimeoutMillis(60 * 1000)
.setConnectTimeoutMillis(60 * 1000) .setReadTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000) .build();
.build(); OAIClient oaiClient = new OAIClient(url)) {
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest(); IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
IdentifyResponse identifyResponse = new IdentifyResponse(); IdentifyResponse identifyResponse = new IdentifyResponse();
Request request = Request.get() Request request = Request.get()
.url(url.resolve(identifyRequest.getURL())) .url(url.resolve(identifyRequest.getURL()))
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.build()
.setResponseListener(resp -> { .setResponseListener(resp -> {
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
identifyResponse.receivedResponse(resp, sw); identifyResponse.receivedResponse(resp, sw);
}); })
.build();
httpClient.execute(request).get(); httpClient.execute(request).get();
String granularity = identifyResponse.getGranularity(); String granularity = identifyResponse.getGranularity();
logger.log(Level.INFO, "granularity = " + granularity); logger.log(Level.INFO, "granularity = " + granularity);
@ -70,17 +70,16 @@ class DOAJClientTest {
request = Request.get() request = Request.get()
.url(url.resolve(listRecordsRequest.getURL())) .url(url.resolve(listRecordsRequest.getURL()))
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.build()
.setResponseListener(resp -> { .setResponseListener(resp -> {
listRecordsResponse.receivedResponse(resp, fileWriter); listRecordsResponse.receivedResponse(resp, fileWriter);
logger.log(Level.FINE, "response headers = " + resp.getHeaders() + logger.log(Level.FINE, "response headers = " + resp.getHeaders() +
" resumption-token = {}" + listRecordsResponse.getResumptionToken()); " resumption-token = {}" + listRecordsResponse.getResumptionToken());
}); })
.build();
httpClient.execute(request).get(); httpClient.execute(request).get();
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} }
fileWriter.close(); fileWriter.close();
httpClient.shutdownGracefully();
logger.log(Level.INFO, "count = " + handler.count()); logger.log(Level.INFO, "count = " + handler.count());
} catch (Exception e) { } catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e); logger.log(Level.SEVERE, e.getMessage(), e);

View file

@ -1,5 +1,3 @@
rootProject.name = name
include 'oai-common' include 'oai-common'
include 'oai-client' include 'oai-client'
include 'oai-server' include 'oai-server'