joerg/oai
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

update to netty 4.1.44

Browse source
This commit is contained in:
Jörg Prante 2020-01-13 15:26:54 +01:00
parent 8efae7e191
commit 3aa1311188
9 changed files with 59 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
build.gradle
View file

@ -1,6 +1,6 @@
plugins {
id "com.github.spotbugs" version "2.0.0"
id "org.sonarqube" version "2.6.1"
id "org.sonarqube" version "2.8"
id "io.codearte.nexus-staging" version "0.11.0"
id "org.xbib.gradle.plugin.asciidoctor" version "1.6.0.1"
}
@ -24,7 +24,6 @@ subprojects {
testCompile "org.junit.jupiter:junit-jupiter-api:${project.property('junit.version')}"
testCompile "org.junit.jupiter:junit-jupiter-params:${project.property('junit.version')}"
testCompile "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}"
//testCompile "org.junit.vintage:junit-vintage-engine:${project.property('junit.version')}"
testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
}

15
gradle.properties
View file

@ -1,15 +1,12 @@
group = org.xbib
name = oai
version = 2.0.2
version = 2.1.0
xbib-content.version = 2.0.1
xbib-netty-http.version = 4.1.41.0
tcnative.version = 2.0.25.Final
xbib-content.version = 2.0.5
xbib-netty-http.version = 4.1.44.2
tcnative.version = 2.0.28.Final
# test
# test only
junit.version = 5.5.1
xbib-bibliographic-character-sets.version = 1.0.0
xbib-marc.version = 2.0.1
# doc
asciidoclet.version = 1.5.4
xbib-marc.version = 2.1.0

4
gradle/wrapper/gradle-wrapper.properties vendored
View file

@ -1,5 +1,5 @@
#Thu Aug 29 18:18:25 CEST 2019
distributionUrl=https\://services.gradle.org/distributions/gradle-5.6-all.zip
#Mon Jan 13 14:23:25 CET 2020
distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-all.zip
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStorePath=wrapper/dists

2
oai-client/build.gradle
View file

@ -2,7 +2,7 @@
dependencies {
compile project(':oai-common')
compile "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}"
testCompile "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}"
compile "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}"
testCompile "org.xbib:marc:${project.property('xbib-marc.version')}"
testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
}

21
oai-client/src/test/java/org/xbib/oai/client/ArxivClientTest.java
View file

@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
import org.xbib.net.URL;
import org.xbib.netty.http.client.Client;
import org.xbib.netty.http.client.Request;
import org.xbib.netty.http.client.api.Request;
import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.identify.IdentifyResponse;
import org.xbib.oai.client.listrecords.ListRecordsRequest;
@ -34,23 +34,23 @@ class ArxivClientTest {
@Test
void testListRecordsArxiv() {
final URL url = URL.create("http://export.arxiv.org/oai2/");
try (OAIClient client = new OAIClient(url)) {
try (Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
OAIClient client = new OAIClient(url)) {
IdentifyRequest identifyRequest = client.newIdentifyRequest();
Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
IdentifyResponse identifyResponse = new IdentifyResponse();
Request request = Request.get()
.url(identifyRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.build()
.setResponseListener(resp -> {
logger.log(Level.INFO,
" body = " + resp.getBodyAsString(StandardCharsets.UTF_8));
StringWriter sw = new StringWriter();
identifyResponse.receivedResponse(resp, sw);
});
})
.build();
httpClient.execute(request).get();
String granularity = identifyResponse.getGranularity();
logger.log(Level.INFO, "granularity = " + granularity);
@ -76,12 +76,12 @@ class ArxivClientTest {
request = Request.get()
.url(listRecordsRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.build()
.setResponseListener(resp -> {
listRecordsResponse.receivedResponse(resp, fileWriter);
logger.log(Level.FINE, "response headers = " + resp.getHeaders() +
" resumption-token = " + listRecordsResponse.getResumptionToken());
});
})
.build();
httpClient.execute(request).get();
listRecordsRequest = client.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} catch (IOException e) {
@ -90,7 +90,6 @@ class ArxivClientTest {
}
}
fileWriter.close();
httpClient.shutdownGracefully();
logger.log(Level.INFO, "count = " + handler.count());
assertTrue(handler.count() > 0L);
} catch (Exception e) {

37
oai-client/src/test/java/org/xbib/oai/client/BundeskunsthalleTest.java
View file

@ -8,7 +8,7 @@ import org.xbib.marc.json.MarcJsonWriter;
import org.xbib.marc.xml.MarcContentHandler;
import org.xbib.net.URL;
import org.xbib.netty.http.client.Client;
import org.xbib.netty.http.client.Request;
import org.xbib.netty.http.client.api.Request;
import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.identify.IdentifyResponse;
import org.xbib.oai.client.listrecords.ListRecordsRequest;
@ -21,13 +21,14 @@ import java.net.ConnectException;
import java.nio.charset.StandardCharsets;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.EnumSet;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
*
*/
@Disabled
@Disabled("takes very long time")
class BundeskunsthalleTest {
private static final Logger logger = Logger.getLogger(BundeskunsthalleTest.class.getName());
@ -35,25 +36,25 @@ class BundeskunsthalleTest {
@Test
void testListRecords() {
URL url = URL.create("https://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh");
try (OAIClient oaiClient = new OAIClient(url)) {
Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
try (Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
OAIClient oaiClient = new OAIClient(url)) {
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
IdentifyResponse identifyResponse = new IdentifyResponse();
Request request = Request.get()
.url(identifyRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.setFollowRedirect(true)
.build()
.setResponseListener(resp -> {
logger.log(Level.INFO,
"status = " + resp.getStatus() +
" body = " + resp.getBodyAsString(StandardCharsets.UTF_8));
StringWriter sw = new StringWriter();
identifyResponse.receivedResponse(resp, sw);
});
})
.build();
httpClient.execute(request).get();
String granularity = identifyResponse.getGranularity();
logger.log(Level.INFO, "granularity = " + granularity);
@ -63,7 +64,7 @@ class BundeskunsthalleTest {
listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
listRecordsRequest.setMetadataPrefix("marcxml");
try (MarcJsonWriter writer = new MarcJsonWriter("bk-bulk%d.jsonl", 1000,
MarcJsonWriter.Style.ELASTICSEARCH_BULK, 65536, false)
EnumSet.of(MarcJsonWriter.Style.ELASTICSEARCH_BULK), 65536, false)
.setIndex("testindex", "testtype")) {
writer.startDocument();
writer.beginCollection();
@ -71,14 +72,18 @@ class BundeskunsthalleTest {
try {
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
logger.log(Level.INFO, "sending " + listRecordsRequest.getURL());
StringWriter sw = new StringWriter();
request = Request.get()
.url(listRecordsRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.setFollowRedirect(true)
.setTimeoutInMillis(60 * 1000)
.build()
.setResponseListener(resp -> {
logger.log(Level.FINE,
"status = " + resp.getStatus() +
" headers = " + resp.getHeaders() +
" resumptiontoken = " + listRecordsResponse.getResumptionToken());
StringWriter sw = new StringWriter();
listRecordsResponse.receivedResponse(resp, sw);
try {
Marc.builder()
.setInputStream(resp.getBodyAsStream())
@ -93,12 +98,8 @@ class BundeskunsthalleTest {
} catch (IOException e) {
throw new OAIException("MARC parser exception: " + e.getMessage(), e);
}
listRecordsResponse.receivedResponse(resp, sw);
logger.log(Level.FINE,
"status = " + resp.getStatus() +
" headers = " + resp.getHeaders() +
" resumptiontoken = " + listRecordsResponse.getResumptionToken());
});
})
.build();
httpClient.execute(request).get();
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} catch (ConnectException e) {

21
oai-client/src/test/java/org/xbib/oai/client/DNBClientTest.java
View file

@ -4,7 +4,7 @@ import io.netty.handler.codec.http.HttpHeaderNames;
import org.junit.jupiter.api.Test;
import org.xbib.net.URL;
import org.xbib.netty.http.client.Client;
import org.xbib.netty.http.client.Request;
import org.xbib.netty.http.client.api.Request;
import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.identify.IdentifyResponse;
import org.xbib.oai.client.listrecords.ListRecordsRequest;
@ -34,21 +34,21 @@ class DNBClientTest {
@Test
void testBibdat() {
URL url = URL.create("http://services.dnb.de/oai/repository");
try (OAIClient oaiClient = new OAIClient(url)) {
Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
try (Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
OAIClient oaiClient = new OAIClient(url)) {
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
IdentifyResponse identifyResponse = new IdentifyResponse();
Request request = Request.get()
.url(identifyRequest.getURL())
.build()
.setResponseListener(resp -> {
logger.log(Level.INFO, resp.getBodyAsString(StandardCharsets.UTF_8));
StringWriter sw = new StringWriter();
identifyResponse.receivedResponse(resp, sw);
});
})
.build();
httpClient.execute(request).get();
String granularity = identifyResponse.getGranularity();
logger.log(Level.INFO, "granularity = " + granularity);
@ -70,8 +70,8 @@ class DNBClientTest {
request = Request.get()
.url(listRecordsRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.build()
.setResponseListener(resp -> listRecordsResponse.receivedResponse(resp, fileWriter));
.setResponseListener(resp -> listRecordsResponse.receivedResponse(resp, fileWriter))
.build();
httpClient.execute(request).get();
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} catch (ConnectException e) {
@ -82,7 +82,6 @@ class DNBClientTest {
}
}
fileWriter.close();
httpClient.shutdownGracefully();
logger.log(Level.INFO, "count=" + handler.count());
} catch (Exception e) {
logger.log(Level.SEVERE, "skipped, HTTP exception");

21
oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java
View file

@ -5,7 +5,7 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.xbib.net.URL;
import org.xbib.netty.http.client.Client;
import org.xbib.netty.http.client.Request;
import org.xbib.netty.http.client.api.Request;
import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.identify.IdentifyResponse;
import org.xbib.oai.client.listrecords.ListRecordsRequest;
@ -33,21 +33,21 @@ class DOAJClientTest {
@Disabled // takes too long time
void testListRecordsDOAJ() {
URL url = URL.create("https://doaj.org/oai");
try (OAIClient oaiClient = new OAIClient(url)) {
Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
try (Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
OAIClient oaiClient = new OAIClient(url)) {
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
IdentifyResponse identifyResponse = new IdentifyResponse();
Request request = Request.get()
.url(url.resolve(identifyRequest.getURL()))
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.build()
.setResponseListener(resp -> {
StringWriter sw = new StringWriter();
identifyResponse.receivedResponse(resp, sw);
});
})
.build();
httpClient.execute(request).get();
String granularity = identifyResponse.getGranularity();
logger.log(Level.INFO, "granularity = " + granularity);
@ -70,17 +70,16 @@ class DOAJClientTest {
request = Request.get()
.url(url.resolve(listRecordsRequest.getURL()))
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.build()
.setResponseListener(resp -> {
listRecordsResponse.receivedResponse(resp, fileWriter);
logger.log(Level.FINE, "response headers = " + resp.getHeaders() +
" resumption-token = {}" + listRecordsResponse.getResumptionToken());
});
})
.build();
httpClient.execute(request).get();
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
}
fileWriter.close();
httpClient.shutdownGracefully();
logger.log(Level.INFO, "count = " + handler.count());
} catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e);

2
settings.gradle
View file

@ -1,5 +1,3 @@
rootProject.name = name
include 'oai-common'
include 'oai-client'
include 'oai-server'