From 270acc90a1d8dfbe2d68e91ccf69f56a7184a40c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Thu, 8 Aug 2019 10:21:04 +0200 Subject: [PATCH] move tests to xbib netty http client --- build.gradle | 8 +-- gradle.properties | 2 +- .../xbib/oai/client/AbstractOAIRequest.java | 52 +--------------- .../java/org/xbib/oai/client/OAIClient.java | 11 ---- .../listrecords/ListRecordsResponse.java | 3 +- .../org/xbib/oai/client/ArxivClientTest.java | 6 +- .../xbib/oai/client/BundeskunsthalleTest.java | 56 ++++++++---------- .../org/xbib/oai/client/DNBClientTest.java | 59 +++++++++++-------- .../org/xbib/oai/client/DOAJClientTest.java | 6 +- 9 files changed, 77 insertions(+), 126 deletions(-) diff --git a/build.gradle b/build.gradle index 0058342..89fc13c 100644 --- a/build.gradle +++ b/build.gradle @@ -31,13 +31,13 @@ subprojects { } compileJava { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 } compileTestJava { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 } diff --git a/gradle.properties b/gradle.properties index ec3522a..c54eba6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group = org.xbib name = oai -version = 1.3.0 +version = 2.0.0 xbib-content.version = 1.3.1 xbib-netty-http.version = 4.1.38.2 diff --git a/oai-client/src/main/java/org/xbib/oai/client/AbstractOAIRequest.java b/oai-client/src/main/java/org/xbib/oai/client/AbstractOAIRequest.java index e40fdef..45ee169 100644 --- a/oai-client/src/main/java/org/xbib/oai/client/AbstractOAIRequest.java +++ b/oai-client/src/main/java/org/xbib/oai/client/AbstractOAIRequest.java @@ -35,7 +35,6 @@ public abstract class AbstractOAIRequest implements OAIRequest { .host(url.getHost()) .port(url.getPort()) .path(url.getPath()); - } public URL getURL() { @@ -98,7 +97,7 @@ public abstract class AbstractOAIRequest implements OAIRequest { public void setResumptionToken(ResumptionToken token) { this.token = token; if (token != null && token.toString() != null) { - // resumption token may have characters that are illegal in URIs like '|' + // note: resumption token may have characters that are illegal in URIs, like '|' addParameter(OAIConstants.RESUMPTION_TOKEN_PARAMETER, token.toString()); } } @@ -124,53 +123,4 @@ public abstract class AbstractOAIRequest implements OAIRequest { + ",until=" + getUntil() + "]"; } - - /*class GetRecord extends AbstractOAIRequest { - - public GetRecord() { - super(url); - addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.GET_RECORD); - } - } - - class Identify extends AbstractOAIRequest { - - public Identify() { - super(url); - addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.IDENTIFY); - } - } - - class ListIdentifiers extends AbstractOAIRequest { - - public ListIdentifiers() { - super(url); - addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_IDENTIFIERS); - } - } - - class ListMetadataFormats extends AbstractOAIRequest { - - public ListMetadataFormats() { - super(url); - addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_METADATA_FORMATS); - } - } - - class ListRecordsRequest extends AbstractOAIRequest { - - public ListRecordsRequest() { - super(url); - addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_RECORDS); - } - - } - - class ListSetsRequest extends AbstractOAIRequest { - - public ListSetsRequest() { - super(url); - addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_SETS); - } - }*/ } diff --git a/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java b/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java index 1b02824..2dc91e4 100644 --- a/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java +++ b/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java @@ -1,7 +1,6 @@ package org.xbib.oai.client; import org.xbib.net.URL; -import org.xbib.netty.http.client.Client; import org.xbib.oai.client.getrecord.GetRecordRequest; import org.xbib.oai.client.identify.IdentifyRequest; import org.xbib.oai.client.listidentifiers.ListIdentifiersRequest; @@ -15,26 +14,16 @@ import org.xbib.oai.util.ResumptionToken; */ public class OAIClient implements AutoCloseable { - private Client client; - private final URL url; public OAIClient(URL url) { this.url = url; - this.client = Client.builder() - .setConnectTimeoutMillis(60 * 1000) - .setReadTimeoutMillis(60 * 1000) - .build(); } public URL getURL() { return url; } - public Client getHttpClient() { - return client; - } - /** * This verb is used to retrieve information about a repository. * Some of the information returned is required as part of the OAI-PMH. diff --git a/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java b/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java index 6a2403b..e51b4c2 100644 --- a/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java +++ b/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java @@ -6,7 +6,6 @@ import org.xbib.content.xml.util.XMLUtil; import org.xbib.netty.http.common.HttpResponse; import org.xbib.oai.client.AbstractOAIResponse; import org.xbib.oai.exceptions.BadVerbException; -import org.xbib.oai.exceptions.TooManyRequestsException; import org.xbib.oai.exceptions.BadArgumentException; import org.xbib.oai.exceptions.BadResumptionTokenException; import org.xbib.oai.exceptions.NoRecordsMatchException; @@ -36,6 +35,7 @@ import javax.xml.transform.stream.StreamResult; public class ListRecordsResponse extends AbstractOAIResponse { private static final Logger logger = Logger.getLogger(ListRecordsResponse.class.getName()); + private static final String[] RETRY_AFTER_HEADERS = { "retry-after", "Retry-after", "Retry-After" }; @@ -112,6 +112,7 @@ public class ListRecordsResponse extends AbstractOAIResponse { } if (status == 429) { try { + logger.log(Level.WARNING, "received 429 Too many requests, waiting 10 seconds..."); Thread.sleep(10000L); } catch (InterruptedException e) { // ignore diff --git a/oai-client/src/test/java/org/xbib/oai/client/ArxivClientTest.java b/oai-client/src/test/java/org/xbib/oai/client/ArxivClientTest.java index f21fe2b..7f89b15 100644 --- a/oai-client/src/test/java/org/xbib/oai/client/ArxivClientTest.java +++ b/oai-client/src/test/java/org/xbib/oai/client/ArxivClientTest.java @@ -37,7 +37,10 @@ public class ArxivClientTest { final URL url = URL.create("http://export.arxiv.org/oai2/"); try (OAIClient client = new OAIClient(url)) { IdentifyRequest identifyRequest = client.newIdentifyRequest(); - Client httpClient = client.getHttpClient(); + Client httpClient = Client.builder() + .setConnectTimeoutMillis(60 * 1000) + .setReadTimeoutMillis(60 * 1000) + .build(); IdentifyResponse identifyResponse = new IdentifyResponse(); Request request = Request.get() .url(identifyRequest.getURL()) @@ -88,6 +91,7 @@ public class ArxivClientTest { } } fileWriter.close(); + httpClient.shutdownGracefully(); logger.log(Level.INFO, "count = " + handler.count()); assertTrue(handler.count() > 0L); } catch (Exception e) { diff --git a/oai-client/src/test/java/org/xbib/oai/client/BundeskunsthalleTest.java b/oai-client/src/test/java/org/xbib/oai/client/BundeskunsthalleTest.java index 7051b5b..3a81c58 100644 --- a/oai-client/src/test/java/org/xbib/oai/client/BundeskunsthalleTest.java +++ b/oai-client/src/test/java/org/xbib/oai/client/BundeskunsthalleTest.java @@ -17,6 +17,7 @@ import org.xbib.oai.exceptions.OAIException; import java.io.IOException; import java.io.StringWriter; +import java.net.ConnectException; import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -26,42 +27,38 @@ import java.util.logging.Logger; /** * */ +@Ignore public class BundeskunsthalleTest { private static final Logger logger = Logger.getLogger(BundeskunsthalleTest.class.getName()); @Test - @Ignore // takes too long time and creates files - public void testListRecords() throws Exception { - URL url = URL.create("http://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh"); + public void testListRecords() { + URL url = URL.create("https://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh"); try (OAIClient oaiClient = new OAIClient(url)) { + Client httpClient = Client.builder() + .setConnectTimeoutMillis(60 * 1000) + .setReadTimeoutMillis(60 * 1000) + .build(); IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest(); - Client httpClient = oaiClient.getHttpClient(); IdentifyResponse identifyResponse = new IdentifyResponse(); Request request = Request.get() - .url(url.resolve(identifyRequest.getURL())) + .url(identifyRequest.getURL()) .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") + .setFollowRedirect(true) .build() .setResponseListener(resp -> { + logger.log(Level.INFO, + "status = " + resp.getStatus() + + " body = " + resp.getBodyAsString(StandardCharsets.UTF_8)); StringWriter sw = new StringWriter(); identifyResponse.receivedResponse(resp, sw); }); httpClient.execute(request).get(); - - /*AggregatedHttpMessage response = client.execute(HttpHeaders.of(HttpMethod.GET, identifyRequest.getPath()) - .set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();*/ - // follow a maximum of 10 HTTP redirects - /*int max = 10; - while (response.followUrl() != null && max-- > 0) { - URI uri = URI.create(response.followUrl()); - client = Clients.newClient(oaiClient.getFactory(), "none+" + uri, HttpClient.class); - response = client.execute(HttpHeaders.of(HttpMethod.GET, response.followUrl()) - .set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get(); - }*/ String granularity = identifyResponse.getGranularity(); logger.log(Level.INFO, "granularity = " + granularity); DateTimeFormatter dateTimeFormatter = "YYYY-MM-DD".equals(granularity) ? - DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("GMT")) : null; + DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")) : null; ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest(); listRecordsRequest.setDateTimeFormatter(dateTimeFormatter); listRecordsRequest.setMetadataPrefix("marcxml"); @@ -73,11 +70,13 @@ public class BundeskunsthalleTest { while (listRecordsRequest != null) { try { ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest); - logger.log(Level.INFO,"sending " + listRecordsRequest.getURL()); + logger.log(Level.INFO, "sending " + listRecordsRequest.getURL()); StringWriter sw = new StringWriter(); request = Request.get() - .url(url.resolve(listRecordsRequest.getURL())) + .url(listRecordsRequest.getURL()) .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") + .setFollowRedirect(true) + .setTimeoutInMillis(60 * 1000) .build() .setResponseListener(resp -> { try { @@ -92,22 +91,18 @@ public class BundeskunsthalleTest { .build() .xmlReader().parse(); } catch (IOException e) { - throw new OAIException(e); + throw new OAIException("MARC parser exception: " + e.getMessage(), e); } listRecordsResponse.receivedResponse(resp, sw); - logger.log(Level.FINE, "response headers = " + resp.getHeaders() + - " resumption-token = {}" + listRecordsResponse.getResumptionToken()); + logger.log(Level.FINE, + "status = " + resp.getStatus() + + " headers = " + resp.getHeaders() + + " resumptiontoken = " + listRecordsResponse.getResumptionToken()); }); httpClient.execute(request).get(); - // follow a maximum of 10 HTTP redirects - /*max = 10; - while (response.followUrl() != null && max-- > 0) { - URI uri = URI.create(response.followUrl()); - client = Clients.newClient(oaiClient.getFactory(), "none+" + uri, HttpClient.class); - response = client.execute(HttpHeaders.of(HttpMethod.GET, response.followUrl()) - .set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get(); - }*/ listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); + } catch (ConnectException e) { + logger.log(Level.WARNING, e.getMessage(), e); } catch (IOException e) { logger.log(Level.SEVERE, e.getMessage(), e); listRecordsRequest = null; @@ -116,6 +111,7 @@ public class BundeskunsthalleTest { writer.endCollection(); writer.endDocument(); } + logger.log(Level.INFO, "completed"); } catch (Exception e) { logger.log(Level.WARNING, e.getMessage(), e); } diff --git a/oai-client/src/test/java/org/xbib/oai/client/DNBClientTest.java b/oai-client/src/test/java/org/xbib/oai/client/DNBClientTest.java index f1cb088..51993fd 100644 --- a/oai-client/src/test/java/org/xbib/oai/client/DNBClientTest.java +++ b/oai-client/src/test/java/org/xbib/oai/client/DNBClientTest.java @@ -1,7 +1,5 @@ package org.xbib.oai.client; -import static org.junit.Assert.assertEquals; - import io.netty.handler.codec.http.HttpHeaderNames; import org.junit.Ignore; import org.junit.Test; @@ -9,6 +7,7 @@ import org.xbib.net.URL; import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Request; import org.xbib.oai.client.identify.IdentifyRequest; +import org.xbib.oai.client.identify.IdentifyResponse; import org.xbib.oai.client.listrecords.ListRecordsRequest; import org.xbib.oai.client.listrecords.ListRecordsResponse; import org.xbib.oai.xml.SimpleMetadataHandler; @@ -16,8 +15,12 @@ import org.xbib.oai.xml.SimpleMetadataHandler; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.StringWriter; +import java.net.ConnectException; import java.nio.charset.StandardCharsets; import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; @@ -25,58 +28,62 @@ import java.util.logging.Logger; /** * */ -@Ignore public class DNBClientTest { private static final Logger logger = Logger.getLogger(DNBClientTest.class.getName()); @Test - public void testIdentify() { + public void testBibdat() { URL url = URL.create("http://services.dnb.de/oai/repository"); - try (OAIClient client = new OAIClient(url)) { - IdentifyRequest identifyRequest = client.newIdentifyRequest(); - Client httpClient = client.getHttpClient(); - assertEquals("/oai/repository?verb=Identify", identifyRequest.getURL().toString()); + try (OAIClient oaiClient = new OAIClient(url)) { + Client httpClient = Client.builder() + .setConnectTimeoutMillis(60 * 1000) + .setReadTimeoutMillis(60 * 1000) + .build(); + IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest(); + IdentifyResponse identifyResponse = new IdentifyResponse(); Request request = Request.get() - .url(url.resolve(identifyRequest.getURL())) + .url(identifyRequest.getURL()) .build() - .setResponseListener(resp -> logger.log(Level.INFO, resp.getBodyAsString(StandardCharsets.UTF_8))); + .setResponseListener(resp -> { + logger.log(Level.INFO, resp.getBodyAsString(StandardCharsets.UTF_8)); + StringWriter sw = new StringWriter(); + identifyResponse.receivedResponse(resp, sw); + }); httpClient.execute(request).get(); - } catch (IOException e) { - logger.log(Level.SEVERE, e.getMessage(), e); - } - } - - @Test - public void testListRecordsDNB() { - URL url = URL.create("http://services.dnb.de/oai/repository"); - try (OAIClient client = new OAIClient(url)) { - ListRecordsRequest listRecordsRequest = client.newListRecordsRequest(); + String granularity = identifyResponse.getGranularity(); + logger.log(Level.INFO, "granularity = " + granularity); + DateTimeFormatter dateTimeFormatter = "YYYY-MM-DD".equals(granularity) ? + DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")) : + DateTimeFormatter.ISO_DATE_TIME; + ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest(); listRecordsRequest.setFrom(Instant.parse("2016-01-01T00:00:00Z")); listRecordsRequest.setUntil(Instant.parse("2016-01-10T00:00:00Z")); listRecordsRequest.setSet("bib"); listRecordsRequest.setMetadataPrefix("PicaPlus-xml"); Handler handler = new Handler(); - File file = File.createTempFile("dnb-bib-pica.", ".xml"); - file.deleteOnExit(); + File file = new File("build/dnb-bib-pica.xml"); FileWriter fileWriter = new FileWriter(file); while (listRecordsRequest != null) { try { ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest); listRecordsRequest.addHandler(handler); - Request request = Request.get() - .url(url.resolve(listRecordsRequest.getURL())) + request = Request.get() + .url(listRecordsRequest.getURL()) .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") .build() .setResponseListener(resp -> listRecordsResponse.receivedResponse(resp, fileWriter)); - client.getHttpClient().execute(request).get(); - listRecordsRequest = client.resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); + httpClient.execute(request).get(); + listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); + } catch (ConnectException e) { + logger.log(Level.WARNING, e.getMessage(), e); } catch (IOException e) { logger.log(Level.SEVERE, e.getMessage(), e); listRecordsRequest = null; } } fileWriter.close(); + httpClient.shutdownGracefully(); logger.log(Level.INFO, "count=" + handler.count()); } catch (Exception e) { logger.log(Level.SEVERE, "skipped, HTTP exception"); diff --git a/oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java b/oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java index 2a98c7f..708a036 100644 --- a/oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java +++ b/oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java @@ -34,8 +34,11 @@ public class DOAJClientTest { public void testListRecordsDOAJ() { URL url = URL.create("https://doaj.org/oai"); try (OAIClient oaiClient = new OAIClient(url)) { + Client httpClient = Client.builder() + .setConnectTimeoutMillis(60 * 1000) + .setReadTimeoutMillis(60 * 1000) + .build(); IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest(); - Client httpClient = oaiClient.getHttpClient(); IdentifyResponse identifyResponse = new IdentifyResponse(); Request request = Request.get() .url(url.resolve(identifyRequest.getURL())) @@ -77,6 +80,7 @@ public class DOAJClientTest { listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); } fileWriter.close(); + httpClient.shutdownGracefully(); logger.log(Level.INFO, "count = " + handler.count()); } catch (Exception e) { logger.log(Level.SEVERE, e.getMessage(), e);