diff --git a/gradle.properties b/gradle.properties index 30fda82..9b2ab37 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group = org.xbib name = oai -version = 4.0.1 +version = 4.1.0 diff --git a/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java b/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java index c9a75b3..32badc2 100644 --- a/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java +++ b/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java @@ -1,12 +1,16 @@ package org.xbib.oai.client; import org.xbib.oai.OAIConstants; +import org.xbib.oai.OAIExceptionHandler; import org.xbib.oai.client.identify.IdentifyRequest; import org.xbib.oai.client.identify.IdentifyResponse; import org.xbib.oai.client.listrecords.ListRecordsRequest; import org.xbib.oai.client.listrecords.ListRecordsResponse; import org.xbib.oai.client.util.UrlBuilder; +import org.xbib.oai.exceptions.BadStatusException; +import org.xbib.oai.exceptions.NoAnswerException; import org.xbib.oai.exceptions.NoRecordsMatchException; +import org.xbib.oai.exceptions.OAIException; import org.xbib.oai.util.ResumptionToken; import org.xbib.oai.xml.MetadataHandler; @@ -70,7 +74,7 @@ public class OAIClient { * descriptive information. * @return identify response */ - public IdentifyResponse identify() throws IOException, InterruptedException { + public IdentifyResponse identify() throws IOException, OAIException, InterruptedException { IdentifyRequest identifyRequest = new IdentifyRequest(); IdentifyResponse identifyResponse = new IdentifyResponse(); UrlBuilder url = UrlBuilder.fromUrl(baseURL); @@ -135,14 +139,15 @@ public class OAIClient { * specified in the request has been deleted. No metadata * will be present for records with deleted status. * - * @param metadataPrefix - * @param set - * @param dateTimeFormatter - * @param from - * @param until - * @param base - * @param handler - * @param consumer + * @param metadataPrefix the metadata prefix + * @param set the set + * @param dateTimeFormatter the date time formatter + * @param from the from date + * @param until the until date + * @param base the base date that defines the date range of the OAI server + * @param handler the content handler + * @param exceptionHandler the exception handler for OAI problems + * @param consumer the input stream consumer */ public void listRecords(String metadataPrefix, String set, @@ -151,7 +156,8 @@ public class OAIClient { Instant until, Instant base, MetadataHandler handler, - Consumer consumer) throws IOException { + OAIExceptionHandler exceptionHandler, + Consumer consumer) throws IOException, OAIException { do { ListRecordsRequest listRecordsRequest = new ListRecordsRequest(); if (metadataPrefix != null) { @@ -213,16 +219,24 @@ public class OAIClient { } if (httpResponse != null) { int status = httpResponse.statusCode(); - String contentType = httpResponse.headers().firstValue("content-type").orElse(null); - String retryAfter = httpResponse.headers().firstValue("retry-after").orElse(null); - String body = new String(httpResponse.body(), StandardCharsets.UTF_8); - listRecordsResponse.receivedResponse(body, status, contentType, retryAfter, splitWriter); - logger.log(Level.FINE, "response headers = " + httpResponse.headers() + - " resumption-token = " + listRecordsResponse.getResumptionToken()); - byte[] b = httpResponse.body(); - if (b.length > 0) { - consumer.accept(new ByteArrayInputStream(b)); + // only process HTTP 200 OK messages + if (status == 200) { + String contentType = httpResponse.headers().firstValue("content-type").orElse(null); + String retryAfter = httpResponse.headers().firstValue("retry-after").orElse(null); + String body = new String(httpResponse.body(), StandardCharsets.UTF_8); + listRecordsResponse.receivedResponse(body, status, contentType, retryAfter, splitWriter); + logger.log(Level.FINE, "response headers = " + httpResponse.headers() + + " resumption-token = " + listRecordsResponse.getResumptionToken()); + byte[] b = httpResponse.body(); + if (b.length > 0) { + consumer.accept(new ByteArrayInputStream(b)); + } + } else { + throw new BadStatusException("unexpected status " + status); } + } else { + // unable to retrieve something even after retrying + throw new NoAnswerException(); } } else { HttpResponse httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); @@ -235,10 +249,18 @@ public class OAIClient { } listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); } catch (NoRecordsMatchException e) { - logger.log(Level.WARNING, "no records match"); + logger.log(Level.WARNING, "no records match, continuing"); listRecordsRequest = null; - } catch (Exception e) { + } catch (OAIException e) { logger.log(Level.SEVERE, e.getMessage(), e); + // logical error, handle it and break out with runtime exception + if (exceptionHandler != null) { + exceptionHandler.handleException(e); + } + throw e; + } catch (Exception e) { + // other unexpected exception, log and repeat request + logger.log(Level.WARNING, "got error, but trying to repeat: " + e.getMessage(), e); listRecordsRequest = null; } } diff --git a/oai-client/src/main/java/org/xbib/oai/client/identify/IdentifyResponse.java b/oai-client/src/main/java/org/xbib/oai/client/identify/IdentifyResponse.java index f8a38a1..6902b45 100644 --- a/oai-client/src/main/java/org/xbib/oai/client/identify/IdentifyResponse.java +++ b/oai-client/src/main/java/org/xbib/oai/client/identify/IdentifyResponse.java @@ -40,7 +40,7 @@ public class IdentifyResponse implements OAIResponse { public IdentifyResponse() { } - public void receivedResponse(String message, int statusCode, String contentType, String retryAfter, Writer writer) { + public void receivedResponse(String message, int statusCode, String contentType, String retryAfter, Writer writer) throws OAIException { try { DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); diff --git a/oai-client/src/test/java/org/xbib/oai/client/test/ArxivClientTest.java b/oai-client/src/test/java/org/xbib/oai/client/test/ArxivClientTest.java index 7595535..fd58c60 100644 --- a/oai-client/src/test/java/org/xbib/oai/client/test/ArxivClientTest.java +++ b/oai-client/src/test/java/org/xbib/oai/client/test/ArxivClientTest.java @@ -1,11 +1,14 @@ package org.xbib.oai.client.test; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; +import org.xbib.oai.OAIExceptionHandler; import org.xbib.oai.client.OAIClient; import org.xbib.oai.client.SplitWriter; import org.xbib.oai.client.identify.IdentifyResponse; +import org.xbib.oai.exceptions.OAIException; import org.xbib.oai.xml.SimpleMetadataHandler; import java.time.Instant; @@ -33,11 +36,13 @@ class ArxivClientTest { logger.log(Level.INFO,"waiting 20 seconds"); Thread.sleep(20 * 1000L); Handler handler = new Handler(); + ExceptionHandler exceptionHandler = new ExceptionHandler(); oaiClient.listRecords("arXiv", null, - dateTimeFormatter, Instant.parse("2016-11-01T00:00:00Z"), Instant.parse("2016-11-02T00:00:00Z"), null, - handler, null); + dateTimeFormatter, Instant.parse("2016-11-01T00:00:00Z"), Instant.parse("2016-11-02T00:00:00Z"), + null, handler, exceptionHandler, null); logger.log(Level.INFO, "count = " + handler.count()); assertTrue(handler.count() > 0L); + assertEquals(0, exceptionHandler.count()); } static class Handler extends SimpleMetadataHandler { @@ -59,4 +64,18 @@ class ArxivClientTest { return count.get(); } } + + static class ExceptionHandler implements OAIExceptionHandler { + + final AtomicLong count = new AtomicLong(0L); + + @Override + public void handleException(OAIException e) { + count.incrementAndGet(); + } + + long count() { + return count.get(); + } + } } diff --git a/oai-client/src/test/java/org/xbib/oai/client/test/BundeskunsthalleTest.java b/oai-client/src/test/java/org/xbib/oai/client/test/BundeskunsthalleTest.java index 5f00934..ad7ca05 100644 --- a/oai-client/src/test/java/org/xbib/oai/client/test/BundeskunsthalleTest.java +++ b/oai-client/src/test/java/org/xbib/oai/client/test/BundeskunsthalleTest.java @@ -7,7 +7,6 @@ import org.xbib.marc.json.MarcJsonWriter; import org.xbib.marc.xml.MarcContentHandler; import org.xbib.oai.client.OAIClient; import org.xbib.oai.client.identify.IdentifyResponse; -import org.xbib.oai.exceptions.OAIException; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -36,7 +35,7 @@ class BundeskunsthalleTest { writer.startDocument(); writer.beginCollection(); oaiClient.listRecords("marcxml", null, - dateTimeFormatter, null, null, null, null, inputStream -> { + dateTimeFormatter, null, null, null, null, null, inputStream -> { try { Marc.builder() .setInputStream(inputStream) @@ -49,7 +48,7 @@ class BundeskunsthalleTest { .build() .xmlReader().parse(); } catch (IOException e) { - throw new OAIException("MARC parser exception: " + e.getMessage(), e); + logger.log(Level.SEVERE, "MARC parser exception: " + e.getMessage(), e); } }); writer.endCollection(); diff --git a/oai-client/src/test/java/org/xbib/oai/client/test/DNBClientTest.java b/oai-client/src/test/java/org/xbib/oai/client/test/DNBClientTest.java index 8f8b584..6c60e56 100644 --- a/oai-client/src/test/java/org/xbib/oai/client/test/DNBClientTest.java +++ b/oai-client/src/test/java/org/xbib/oai/client/test/DNBClientTest.java @@ -1,6 +1,5 @@ package org.xbib.oai.client.test; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.xbib.oai.client.OAIClient; import org.xbib.oai.client.SplitWriter; @@ -19,7 +18,7 @@ class DNBClientTest { oaiClient.setSplitWriter(splitWriter); oaiClient.listRecords("PicaPlus-xml", "bib", null, from, until, base, - null, null); + null, null, null); } @Test @@ -33,7 +32,7 @@ class DNBClientTest { oaiClient.listRecords("PicaPlus-xml", "bib", null, Instant.parse("2016-01-01T00:00:00Z"), Instant.parse("2016-02-01T00:00:00Z"), - base, null, null); + base, null, null, null); } } diff --git a/oai-client/src/test/java/org/xbib/oai/client/test/DOAJClientTest.java b/oai-client/src/test/java/org/xbib/oai/client/test/DOAJClientTest.java index 3f0b790..80915a4 100644 --- a/oai-client/src/test/java/org/xbib/oai/client/test/DOAJClientTest.java +++ b/oai-client/src/test/java/org/xbib/oai/client/test/DOAJClientTest.java @@ -1,9 +1,11 @@ package org.xbib.oai.client.test; import org.junit.jupiter.api.Test; +import org.xbib.oai.OAIExceptionHandler; import org.xbib.oai.client.OAIClient; import org.xbib.oai.client.SplitWriter; import org.xbib.oai.client.identify.IdentifyResponse; +import org.xbib.oai.exceptions.OAIException; import org.xbib.oai.xml.SimpleMetadataHandler; import java.time.Instant; @@ -13,6 +15,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; class DOAJClientTest { @@ -28,15 +31,17 @@ class DOAJClientTest { // override granularity because of "bad arguments" error. Seems DOAJ is unable to manage it's own declared granularity. DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("GMT")); Handler handler = new Handler(); + ExceptionHandler exceptionHandler = new ExceptionHandler(); SplitWriter splitWriter = new SplitWriter("build/doaj-%d.xml", -1, 8192, false); oaiClient.setSplitWriter(splitWriter); Instant to = Instant.now(); Instant from = to.atZone(ZoneId.systemDefault()).minusMonths(1).toInstant(); oaiClient.listRecords("oai_dc", null, dateTimeFormatter, from, to, null, - handler, null); + handler, exceptionHandler, null); logger.log(Level.INFO, "count = " + handler.count()); assertTrue(handler.count() > 0); + assertEquals(0, exceptionHandler.count()); } static class Handler extends SimpleMetadataHandler { @@ -58,4 +63,18 @@ class DOAJClientTest { return count.get(); } } + + static class ExceptionHandler implements OAIExceptionHandler { + + final AtomicLong count = new AtomicLong(0L); + + @Override + public void handleException(OAIException e) { + count.incrementAndGet(); + } + + long count() { + return count.get(); + } + } } diff --git a/oai-common/src/main/java/org/xbib/oai/OAIExceptionHandler.java b/oai-common/src/main/java/org/xbib/oai/OAIExceptionHandler.java new file mode 100644 index 0000000..97445f2 --- /dev/null +++ b/oai-common/src/main/java/org/xbib/oai/OAIExceptionHandler.java @@ -0,0 +1,9 @@ +package org.xbib.oai; + +import org.xbib.oai.exceptions.OAIException; + +public interface OAIExceptionHandler { + + void handleException(OAIException e); + +} diff --git a/oai-common/src/main/java/org/xbib/oai/exceptions/BadArgumentException.java b/oai-common/src/main/java/org/xbib/oai/exceptions/BadArgumentException.java index ac7d443..16cfac7 100644 --- a/oai-common/src/main/java/org/xbib/oai/exceptions/BadArgumentException.java +++ b/oai-common/src/main/java/org/xbib/oai/exceptions/BadArgumentException.java @@ -1,12 +1,8 @@ package org.xbib.oai.exceptions; -/** - * - */ +@SuppressWarnings("serial") public class BadArgumentException extends OAIException { - private static final long serialVersionUID = -6647892792394074500L; - public BadArgumentException() { this(null); } diff --git a/oai-common/src/main/java/org/xbib/oai/exceptions/BadResumptionTokenException.java b/oai-common/src/main/java/org/xbib/oai/exceptions/BadResumptionTokenException.java index 7a33197..8b3b081 100644 --- a/oai-common/src/main/java/org/xbib/oai/exceptions/BadResumptionTokenException.java +++ b/oai-common/src/main/java/org/xbib/oai/exceptions/BadResumptionTokenException.java @@ -2,13 +2,9 @@ package org.xbib.oai.exceptions; import org.xbib.oai.util.ResumptionToken; -/** - * - */ +@SuppressWarnings("serial") public class BadResumptionTokenException extends OAIException { - private static final long serialVersionUID = 7384401627260164303L; - public BadResumptionTokenException(ResumptionToken token) { super(token != null ? token.toString() : null); } diff --git a/oai-common/src/main/java/org/xbib/oai/exceptions/BadStatusException.java b/oai-common/src/main/java/org/xbib/oai/exceptions/BadStatusException.java new file mode 100644 index 0000000..343dbe4 --- /dev/null +++ b/oai-common/src/main/java/org/xbib/oai/exceptions/BadStatusException.java @@ -0,0 +1,9 @@ +package org.xbib.oai.exceptions; + +@SuppressWarnings("serial") +public class BadStatusException extends OAIException { + + public BadStatusException(String message) { + super(message); + } +} diff --git a/oai-common/src/main/java/org/xbib/oai/exceptions/BadVerbException.java b/oai-common/src/main/java/org/xbib/oai/exceptions/BadVerbException.java index 47fcb42..21e1e63 100644 --- a/oai-common/src/main/java/org/xbib/oai/exceptions/BadVerbException.java +++ b/oai-common/src/main/java/org/xbib/oai/exceptions/BadVerbException.java @@ -1,12 +1,8 @@ package org.xbib.oai.exceptions; -/** - * - */ +@SuppressWarnings("serial") public class BadVerbException extends OAIException { - private static final long serialVersionUID = 1642129565793325510L; - public BadVerbException(String message) { super(message); } diff --git a/oai-common/src/main/java/org/xbib/oai/exceptions/CannotDisseminateFormatException.java b/oai-common/src/main/java/org/xbib/oai/exceptions/CannotDisseminateFormatException.java index 12a9b0e..a21a99b 100644 --- a/oai-common/src/main/java/org/xbib/oai/exceptions/CannotDisseminateFormatException.java +++ b/oai-common/src/main/java/org/xbib/oai/exceptions/CannotDisseminateFormatException.java @@ -1,12 +1,8 @@ package org.xbib.oai.exceptions; -/** - * - */ +@SuppressWarnings("serial") public class CannotDisseminateFormatException extends OAIException { - private static final long serialVersionUID = 154900133710811545L; - public CannotDisseminateFormatException(String message) { super(message); } diff --git a/oai-common/src/main/java/org/xbib/oai/exceptions/IdDoesNotExistException.java b/oai-common/src/main/java/org/xbib/oai/exceptions/IdDoesNotExistException.java index a9025a9..2b86095 100644 --- a/oai-common/src/main/java/org/xbib/oai/exceptions/IdDoesNotExistException.java +++ b/oai-common/src/main/java/org/xbib/oai/exceptions/IdDoesNotExistException.java @@ -1,12 +1,8 @@ package org.xbib.oai.exceptions; -/** - * - */ +@SuppressWarnings("serial") public class IdDoesNotExistException extends OAIException { - private static final long serialVersionUID = 9201985582562843506L; - public IdDoesNotExistException(String message) { super(message); } diff --git a/oai-common/src/main/java/org/xbib/oai/exceptions/NoAnswerException.java b/oai-common/src/main/java/org/xbib/oai/exceptions/NoAnswerException.java new file mode 100644 index 0000000..a1f4023 --- /dev/null +++ b/oai-common/src/main/java/org/xbib/oai/exceptions/NoAnswerException.java @@ -0,0 +1,13 @@ +package org.xbib.oai.exceptions; + +@SuppressWarnings("serial") +public class NoAnswerException extends OAIException { + + public NoAnswerException() { + this(null); + } + + public NoAnswerException(String message) { + super(message); + } +} diff --git a/oai-common/src/main/java/org/xbib/oai/exceptions/NoRecordsMatchException.java b/oai-common/src/main/java/org/xbib/oai/exceptions/NoRecordsMatchException.java index 708c343..796197c 100644 --- a/oai-common/src/main/java/org/xbib/oai/exceptions/NoRecordsMatchException.java +++ b/oai-common/src/main/java/org/xbib/oai/exceptions/NoRecordsMatchException.java @@ -1,12 +1,8 @@ package org.xbib.oai.exceptions; -/** - * - */ +@SuppressWarnings("serial") public class NoRecordsMatchException extends OAIException { - private static final long serialVersionUID = 5201331168058463772L; - public NoRecordsMatchException(String message) { super(message); } diff --git a/oai-common/src/main/java/org/xbib/oai/exceptions/NoSetHierarchyException.java b/oai-common/src/main/java/org/xbib/oai/exceptions/NoSetHierarchyException.java index dfa1abd..d20e305 100644 --- a/oai-common/src/main/java/org/xbib/oai/exceptions/NoSetHierarchyException.java +++ b/oai-common/src/main/java/org/xbib/oai/exceptions/NoSetHierarchyException.java @@ -1,12 +1,8 @@ package org.xbib.oai.exceptions; -/** - * - */ +@SuppressWarnings("serial") public class NoSetHierarchyException extends OAIException { - private static final long serialVersionUID = 6275260694745177314L; - public NoSetHierarchyException(String message) { super(message); } diff --git a/oai-common/src/main/java/org/xbib/oai/exceptions/OAIException.java b/oai-common/src/main/java/org/xbib/oai/exceptions/OAIException.java index e13fc6f..4951776 100644 --- a/oai-common/src/main/java/org/xbib/oai/exceptions/OAIException.java +++ b/oai-common/src/main/java/org/xbib/oai/exceptions/OAIException.java @@ -1,11 +1,7 @@ package org.xbib.oai.exceptions; -/** - * - */ -public class OAIException extends RuntimeException { - - private static final long serialVersionUID = -1890146067179892744L; +@SuppressWarnings("serial") +public class OAIException extends Exception { public OAIException(String message) { super(message); diff --git a/oai-common/src/main/java/org/xbib/oai/exceptions/TooManyRequestsException.java b/oai-common/src/main/java/org/xbib/oai/exceptions/TooManyRequestsException.java index e1dd53f..810df1a 100644 --- a/oai-common/src/main/java/org/xbib/oai/exceptions/TooManyRequestsException.java +++ b/oai-common/src/main/java/org/xbib/oai/exceptions/TooManyRequestsException.java @@ -2,6 +2,7 @@ package org.xbib.oai.exceptions; @SuppressWarnings("serial") public class TooManyRequestsException extends OAIException { + public TooManyRequestsException(String message) { super(message); } diff --git a/oai-common/src/main/java/org/xbib/oai/exceptions/package-info.java b/oai-common/src/main/java/org/xbib/oai/exceptions/package-info.java deleted file mode 100644 index e05b334..0000000 --- a/oai-common/src/main/java/org/xbib/oai/exceptions/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * OAI exceptions. - */ -package org.xbib.oai.exceptions;