move OAIException from unchecked to checked exception, add exception handler to listRecords for better error handling

gradle.properties

@@ -1,3 +1,3 @@
 group = org.xbib
 name = oai
-version = 4.0.1
+version = 4.1.0

oai-client/src/main/java/org/xbib/oai/client/OAIClient.java

@@ -1,12 +1,16 @@
 package org.xbib.oai.client;
 
 import org.xbib.oai.OAIConstants;
+import org.xbib.oai.OAIExceptionHandler;
 import org.xbib.oai.client.identify.IdentifyRequest;
 import org.xbib.oai.client.identify.IdentifyResponse;
 import org.xbib.oai.client.listrecords.ListRecordsRequest;
 import org.xbib.oai.client.listrecords.ListRecordsResponse;
 import org.xbib.oai.client.util.UrlBuilder;
+import org.xbib.oai.exceptions.BadStatusException;
+import org.xbib.oai.exceptions.NoAnswerException;
 import org.xbib.oai.exceptions.NoRecordsMatchException;
+import org.xbib.oai.exceptions.OAIException;
 import org.xbib.oai.util.ResumptionToken;
 import org.xbib.oai.xml.MetadataHandler;
 
@@ -70,7 +74,7 @@ public class OAIClient {
      * descriptive information.
      * @return identify response
      */
-    public IdentifyResponse identify() throws IOException, InterruptedException {
+    public IdentifyResponse identify() throws IOException, OAIException, InterruptedException {
         IdentifyRequest identifyRequest = new IdentifyRequest();
         IdentifyResponse identifyResponse = new IdentifyResponse();
         UrlBuilder url = UrlBuilder.fromUrl(baseURL);
@@ -135,14 +139,15 @@ public class OAIClient {
      * specified in the request has been deleted. No metadata
      * will be present for records with deleted status.
      *
-     * @param metadataPrefix
+     * @param metadataPrefix the metadata prefix
-     * @param set
+     * @param set the set
-     * @param dateTimeFormatter
+     * @param dateTimeFormatter the date time formatter
-     * @param from
+     * @param from the from date
-     * @param until
+     * @param until the until date
-     * @param base
+     * @param base the base date that defines the date range of the OAI server
-     * @param handler
+     * @param handler the content handler
-     * @param consumer
+     * @param exceptionHandler the exception handler for OAI problems
+     * @param consumer the input stream consumer
      */
     public void listRecords(String metadataPrefix,
                             String set,
@@ -151,7 +156,8 @@ public class OAIClient {
                             Instant until,
                             Instant base,
                             MetadataHandler handler,
-                            Consumer<InputStream> consumer) throws IOException {
+                            OAIExceptionHandler exceptionHandler,
+                            Consumer<InputStream> consumer) throws IOException, OAIException {
         do {
             ListRecordsRequest listRecordsRequest = new ListRecordsRequest();
             if (metadataPrefix != null) {
@@ -213,16 +219,24 @@ public class OAIClient {
                         }
                         if (httpResponse != null) {
                             int status = httpResponse.statusCode();
-                            String contentType = httpResponse.headers().firstValue("content-type").orElse(null);
+                            // only process HTTP 200 OK messages
-                            String retryAfter = httpResponse.headers().firstValue("retry-after").orElse(null);
+                            if (status == 200) {
-                            String body = new String(httpResponse.body(), StandardCharsets.UTF_8);
+                                String contentType = httpResponse.headers().firstValue("content-type").orElse(null);
-                            listRecordsResponse.receivedResponse(body, status, contentType, retryAfter, splitWriter);
+                                String retryAfter = httpResponse.headers().firstValue("retry-after").orElse(null);
-                            logger.log(Level.FINE, "response headers = " + httpResponse.headers() +
+                                String body = new String(httpResponse.body(), StandardCharsets.UTF_8);
-                                    " resumption-token = " + listRecordsResponse.getResumptionToken());
+                                listRecordsResponse.receivedResponse(body, status, contentType, retryAfter, splitWriter);
-                            byte[] b = httpResponse.body();
+                                logger.log(Level.FINE, "response headers = " + httpResponse.headers() +
-                            if (b.length > 0) {
+                                        " resumption-token = " + listRecordsResponse.getResumptionToken());
-                                consumer.accept(new ByteArrayInputStream(b));
+                                byte[] b = httpResponse.body();
+                                if (b.length > 0) {
+                                    consumer.accept(new ByteArrayInputStream(b));
+                                }
+                            } else {
+                                throw new BadStatusException("unexpected status " + status);
                             }
+                        } else {
+                            // unable to retrieve something even after retrying
+                            throw new NoAnswerException();
                         }
                     } else {
                         HttpResponse<String> httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString());
@@ -235,10 +249,18 @@ public class OAIClient {
                     }
                     listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
                 } catch (NoRecordsMatchException e) {
-                    logger.log(Level.WARNING, "no records match");
+                    logger.log(Level.WARNING, "no records match, continuing");
                     listRecordsRequest = null;
-                } catch (Exception e) {
+                } catch (OAIException e) {
                     logger.log(Level.SEVERE, e.getMessage(), e);
+                    // logical error, handle it and break out with runtime exception
+                    if (exceptionHandler != null) {
+                        exceptionHandler.handleException(e);
+                    }
+                    throw e;
+                } catch (Exception e) {
+                    // other unexpected exception, log and repeat request
+                    logger.log(Level.WARNING, "got error, but trying to repeat: " + e.getMessage(), e);
                     listRecordsRequest = null;
                 }
             }

oai-client/src/main/java/org/xbib/oai/client/identify/IdentifyResponse.java

@@ -40,7 +40,7 @@ public class IdentifyResponse implements OAIResponse {
     public IdentifyResponse() {
     }
 
-    public void receivedResponse(String message, int statusCode, String contentType, String retryAfter, Writer writer) {
+    public void receivedResponse(String message, int statusCode, String contentType, String retryAfter, Writer writer) throws OAIException {
         try {
             DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
             DocumentBuilder db = dbf.newDocumentBuilder();

oai-client/src/test/java/org/xbib/oai/client/test/ArxivClientTest.java

@@ -1,11 +1,14 @@
 package org.xbib.oai.client.test;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import org.junit.jupiter.api.Test;
+import org.xbib.oai.OAIExceptionHandler;
 import org.xbib.oai.client.OAIClient;
 import org.xbib.oai.client.SplitWriter;
 import org.xbib.oai.client.identify.IdentifyResponse;
+import org.xbib.oai.exceptions.OAIException;
 import org.xbib.oai.xml.SimpleMetadataHandler;
 
 import java.time.Instant;
@@ -33,11 +36,13 @@ class ArxivClientTest {
         logger.log(Level.INFO,"waiting 20 seconds");
         Thread.sleep(20 * 1000L);
         Handler handler  = new Handler();
+        ExceptionHandler exceptionHandler = new ExceptionHandler();
         oaiClient.listRecords("arXiv", null,
-                dateTimeFormatter, Instant.parse("2016-11-01T00:00:00Z"), Instant.parse("2016-11-02T00:00:00Z"), null,
+                dateTimeFormatter, Instant.parse("2016-11-01T00:00:00Z"), Instant.parse("2016-11-02T00:00:00Z"),
-                handler, null);
+                null, handler, exceptionHandler, null);
         logger.log(Level.INFO, "count = " + handler.count());
         assertTrue(handler.count() > 0L);
+        assertEquals(0, exceptionHandler.count());
     }
 
     static class Handler extends SimpleMetadataHandler {
@@ -59,4 +64,18 @@ class ArxivClientTest {
             return count.get();
         }
     }
+
+    static class ExceptionHandler implements OAIExceptionHandler {
+
+        final AtomicLong count = new AtomicLong(0L);
+
+        @Override
+        public void handleException(OAIException e) {
+            count.incrementAndGet();
+        }
+
+        long count() {
+            return count.get();
+        }
+    }
 }

oai-client/src/test/java/org/xbib/oai/client/test/BundeskunsthalleTest.java

@@ -7,7 +7,6 @@ import org.xbib.marc.json.MarcJsonWriter;
 import org.xbib.marc.xml.MarcContentHandler;
 import org.xbib.oai.client.OAIClient;
 import org.xbib.oai.client.identify.IdentifyResponse;
-import org.xbib.oai.exceptions.OAIException;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -36,7 +35,7 @@ class BundeskunsthalleTest {
             writer.startDocument();
             writer.beginCollection();
             oaiClient.listRecords("marcxml", null,
-                    dateTimeFormatter, null, null, null, null, inputStream -> {
+                    dateTimeFormatter, null, null, null, null, null, inputStream -> {
                     try {
                         Marc.builder()
                                 .setInputStream(inputStream)
@@ -49,7 +48,7 @@ class BundeskunsthalleTest {
                                 .build()
                                 .xmlReader().parse();
                     } catch (IOException e) {
-                        throw new OAIException("MARC parser exception: " + e.getMessage(), e);
+                        logger.log(Level.SEVERE, "MARC parser exception: " + e.getMessage(), e);
                     }
                 });
             writer.endCollection();

oai-client/src/test/java/org/xbib/oai/client/test/DNBClientTest.java

@@ -1,6 +1,5 @@
 package org.xbib.oai.client.test;
 
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.xbib.oai.client.OAIClient;
 import org.xbib.oai.client.SplitWriter;
@@ -19,7 +18,7 @@ class DNBClientTest {
         oaiClient.setSplitWriter(splitWriter);
         oaiClient.listRecords("PicaPlus-xml", "bib",
                     null, from, until, base,
-                    null, null);
+                    null, null, null);
     }
 
     @Test
@@ -33,7 +32,7 @@ class DNBClientTest {
         oaiClient.listRecords("PicaPlus-xml", "bib", null,
                 Instant.parse("2016-01-01T00:00:00Z"),
                 Instant.parse("2016-02-01T00:00:00Z"),
-                base, null, null);
+                base, null, null, null);
     }
 
 }

oai-client/src/test/java/org/xbib/oai/client/test/DOAJClientTest.java

@@ -1,9 +1,11 @@
 package org.xbib.oai.client.test;
 
 import org.junit.jupiter.api.Test;
+import org.xbib.oai.OAIExceptionHandler;
 import org.xbib.oai.client.OAIClient;
 import org.xbib.oai.client.SplitWriter;
 import org.xbib.oai.client.identify.IdentifyResponse;
+import org.xbib.oai.exceptions.OAIException;
 import org.xbib.oai.xml.SimpleMetadataHandler;
 
 import java.time.Instant;
@@ -13,6 +15,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 class DOAJClientTest {
@@ -28,15 +31,17 @@ class DOAJClientTest {
         // override granularity because of "bad arguments" error. Seems DOAJ is unable to manage it's own declared granularity.
         DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("GMT"));
         Handler handler = new Handler();
+        ExceptionHandler exceptionHandler = new ExceptionHandler();
         SplitWriter splitWriter = new SplitWriter("build/doaj-%d.xml", -1, 8192, false);
         oaiClient.setSplitWriter(splitWriter);
         Instant to = Instant.now();
         Instant from = to.atZone(ZoneId.systemDefault()).minusMonths(1).toInstant();
         oaiClient.listRecords("oai_dc", null,
                     dateTimeFormatter, from, to, null,
-                    handler, null);
+                    handler, exceptionHandler, null);
         logger.log(Level.INFO, "count = " + handler.count());
         assertTrue(handler.count() > 0);
+        assertEquals(0, exceptionHandler.count());
     }
 
     static class Handler extends SimpleMetadataHandler {
@@ -58,4 +63,18 @@ class DOAJClientTest {
             return count.get();
         }
     }
+
+    static class ExceptionHandler implements OAIExceptionHandler {
+
+        final AtomicLong count = new AtomicLong(0L);
+
+        @Override
+        public void handleException(OAIException e) {
+            count.incrementAndGet();
+        }
+
+        long count() {
+            return count.get();
+        }
+    }
 }

oai-common/src/main/java/org/xbib/oai/OAIExceptionHandler.java

@@ -0,0 +1,9 @@
+package org.xbib.oai;
+
+import org.xbib.oai.exceptions.OAIException;
+
+public interface OAIExceptionHandler {
+
+    void handleException(OAIException e);
+
+}

oai-common/src/main/java/org/xbib/oai/exceptions/BadArgumentException.java

@@ -1,12 +1,8 @@
 package org.xbib.oai.exceptions;
 
-/**
+@SuppressWarnings("serial")
- *
- */
 public class BadArgumentException extends OAIException {
 
-    private static final long serialVersionUID = -6647892792394074500L;
-
     public BadArgumentException() {
         this(null);
     }

oai-common/src/main/java/org/xbib/oai/exceptions/BadResumptionTokenException.java

@@ -2,13 +2,9 @@ package org.xbib.oai.exceptions;
 
 import org.xbib.oai.util.ResumptionToken;
 
-/**
+@SuppressWarnings("serial")
- *
- */
 public class BadResumptionTokenException extends OAIException {
 
-    private static final long serialVersionUID = 7384401627260164303L;
-
     public BadResumptionTokenException(ResumptionToken<?> token) {
         super(token != null ? token.toString() : null);
     }   

oai-common/src/main/java/org/xbib/oai/exceptions/BadStatusException.java

@@ -0,0 +1,9 @@
+package org.xbib.oai.exceptions;
+
+@SuppressWarnings("serial")
+public class BadStatusException extends OAIException {
+
+    public BadStatusException(String message) {
+        super(message);
+    }   
+}

oai-common/src/main/java/org/xbib/oai/exceptions/BadVerbException.java

@@ -1,12 +1,8 @@
 package org.xbib.oai.exceptions;
 
-/**
+@SuppressWarnings("serial")
- *
- */
 public class BadVerbException extends OAIException {
 
-    private static final long serialVersionUID = 1642129565793325510L;
-
     public BadVerbException(String message) {
         super(message);
     }   

oai-common/src/main/java/org/xbib/oai/exceptions/CannotDisseminateFormatException.java

@@ -1,12 +1,8 @@
 package org.xbib.oai.exceptions;
 
-/**
+@SuppressWarnings("serial")
- *
- */
 public class CannotDisseminateFormatException extends OAIException {
 
-    private static final long serialVersionUID = 154900133710811545L;
-
     public CannotDisseminateFormatException(String message) {
         super(message);
     }

oai-common/src/main/java/org/xbib/oai/exceptions/IdDoesNotExistException.java

@@ -1,12 +1,8 @@
 package org.xbib.oai.exceptions;
 
-/**
+@SuppressWarnings("serial")
- *
- */
 public class IdDoesNotExistException extends OAIException {
 
-    private static final long serialVersionUID = 9201985582562843506L;
-
     public IdDoesNotExistException(String message) {
         super(message);
     }

oai-common/src/main/java/org/xbib/oai/exceptions/NoAnswerException.java

@@ -0,0 +1,13 @@
+package org.xbib.oai.exceptions;
+
+@SuppressWarnings("serial")
+public class NoAnswerException extends OAIException {
+
+    public NoAnswerException() {
+        this(null);
+    }
+
+    public NoAnswerException(String message) {
+        super(message);
+    }   
+}

oai-common/src/main/java/org/xbib/oai/exceptions/NoRecordsMatchException.java

@@ -1,12 +1,8 @@
 package org.xbib.oai.exceptions;
 
-/**
+@SuppressWarnings("serial")
- *
- */
 public class NoRecordsMatchException extends OAIException {
 
-    private static final long serialVersionUID = 5201331168058463772L;
-
     public NoRecordsMatchException(String message) {
         super(message);
     }

oai-common/src/main/java/org/xbib/oai/exceptions/NoSetHierarchyException.java

@@ -1,12 +1,8 @@
 package org.xbib.oai.exceptions;
 
-/**
+@SuppressWarnings("serial")
- *
- */
 public class NoSetHierarchyException extends OAIException {
 
-    private static final long serialVersionUID = 6275260694745177314L;
-
     public NoSetHierarchyException(String message) {
         super(message);
     }

oai-common/src/main/java/org/xbib/oai/exceptions/OAIException.java

@@ -1,11 +1,7 @@
 package org.xbib.oai.exceptions;
 
-/**
+@SuppressWarnings("serial")
- *
+public class OAIException extends Exception {
- */
-public class OAIException extends RuntimeException {
-
-    private static final long serialVersionUID = -1890146067179892744L;
 
     public OAIException(String message) {
         super(message);

oai-common/src/main/java/org/xbib/oai/exceptions/TooManyRequestsException.java

@@ -2,6 +2,7 @@ package org.xbib.oai.exceptions;
 
 @SuppressWarnings("serial")
 public class TooManyRequestsException extends OAIException {
+
     public TooManyRequestsException(String message) {
         super(message);
     }

oai-common/src/main/java/org/xbib/oai/exceptions/package-info.java

@@ -1,4 +0,0 @@
-/**
- * OAI exceptions.
- */
-package org.xbib.oai.exceptions;