fixes for listing records

gradle.properties

@@ -1,6 +1,6 @@
 group = org.xbib
 name = oai
-version = 2.5.1
+version = 2.5.2
 
 gradle.wrapper.version = 6.6.1
 xbib-content.version = 2.6.2

oai-client/src/main/java/org/xbib/oai/client/OAIClient.java

@@ -9,8 +9,9 @@ import org.xbib.oai.client.listmetadataformats.ListMetadataFormatsRequest;
 import org.xbib.oai.client.listrecords.ListRecordsRequest;
 import org.xbib.oai.client.listrecords.ListRecordsResponse;
 import org.xbib.oai.client.listsets.ListSetsRequest;
+import org.xbib.oai.exceptions.NoRecordsMatchException;
 import org.xbib.oai.util.ResumptionToken;
-import org.xbib.oai.xml.SimpleMetadataHandler;
+import org.xbib.oai.xml.MetadataHandler;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -23,6 +24,10 @@ import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.Period;
+import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.util.function.Consumer;
 import java.util.logging.Level;
@@ -48,7 +53,7 @@ public class OAIClient {
                 .build();
     }
 
-    void setUserAgent(String userAgent) {
+    public void setUserAgent(String userAgent) {
         this.userAgent = userAgent;
     }
 
@@ -142,7 +147,19 @@ public class OAIClient {
                             Instant from,
                             Instant until,
                             Writer writer,
-                            SimpleMetadataHandler handler) {
+                            MetadataHandler handler) {
+        listRecords(metadataPrefix, set, dateTimeFormatter, from, until, null, writer, handler);
+    }
+
+    public void listRecords(String metadataPrefix,
+                            String set,
+                            DateTimeFormatter dateTimeFormatter,
+                            Instant from,
+                            Instant until,
+                            Instant base,
+                            Writer writer,
+                            MetadataHandler handler) {
+        do {
             ListRecordsRequest listRecordsRequest = new ListRecordsRequest();
             if (metadataPrefix != null) {
                 listRecordsRequest.setMetadataPrefix(metadataPrefix);
@@ -159,6 +176,11 @@ public class OAIClient {
             if (until != null) {
                 listRecordsRequest.setUntil(until);
             }
+            if (from != null && until != null) {
+                if (until.isBefore(from)) {
+                    throw new IllegalArgumentException("until must not be before from");
+                }
+            }
             while (listRecordsRequest != null) {
                 try {
                     if (handler != null) {
@@ -170,6 +192,7 @@ public class OAIClient {
                     HttpRequest httpRequest = HttpRequest.newBuilder()
                             .uri(URI.create(url.build().toExternalForm()))
                             .header("accept", "utf-8")
+                            .header("user-agent", userAgent != null ? userAgent : "xbib OAI client")
                             .GET()
                             .build();
                     logger.log(Level.INFO, "sending " + httpRequest);
@@ -181,11 +204,34 @@ public class OAIClient {
                     logger.log(Level.FINE, "response headers = " + httpResponse.headers() +
                             " resumption-token = " + listRecordsResponse.getResumptionToken());
                     listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
+                } catch (NoRecordsMatchException e) {
+                    logger.log(Level.WARNING, "no records match");
                 } catch (Exception e) {
                     logger.log(Level.SEVERE, e.getMessage(), e);
                     listRecordsRequest = null;
                 }
             }
+            if (base != null && from != null && until != null) {
+                LocalDate fromLocalDate = LocalDate.ofInstant(from, ZoneOffset.UTC);
+                LocalDate untilLocalDate = LocalDate.ofInstant(until, ZoneOffset.UTC);
+                Period period =  Period.between(fromLocalDate, untilLocalDate);
+                logger.log(Level.INFO, "from = " + fromLocalDate + " until = " + untilLocalDate + " period = " + period);
+                if (period.getYears() > 0 || period.getMonths() > 0 || period.getDays() > 0) {
+                    from = LocalDateTime.ofInstant(from, ZoneOffset.UTC)
+                            .plusYears(-period.getYears())
+                            .plusMonths(-period.getMonths())
+                            .plusDays(-period.getDays())
+                            .toInstant(ZoneOffset.UTC);
+                    until = LocalDateTime.ofInstant(until, ZoneOffset.UTC)
+                            .plusYears(-period.getYears())
+                            .plusMonths(-period.getMonths())
+                            .plusDays(-period.getDays())
+                            .toInstant(ZoneOffset.UTC);
+                } else {
+                    throw new IllegalStateException("from = " + from + " until = " + until + ": period is zero");
+                }
+            }
+        } while (base != null && from != null && until != null && from.isAfter(base) && until.isAfter(base));
     }
 
     public void listRecords(String metadataPrefix,
@@ -194,6 +240,17 @@ public class OAIClient {
                             Instant from,
                             Instant until,
                             Consumer<InputStream> consumer) {
+        listRecords(metadataPrefix, set, dateTimeFormatter, from, until, null, consumer);
+    }
+
+    public void listRecords(String metadataPrefix,
+                            String set,
+                            DateTimeFormatter dateTimeFormatter,
+                            Instant from,
+                            Instant until,
+                            Instant base,
+                            Consumer<InputStream> consumer) {
+        do {
             ListRecordsRequest listRecordsRequest = new ListRecordsRequest();
             if (metadataPrefix != null) {
                 listRecordsRequest.setMetadataPrefix(metadataPrefix);
@@ -210,6 +267,11 @@ public class OAIClient {
             if (until != null) {
                 listRecordsRequest.setUntil(until);
             }
+            if (from != null && until != null) {
+                if (until.isBefore(from)) {
+                    throw new IllegalArgumentException("until must not be before from");
+                }
+            }
             while (listRecordsRequest != null) {
                 try {
                     StringWriter sw = new StringWriter();
@@ -219,6 +281,7 @@ public class OAIClient {
                     HttpRequest httpRequest = HttpRequest.newBuilder()
                             .uri(URI.create(url.build().toExternalForm()))
                             .header("accept", "utf-8")
+                            .header("user-agent", userAgent != null ? userAgent : "xbib OAI client")
                             .GET()
                             .build();
                     logger.log(Level.INFO, "sending " + httpRequest);
@@ -233,11 +296,34 @@ public class OAIClient {
                     logger.log(Level.FINE, "response headers = " + httpResponse.headers() +
                             " resumption-token = " + listRecordsResponse.getResumptionToken());
                     listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
+                } catch (NoRecordsMatchException e) {
+                    logger.log(Level.WARNING, "no records match");
                 } catch (Exception e) {
                     logger.log(Level.SEVERE, e.getMessage(), e);
                     listRecordsRequest = null;
                 }
             }
+            if (base != null && from != null && until != null) {
+                LocalDate fromLocalDate = LocalDate.ofInstant(from, ZoneOffset.UTC);
+                LocalDate untilLocalDate = LocalDate.ofInstant(until, ZoneOffset.UTC);
+                Period period =  Period.between(fromLocalDate, untilLocalDate);
+                logger.log(Level.INFO, "from = " + fromLocalDate + " until = " + untilLocalDate + " period = " + period);
+                if (period.getYears() > 0 || period.getMonths() > 0 || period.getDays() > 0) {
+                    from = LocalDateTime.ofInstant(from, ZoneOffset.UTC)
+                            .plusYears(-period.getYears())
+                            .plusMonths(-period.getMonths())
+                            .plusDays(-period.getDays())
+                            .toInstant(ZoneOffset.UTC);
+                    until = LocalDateTime.ofInstant(until, ZoneOffset.UTC)
+                            .plusYears(-period.getYears())
+                            .plusMonths(-period.getMonths())
+                            .plusDays(-period.getDays())
+                            .toInstant(ZoneOffset.UTC);
+                } else {
+                    throw new IllegalStateException("from = " + from + " until = " + until + ": period is zero");
+                }
+            }
+        } while (base != null && from != null && until != null && from.isAfter(base) && until.isAfter(base));
     }
 
     public IdentifyRequest resume(IdentifyRequest request, ResumptionToken<?> token) {

oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java

@@ -12,6 +12,7 @@ import org.xbib.oai.util.ResumptionToken;
 import org.xml.sax.InputSource;
 
 import java.io.StringReader;
+import java.io.StringWriter;
 import java.io.Writer;
 import java.time.Instant;
 import java.time.format.DateTimeFormatter;
@@ -28,10 +29,6 @@ import javax.xml.transform.stream.StreamResult;
  */
 public class ListRecordsResponse extends AbstractOAIResponse {
 
-    private static final String[] RETRY_AFTER_HEADERS = {
-            "retry-after", "Retry-after", "Retry-After"
-    };
-
     private final ListRecordsRequest request;
 
     private ListRecordsFilterReader filterreader;
@@ -112,7 +109,7 @@ public class ListRecordsResponse extends AbstractOAIResponse {
                 transformerFactory.setURIResolver(new TransformerURIResolver("xsl"));
                 Transformer transformer = transformerFactory.newTransformer();
                 Source source = new SAXSource(filterreader, new InputSource(new StringReader(XMLUtil.sanitize(message))));
-                StreamResult streamResult = new StreamResult(writer);
+                StreamResult streamResult = writer != null ? new StreamResult(writer) : new StreamResult(new StringWriter());
                 transformer.transform(source, streamResult);
                 if ("noRecordsMatch".equals(error)) {
                     throw new NoRecordsMatchException("metadataPrefix=" + request.getMetadataPrefix()