fixes for listing records

2021-02-03 18:45:56 +01:00 · 2021-02-03 18:45:56 +01:00 · 37ca0ce41f
commit 37ca0ce41f
parent fb442a16fd
3 changed files with 177 additions and 94 deletions
--- a/gradle.properties
+++ b/gradle.properties
@ -1,6 +1,6 @@
 group = org.xbib
 name = oai
-version = 2.5.1
+version = 2.5.2

 gradle.wrapper.version = 6.6.1
 xbib-content.version = 2.6.2
--- a/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java
+++ b/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java
@ -9,8 +9,9 @@ import org.xbib.oai.client.listmetadataformats.ListMetadataFormatsRequest;
 import org.xbib.oai.client.listrecords.ListRecordsRequest;
 import org.xbib.oai.client.listrecords.ListRecordsResponse;
 import org.xbib.oai.client.listsets.ListSetsRequest;
+import org.xbib.oai.exceptions.NoRecordsMatchException;
 import org.xbib.oai.util.ResumptionToken;
-import org.xbib.oai.xml.SimpleMetadataHandler;
+import org.xbib.oai.xml.MetadataHandler;

 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@ -23,6 +24,10 @@ import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.Period;
+import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.util.function.Consumer;
 import java.util.logging.Level;
@ -48,7 +53,7 @@ public class OAIClient {
                .build();
    }

-    void setUserAgent(String userAgent) {
+    public void setUserAgent(String userAgent) {
        this.userAgent = userAgent;
    }

@ -142,50 +147,91 @@ public class OAIClient {
                            Instant from,
                            Instant until,
                            Writer writer,
-                            SimpleMetadataHandler handler) {
-        ListRecordsRequest listRecordsRequest = new ListRecordsRequest();
-        if (metadataPrefix != null) {
-            listRecordsRequest.setMetadataPrefix(metadataPrefix);
-        }
-        if (set != null) {
-            listRecordsRequest.setSet(set);
-        }
-        if (dateTimeFormatter != null) {
-            listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
-        }
-        if (from != null) {
-            listRecordsRequest.setFrom(from);
-        }
-        if (until != null) {
-            listRecordsRequest.setUntil(until);
-        }
-        while (listRecordsRequest != null) {
-            try {
-                if (handler != null) {
-                    listRecordsRequest.addHandler(handler);
-                }
-                ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
-                URL.Builder url = URL.from(baseURL).mutator();
-                listRecordsRequest.getParams().forEach(url::queryParam);
-                HttpRequest httpRequest = HttpRequest.newBuilder()
-                        .uri(URI.create(url.build().toExternalForm()))
-                        .header("accept", "utf-8")
-                        .GET()
-                        .build();
-                logger.log(Level.INFO,"sending " + httpRequest);
-                HttpResponse<String> httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString());
-                int status = httpResponse.statusCode();
-                String contentType = httpResponse.headers().firstValue("content-type").orElse(null);
-                String retryAfter = httpResponse.headers().firstValue("retry-after").orElse(null);
-                listRecordsResponse.receivedResponse(httpResponse.body(), status, contentType, retryAfter, writer);
-                logger.log(Level.FINE, "response headers = " + httpResponse.headers() +
-                        " resumption-token = " + listRecordsResponse.getResumptionToken());
-                listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
-            } catch (Exception e) {
-                logger.log(Level.SEVERE, e.getMessage(), e);
-                listRecordsRequest = null;
+                            MetadataHandler handler) {
+        listRecords(metadataPrefix, set, dateTimeFormatter, from, until, null, writer, handler);
+    }
+
+    public void listRecords(String metadataPrefix,
+                            String set,
+                            DateTimeFormatter dateTimeFormatter,
+                            Instant from,
+                            Instant until,
+                            Instant base,
+                            Writer writer,
+                            MetadataHandler handler) {
+        do {
+            ListRecordsRequest listRecordsRequest = new ListRecordsRequest();
+            if (metadataPrefix != null) {
+                listRecordsRequest.setMetadataPrefix(metadataPrefix);
            }
-        }
+            if (set != null) {
+                listRecordsRequest.setSet(set);
+            }
+            if (dateTimeFormatter != null) {
+                listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
+            }
+            if (from != null) {
+                listRecordsRequest.setFrom(from);
+            }
+            if (until != null) {
+                listRecordsRequest.setUntil(until);
+            }
+            if (from != null && until != null) {
+                if (until.isBefore(from)) {
+                    throw new IllegalArgumentException("until must not be before from");
+                }
+            }
+            while (listRecordsRequest != null) {
+                try {
+                    if (handler != null) {
+                        listRecordsRequest.addHandler(handler);
+                    }
+                    ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
+                    URL.Builder url = URL.from(baseURL).mutator();
+                    listRecordsRequest.getParams().forEach(url::queryParam);
+                    HttpRequest httpRequest = HttpRequest.newBuilder()
+                            .uri(URI.create(url.build().toExternalForm()))
+                            .header("accept", "utf-8")
+                            .header("user-agent", userAgent != null ? userAgent : "xbib OAI client")
+                            .GET()
+                            .build();
+                    logger.log(Level.INFO, "sending " + httpRequest);
+                    HttpResponse<String> httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString());
+                    int status = httpResponse.statusCode();
+                    String contentType = httpResponse.headers().firstValue("content-type").orElse(null);
+                    String retryAfter = httpResponse.headers().firstValue("retry-after").orElse(null);
+                    listRecordsResponse.receivedResponse(httpResponse.body(), status, contentType, retryAfter, writer);
+                    logger.log(Level.FINE, "response headers = " + httpResponse.headers() +
+                            " resumption-token = " + listRecordsResponse.getResumptionToken());
+                    listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
+                } catch (NoRecordsMatchException e) {
+                    logger.log(Level.WARNING, "no records match");
+                } catch (Exception e) {
+                    logger.log(Level.SEVERE, e.getMessage(), e);
+                    listRecordsRequest = null;
+                }
+            }
+            if (base != null && from != null && until != null) {
+                LocalDate fromLocalDate = LocalDate.ofInstant(from, ZoneOffset.UTC);
+                LocalDate untilLocalDate = LocalDate.ofInstant(until, ZoneOffset.UTC);
+                Period period =  Period.between(fromLocalDate, untilLocalDate);
+                logger.log(Level.INFO, "from = " + fromLocalDate + " until = " + untilLocalDate + " period = " + period);
+                if (period.getYears() > 0 || period.getMonths() > 0 || period.getDays() > 0) {
+                    from = LocalDateTime.ofInstant(from, ZoneOffset.UTC)
+                            .plusYears(-period.getYears())
+                            .plusMonths(-period.getMonths())
+                            .plusDays(-period.getDays())
+                            .toInstant(ZoneOffset.UTC);
+                    until = LocalDateTime.ofInstant(until, ZoneOffset.UTC)
+                            .plusYears(-period.getYears())
+                            .plusMonths(-period.getMonths())
+                            .plusDays(-period.getDays())
+                            .toInstant(ZoneOffset.UTC);
+                } else {
+                    throw new IllegalStateException("from = " + from + " until = " + until + ": period is zero");
+                }
+            }
+        } while (base != null && from != null && until != null && from.isAfter(base) && until.isAfter(base));
    }

    public void listRecords(String metadataPrefix,
@ -194,50 +240,90 @@ public class OAIClient {
                            Instant from,
                            Instant until,
                            Consumer<InputStream> consumer) {
-        ListRecordsRequest listRecordsRequest = new ListRecordsRequest();
-        if (metadataPrefix != null) {
-            listRecordsRequest.setMetadataPrefix(metadataPrefix);
-        }
-        if (set != null) {
-            listRecordsRequest.setSet(set);
-        }
-        if (dateTimeFormatter != null) {
-            listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
-        }
-        if (from != null) {
-            listRecordsRequest.setFrom(from);
-        }
-        if (until != null) {
-            listRecordsRequest.setUntil(until);
-        }
-        while (listRecordsRequest != null) {
-            try {
-                StringWriter sw = new StringWriter();
-                ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
-                URL.Builder url = URL.from(baseURL).mutator();
-                listRecordsRequest.getParams().forEach(url::queryParam);
-                HttpRequest httpRequest = HttpRequest.newBuilder()
-                        .uri(URI.create(url.build().toExternalForm()))
-                        .header("accept", "utf-8")
-                        .GET()
-                        .build();
-                logger.log(Level.INFO,"sending " + httpRequest);
-                HttpResponse<byte[]> httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofByteArray());
-                int status = httpResponse.statusCode();
-                String contentType = httpResponse.headers().firstValue("content-type").orElse(null);
-                String retryAfter = httpResponse.headers().firstValue("retry-after").orElse(null);
-                listRecordsResponse.receivedResponse(new String(httpResponse.body(), StandardCharsets.UTF_8), status, contentType, retryAfter, sw);
-                if (consumer != null) {
-                    consumer.accept(new ByteArrayInputStream(httpResponse.body()));
-                }
-                logger.log(Level.FINE, "response headers = " + httpResponse.headers() +
-                        " resumption-token = " + listRecordsResponse.getResumptionToken());
-                listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
-            } catch (Exception e) {
-                logger.log(Level.SEVERE, e.getMessage(), e);
-                listRecordsRequest = null;
+        listRecords(metadataPrefix, set, dateTimeFormatter, from, until, null, consumer);
+    }
+
+    public void listRecords(String metadataPrefix,
+                            String set,
+                            DateTimeFormatter dateTimeFormatter,
+                            Instant from,
+                            Instant until,
+                            Instant base,
+                            Consumer<InputStream> consumer) {
+        do {
+            ListRecordsRequest listRecordsRequest = new ListRecordsRequest();
+            if (metadataPrefix != null) {
+                listRecordsRequest.setMetadataPrefix(metadataPrefix);
            }
-        }
+            if (set != null) {
+                listRecordsRequest.setSet(set);
+            }
+            if (dateTimeFormatter != null) {
+                listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
+            }
+            if (from != null) {
+                listRecordsRequest.setFrom(from);
+            }
+            if (until != null) {
+                listRecordsRequest.setUntil(until);
+            }
+            if (from != null && until != null) {
+                if (until.isBefore(from)) {
+                    throw new IllegalArgumentException("until must not be before from");
+                }
+            }
+            while (listRecordsRequest != null) {
+                try {
+                    StringWriter sw = new StringWriter();
+                    ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
+                    URL.Builder url = URL.from(baseURL).mutator();
+                    listRecordsRequest.getParams().forEach(url::queryParam);
+                    HttpRequest httpRequest = HttpRequest.newBuilder()
+                            .uri(URI.create(url.build().toExternalForm()))
+                            .header("accept", "utf-8")
+                            .header("user-agent", userAgent != null ? userAgent : "xbib OAI client")
+                            .GET()
+                            .build();
+                    logger.log(Level.INFO, "sending " + httpRequest);
+                    HttpResponse<byte[]> httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofByteArray());
+                    int status = httpResponse.statusCode();
+                    String contentType = httpResponse.headers().firstValue("content-type").orElse(null);
+                    String retryAfter = httpResponse.headers().firstValue("retry-after").orElse(null);
+                    listRecordsResponse.receivedResponse(new String(httpResponse.body(), StandardCharsets.UTF_8), status, contentType, retryAfter, sw);
+                    if (consumer != null) {
+                        consumer.accept(new ByteArrayInputStream(httpResponse.body()));
+                    }
+                    logger.log(Level.FINE, "response headers = " + httpResponse.headers() +
+                            " resumption-token = " + listRecordsResponse.getResumptionToken());
+                    listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
+                } catch (NoRecordsMatchException e) {
+                    logger.log(Level.WARNING, "no records match");
+                } catch (Exception e) {
+                    logger.log(Level.SEVERE, e.getMessage(), e);
+                    listRecordsRequest = null;
+                }
+            }
+            if (base != null && from != null && until != null) {
+                LocalDate fromLocalDate = LocalDate.ofInstant(from, ZoneOffset.UTC);
+                LocalDate untilLocalDate = LocalDate.ofInstant(until, ZoneOffset.UTC);
+                Period period =  Period.between(fromLocalDate, untilLocalDate);
+                logger.log(Level.INFO, "from = " + fromLocalDate + " until = " + untilLocalDate + " period = " + period);
+                if (period.getYears() > 0 || period.getMonths() > 0 || period.getDays() > 0) {
+                    from = LocalDateTime.ofInstant(from, ZoneOffset.UTC)
+                            .plusYears(-period.getYears())
+                            .plusMonths(-period.getMonths())
+                            .plusDays(-period.getDays())
+                            .toInstant(ZoneOffset.UTC);
+                    until = LocalDateTime.ofInstant(until, ZoneOffset.UTC)
+                            .plusYears(-period.getYears())
+                            .plusMonths(-period.getMonths())
+                            .plusDays(-period.getDays())
+                            .toInstant(ZoneOffset.UTC);
+                } else {
+                    throw new IllegalStateException("from = " + from + " until = " + until + ": period is zero");
+                }
+            }
+        } while (base != null && from != null && until != null && from.isAfter(base) && until.isAfter(base));
    }

    public IdentifyRequest resume(IdentifyRequest request, ResumptionToken<?> token) {
--- a/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java
+++ b/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java
@ -12,6 +12,7 @@ import org.xbib.oai.util.ResumptionToken;
 import org.xml.sax.InputSource;

 import java.io.StringReader;
+import java.io.StringWriter;
 import java.io.Writer;
 import java.time.Instant;
 import java.time.format.DateTimeFormatter;
@ -28,10 +29,6 @@ import javax.xml.transform.stream.StreamResult;
 */
 public class ListRecordsResponse extends AbstractOAIResponse {

-    private static final String[] RETRY_AFTER_HEADERS = {
-            "retry-after", "Retry-after", "Retry-After"
-    };
-
    private final ListRecordsRequest request;

    private ListRecordsFilterReader filterreader;
@ -112,7 +109,7 @@ public class ListRecordsResponse extends AbstractOAIResponse {
                transformerFactory.setURIResolver(new TransformerURIResolver("xsl"));
                Transformer transformer = transformerFactory.newTransformer();
                Source source = new SAXSource(filterreader, new InputSource(new StringReader(XMLUtil.sanitize(message))));
-                StreamResult streamResult = new StreamResult(writer);
+                StreamResult streamResult = writer != null ? new StreamResult(writer) : new StreamResult(new StringWriter());
                transformer.transform(source, streamResult);
                if ("noRecordsMatch".equals(error)) {
                    throw new NoRecordsMatchException("metadataPrefix=" + request.getMetadataPrefix()