diff --git a/gradle.properties b/gradle.properties index b27d144..c6c6ed8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group = org.xbib name = oai -version = 2.5.1 +version = 2.5.2 gradle.wrapper.version = 6.6.1 xbib-content.version = 2.6.2 diff --git a/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java b/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java index 015a018..ac8da82 100644 --- a/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java +++ b/oai-client/src/main/java/org/xbib/oai/client/OAIClient.java @@ -9,8 +9,9 @@ import org.xbib.oai.client.listmetadataformats.ListMetadataFormatsRequest; import org.xbib.oai.client.listrecords.ListRecordsRequest; import org.xbib.oai.client.listrecords.ListRecordsResponse; import org.xbib.oai.client.listsets.ListSetsRequest; +import org.xbib.oai.exceptions.NoRecordsMatchException; import org.xbib.oai.util.ResumptionToken; -import org.xbib.oai.xml.SimpleMetadataHandler; +import org.xbib.oai.xml.MetadataHandler; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -23,6 +24,10 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.Period; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.function.Consumer; import java.util.logging.Level; @@ -48,7 +53,7 @@ public class OAIClient { .build(); } - void setUserAgent(String userAgent) { + public void setUserAgent(String userAgent) { this.userAgent = userAgent; } @@ -142,50 +147,91 @@ public class OAIClient { Instant from, Instant until, Writer writer, - SimpleMetadataHandler handler) { - ListRecordsRequest listRecordsRequest = new ListRecordsRequest(); - if (metadataPrefix != null) { - listRecordsRequest.setMetadataPrefix(metadataPrefix); - } - if (set != null) { - listRecordsRequest.setSet(set); - } - if (dateTimeFormatter != null) { - listRecordsRequest.setDateTimeFormatter(dateTimeFormatter); - } - if (from != null) { - listRecordsRequest.setFrom(from); - } - if (until != null) { - listRecordsRequest.setUntil(until); - } - while (listRecordsRequest != null) { - try { - if (handler != null) { - listRecordsRequest.addHandler(handler); - } - ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest); - URL.Builder url = URL.from(baseURL).mutator(); - listRecordsRequest.getParams().forEach(url::queryParam); - HttpRequest httpRequest = HttpRequest.newBuilder() - .uri(URI.create(url.build().toExternalForm())) - .header("accept", "utf-8") - .GET() - .build(); - logger.log(Level.INFO,"sending " + httpRequest); - HttpResponse httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); - int status = httpResponse.statusCode(); - String contentType = httpResponse.headers().firstValue("content-type").orElse(null); - String retryAfter = httpResponse.headers().firstValue("retry-after").orElse(null); - listRecordsResponse.receivedResponse(httpResponse.body(), status, contentType, retryAfter, writer); - logger.log(Level.FINE, "response headers = " + httpResponse.headers() + - " resumption-token = " + listRecordsResponse.getResumptionToken()); - listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); - } catch (Exception e) { - logger.log(Level.SEVERE, e.getMessage(), e); - listRecordsRequest = null; + MetadataHandler handler) { + listRecords(metadataPrefix, set, dateTimeFormatter, from, until, null, writer, handler); + } + + public void listRecords(String metadataPrefix, + String set, + DateTimeFormatter dateTimeFormatter, + Instant from, + Instant until, + Instant base, + Writer writer, + MetadataHandler handler) { + do { + ListRecordsRequest listRecordsRequest = new ListRecordsRequest(); + if (metadataPrefix != null) { + listRecordsRequest.setMetadataPrefix(metadataPrefix); } - } + if (set != null) { + listRecordsRequest.setSet(set); + } + if (dateTimeFormatter != null) { + listRecordsRequest.setDateTimeFormatter(dateTimeFormatter); + } + if (from != null) { + listRecordsRequest.setFrom(from); + } + if (until != null) { + listRecordsRequest.setUntil(until); + } + if (from != null && until != null) { + if (until.isBefore(from)) { + throw new IllegalArgumentException("until must not be before from"); + } + } + while (listRecordsRequest != null) { + try { + if (handler != null) { + listRecordsRequest.addHandler(handler); + } + ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest); + URL.Builder url = URL.from(baseURL).mutator(); + listRecordsRequest.getParams().forEach(url::queryParam); + HttpRequest httpRequest = HttpRequest.newBuilder() + .uri(URI.create(url.build().toExternalForm())) + .header("accept", "utf-8") + .header("user-agent", userAgent != null ? userAgent : "xbib OAI client") + .GET() + .build(); + logger.log(Level.INFO, "sending " + httpRequest); + HttpResponse httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); + int status = httpResponse.statusCode(); + String contentType = httpResponse.headers().firstValue("content-type").orElse(null); + String retryAfter = httpResponse.headers().firstValue("retry-after").orElse(null); + listRecordsResponse.receivedResponse(httpResponse.body(), status, contentType, retryAfter, writer); + logger.log(Level.FINE, "response headers = " + httpResponse.headers() + + " resumption-token = " + listRecordsResponse.getResumptionToken()); + listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); + } catch (NoRecordsMatchException e) { + logger.log(Level.WARNING, "no records match"); + } catch (Exception e) { + logger.log(Level.SEVERE, e.getMessage(), e); + listRecordsRequest = null; + } + } + if (base != null && from != null && until != null) { + LocalDate fromLocalDate = LocalDate.ofInstant(from, ZoneOffset.UTC); + LocalDate untilLocalDate = LocalDate.ofInstant(until, ZoneOffset.UTC); + Period period = Period.between(fromLocalDate, untilLocalDate); + logger.log(Level.INFO, "from = " + fromLocalDate + " until = " + untilLocalDate + " period = " + period); + if (period.getYears() > 0 || period.getMonths() > 0 || period.getDays() > 0) { + from = LocalDateTime.ofInstant(from, ZoneOffset.UTC) + .plusYears(-period.getYears()) + .plusMonths(-period.getMonths()) + .plusDays(-period.getDays()) + .toInstant(ZoneOffset.UTC); + until = LocalDateTime.ofInstant(until, ZoneOffset.UTC) + .plusYears(-period.getYears()) + .plusMonths(-period.getMonths()) + .plusDays(-period.getDays()) + .toInstant(ZoneOffset.UTC); + } else { + throw new IllegalStateException("from = " + from + " until = " + until + ": period is zero"); + } + } + } while (base != null && from != null && until != null && from.isAfter(base) && until.isAfter(base)); } public void listRecords(String metadataPrefix, @@ -194,50 +240,90 @@ public class OAIClient { Instant from, Instant until, Consumer consumer) { - ListRecordsRequest listRecordsRequest = new ListRecordsRequest(); - if (metadataPrefix != null) { - listRecordsRequest.setMetadataPrefix(metadataPrefix); - } - if (set != null) { - listRecordsRequest.setSet(set); - } - if (dateTimeFormatter != null) { - listRecordsRequest.setDateTimeFormatter(dateTimeFormatter); - } - if (from != null) { - listRecordsRequest.setFrom(from); - } - if (until != null) { - listRecordsRequest.setUntil(until); - } - while (listRecordsRequest != null) { - try { - StringWriter sw = new StringWriter(); - ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest); - URL.Builder url = URL.from(baseURL).mutator(); - listRecordsRequest.getParams().forEach(url::queryParam); - HttpRequest httpRequest = HttpRequest.newBuilder() - .uri(URI.create(url.build().toExternalForm())) - .header("accept", "utf-8") - .GET() - .build(); - logger.log(Level.INFO,"sending " + httpRequest); - HttpResponse httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofByteArray()); - int status = httpResponse.statusCode(); - String contentType = httpResponse.headers().firstValue("content-type").orElse(null); - String retryAfter = httpResponse.headers().firstValue("retry-after").orElse(null); - listRecordsResponse.receivedResponse(new String(httpResponse.body(), StandardCharsets.UTF_8), status, contentType, retryAfter, sw); - if (consumer != null) { - consumer.accept(new ByteArrayInputStream(httpResponse.body())); - } - logger.log(Level.FINE, "response headers = " + httpResponse.headers() + - " resumption-token = " + listRecordsResponse.getResumptionToken()); - listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); - } catch (Exception e) { - logger.log(Level.SEVERE, e.getMessage(), e); - listRecordsRequest = null; + listRecords(metadataPrefix, set, dateTimeFormatter, from, until, null, consumer); + } + + public void listRecords(String metadataPrefix, + String set, + DateTimeFormatter dateTimeFormatter, + Instant from, + Instant until, + Instant base, + Consumer consumer) { + do { + ListRecordsRequest listRecordsRequest = new ListRecordsRequest(); + if (metadataPrefix != null) { + listRecordsRequest.setMetadataPrefix(metadataPrefix); } - } + if (set != null) { + listRecordsRequest.setSet(set); + } + if (dateTimeFormatter != null) { + listRecordsRequest.setDateTimeFormatter(dateTimeFormatter); + } + if (from != null) { + listRecordsRequest.setFrom(from); + } + if (until != null) { + listRecordsRequest.setUntil(until); + } + if (from != null && until != null) { + if (until.isBefore(from)) { + throw new IllegalArgumentException("until must not be before from"); + } + } + while (listRecordsRequest != null) { + try { + StringWriter sw = new StringWriter(); + ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest); + URL.Builder url = URL.from(baseURL).mutator(); + listRecordsRequest.getParams().forEach(url::queryParam); + HttpRequest httpRequest = HttpRequest.newBuilder() + .uri(URI.create(url.build().toExternalForm())) + .header("accept", "utf-8") + .header("user-agent", userAgent != null ? userAgent : "xbib OAI client") + .GET() + .build(); + logger.log(Level.INFO, "sending " + httpRequest); + HttpResponse httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofByteArray()); + int status = httpResponse.statusCode(); + String contentType = httpResponse.headers().firstValue("content-type").orElse(null); + String retryAfter = httpResponse.headers().firstValue("retry-after").orElse(null); + listRecordsResponse.receivedResponse(new String(httpResponse.body(), StandardCharsets.UTF_8), status, contentType, retryAfter, sw); + if (consumer != null) { + consumer.accept(new ByteArrayInputStream(httpResponse.body())); + } + logger.log(Level.FINE, "response headers = " + httpResponse.headers() + + " resumption-token = " + listRecordsResponse.getResumptionToken()); + listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); + } catch (NoRecordsMatchException e) { + logger.log(Level.WARNING, "no records match"); + } catch (Exception e) { + logger.log(Level.SEVERE, e.getMessage(), e); + listRecordsRequest = null; + } + } + if (base != null && from != null && until != null) { + LocalDate fromLocalDate = LocalDate.ofInstant(from, ZoneOffset.UTC); + LocalDate untilLocalDate = LocalDate.ofInstant(until, ZoneOffset.UTC); + Period period = Period.between(fromLocalDate, untilLocalDate); + logger.log(Level.INFO, "from = " + fromLocalDate + " until = " + untilLocalDate + " period = " + period); + if (period.getYears() > 0 || period.getMonths() > 0 || period.getDays() > 0) { + from = LocalDateTime.ofInstant(from, ZoneOffset.UTC) + .plusYears(-period.getYears()) + .plusMonths(-period.getMonths()) + .plusDays(-period.getDays()) + .toInstant(ZoneOffset.UTC); + until = LocalDateTime.ofInstant(until, ZoneOffset.UTC) + .plusYears(-period.getYears()) + .plusMonths(-period.getMonths()) + .plusDays(-period.getDays()) + .toInstant(ZoneOffset.UTC); + } else { + throw new IllegalStateException("from = " + from + " until = " + until + ": period is zero"); + } + } + } while (base != null && from != null && until != null && from.isAfter(base) && until.isAfter(base)); } public IdentifyRequest resume(IdentifyRequest request, ResumptionToken token) { diff --git a/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java b/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java index 142d450..0be3a0a 100644 --- a/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java +++ b/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java @@ -12,6 +12,7 @@ import org.xbib.oai.util.ResumptionToken; import org.xml.sax.InputSource; import java.io.StringReader; +import java.io.StringWriter; import java.io.Writer; import java.time.Instant; import java.time.format.DateTimeFormatter; @@ -28,10 +29,6 @@ import javax.xml.transform.stream.StreamResult; */ public class ListRecordsResponse extends AbstractOAIResponse { - private static final String[] RETRY_AFTER_HEADERS = { - "retry-after", "Retry-after", "Retry-After" - }; - private final ListRecordsRequest request; private ListRecordsFilterReader filterreader; @@ -112,7 +109,7 @@ public class ListRecordsResponse extends AbstractOAIResponse { transformerFactory.setURIResolver(new TransformerURIResolver("xsl")); Transformer transformer = transformerFactory.newTransformer(); Source source = new SAXSource(filterreader, new InputSource(new StringReader(XMLUtil.sanitize(message)))); - StreamResult streamResult = new StreamResult(writer); + StreamResult streamResult = writer != null ? new StreamResult(writer) : new StreamResult(new StringWriter()); transformer.transform(source, streamResult); if ("noRecordsMatch".equals(error)) { throw new NoRecordsMatchException("metadataPrefix=" + request.getMetadataPrefix()