joerg/oai
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixes for listing records

Browse source
This commit is contained in:
Jörg Prante 2021-02-03 18:45:56 +01:00
parent fb442a16fd
commit 37ca0ce41f
3 changed files with 177 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
gradle.properties
View file

@ -1,6 +1,6 @@
group = org.xbib
name = oai
version = 2.5.1
version = 2.5.2
gradle.wrapper.version = 6.6.1
xbib-content.version = 2.6.2

96
oai-client/src/main/java/org/xbib/oai/client/OAIClient.java
View file

@ -9,8 +9,9 @@ import org.xbib.oai.client.listmetadataformats.ListMetadataFormatsRequest;
import org.xbib.oai.client.listrecords.ListRecordsRequest;
import org.xbib.oai.client.listrecords.ListRecordsResponse;
import org.xbib.oai.client.listsets.ListSetsRequest;
import org.xbib.oai.exceptions.NoRecordsMatchException;
import org.xbib.oai.util.ResumptionToken;
import org.xbib.oai.xml.SimpleMetadataHandler;
import org.xbib.oai.xml.MetadataHandler;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@ -23,6 +24,10 @@ import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.Period;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.function.Consumer;
import java.util.logging.Level;
@ -48,7 +53,7 @@ public class OAIClient {
.build();
}
void setUserAgent(String userAgent) {
public void setUserAgent(String userAgent) {
this.userAgent = userAgent;
}
@ -142,7 +147,19 @@ public class OAIClient {
Instant from,
Instant until,
Writer writer,
SimpleMetadataHandler handler) {
MetadataHandler handler) {
listRecords(metadataPrefix, set, dateTimeFormatter, from, until, null, writer, handler);
}
public void listRecords(String metadataPrefix,
String set,
DateTimeFormatter dateTimeFormatter,
Instant from,
Instant until,
Instant base,
Writer writer,
MetadataHandler handler) {
do {
ListRecordsRequest listRecordsRequest = new ListRecordsRequest();
if (metadataPrefix != null) {
listRecordsRequest.setMetadataPrefix(metadataPrefix);
@ -159,6 +176,11 @@ public class OAIClient {
if (until != null) {
listRecordsRequest.setUntil(until);
}
if (from != null && until != null) {
if (until.isBefore(from)) {
throw new IllegalArgumentException("until must not be before from");
}
}
while (listRecordsRequest != null) {
try {
if (handler != null) {
@ -170,9 +192,10 @@ public class OAIClient {
HttpRequest httpRequest = HttpRequest.newBuilder()
.uri(URI.create(url.build().toExternalForm()))
.header("accept", "utf-8")
.header("user-agent", userAgent != null ? userAgent : "xbib OAI client")
.GET()
.build();
logger.log(Level.INFO,"sending " + httpRequest);
logger.log(Level.INFO, "sending " + httpRequest);
HttpResponse<String> httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString());
int status = httpResponse.statusCode();
String contentType = httpResponse.headers().firstValue("content-type").orElse(null);
@ -181,11 +204,34 @@ public class OAIClient {
logger.log(Level.FINE, "response headers = " + httpResponse.headers() +
" resumption-token = " + listRecordsResponse.getResumptionToken());
listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} catch (NoRecordsMatchException e) {
logger.log(Level.WARNING, "no records match");
} catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e);
listRecordsRequest = null;
}
}
if (base != null && from != null && until != null) {
LocalDate fromLocalDate = LocalDate.ofInstant(from, ZoneOffset.UTC);
LocalDate untilLocalDate = LocalDate.ofInstant(until, ZoneOffset.UTC);
Period period = Period.between(fromLocalDate, untilLocalDate);
logger.log(Level.INFO, "from = " + fromLocalDate + " until = " + untilLocalDate + " period = " + period);
if (period.getYears() > 0 || period.getMonths() > 0 || period.getDays() > 0) {
from = LocalDateTime.ofInstant(from, ZoneOffset.UTC)
.plusYears(-period.getYears())
.plusMonths(-period.getMonths())
.plusDays(-period.getDays())
.toInstant(ZoneOffset.UTC);
until = LocalDateTime.ofInstant(until, ZoneOffset.UTC)
.plusYears(-period.getYears())
.plusMonths(-period.getMonths())
.plusDays(-period.getDays())
.toInstant(ZoneOffset.UTC);
} else {
throw new IllegalStateException("from = " + from + " until = " + until + ": period is zero");
}
}
} while (base != null && from != null && until != null && from.isAfter(base) && until.isAfter(base));
}
public void listRecords(String metadataPrefix,
@ -194,6 +240,17 @@ public class OAIClient {
Instant from,
Instant until,
Consumer<InputStream> consumer) {
listRecords(metadataPrefix, set, dateTimeFormatter, from, until, null, consumer);
}
public void listRecords(String metadataPrefix,
String set,
DateTimeFormatter dateTimeFormatter,
Instant from,
Instant until,
Instant base,
Consumer<InputStream> consumer) {
do {
ListRecordsRequest listRecordsRequest = new ListRecordsRequest();
if (metadataPrefix != null) {
listRecordsRequest.setMetadataPrefix(metadataPrefix);
@ -210,6 +267,11 @@ public class OAIClient {
if (until != null) {
listRecordsRequest.setUntil(until);
}
if (from != null && until != null) {
if (until.isBefore(from)) {
throw new IllegalArgumentException("until must not be before from");
}
}
while (listRecordsRequest != null) {
try {
StringWriter sw = new StringWriter();
@ -219,9 +281,10 @@ public class OAIClient {
HttpRequest httpRequest = HttpRequest.newBuilder()
.uri(URI.create(url.build().toExternalForm()))
.header("accept", "utf-8")
.header("user-agent", userAgent != null ? userAgent : "xbib OAI client")
.GET()
.build();
logger.log(Level.INFO,"sending " + httpRequest);
logger.log(Level.INFO, "sending " + httpRequest);
HttpResponse<byte[]> httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofByteArray());
int status = httpResponse.statusCode();
String contentType = httpResponse.headers().firstValue("content-type").orElse(null);
@ -233,11 +296,34 @@ public class OAIClient {
logger.log(Level.FINE, "response headers = " + httpResponse.headers() +
" resumption-token = " + listRecordsResponse.getResumptionToken());
listRecordsRequest = resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} catch (NoRecordsMatchException e) {
logger.log(Level.WARNING, "no records match");
} catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e);
listRecordsRequest = null;
}
}
if (base != null && from != null && until != null) {
LocalDate fromLocalDate = LocalDate.ofInstant(from, ZoneOffset.UTC);
LocalDate untilLocalDate = LocalDate.ofInstant(until, ZoneOffset.UTC);
Period period = Period.between(fromLocalDate, untilLocalDate);
logger.log(Level.INFO, "from = " + fromLocalDate + " until = " + untilLocalDate + " period = " + period);
if (period.getYears() > 0 || period.getMonths() > 0 || period.getDays() > 0) {
from = LocalDateTime.ofInstant(from, ZoneOffset.UTC)
.plusYears(-period.getYears())
.plusMonths(-period.getMonths())
.plusDays(-period.getDays())
.toInstant(ZoneOffset.UTC);
until = LocalDateTime.ofInstant(until, ZoneOffset.UTC)
.plusYears(-period.getYears())
.plusMonths(-period.getMonths())
.plusDays(-period.getDays())
.toInstant(ZoneOffset.UTC);
} else {
throw new IllegalStateException("from = " + from + " until = " + until + ": period is zero");
}
}
} while (base != null && from != null && until != null && from.isAfter(base) && until.isAfter(base));
}
public IdentifyRequest resume(IdentifyRequest request, ResumptionToken<?> token) {

7
oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java
View file

@ -12,6 +12,7 @@ import org.xbib.oai.util.ResumptionToken;
import org.xml.sax.InputSource;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.Writer;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
@ -28,10 +29,6 @@ import javax.xml.transform.stream.StreamResult;
*/
public class ListRecordsResponse extends AbstractOAIResponse {
private static final String[] RETRY_AFTER_HEADERS = {
"retry-after", "Retry-after", "Retry-After"
};
private final ListRecordsRequest request;
private ListRecordsFilterReader filterreader;
@ -112,7 +109,7 @@ public class ListRecordsResponse extends AbstractOAIResponse {
transformerFactory.setURIResolver(new TransformerURIResolver("xsl"));
Transformer transformer = transformerFactory.newTransformer();
Source source = new SAXSource(filterreader, new InputSource(new StringReader(XMLUtil.sanitize(message))));
StreamResult streamResult = new StreamResult(writer);
StreamResult streamResult = writer != null ? new StreamResult(writer) : new StreamResult(new StringWriter());
transformer.transform(source, streamResult);
if ("noRecordsMatch".equals(error)) {
throw new NoRecordsMatchException("metadataPrefix=" + request.getMetadataPrefix()