diff --git a/gradle.properties b/gradle.properties index c6863a3..eecb243 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group = org.xbib name = oai -version = 1.2.0 +version = 1.2.1 xbib-content.version = 1.3.0 xbib-marc.version = 1.1.0 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index c44b679..f6b961f 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index cad203e..bf3de21 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,5 @@ -#Wed Feb 07 09:29:06 CET 2018 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-4.6-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.5.1-all.zip diff --git a/oai-client/src/main/java/org/xbib/oai/client/AbstractOAIRequest.java b/oai-client/src/main/java/org/xbib/oai/client/AbstractOAIRequest.java index b9d7caf..03647b8 100644 --- a/oai-client/src/main/java/org/xbib/oai/client/AbstractOAIRequest.java +++ b/oai-client/src/main/java/org/xbib/oai/client/AbstractOAIRequest.java @@ -135,6 +135,15 @@ public abstract class AbstractOAIRequest implements OAIRequest { return retry; } + @Override + public String toString() { + return "[request:metadataPrefix=" + getMetadataPrefix() + + ",set=" + getSet() + + ",from=" + getFrom() + + ",until=" + getUntil() + + "]"; + } + class GetRecord extends AbstractOAIRequest { public GetRecord() { diff --git a/oai-client/src/main/java/org/xbib/oai/client/TooManyRequestsException.java b/oai-client/src/main/java/org/xbib/oai/client/TooManyRequestsException.java new file mode 100644 index 0000000..0669569 --- /dev/null +++ b/oai-client/src/main/java/org/xbib/oai/client/TooManyRequestsException.java @@ -0,0 +1,7 @@ +package org.xbib.oai.client; + +import java.io.IOException; + +@SuppressWarnings("serial") +public class TooManyRequestsException extends IOException { +} diff --git a/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java b/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java index 86d2d67..e6e5052 100644 --- a/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java +++ b/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java @@ -6,6 +6,7 @@ import org.xbib.content.xml.transform.TransformerURIResolver; import org.xbib.content.xml.util.XMLUtil; import org.xbib.helianthus.common.http.AggregatedHttpMessage; import org.xbib.oai.client.AbstractOAIResponse; +import org.xbib.oai.client.TooManyRequestsException; import org.xbib.oai.exceptions.BadArgumentException; import org.xbib.oai.exceptions.BadResumptionTokenException; import org.xbib.oai.exceptions.NoRecordsMatchException; @@ -109,6 +110,9 @@ public class ListRecordsResponse extends AbstractOAIResponse { } return; } + if (status == 429) { + throw new TooManyRequestsException(); + } if (status != 200) { throw new IOException("status = " + status + " response = " + content); } diff --git a/oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java b/oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java index 4d14a79..ecf57ac 100644 --- a/oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java +++ b/oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java @@ -32,7 +32,6 @@ import java.util.concurrent.atomic.AtomicLong; /** * */ -@Ignore public class DOAJClientTest { private static final Logger logger = LogManager.getLogger(DOAJClientTest.class.getName()); @@ -64,21 +63,23 @@ public class DOAJClientTest { DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("GMT")) : null; ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest(); listRecordsRequest.setDateTimeFormatter(dateTimeFormatter); - listRecordsRequest.setFrom(Instant.parse("2017-01-01T00:00:00Z")); + listRecordsRequest.setFrom(Instant.parse("2008-01-01T00:00:00Z")); listRecordsRequest.setUntil(Instant.parse("2018-01-01T00:00:00Z")); listRecordsRequest.setMetadataPrefix("oai_dc"); Handler handler = new Handler(); - File file = File.createTempFile("doaj.", ".xml"); + File file = File.createTempFile("doaj.", ".xml"); file.deleteOnExit(); FileWriter fileWriter = new FileWriter(file); + ListRecordsResponse listRecordsResponse = null; while (listRecordsRequest != null) { try { - ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest); + logger.debug("request = {}", listRecordsRequest); + listRecordsResponse = new ListRecordsResponse(listRecordsRequest); logger.debug("response = {}", response.headers()); listRecordsRequest.addHandler(handler); client = oaiClient.getHttpClient(); response = client.execute(HttpHeaders.of(HttpMethod.GET, listRecordsRequest.getPath()) - .set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get(); + .set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get(); // follow a maximum of 10 HTTP redirects max = 10; while (response.followUrl() != null && max-- > 0) { @@ -89,6 +90,10 @@ public class DOAJClientTest { } listRecordsResponse.receivedResponse(response, fileWriter); listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); + } catch (TooManyRequestsException e) { + logger.error(e.getMessage(), e); + Thread.sleep(10000L); + listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); } catch (IOException e) { logger.error(e.getMessage(), e); listRecordsRequest = null;