better resuming

This commit is contained in:
Jörg Prante 2021-02-04 19:10:32 +01:00
parent 560bd890f0
commit e1369a2413
3 changed files with 23 additions and 93 deletions

View file

@ -1,6 +1,6 @@
group = org.xbib group = org.xbib
name = oai name = oai
version = 2.5.3 version = 2.5.4
gradle.wrapper.version = 6.6.1 gradle.wrapper.version = 6.6.1
xbib-content.version = 2.6.2 xbib-content.version = 2.6.2

View file

@ -1,6 +1,7 @@
package org.xbib.oai.client; package org.xbib.oai.client;
import org.xbib.net.URL; import org.xbib.net.URL;
import org.xbib.oai.OAIConstants;
import org.xbib.oai.client.getrecord.GetRecordRequest; import org.xbib.oai.client.getrecord.GetRecordRequest;
import org.xbib.oai.client.identify.IdentifyRequest; import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.identify.IdentifyResponse; import org.xbib.oai.client.identify.IdentifyResponse;
@ -99,18 +100,12 @@ public class OAIClient {
* to the formats available for a specific item. * to the formats available for a specific item.
* @return list metadata formats request * @return list metadata formats request
*/ */
public ListMetadataFormatsRequest newListMetadataFormatsRequest() {
return new ListMetadataFormatsRequest();
}
/** /**
* This verb is used to retrieve the set structure of a repository, * This verb is used to retrieve the set structure of a repository,
* useful for selective harvesting. * useful for selective harvesting.
* @return list sets request * @return list sets request
*/ */
public ListSetsRequest newListSetsRequest() {
return new ListSetsRequest();
}
/** /**
* This verb is an abbreviated form of ListRecords, retrieving only * This verb is an abbreviated form of ListRecords, retrieving only
@ -122,9 +117,6 @@ public class OAIClient {
* @return list identifiers request * @return list identifiers request
* *
*/ */
public ListIdentifiersRequest newListIdentifiersRequest() {
return new ListIdentifiersRequest();
}
/** /**
* This verb is used to retrieve an individual metadata record from * This verb is used to retrieve an individual metadata record from
@ -137,9 +129,6 @@ public class OAIClient {
* repository or from the specified item. * repository or from the specified item.
* @return get record request * @return get record request
*/ */
public GetRecordRequest newGetRecordRequest() {
return new GetRecordRequest();
}
/** /**
* This verb is used to harvest records from a repository. * This verb is used to harvest records from a repository.
@ -196,9 +185,20 @@ public class OAIClient {
} }
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest); ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
URL.Builder url = URL.from(baseURL).mutator(); URL.Builder url = URL.from(baseURL).mutator();
logger.log(Level.FINE, "before: baseURL = " + baseURL + " url = " + url.build() + " uri = ");
// kind of hacky here - suppress all OAI params if resumption token is present
if (listRecordsRequest.getResumptionToken() == null) {
logger.log(Level.FINE, "no resumption token, params are " + listRecordsRequest.getParams() );
listRecordsRequest.getParams().forEach(url::queryParam); listRecordsRequest.getParams().forEach(url::queryParam);
} else {
logger.log(Level.FINE, "no params because of resumption token " + listRecordsRequest.getResumptionToken());
url.queryParam(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_RECORDS);
url.queryParam(OAIConstants.RESUMPTION_TOKEN_PARAMETER, listRecordsRequest.getResumptionToken().toString());
}
URI uri = URI.create(url.build().toExternalForm());
logger.log(Level.FINE, "after: baseURL = " + baseURL + " url = " + url.build() + " uri = " + uri);
HttpRequest httpRequest = HttpRequest.newBuilder() HttpRequest httpRequest = HttpRequest.newBuilder()
.uri(URI.create(url.build().toExternalForm())) .uri(uri)
.header("accept", "utf-8") .header("accept", "utf-8")
.header("user-agent", userAgent != null ? userAgent : "xbib OAI client") .header("user-agent", userAgent != null ? userAgent : "xbib OAI client")
.GET() .GET()
@ -216,7 +216,6 @@ public class OAIClient {
byte[] b = httpResponse.body(); byte[] b = httpResponse.body();
if (b.length > 0) { if (b.length > 0) {
consumer.accept(new ByteArrayInputStream(b)); consumer.accept(new ByteArrayInputStream(b));
logger.log(Level.FINE, "body consumed: " + body);
} }
} else { } else {
HttpResponse<String> httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); HttpResponse<String> httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString());
@ -262,81 +261,15 @@ public class OAIClient {
} }
} }
public IdentifyRequest resume(IdentifyRequest request, ResumptionToken<?> token) {
if (request.isRetry()) {
request.setRetry(false);
return request;
}
if (token == null) {
return null;
}
IdentifyRequest nextRequest = new IdentifyRequest();
nextRequest.setResumptionToken(token);
return nextRequest;
}
public ListRecordsRequest resume(ListRecordsRequest request, ResumptionToken<?> token) { public ListRecordsRequest resume(ListRecordsRequest request, ResumptionToken<?> token) {
if (request.isRetry()) { ListRecordsRequest nextRequest = token != null ? new ListRecordsRequest() : request;
request.setRetry(false); if (token != null) {
return request;
}
if (token == null) {
return null;
}
ListRecordsRequest nextRequest = new ListRecordsRequest();
nextRequest.setResumptionToken(token); nextRequest.setResumptionToken(token);
}
if (request.isRetry()) {
nextRequest.setRetry(false);
return nextRequest; return nextRequest;
} }
return token != null ? nextRequest : null;
public ListIdentifiersRequest resume(ListIdentifiersRequest request, ResumptionToken<?> token) {
if (request.isRetry()) {
request.setRetry(false);
return request;
}
if (token == null) {
return null;
}
ListIdentifiersRequest nextRequest = newListIdentifiersRequest();
nextRequest.setResumptionToken(token);
return nextRequest;
}
public ListMetadataFormatsRequest resume(ListMetadataFormatsRequest request, ResumptionToken<?> token) {
if (request.isRetry()) {
request.setRetry(false);
return request;
}
if (token == null) {
return null;
}
ListMetadataFormatsRequest nextRequest = newListMetadataFormatsRequest();
nextRequest.setResumptionToken(token);
return nextRequest;
}
public ListSetsRequest resume(ListSetsRequest request, ResumptionToken<?> token) {
if (request.isRetry()) {
request.setRetry(false);
return request;
}
if (token == null) {
return null;
}
ListSetsRequest nextRequest = newListSetsRequest();
nextRequest.setResumptionToken(token);
return nextRequest;
}
public GetRecordRequest resume(GetRecordRequest request, ResumptionToken<?> token) {
if (request.isRetry()) {
request.setRetry(false);
return request;
}
if (token == null) {
return null;
}
GetRecordRequest nextRequest = newGetRecordRequest();
nextRequest.setResumptionToken(token);
return nextRequest;
} }
} }

View file

@ -7,12 +7,9 @@ import org.xbib.oai.xml.MetadataHandler;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
/**
*
*/
public class ListRecordsRequest extends AbstractOAIRequest { public class ListRecordsRequest extends AbstractOAIRequest {
private List<MetadataHandler> handlers = new LinkedList<>(); private final List<MetadataHandler> handlers = new LinkedList<>();
public ListRecordsRequest() { public ListRecordsRequest() {
super(); super();