move tests to xbib netty http client
This commit is contained in:
parent
234edaf99c
commit
270acc90a1
9 changed files with 77 additions and 126 deletions
|
|
@ -31,13 +31,13 @@ subprojects {
|
|||
}
|
||||
|
||||
compileJava {
|
||||
sourceCompatibility = JavaVersion.VERSION_11
|
||||
targetCompatibility = JavaVersion.VERSION_11
|
||||
sourceCompatibility = JavaVersion.VERSION_1_8
|
||||
targetCompatibility = JavaVersion.VERSION_1_8
|
||||
}
|
||||
|
||||
compileTestJava {
|
||||
sourceCompatibility = JavaVersion.VERSION_11
|
||||
targetCompatibility = JavaVersion.VERSION_11
|
||||
sourceCompatibility = JavaVersion.VERSION_1_8
|
||||
targetCompatibility = JavaVersion.VERSION_1_8
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
group = org.xbib
|
||||
name = oai
|
||||
version = 1.3.0
|
||||
version = 2.0.0
|
||||
|
||||
xbib-content.version = 1.3.1
|
||||
xbib-netty-http.version = 4.1.38.2
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ public abstract class AbstractOAIRequest implements OAIRequest {
|
|||
.host(url.getHost())
|
||||
.port(url.getPort())
|
||||
.path(url.getPath());
|
||||
|
||||
}
|
||||
|
||||
public URL getURL() {
|
||||
|
|
@ -98,7 +97,7 @@ public abstract class AbstractOAIRequest implements OAIRequest {
|
|||
public void setResumptionToken(ResumptionToken<?> token) {
|
||||
this.token = token;
|
||||
if (token != null && token.toString() != null) {
|
||||
// resumption token may have characters that are illegal in URIs like '|'
|
||||
// note: resumption token may have characters that are illegal in URIs, like '|'
|
||||
addParameter(OAIConstants.RESUMPTION_TOKEN_PARAMETER, token.toString());
|
||||
}
|
||||
}
|
||||
|
|
@ -124,53 +123,4 @@ public abstract class AbstractOAIRequest implements OAIRequest {
|
|||
+ ",until=" + getUntil()
|
||||
+ "]";
|
||||
}
|
||||
|
||||
/*class GetRecord extends AbstractOAIRequest {
|
||||
|
||||
public GetRecord() {
|
||||
super(url);
|
||||
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.GET_RECORD);
|
||||
}
|
||||
}
|
||||
|
||||
class Identify extends AbstractOAIRequest {
|
||||
|
||||
public Identify() {
|
||||
super(url);
|
||||
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.IDENTIFY);
|
||||
}
|
||||
}
|
||||
|
||||
class ListIdentifiers extends AbstractOAIRequest {
|
||||
|
||||
public ListIdentifiers() {
|
||||
super(url);
|
||||
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_IDENTIFIERS);
|
||||
}
|
||||
}
|
||||
|
||||
class ListMetadataFormats extends AbstractOAIRequest {
|
||||
|
||||
public ListMetadataFormats() {
|
||||
super(url);
|
||||
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_METADATA_FORMATS);
|
||||
}
|
||||
}
|
||||
|
||||
class ListRecordsRequest extends AbstractOAIRequest {
|
||||
|
||||
public ListRecordsRequest() {
|
||||
super(url);
|
||||
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_RECORDS);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
class ListSetsRequest extends AbstractOAIRequest {
|
||||
|
||||
public ListSetsRequest() {
|
||||
super(url);
|
||||
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_SETS);
|
||||
}
|
||||
}*/
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
package org.xbib.oai.client;
|
||||
|
||||
import org.xbib.net.URL;
|
||||
import org.xbib.netty.http.client.Client;
|
||||
import org.xbib.oai.client.getrecord.GetRecordRequest;
|
||||
import org.xbib.oai.client.identify.IdentifyRequest;
|
||||
import org.xbib.oai.client.listidentifiers.ListIdentifiersRequest;
|
||||
|
|
@ -15,26 +14,16 @@ import org.xbib.oai.util.ResumptionToken;
|
|||
*/
|
||||
public class OAIClient implements AutoCloseable {
|
||||
|
||||
private Client client;
|
||||
|
||||
private final URL url;
|
||||
|
||||
public OAIClient(URL url) {
|
||||
this.url = url;
|
||||
this.client = Client.builder()
|
||||
.setConnectTimeoutMillis(60 * 1000)
|
||||
.setReadTimeoutMillis(60 * 1000)
|
||||
.build();
|
||||
}
|
||||
|
||||
public URL getURL() {
|
||||
return url;
|
||||
}
|
||||
|
||||
public Client getHttpClient() {
|
||||
return client;
|
||||
}
|
||||
|
||||
/**
|
||||
* This verb is used to retrieve information about a repository.
|
||||
* Some of the information returned is required as part of the OAI-PMH.
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import org.xbib.content.xml.util.XMLUtil;
|
|||
import org.xbib.netty.http.common.HttpResponse;
|
||||
import org.xbib.oai.client.AbstractOAIResponse;
|
||||
import org.xbib.oai.exceptions.BadVerbException;
|
||||
import org.xbib.oai.exceptions.TooManyRequestsException;
|
||||
import org.xbib.oai.exceptions.BadArgumentException;
|
||||
import org.xbib.oai.exceptions.BadResumptionTokenException;
|
||||
import org.xbib.oai.exceptions.NoRecordsMatchException;
|
||||
|
|
@ -36,6 +35,7 @@ import javax.xml.transform.stream.StreamResult;
|
|||
public class ListRecordsResponse extends AbstractOAIResponse {
|
||||
|
||||
private static final Logger logger = Logger.getLogger(ListRecordsResponse.class.getName());
|
||||
|
||||
private static final String[] RETRY_AFTER_HEADERS = {
|
||||
"retry-after", "Retry-after", "Retry-After"
|
||||
};
|
||||
|
|
@ -112,6 +112,7 @@ public class ListRecordsResponse extends AbstractOAIResponse {
|
|||
}
|
||||
if (status == 429) {
|
||||
try {
|
||||
logger.log(Level.WARNING, "received 429 Too many requests, waiting 10 seconds...");
|
||||
Thread.sleep(10000L);
|
||||
} catch (InterruptedException e) {
|
||||
// ignore
|
||||
|
|
|
|||
|
|
@ -37,7 +37,10 @@ public class ArxivClientTest {
|
|||
final URL url = URL.create("http://export.arxiv.org/oai2/");
|
||||
try (OAIClient client = new OAIClient(url)) {
|
||||
IdentifyRequest identifyRequest = client.newIdentifyRequest();
|
||||
Client httpClient = client.getHttpClient();
|
||||
Client httpClient = Client.builder()
|
||||
.setConnectTimeoutMillis(60 * 1000)
|
||||
.setReadTimeoutMillis(60 * 1000)
|
||||
.build();
|
||||
IdentifyResponse identifyResponse = new IdentifyResponse();
|
||||
Request request = Request.get()
|
||||
.url(identifyRequest.getURL())
|
||||
|
|
@ -88,6 +91,7 @@ public class ArxivClientTest {
|
|||
}
|
||||
}
|
||||
fileWriter.close();
|
||||
httpClient.shutdownGracefully();
|
||||
logger.log(Level.INFO, "count = " + handler.count());
|
||||
assertTrue(handler.count() > 0L);
|
||||
} catch (Exception e) {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import org.xbib.oai.exceptions.OAIException;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.net.ConnectException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
|
@ -26,42 +27,38 @@ import java.util.logging.Logger;
|
|||
/**
|
||||
*
|
||||
*/
|
||||
@Ignore
|
||||
public class BundeskunsthalleTest {
|
||||
|
||||
private static final Logger logger = Logger.getLogger(BundeskunsthalleTest.class.getName());
|
||||
|
||||
@Test
|
||||
@Ignore // takes too long time and creates files
|
||||
public void testListRecords() throws Exception {
|
||||
URL url = URL.create("http://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh");
|
||||
public void testListRecords() {
|
||||
URL url = URL.create("https://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh");
|
||||
try (OAIClient oaiClient = new OAIClient(url)) {
|
||||
Client httpClient = Client.builder()
|
||||
.setConnectTimeoutMillis(60 * 1000)
|
||||
.setReadTimeoutMillis(60 * 1000)
|
||||
.build();
|
||||
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
|
||||
Client httpClient = oaiClient.getHttpClient();
|
||||
IdentifyResponse identifyResponse = new IdentifyResponse();
|
||||
Request request = Request.get()
|
||||
.url(url.resolve(identifyRequest.getURL()))
|
||||
.url(identifyRequest.getURL())
|
||||
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
|
||||
.setFollowRedirect(true)
|
||||
.build()
|
||||
.setResponseListener(resp -> {
|
||||
logger.log(Level.INFO,
|
||||
"status = " + resp.getStatus() +
|
||||
" body = " + resp.getBodyAsString(StandardCharsets.UTF_8));
|
||||
StringWriter sw = new StringWriter();
|
||||
identifyResponse.receivedResponse(resp, sw);
|
||||
});
|
||||
httpClient.execute(request).get();
|
||||
|
||||
/*AggregatedHttpMessage response = client.execute(HttpHeaders.of(HttpMethod.GET, identifyRequest.getPath())
|
||||
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();*/
|
||||
// follow a maximum of 10 HTTP redirects
|
||||
/*int max = 10;
|
||||
while (response.followUrl() != null && max-- > 0) {
|
||||
URI uri = URI.create(response.followUrl());
|
||||
client = Clients.newClient(oaiClient.getFactory(), "none+" + uri, HttpClient.class);
|
||||
response = client.execute(HttpHeaders.of(HttpMethod.GET, response.followUrl())
|
||||
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
|
||||
}*/
|
||||
String granularity = identifyResponse.getGranularity();
|
||||
logger.log(Level.INFO, "granularity = " + granularity);
|
||||
DateTimeFormatter dateTimeFormatter = "YYYY-MM-DD".equals(granularity) ?
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("GMT")) : null;
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")) : null;
|
||||
ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest();
|
||||
listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
|
||||
listRecordsRequest.setMetadataPrefix("marcxml");
|
||||
|
|
@ -73,11 +70,13 @@ public class BundeskunsthalleTest {
|
|||
while (listRecordsRequest != null) {
|
||||
try {
|
||||
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
|
||||
logger.log(Level.INFO,"sending " + listRecordsRequest.getURL());
|
||||
logger.log(Level.INFO, "sending " + listRecordsRequest.getURL());
|
||||
StringWriter sw = new StringWriter();
|
||||
request = Request.get()
|
||||
.url(url.resolve(listRecordsRequest.getURL()))
|
||||
.url(listRecordsRequest.getURL())
|
||||
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
|
||||
.setFollowRedirect(true)
|
||||
.setTimeoutInMillis(60 * 1000)
|
||||
.build()
|
||||
.setResponseListener(resp -> {
|
||||
try {
|
||||
|
|
@ -92,22 +91,18 @@ public class BundeskunsthalleTest {
|
|||
.build()
|
||||
.xmlReader().parse();
|
||||
} catch (IOException e) {
|
||||
throw new OAIException(e);
|
||||
throw new OAIException("MARC parser exception: " + e.getMessage(), e);
|
||||
}
|
||||
listRecordsResponse.receivedResponse(resp, sw);
|
||||
logger.log(Level.FINE, "response headers = " + resp.getHeaders() +
|
||||
" resumption-token = {}" + listRecordsResponse.getResumptionToken());
|
||||
logger.log(Level.FINE,
|
||||
"status = " + resp.getStatus() +
|
||||
" headers = " + resp.getHeaders() +
|
||||
" resumptiontoken = " + listRecordsResponse.getResumptionToken());
|
||||
});
|
||||
httpClient.execute(request).get();
|
||||
// follow a maximum of 10 HTTP redirects
|
||||
/*max = 10;
|
||||
while (response.followUrl() != null && max-- > 0) {
|
||||
URI uri = URI.create(response.followUrl());
|
||||
client = Clients.newClient(oaiClient.getFactory(), "none+" + uri, HttpClient.class);
|
||||
response = client.execute(HttpHeaders.of(HttpMethod.GET, response.followUrl())
|
||||
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
|
||||
}*/
|
||||
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
|
||||
} catch (ConnectException e) {
|
||||
logger.log(Level.WARNING, e.getMessage(), e);
|
||||
} catch (IOException e) {
|
||||
logger.log(Level.SEVERE, e.getMessage(), e);
|
||||
listRecordsRequest = null;
|
||||
|
|
@ -116,6 +111,7 @@ public class BundeskunsthalleTest {
|
|||
writer.endCollection();
|
||||
writer.endDocument();
|
||||
}
|
||||
logger.log(Level.INFO, "completed");
|
||||
} catch (Exception e) {
|
||||
logger.log(Level.WARNING, e.getMessage(), e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
package org.xbib.oai.client;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import io.netty.handler.codec.http.HttpHeaderNames;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
|
@ -9,6 +7,7 @@ import org.xbib.net.URL;
|
|||
import org.xbib.netty.http.client.Client;
|
||||
import org.xbib.netty.http.client.Request;
|
||||
import org.xbib.oai.client.identify.IdentifyRequest;
|
||||
import org.xbib.oai.client.identify.IdentifyResponse;
|
||||
import org.xbib.oai.client.listrecords.ListRecordsRequest;
|
||||
import org.xbib.oai.client.listrecords.ListRecordsResponse;
|
||||
import org.xbib.oai.xml.SimpleMetadataHandler;
|
||||
|
|
@ -16,8 +15,12 @@ import org.xbib.oai.xml.SimpleMetadataHandler;
|
|||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.net.ConnectException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
|
@ -25,58 +28,62 @@ import java.util.logging.Logger;
|
|||
/**
|
||||
*
|
||||
*/
|
||||
@Ignore
|
||||
public class DNBClientTest {
|
||||
|
||||
private static final Logger logger = Logger.getLogger(DNBClientTest.class.getName());
|
||||
|
||||
@Test
|
||||
public void testIdentify() {
|
||||
public void testBibdat() {
|
||||
URL url = URL.create("http://services.dnb.de/oai/repository");
|
||||
try (OAIClient client = new OAIClient(url)) {
|
||||
IdentifyRequest identifyRequest = client.newIdentifyRequest();
|
||||
Client httpClient = client.getHttpClient();
|
||||
assertEquals("/oai/repository?verb=Identify", identifyRequest.getURL().toString());
|
||||
try (OAIClient oaiClient = new OAIClient(url)) {
|
||||
Client httpClient = Client.builder()
|
||||
.setConnectTimeoutMillis(60 * 1000)
|
||||
.setReadTimeoutMillis(60 * 1000)
|
||||
.build();
|
||||
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
|
||||
IdentifyResponse identifyResponse = new IdentifyResponse();
|
||||
Request request = Request.get()
|
||||
.url(url.resolve(identifyRequest.getURL()))
|
||||
.url(identifyRequest.getURL())
|
||||
.build()
|
||||
.setResponseListener(resp -> logger.log(Level.INFO, resp.getBodyAsString(StandardCharsets.UTF_8)));
|
||||
.setResponseListener(resp -> {
|
||||
logger.log(Level.INFO, resp.getBodyAsString(StandardCharsets.UTF_8));
|
||||
StringWriter sw = new StringWriter();
|
||||
identifyResponse.receivedResponse(resp, sw);
|
||||
});
|
||||
httpClient.execute(request).get();
|
||||
} catch (IOException e) {
|
||||
logger.log(Level.SEVERE, e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testListRecordsDNB() {
|
||||
URL url = URL.create("http://services.dnb.de/oai/repository");
|
||||
try (OAIClient client = new OAIClient(url)) {
|
||||
ListRecordsRequest listRecordsRequest = client.newListRecordsRequest();
|
||||
String granularity = identifyResponse.getGranularity();
|
||||
logger.log(Level.INFO, "granularity = " + granularity);
|
||||
DateTimeFormatter dateTimeFormatter = "YYYY-MM-DD".equals(granularity) ?
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")) :
|
||||
DateTimeFormatter.ISO_DATE_TIME;
|
||||
ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest();
|
||||
listRecordsRequest.setFrom(Instant.parse("2016-01-01T00:00:00Z"));
|
||||
listRecordsRequest.setUntil(Instant.parse("2016-01-10T00:00:00Z"));
|
||||
listRecordsRequest.setSet("bib");
|
||||
listRecordsRequest.setMetadataPrefix("PicaPlus-xml");
|
||||
Handler handler = new Handler();
|
||||
File file = File.createTempFile("dnb-bib-pica.", ".xml");
|
||||
file.deleteOnExit();
|
||||
File file = new File("build/dnb-bib-pica.xml");
|
||||
FileWriter fileWriter = new FileWriter(file);
|
||||
while (listRecordsRequest != null) {
|
||||
try {
|
||||
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
|
||||
listRecordsRequest.addHandler(handler);
|
||||
Request request = Request.get()
|
||||
.url(url.resolve(listRecordsRequest.getURL()))
|
||||
request = Request.get()
|
||||
.url(listRecordsRequest.getURL())
|
||||
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
|
||||
.build()
|
||||
.setResponseListener(resp -> listRecordsResponse.receivedResponse(resp, fileWriter));
|
||||
client.getHttpClient().execute(request).get();
|
||||
listRecordsRequest = client.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
|
||||
httpClient.execute(request).get();
|
||||
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
|
||||
} catch (ConnectException e) {
|
||||
logger.log(Level.WARNING, e.getMessage(), e);
|
||||
} catch (IOException e) {
|
||||
logger.log(Level.SEVERE, e.getMessage(), e);
|
||||
listRecordsRequest = null;
|
||||
}
|
||||
}
|
||||
fileWriter.close();
|
||||
httpClient.shutdownGracefully();
|
||||
logger.log(Level.INFO, "count=" + handler.count());
|
||||
} catch (Exception e) {
|
||||
logger.log(Level.SEVERE, "skipped, HTTP exception");
|
||||
|
|
|
|||
|
|
@ -34,8 +34,11 @@ public class DOAJClientTest {
|
|||
public void testListRecordsDOAJ() {
|
||||
URL url = URL.create("https://doaj.org/oai");
|
||||
try (OAIClient oaiClient = new OAIClient(url)) {
|
||||
Client httpClient = Client.builder()
|
||||
.setConnectTimeoutMillis(60 * 1000)
|
||||
.setReadTimeoutMillis(60 * 1000)
|
||||
.build();
|
||||
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
|
||||
Client httpClient = oaiClient.getHttpClient();
|
||||
IdentifyResponse identifyResponse = new IdentifyResponse();
|
||||
Request request = Request.get()
|
||||
.url(url.resolve(identifyRequest.getURL()))
|
||||
|
|
@ -77,6 +80,7 @@ public class DOAJClientTest {
|
|||
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
|
||||
}
|
||||
fileWriter.close();
|
||||
httpClient.shutdownGracefully();
|
||||
logger.log(Level.INFO, "count = " + handler.count());
|
||||
} catch (Exception e) {
|
||||
logger.log(Level.SEVERE, e.getMessage(), e);
|
||||
|
|
|
|||
Loading…
Reference in a new issue