move tests to xbib netty http client

This commit is contained in:
Jörg Prante 2019-08-08 10:21:04 +02:00
parent 234edaf99c
commit 270acc90a1
9 changed files with 77 additions and 126 deletions

View file

@ -31,13 +31,13 @@ subprojects {
}
compileJava {
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
sourceCompatibility = JavaVersion.VERSION_1_8
targetCompatibility = JavaVersion.VERSION_1_8
}
compileTestJava {
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
sourceCompatibility = JavaVersion.VERSION_1_8
targetCompatibility = JavaVersion.VERSION_1_8
}

View file

@ -1,6 +1,6 @@
group = org.xbib
name = oai
version = 1.3.0
version = 2.0.0
xbib-content.version = 1.3.1
xbib-netty-http.version = 4.1.38.2

View file

@ -35,7 +35,6 @@ public abstract class AbstractOAIRequest implements OAIRequest {
.host(url.getHost())
.port(url.getPort())
.path(url.getPath());
}
public URL getURL() {
@ -98,7 +97,7 @@ public abstract class AbstractOAIRequest implements OAIRequest {
public void setResumptionToken(ResumptionToken<?> token) {
this.token = token;
if (token != null && token.toString() != null) {
// resumption token may have characters that are illegal in URIs like '|'
// note: resumption token may have characters that are illegal in URIs, like '|'
addParameter(OAIConstants.RESUMPTION_TOKEN_PARAMETER, token.toString());
}
}
@ -124,53 +123,4 @@ public abstract class AbstractOAIRequest implements OAIRequest {
+ ",until=" + getUntil()
+ "]";
}
/*class GetRecord extends AbstractOAIRequest {
public GetRecord() {
super(url);
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.GET_RECORD);
}
}
class Identify extends AbstractOAIRequest {
public Identify() {
super(url);
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.IDENTIFY);
}
}
class ListIdentifiers extends AbstractOAIRequest {
public ListIdentifiers() {
super(url);
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_IDENTIFIERS);
}
}
class ListMetadataFormats extends AbstractOAIRequest {
public ListMetadataFormats() {
super(url);
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_METADATA_FORMATS);
}
}
class ListRecordsRequest extends AbstractOAIRequest {
public ListRecordsRequest() {
super(url);
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_RECORDS);
}
}
class ListSetsRequest extends AbstractOAIRequest {
public ListSetsRequest() {
super(url);
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_SETS);
}
}*/
}

View file

@ -1,7 +1,6 @@
package org.xbib.oai.client;
import org.xbib.net.URL;
import org.xbib.netty.http.client.Client;
import org.xbib.oai.client.getrecord.GetRecordRequest;
import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.listidentifiers.ListIdentifiersRequest;
@ -15,26 +14,16 @@ import org.xbib.oai.util.ResumptionToken;
*/
public class OAIClient implements AutoCloseable {
private Client client;
private final URL url;
public OAIClient(URL url) {
this.url = url;
this.client = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
}
public URL getURL() {
return url;
}
public Client getHttpClient() {
return client;
}
/**
* This verb is used to retrieve information about a repository.
* Some of the information returned is required as part of the OAI-PMH.

View file

@ -6,7 +6,6 @@ import org.xbib.content.xml.util.XMLUtil;
import org.xbib.netty.http.common.HttpResponse;
import org.xbib.oai.client.AbstractOAIResponse;
import org.xbib.oai.exceptions.BadVerbException;
import org.xbib.oai.exceptions.TooManyRequestsException;
import org.xbib.oai.exceptions.BadArgumentException;
import org.xbib.oai.exceptions.BadResumptionTokenException;
import org.xbib.oai.exceptions.NoRecordsMatchException;
@ -36,6 +35,7 @@ import javax.xml.transform.stream.StreamResult;
public class ListRecordsResponse extends AbstractOAIResponse {
private static final Logger logger = Logger.getLogger(ListRecordsResponse.class.getName());
private static final String[] RETRY_AFTER_HEADERS = {
"retry-after", "Retry-after", "Retry-After"
};
@ -112,6 +112,7 @@ public class ListRecordsResponse extends AbstractOAIResponse {
}
if (status == 429) {
try {
logger.log(Level.WARNING, "received 429 Too many requests, waiting 10 seconds...");
Thread.sleep(10000L);
} catch (InterruptedException e) {
// ignore

View file

@ -37,7 +37,10 @@ public class ArxivClientTest {
final URL url = URL.create("http://export.arxiv.org/oai2/");
try (OAIClient client = new OAIClient(url)) {
IdentifyRequest identifyRequest = client.newIdentifyRequest();
Client httpClient = client.getHttpClient();
Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
IdentifyResponse identifyResponse = new IdentifyResponse();
Request request = Request.get()
.url(identifyRequest.getURL())
@ -88,6 +91,7 @@ public class ArxivClientTest {
}
}
fileWriter.close();
httpClient.shutdownGracefully();
logger.log(Level.INFO, "count = " + handler.count());
assertTrue(handler.count() > 0L);
} catch (Exception e) {

View file

@ -17,6 +17,7 @@ import org.xbib.oai.exceptions.OAIException;
import java.io.IOException;
import java.io.StringWriter;
import java.net.ConnectException;
import java.nio.charset.StandardCharsets;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
@ -26,42 +27,38 @@ import java.util.logging.Logger;
/**
*
*/
@Ignore
public class BundeskunsthalleTest {
private static final Logger logger = Logger.getLogger(BundeskunsthalleTest.class.getName());
@Test
@Ignore // takes too long time and creates files
public void testListRecords() throws Exception {
URL url = URL.create("http://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh");
public void testListRecords() {
URL url = URL.create("https://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh");
try (OAIClient oaiClient = new OAIClient(url)) {
Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
Client httpClient = oaiClient.getHttpClient();
IdentifyResponse identifyResponse = new IdentifyResponse();
Request request = Request.get()
.url(url.resolve(identifyRequest.getURL()))
.url(identifyRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.setFollowRedirect(true)
.build()
.setResponseListener(resp -> {
logger.log(Level.INFO,
"status = " + resp.getStatus() +
" body = " + resp.getBodyAsString(StandardCharsets.UTF_8));
StringWriter sw = new StringWriter();
identifyResponse.receivedResponse(resp, sw);
});
httpClient.execute(request).get();
/*AggregatedHttpMessage response = client.execute(HttpHeaders.of(HttpMethod.GET, identifyRequest.getPath())
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();*/
// follow a maximum of 10 HTTP redirects
/*int max = 10;
while (response.followUrl() != null && max-- > 0) {
URI uri = URI.create(response.followUrl());
client = Clients.newClient(oaiClient.getFactory(), "none+" + uri, HttpClient.class);
response = client.execute(HttpHeaders.of(HttpMethod.GET, response.followUrl())
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
}*/
String granularity = identifyResponse.getGranularity();
logger.log(Level.INFO, "granularity = " + granularity);
DateTimeFormatter dateTimeFormatter = "YYYY-MM-DD".equals(granularity) ?
DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("GMT")) : null;
DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")) : null;
ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest();
listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
listRecordsRequest.setMetadataPrefix("marcxml");
@ -73,11 +70,13 @@ public class BundeskunsthalleTest {
while (listRecordsRequest != null) {
try {
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
logger.log(Level.INFO,"sending " + listRecordsRequest.getURL());
logger.log(Level.INFO, "sending " + listRecordsRequest.getURL());
StringWriter sw = new StringWriter();
request = Request.get()
.url(url.resolve(listRecordsRequest.getURL()))
.url(listRecordsRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.setFollowRedirect(true)
.setTimeoutInMillis(60 * 1000)
.build()
.setResponseListener(resp -> {
try {
@ -92,22 +91,18 @@ public class BundeskunsthalleTest {
.build()
.xmlReader().parse();
} catch (IOException e) {
throw new OAIException(e);
throw new OAIException("MARC parser exception: " + e.getMessage(), e);
}
listRecordsResponse.receivedResponse(resp, sw);
logger.log(Level.FINE, "response headers = " + resp.getHeaders() +
" resumption-token = {}" + listRecordsResponse.getResumptionToken());
logger.log(Level.FINE,
"status = " + resp.getStatus() +
" headers = " + resp.getHeaders() +
" resumptiontoken = " + listRecordsResponse.getResumptionToken());
});
httpClient.execute(request).get();
// follow a maximum of 10 HTTP redirects
/*max = 10;
while (response.followUrl() != null && max-- > 0) {
URI uri = URI.create(response.followUrl());
client = Clients.newClient(oaiClient.getFactory(), "none+" + uri, HttpClient.class);
response = client.execute(HttpHeaders.of(HttpMethod.GET, response.followUrl())
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
}*/
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} catch (ConnectException e) {
logger.log(Level.WARNING, e.getMessage(), e);
} catch (IOException e) {
logger.log(Level.SEVERE, e.getMessage(), e);
listRecordsRequest = null;
@ -116,6 +111,7 @@ public class BundeskunsthalleTest {
writer.endCollection();
writer.endDocument();
}
logger.log(Level.INFO, "completed");
} catch (Exception e) {
logger.log(Level.WARNING, e.getMessage(), e);
}

View file

@ -1,7 +1,5 @@
package org.xbib.oai.client;
import static org.junit.Assert.assertEquals;
import io.netty.handler.codec.http.HttpHeaderNames;
import org.junit.Ignore;
import org.junit.Test;
@ -9,6 +7,7 @@ import org.xbib.net.URL;
import org.xbib.netty.http.client.Client;
import org.xbib.netty.http.client.Request;
import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.identify.IdentifyResponse;
import org.xbib.oai.client.listrecords.ListRecordsRequest;
import org.xbib.oai.client.listrecords.ListRecordsResponse;
import org.xbib.oai.xml.SimpleMetadataHandler;
@ -16,8 +15,12 @@ import org.xbib.oai.xml.SimpleMetadataHandler;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringWriter;
import java.net.ConnectException;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
@ -25,58 +28,62 @@ import java.util.logging.Logger;
/**
*
*/
@Ignore
public class DNBClientTest {
private static final Logger logger = Logger.getLogger(DNBClientTest.class.getName());
@Test
public void testIdentify() {
public void testBibdat() {
URL url = URL.create("http://services.dnb.de/oai/repository");
try (OAIClient client = new OAIClient(url)) {
IdentifyRequest identifyRequest = client.newIdentifyRequest();
Client httpClient = client.getHttpClient();
assertEquals("/oai/repository?verb=Identify", identifyRequest.getURL().toString());
try (OAIClient oaiClient = new OAIClient(url)) {
Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
IdentifyResponse identifyResponse = new IdentifyResponse();
Request request = Request.get()
.url(url.resolve(identifyRequest.getURL()))
.url(identifyRequest.getURL())
.build()
.setResponseListener(resp -> logger.log(Level.INFO, resp.getBodyAsString(StandardCharsets.UTF_8)));
.setResponseListener(resp -> {
logger.log(Level.INFO, resp.getBodyAsString(StandardCharsets.UTF_8));
StringWriter sw = new StringWriter();
identifyResponse.receivedResponse(resp, sw);
});
httpClient.execute(request).get();
} catch (IOException e) {
logger.log(Level.SEVERE, e.getMessage(), e);
}
}
@Test
public void testListRecordsDNB() {
URL url = URL.create("http://services.dnb.de/oai/repository");
try (OAIClient client = new OAIClient(url)) {
ListRecordsRequest listRecordsRequest = client.newListRecordsRequest();
String granularity = identifyResponse.getGranularity();
logger.log(Level.INFO, "granularity = " + granularity);
DateTimeFormatter dateTimeFormatter = "YYYY-MM-DD".equals(granularity) ?
DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")) :
DateTimeFormatter.ISO_DATE_TIME;
ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest();
listRecordsRequest.setFrom(Instant.parse("2016-01-01T00:00:00Z"));
listRecordsRequest.setUntil(Instant.parse("2016-01-10T00:00:00Z"));
listRecordsRequest.setSet("bib");
listRecordsRequest.setMetadataPrefix("PicaPlus-xml");
Handler handler = new Handler();
File file = File.createTempFile("dnb-bib-pica.", ".xml");
file.deleteOnExit();
File file = new File("build/dnb-bib-pica.xml");
FileWriter fileWriter = new FileWriter(file);
while (listRecordsRequest != null) {
try {
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
listRecordsRequest.addHandler(handler);
Request request = Request.get()
.url(url.resolve(listRecordsRequest.getURL()))
request = Request.get()
.url(listRecordsRequest.getURL())
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.build()
.setResponseListener(resp -> listRecordsResponse.receivedResponse(resp, fileWriter));
client.getHttpClient().execute(request).get();
listRecordsRequest = client.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
httpClient.execute(request).get();
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} catch (ConnectException e) {
logger.log(Level.WARNING, e.getMessage(), e);
} catch (IOException e) {
logger.log(Level.SEVERE, e.getMessage(), e);
listRecordsRequest = null;
}
}
fileWriter.close();
httpClient.shutdownGracefully();
logger.log(Level.INFO, "count=" + handler.count());
} catch (Exception e) {
logger.log(Level.SEVERE, "skipped, HTTP exception");

View file

@ -34,8 +34,11 @@ public class DOAJClientTest {
public void testListRecordsDOAJ() {
URL url = URL.create("https://doaj.org/oai");
try (OAIClient oaiClient = new OAIClient(url)) {
Client httpClient = Client.builder()
.setConnectTimeoutMillis(60 * 1000)
.setReadTimeoutMillis(60 * 1000)
.build();
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
Client httpClient = oaiClient.getHttpClient();
IdentifyResponse identifyResponse = new IdentifyResponse();
Request request = Request.get()
.url(url.resolve(identifyRequest.getURL()))
@ -77,6 +80,7 @@ public class DOAJClientTest {
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
}
fileWriter.close();
httpClient.shutdownGracefully();
logger.log(Level.INFO, "count = " + handler.count());
} catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e);