move tests to xbib netty http client

build.gradle

@@ -31,13 +31,13 @@ subprojects {
     }
 
     compileJava {
-        sourceCompatibility = JavaVersion.VERSION_11
+        sourceCompatibility = JavaVersion.VERSION_1_8
-        targetCompatibility = JavaVersion.VERSION_11
+        targetCompatibility = JavaVersion.VERSION_1_8
     }
 
     compileTestJava {
-        sourceCompatibility = JavaVersion.VERSION_11
+        sourceCompatibility = JavaVersion.VERSION_1_8
-        targetCompatibility = JavaVersion.VERSION_11
+        targetCompatibility = JavaVersion.VERSION_1_8
 
     }
 

gradle.properties

@@ -1,6 +1,6 @@
 group = org.xbib
 name = oai
-version = 1.3.0
+version = 2.0.0
 
 xbib-content.version = 1.3.1
 xbib-netty-http.version = 4.1.38.2

oai-client/src/main/java/org/xbib/oai/client/AbstractOAIRequest.java

@@ -35,7 +35,6 @@ public abstract class AbstractOAIRequest implements OAIRequest {
                 .host(url.getHost())
                 .port(url.getPort())
                 .path(url.getPath());
-
     }
 
     public URL getURL() {
@@ -98,7 +97,7 @@ public abstract class AbstractOAIRequest implements OAIRequest {
     public void setResumptionToken(ResumptionToken<?> token) {
         this.token = token;
         if (token != null && token.toString() != null) {
-            // resumption token may have characters that are illegal in URIs like '|'
+            // note: resumption token may have characters that are illegal in URIs, like '|'
             addParameter(OAIConstants.RESUMPTION_TOKEN_PARAMETER, token.toString());
         }
     }
@@ -124,53 +123,4 @@ public abstract class AbstractOAIRequest implements OAIRequest {
                 + ",until=" + getUntil()
                 + "]";
     }
-
-    /*class GetRecord extends AbstractOAIRequest {
-
-        public GetRecord() {
-            super(url);
-            addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.GET_RECORD);
-        }
-    }
-
-    class Identify extends AbstractOAIRequest {
-
-        public Identify() {
-            super(url);
-            addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.IDENTIFY);
-        }
-    }
-
-    class ListIdentifiers extends AbstractOAIRequest {
-
-        public ListIdentifiers() {
-            super(url);
-            addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_IDENTIFIERS);
-        }
-    }
-
-    class ListMetadataFormats extends AbstractOAIRequest {
-
-        public ListMetadataFormats() {
-            super(url);
-            addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_METADATA_FORMATS);
-        }
-    }
-
-    class ListRecordsRequest extends AbstractOAIRequest {
-
-        public ListRecordsRequest() {
-            super(url);
-            addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_RECORDS);
-        }
-
-    }
-
-    class ListSetsRequest extends AbstractOAIRequest {
-
-        public ListSetsRequest() {
-            super(url);
-            addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_SETS);
-        }
-    }*/
 }

oai-client/src/main/java/org/xbib/oai/client/OAIClient.java

@@ -1,7 +1,6 @@
 package org.xbib.oai.client;
 
 import org.xbib.net.URL;
-import org.xbib.netty.http.client.Client;
 import org.xbib.oai.client.getrecord.GetRecordRequest;
 import org.xbib.oai.client.identify.IdentifyRequest;
 import org.xbib.oai.client.listidentifiers.ListIdentifiersRequest;
@@ -15,26 +14,16 @@ import org.xbib.oai.util.ResumptionToken;
  */
 public class OAIClient implements AutoCloseable {
 
-    private Client client;
-
     private final URL url;
 
     public OAIClient(URL url) {
         this.url = url;
-        this.client = Client.builder()
-                .setConnectTimeoutMillis(60 * 1000)
-                .setReadTimeoutMillis(60 * 1000)
-                .build();
     }
 
     public URL getURL() {
         return url;
     }
 
-    public Client getHttpClient() {
-        return client;
-    }
-
     /**
      * This verb is used to retrieve information about a repository.
      * Some of the information returned is required as part of the OAI-PMH.

oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java

@@ -6,7 +6,6 @@ import org.xbib.content.xml.util.XMLUtil;
 import org.xbib.netty.http.common.HttpResponse;
 import org.xbib.oai.client.AbstractOAIResponse;
 import org.xbib.oai.exceptions.BadVerbException;
-import org.xbib.oai.exceptions.TooManyRequestsException;
 import org.xbib.oai.exceptions.BadArgumentException;
 import org.xbib.oai.exceptions.BadResumptionTokenException;
 import org.xbib.oai.exceptions.NoRecordsMatchException;
@@ -36,6 +35,7 @@ import javax.xml.transform.stream.StreamResult;
 public class ListRecordsResponse extends AbstractOAIResponse {
 
     private static final Logger logger = Logger.getLogger(ListRecordsResponse.class.getName());
+
     private static final String[] RETRY_AFTER_HEADERS = {
             "retry-after", "Retry-after", "Retry-After"
     };
@@ -112,6 +112,7 @@ public class ListRecordsResponse extends AbstractOAIResponse {
         }
         if (status == 429) {
             try {
+                logger.log(Level.WARNING, "received 429 Too many requests, waiting 10 seconds...");
                 Thread.sleep(10000L);
             } catch (InterruptedException e) {
                 // ignore

oai-client/src/test/java/org/xbib/oai/client/ArxivClientTest.java

@@ -37,7 +37,10 @@ public class ArxivClientTest {
         final URL url = URL.create("http://export.arxiv.org/oai2/");
         try (OAIClient client = new OAIClient(url)) {
             IdentifyRequest identifyRequest = client.newIdentifyRequest();
-            Client httpClient = client.getHttpClient();
+            Client httpClient  = Client.builder()
+                    .setConnectTimeoutMillis(60 * 1000)
+                    .setReadTimeoutMillis(60 * 1000)
+                    .build();
             IdentifyResponse identifyResponse = new IdentifyResponse();
             Request request = Request.get()
                     .url(identifyRequest.getURL())
@@ -88,6 +91,7 @@ public class ArxivClientTest {
                 }
             }
             fileWriter.close();
+            httpClient.shutdownGracefully();
             logger.log(Level.INFO, "count = " + handler.count());
             assertTrue(handler.count() > 0L);
         } catch (Exception e) {

oai-client/src/test/java/org/xbib/oai/client/BundeskunsthalleTest.java

@@ -17,6 +17,7 @@ import org.xbib.oai.exceptions.OAIException;
 
 import java.io.IOException;
 import java.io.StringWriter;
+import java.net.ConnectException;
 import java.nio.charset.StandardCharsets;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
@@ -26,42 +27,38 @@ import java.util.logging.Logger;
 /**
  *
  */
+@Ignore
 public class BundeskunsthalleTest {
 
     private static final Logger logger = Logger.getLogger(BundeskunsthalleTest.class.getName());
 
     @Test
-    @Ignore // takes too long time and creates files
+    public void testListRecords() {
-    public void testListRecords() throws Exception {
+        URL url = URL.create("https://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh");
-        URL url = URL.create("http://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh");
         try (OAIClient oaiClient = new OAIClient(url)) {
+            Client httpClient  = Client.builder()
+                    .setConnectTimeoutMillis(60 * 1000)
+                    .setReadTimeoutMillis(60 * 1000)
+                    .build();
             IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
-            Client httpClient = oaiClient.getHttpClient();
             IdentifyResponse identifyResponse = new IdentifyResponse();
             Request request = Request.get()
-                    .url(url.resolve(identifyRequest.getURL()))
+                    .url(identifyRequest.getURL())
                     .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
+                    .setFollowRedirect(true)
                     .build()
                     .setResponseListener(resp -> {
+                        logger.log(Level.INFO,
+                                "status = " + resp.getStatus() +
+                                " body = " + resp.getBodyAsString(StandardCharsets.UTF_8));
                         StringWriter sw = new StringWriter();
                         identifyResponse.receivedResponse(resp, sw);
                     });
             httpClient.execute(request).get();
-
-            /*AggregatedHttpMessage response = client.execute(HttpHeaders.of(HttpMethod.GET, identifyRequest.getPath())
-                    .set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();*/
-            // follow a maximum of 10 HTTP redirects
-            /*int max = 10;
-            while (response.followUrl() != null && max-- > 0) {
-                URI uri = URI.create(response.followUrl());
-                client = Clients.newClient(oaiClient.getFactory(), "none+" + uri, HttpClient.class);
-                response = client.execute(HttpHeaders.of(HttpMethod.GET, response.followUrl())
-                        .set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
-            }*/
             String granularity = identifyResponse.getGranularity();
             logger.log(Level.INFO, "granularity = " + granularity);
             DateTimeFormatter dateTimeFormatter = "YYYY-MM-DD".equals(granularity) ?
-                    DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("GMT")) : null;
+                    DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")) : null;
             ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest();
             listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
             listRecordsRequest.setMetadataPrefix("marcxml");
@@ -73,11 +70,13 @@ public class BundeskunsthalleTest {
                 while (listRecordsRequest != null) {
                     try {
                         ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
-                        logger.log(Level.INFO,"sending " + listRecordsRequest.getURL());
+                        logger.log(Level.INFO, "sending " + listRecordsRequest.getURL());
                         StringWriter sw = new StringWriter();
                         request = Request.get()
-                                .url(url.resolve(listRecordsRequest.getURL()))
+                                .url(listRecordsRequest.getURL())
                                 .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
+                                .setFollowRedirect(true)
+                                .setTimeoutInMillis(60 * 1000)
                                 .build()
                                 .setResponseListener(resp -> {
                                     try {
@@ -92,22 +91,18 @@ public class BundeskunsthalleTest {
                                                 .build()
                                                 .xmlReader().parse();
                                     } catch (IOException e) {
-                                        throw new OAIException(e);
+                                        throw new OAIException("MARC parser exception: " + e.getMessage(), e);
                                     }
                                     listRecordsResponse.receivedResponse(resp, sw);
-                                    logger.log(Level.FINE, "response headers = " + resp.getHeaders() +
+                                    logger.log(Level.FINE,
-                                            " resumption-token = {}" + listRecordsResponse.getResumptionToken());
+                                            "status = " + resp.getStatus() +
+                                                    " headers = " + resp.getHeaders() +
+                                                    " resumptiontoken = " + listRecordsResponse.getResumptionToken());
                                 });
                         httpClient.execute(request).get();
-                        // follow a maximum of 10 HTTP redirects
-                        /*max = 10;
-                        while (response.followUrl() != null && max-- > 0) {
-                            URI uri = URI.create(response.followUrl());
-                            client = Clients.newClient(oaiClient.getFactory(), "none+" + uri, HttpClient.class);
-                            response = client.execute(HttpHeaders.of(HttpMethod.GET, response.followUrl())
-                                    .set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
-                        }*/
                         listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
+                    } catch (ConnectException e) {
+                        logger.log(Level.WARNING, e.getMessage(), e);
                     } catch (IOException e) {
                         logger.log(Level.SEVERE, e.getMessage(), e);
                         listRecordsRequest = null;
@@ -116,6 +111,7 @@ public class BundeskunsthalleTest {
                 writer.endCollection();
                 writer.endDocument();
             }
+            logger.log(Level.INFO, "completed");
         } catch (Exception e) {
             logger.log(Level.WARNING, e.getMessage(), e);
         }

oai-client/src/test/java/org/xbib/oai/client/DNBClientTest.java

@@ -1,7 +1,5 @@
 package org.xbib.oai.client;
 
-import static org.junit.Assert.assertEquals;
-
 import io.netty.handler.codec.http.HttpHeaderNames;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -9,6 +7,7 @@ import org.xbib.net.URL;
 import org.xbib.netty.http.client.Client;
 import org.xbib.netty.http.client.Request;
 import org.xbib.oai.client.identify.IdentifyRequest;
+import org.xbib.oai.client.identify.IdentifyResponse;
 import org.xbib.oai.client.listrecords.ListRecordsRequest;
 import org.xbib.oai.client.listrecords.ListRecordsResponse;
 import org.xbib.oai.xml.SimpleMetadataHandler;
@@ -16,8 +15,12 @@ import org.xbib.oai.xml.SimpleMetadataHandler;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.StringWriter;
+import java.net.ConnectException;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -25,58 +28,62 @@ import java.util.logging.Logger;
 /**
  *
  */
-@Ignore
 public class DNBClientTest {
 
     private static final Logger logger = Logger.getLogger(DNBClientTest.class.getName());
 
     @Test
-    public void testIdentify() {
+    public void testBibdat() {
         URL url = URL.create("http://services.dnb.de/oai/repository");
-        try (OAIClient client = new OAIClient(url)) {
+        try (OAIClient oaiClient = new OAIClient(url)) {
-            IdentifyRequest identifyRequest = client.newIdentifyRequest();
+            Client httpClient  = Client.builder()
-            Client httpClient = client.getHttpClient();
+                    .setConnectTimeoutMillis(60 * 1000)
-            assertEquals("/oai/repository?verb=Identify", identifyRequest.getURL().toString());
+                    .setReadTimeoutMillis(60 * 1000)
+                    .build();
+            IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
+            IdentifyResponse identifyResponse = new IdentifyResponse();
             Request request = Request.get()
-                    .url(url.resolve(identifyRequest.getURL()))
+                    .url(identifyRequest.getURL())
                     .build()
-                    .setResponseListener(resp -> logger.log(Level.INFO, resp.getBodyAsString(StandardCharsets.UTF_8)));
+                    .setResponseListener(resp -> {
+                        logger.log(Level.INFO, resp.getBodyAsString(StandardCharsets.UTF_8));
+                        StringWriter sw = new StringWriter();
+                        identifyResponse.receivedResponse(resp, sw);
+                    });
             httpClient.execute(request).get();
-        } catch (IOException e) {
+            String granularity = identifyResponse.getGranularity();
-            logger.log(Level.SEVERE, e.getMessage(), e);
+            logger.log(Level.INFO, "granularity = " + granularity);
-        }
+            DateTimeFormatter dateTimeFormatter = "YYYY-MM-DD".equals(granularity) ?
-    }
+                    DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")) :
-
+                    DateTimeFormatter.ISO_DATE_TIME;
-    @Test
+            ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest();
-    public void testListRecordsDNB() {
-        URL url = URL.create("http://services.dnb.de/oai/repository");
-        try (OAIClient client = new OAIClient(url)) {
-            ListRecordsRequest listRecordsRequest = client.newListRecordsRequest();
             listRecordsRequest.setFrom(Instant.parse("2016-01-01T00:00:00Z"));
             listRecordsRequest.setUntil(Instant.parse("2016-01-10T00:00:00Z"));
             listRecordsRequest.setSet("bib");
             listRecordsRequest.setMetadataPrefix("PicaPlus-xml");
             Handler handler = new Handler();
-            File file = File.createTempFile("dnb-bib-pica.", ".xml");
+            File file =  new File("build/dnb-bib-pica.xml");
-            file.deleteOnExit();
             FileWriter fileWriter = new FileWriter(file);
             while (listRecordsRequest != null) {
                 try {
                     ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
                     listRecordsRequest.addHandler(handler);
-                    Request request = Request.get()
+                    request = Request.get()
-                            .url(url.resolve(listRecordsRequest.getURL()))
+                            .url(listRecordsRequest.getURL())
                             .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
                             .build()
                             .setResponseListener(resp -> listRecordsResponse.receivedResponse(resp, fileWriter));
-                    client.getHttpClient().execute(request).get();
+                    httpClient.execute(request).get();
-                    listRecordsRequest = client.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
+                    listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
+                } catch (ConnectException e) {
+                    logger.log(Level.WARNING, e.getMessage(), e);
                 } catch (IOException e) {
                     logger.log(Level.SEVERE, e.getMessage(), e);
                     listRecordsRequest = null;
                 }
             }
             fileWriter.close();
+            httpClient.shutdownGracefully();
             logger.log(Level.INFO, "count=" + handler.count());
         } catch (Exception e) {
             logger.log(Level.SEVERE, "skipped, HTTP exception");

oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java

@@ -34,8 +34,11 @@ public class DOAJClientTest {
     public void testListRecordsDOAJ() {
         URL url = URL.create("https://doaj.org/oai");
         try (OAIClient oaiClient = new OAIClient(url)) {
+            Client httpClient  = Client.builder()
+                    .setConnectTimeoutMillis(60 * 1000)
+                    .setReadTimeoutMillis(60 * 1000)
+                    .build();
             IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
-            Client httpClient = oaiClient.getHttpClient();
             IdentifyResponse identifyResponse = new IdentifyResponse();
             Request request = Request.get()
                     .url(url.resolve(identifyRequest.getURL()))
@@ -77,6 +80,7 @@ public class DOAJClientTest {
                 listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
             }
             fileWriter.close();
+            httpClient.shutdownGracefully();
             logger.log(Level.INFO, "count = " + handler.count());
         } catch (Exception e) {
             logger.log(Level.SEVERE, e.getMessage(), e);