diff --git a/build.gradle b/build.gradle index e6f0506..c3b4380 100644 --- a/build.gradle +++ b/build.gradle @@ -1,179 +1,39 @@ plugins { - id "com.github.spotbugs" version "2.0.0" - id "org.sonarqube" version "2.8" + id "de.marcphilipp.nexus-publish" version "0.4.0" id "io.codearte.nexus-staging" version "0.21.1" - id "org.xbib.gradle.plugin.asciidoctor" version "1.6.0.1" } -apply plugin: "io.codearte.nexus-staging" +wrapper { + gradleVersion = "${project.property('gradle.wrapper.version')}" + distributionType = Wrapper.DistributionType.ALL +} + +ext { + user = 'xbib' + name = 'oai' + description = 'Open Archive Initiative library for Java' + inceptionYear = '2016' + url = 'https://github.com/' + user + '/' + name + scmUrl = 'https://github.com/' + user + '/' + name + scmConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git' + scmDeveloperConnection = 'scm:git:ssh://git@github.com:' + user + '/' + name + '.git' + issueManagementSystem = 'Github' + issueManagementUrl = ext.scmUrl + '/issues' + licenseName = 'The Apache License, Version 2.0' + licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt' +} subprojects { - apply plugin: 'java-library' - apply plugin: 'maven' - apply plugin: 'pmd' - apply plugin: 'checkstyle' - apply plugin: "com.github.spotbugs" - apply plugin: 'org.xbib.gradle.plugin.asciidoctor' - - repositories { - mavenCentral() - } dependencies { - testCompile "org.junit.jupiter:junit-jupiter-api:${project.property('junit.version')}" - testCompile "org.junit.jupiter:junit-jupiter-params:${project.property('junit.version')}" - testCompile "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}" - testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}" + testImplementation "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}" } - compileJava { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 - } - - compileTestJava { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 - - } - - tasks.withType(JavaCompile) { - options.compilerArgs << "-Xlint:all" - } - - clean { - delete 'out' - } - - test { - useJUnitPlatform() - failFast = false - testLogging { - events 'STARTED', 'PASSED', 'FAILED', 'SKIPPED' - showStandardStreams = false - } - afterSuite { desc, result -> - if (!desc.parent) { - println "\nTest result: ${result.resultType}" - println "Test summary: ${result.testCount} tests, " + - "${result.successfulTestCount} succeeded, " + - "${result.failedTestCount} failed, " + - "${result.skippedTestCount} skipped" - } - } - } - - task sourcesJar(type: Jar, dependsOn: classes) { - classifier 'sources' - from sourceSets.main.allSource - } - - task javadocJar(type: Jar, dependsOn: javadoc) { - classifier 'javadoc' - } - - artifacts { - archives sourcesJar, javadocJar - } - - ext { - user = 'xbib' - projectName = 'oai' - projectDescription = 'Open Archive Initiative library for Java' - scmUrl = 'https://github.com/xbib/oai' - scmConnection = 'scm:git:git://github.com/xbib/oai.git' - scmDeveloperConnection = 'scm:git:git://github.com/xbib/oai.git' - } - - task sonatypeUpload(type: Upload, dependsOn: build) { - group = 'publish' - configuration = configurations.archives - uploadDescriptor = true - repositories { - if (project.hasProperty('ossrhUsername')) { - mavenDeployer { - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - repository(url: uri(ossrhReleaseUrl)) { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - snapshotRepository(url: uri(ossrhSnapshotUrl)) { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - pom.project { - groupId project.group - artifactId project.name - version project.version - name project.name - description projectDescription - packaging 'jar' - inceptionYear '2016' - url scmUrl - organization { - name 'xbib' - url 'http://xbib.org' - } - developers { - developer { - id user - name 'Jörg Prante' - email 'joergprante@gmail.com' - url 'https://github.com/jprante' - } - } - scm { - url scmUrl - connection scmConnection - developerConnection scmDeveloperConnection - } - licenses { - license { - name 'The Apache License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } - } - } - } - } - } - } - - nexusStaging { - packageGroup = "org.xbib" - } - - spotbugs { - effort = "max" - reportLevel = "low" - } - - tasks.withType(com.github.spotbugs.SpotBugsTask) { - ignoreFailures = true - reports { - xml.enabled = false - html.enabled = true - } - } - - pmd { - toolVersion = '6.11.0' - ruleSets = ['category/java/bestpractices.xml'] - ignoreFailures = true - } - - checkstyle { - configFile = rootProject.file('config/checkstyle/checkstyle.xml') - ignoreFailures = true - showViolations = true - } - - sonarqube { - properties { - property "sonar.projectName", "${project.group} ${project.name}" - property "sonar.sourceEncoding", "UTF-8" - property "sonar.tests", "src/test/java" - property "sonar.scm.provider", "git" - property "sonar.junit.reportsPath", "build/test-results/test/" - } - } + apply from: rootProject.file('gradle/ide/idea.gradle') + apply from: rootProject.file('gradle/compile/java.gradle') + apply from: rootProject.file('gradle/test/junit5.gradle') + apply from: rootProject.file('gradle/publishing/publication.gradle') } + +apply from: rootProject.file('gradle/publishing/sonatype.gradle') diff --git a/gradle.properties b/gradle.properties index e7cf6bd..65a8d45 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,12 +1,10 @@ group = org.xbib name = oai -version = 2.2.2 +version = 2.3.0 -xbib-content.version = 2.0.5 -xbib-netty-http.version = 4.1.48.0 +gradle.wrapper.version = 6.4.1 +xbib-content.version = 2.2.0 +xbib-netty-http.version = 4.1.49.1 tcnative.version = 2.0.28.Final - -# test only -junit.version = 5.5.1 -xbib-bibliographic-character-sets.version = 1.0.0 -xbib-marc.version = 2.1.0 +xbib-marc.version = 2.3.0 +xbib-bibliographic-character-sets.version = 2.0.0 diff --git a/gradle/compile/java.gradle b/gradle/compile/java.gradle new file mode 100644 index 0000000..a9d76ce --- /dev/null +++ b/gradle/compile/java.gradle @@ -0,0 +1,35 @@ + +apply plugin: 'java-library' + +java { + modularity.inferModulePath.set(true) +} + +compileJava { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + +compileTestJava { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + +jar { + manifest { + attributes('Implementation-Version': project.version) + } +} + +task sourcesJar(type: Jar, dependsOn: classes) { + classifier 'sources' + from sourceSets.main.allSource +} + +task javadocJar(type: Jar, dependsOn: javadoc) { + classifier 'javadoc' +} + +artifacts { + archives sourcesJar, javadocJar +} \ No newline at end of file diff --git a/gradle/documentation/asciidoc.gradle b/gradle/documentation/asciidoc.gradle new file mode 100644 index 0000000..87ba22e --- /dev/null +++ b/gradle/documentation/asciidoc.gradle @@ -0,0 +1,55 @@ +apply plugin: 'org.xbib.gradle.plugin.asciidoctor' + +configurations { + asciidoclet +} + +dependencies { + asciidoclet "org.asciidoctor:asciidoclet:${project.property('asciidoclet.version')}" +} + + +asciidoctor { + backends 'html5' + outputDir = file("${rootProject.projectDir}/docs") + separateOutputDirs = false + attributes 'source-highlighter': 'coderay', + idprefix: '', + idseparator: '-', + toc: 'left', + doctype: 'book', + icons: 'font', + encoding: 'utf-8', + sectlink: true, + sectanchors: true, + linkattrs: true, + imagesdir: 'img', + stylesheet: "${projectDir}/src/docs/asciidoc/css/foundation.css" +} + + +/*javadoc { +options.docletpath = configurations.asciidoclet.files.asType(List) +options.doclet = 'org.asciidoctor.Asciidoclet' +//options.overview = "src/docs/asciidoclet/overview.adoc" +options.addStringOption "-base-dir", "${projectDir}" +options.addStringOption "-attribute", + "name=${project.name},version=${project.version},title-link=https://github.com/xbib/${project.name}" +configure(options) { + noTimestamp = true +} +}*/ + + +/*javadoc { + options.docletpath = configurations.asciidoclet.files.asType(List) + options.doclet = 'org.asciidoctor.Asciidoclet' + options.overview = "${rootProject.projectDir}/src/docs/asciidoclet/overview.adoc" + options.addStringOption "-base-dir", "${projectDir}" + options.addStringOption "-attribute", + "name=${project.name},version=${project.version},title-link=https://github.com/xbib/${project.name}" + options.destinationDirectory(file("${projectDir}/docs/javadoc")) + configure(options) { + noTimestamp = true + } +}*/ diff --git a/gradle/ide/idea.gradle b/gradle/ide/idea.gradle new file mode 100644 index 0000000..64e2167 --- /dev/null +++ b/gradle/ide/idea.gradle @@ -0,0 +1,13 @@ +apply plugin: 'idea' + +idea { + module { + outputDir file('build/classes/java/main') + testOutputDir file('build/classes/java/test') + } +} + +if (project.convention.findPlugin(JavaPluginConvention)) { + //sourceSets.main.output.classesDirs = file("build/classes/java/main") + //sourceSets.test.output.classesDirs = file("build/classes/java/test") +} diff --git a/gradle/publishing/publication.gradle b/gradle/publishing/publication.gradle new file mode 100644 index 0000000..c35fcb9 --- /dev/null +++ b/gradle/publishing/publication.gradle @@ -0,0 +1,64 @@ + +apply plugin: "de.marcphilipp.nexus-publish" + +publishing { + publications { + mavenJava(MavenPublication) { + from components.java + artifact sourcesJar + artifact javadocJar + pom { + name = project.name + description = rootProject.ext.description + url = rootProject.ext.url + inceptionYear = rootProject.ext.inceptionYear + packaging = 'jar' + organization { + name = 'xbib' + url = 'https://xbib.org' + } + developers { + developer { + id = 'jprante' + name = 'Jörg Prante' + email = 'joergprante@gmail.com' + url = 'https://github.com/jprante' + } + } + scm { + url = rootProject.ext.scmUrl + connection = rootProject.ext.scmConnection + developerConnection = rootProject.ext.scmDeveloperConnection + } + issueManagement { + system = rootProject.ext.issueManagementSystem + url = rootProject.ext.issueManagementUrl + } + licenses { + license { + name = rootProject.ext.licenseName + url = rootProject.ext.licenseUrl + distribution = 'repo' + } + } + } + } + } +} + +if (project.hasProperty("signing.keyId")) { + apply plugin: 'signing' + signing { + sign publishing.publications.mavenJava + } +} + +nexusPublishing { + repositories { + sonatype { + username = project.property('ossrhUsername') + password = project.property('ossrhPassword') + packageGroup = "org.xbib" + } + } +} diff --git a/gradle/publishing/sonatype.gradle b/gradle/publishing/sonatype.gradle new file mode 100644 index 0000000..e1813f3 --- /dev/null +++ b/gradle/publishing/sonatype.gradle @@ -0,0 +1,11 @@ + +if (project.hasProperty('ossrhUsername') && project.hasProperty('ossrhPassword')) { + + apply plugin: 'io.codearte.nexus-staging' + + nexusStaging { + username = project.property('ossrhUsername') + password = project.property('ossrhPassword') + packageGroup = "org.xbib" + } +} diff --git a/gradle/test/junit5.gradle b/gradle/test/junit5.gradle new file mode 100644 index 0000000..cfef972 --- /dev/null +++ b/gradle/test/junit5.gradle @@ -0,0 +1,27 @@ + +def junitVersion = project.hasProperty('junit.version')?project.property('junit.version'):'5.6.2' +def hamcrestVersion = project.hasProperty('hamcrest.version')?project.property('hamcrest.version'):'2.2' + +dependencies { + testImplementation "org.junit.jupiter:junit-jupiter-api:${junitVersion}" + testImplementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}" + testImplementation "org.hamcrest:hamcrest-library:${hamcrestVersion}" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}" +} + +test { + useJUnitPlatform() + failFast = true + testLogging { + events 'STARTED', 'PASSED', 'FAILED', 'SKIPPED' + } + afterSuite { desc, result -> + if (!desc.parent) { + println "\nTest result: ${result.resultType}" + println "Test summary: ${result.testCount} tests, " + + "${result.successfulTestCount} succeeded, " + + "${result.failedTestCount} failed, " + + "${result.skippedTestCount} skipped" + } + } +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 5c2d1cf..62d4c05 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index bb90aa9..21e622d 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,5 @@ -#Mon Jan 13 14:23:25 CET 2020 -distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.4.1-all.zip zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 83f2acf..fbd7c51 100755 --- a/gradlew +++ b/gradlew @@ -82,6 +82,7 @@ esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then @@ -129,6 +130,7 @@ fi if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath @@ -154,19 +156,19 @@ if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then else eval `echo args$i`="\"$arg\"" fi - i=$((i+1)) + i=`expr $i + 1` done case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; esac fi @@ -175,14 +177,9 @@ save () { for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done echo " " } -APP_ARGS=$(save "$@") +APP_ARGS=`save "$@"` # Collect all arguments for the java command, following the shell quoting and substitution rules eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index 24467a1..a9f778a 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -29,6 +29,9 @@ if "%DIRNAME%" == "" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @@ -81,6 +84,7 @@ set CMD_LINE_ARGS=%* set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + @rem Execute Gradle "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% diff --git a/oai-client/build.gradle b/oai-client/build.gradle index 311a552..117ca51 100644 --- a/oai-client/build.gradle +++ b/oai-client/build.gradle @@ -1,8 +1,7 @@ - dependencies { - compile project(':oai-common') - compile "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}" - compile "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}" - testCompile "org.xbib:marc:${project.property('xbib-marc.version')}" - testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}" + api project(':oai-common') + api "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}" + implementation "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}" + testImplementation "org.xbib:marc:${project.property('xbib-marc.version')}" + testImplementation "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}" } diff --git a/oai-client/src/main/java/module-info.java b/oai-client/src/main/java/module-info.java new file mode 100644 index 0000000..24b5501 --- /dev/null +++ b/oai-client/src/main/java/module-info.java @@ -0,0 +1,13 @@ +module org.xbib.oai.client { + exports org.xbib.oai.client; + exports org.xbib.oai.client.getrecord; + exports org.xbib.oai.client.identify; + exports org.xbib.oai.client.listidentifiers; + exports org.xbib.oai.client.listrecords; + exports org.xbib.oai.client.listsets; + requires org.xbib.oai; + requires org.xbib.net.url; + requires org.xbib.netty.http.common; + requires org.xbib.content.xml; + requires java.xml; +} diff --git a/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsFilterReader.java b/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsFilterReader.java index 29d00d1..6d8d3b7 100644 --- a/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsFilterReader.java +++ b/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsFilterReader.java @@ -7,6 +7,7 @@ import org.xbib.oai.util.ResumptionToken; import org.xbib.oai.xml.MetadataHandler; import org.xml.sax.Attributes; import org.xml.sax.SAXException; +import org.xml.sax.ContentHandler; import java.time.Instant; import java.time.LocalDate; @@ -14,21 +15,17 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; -import java.util.logging.Level; -import java.util.logging.Logger; /** * */ public class ListRecordsFilterReader extends XMLFilterReader { - private static final Logger logger = Logger.getLogger(ListRecordsFilterReader.class.getName()); - private final ListRecordsRequest request; private final ListRecordsResponse response; - private StringBuilder content; + private final StringBuilder content; private RecordHeader header; @@ -54,14 +51,12 @@ public class ListRecordsFilterReader extends XMLFilterReader { @Override public void startDocument() throws SAXException { - logger.log(Level.FINE, "start of document"); super.startDocument(); request.setResumptionToken(null); } @Override public void endDocument() throws SAXException { - logger.log(Level.FINE, "end of document"); super.endDocument(); } @@ -97,7 +92,6 @@ public class ListRecordsFilterReader extends XMLFilterReader { request.setResumptionToken(token); } } catch (Exception e) { - logger.log(Level.FINE, e.getMessage(), e); throw new SAXException(e); } break; @@ -139,7 +133,6 @@ public class ListRecordsFilterReader extends XMLFilterReader { // feedback to request request.setResumptionToken(token); } else { - logger.log(Level.WARNING, "empty resumption token value"); // some servers send a null or an empty token as last token token = null; request.setResumptionToken(null); @@ -167,9 +160,7 @@ public class ListRecordsFilterReader extends XMLFilterReader { ZonedDateTime zonedDateTime = localDate.atStartOfDay(ZoneId.of("UTC")); header.setDate(Instant.from(zonedDateTime)); } catch (DateTimeParseException e2) { - if (logger.isLoggable(Level.WARNING)) { - logger.log(Level.WARNING, "unable to parse date: " + s + " reason = " + e2.getMessage()); - } + throw new IllegalArgumentException("unable to parse: " + e2.getMessage(), e2); } } } diff --git a/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java b/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java index e51b4c2..fde1ab1 100644 --- a/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java +++ b/oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java @@ -1,6 +1,5 @@ package org.xbib.oai.client.listrecords; -import io.netty.handler.codec.http.HttpHeaderNames; import org.xbib.content.xml.transform.TransformerURIResolver; import org.xbib.content.xml.util.XMLUtil; import org.xbib.netty.http.common.HttpResponse; @@ -16,12 +15,9 @@ import org.xml.sax.InputSource; import java.io.StringReader; import java.io.Writer; import java.nio.charset.StandardCharsets; -import java.text.MessageFormat; import java.time.Instant; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; -import java.util.logging.Level; -import java.util.logging.Logger; import javax.xml.transform.Source; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; @@ -34,8 +30,6 @@ import javax.xml.transform.stream.StreamResult; */ public class ListRecordsResponse extends AbstractOAIResponse { - private static final Logger logger = Logger.getLogger(ListRecordsResponse.class.getName()); - private static final String[] RETRY_AFTER_HEADERS = { "retry-after", "Retry-after", "Retry-After" }; @@ -93,26 +87,21 @@ public class ListRecordsResponse extends AbstractOAIResponse { // parse RFC date, e.g. Fri, 31 Dec 1999 23:59:59 GMT Instant instant = Instant.from(DateTimeFormatter.RFC_1123_DATE_TIME.parse(retryAfter)); secs = ChronoUnit.SECONDS.between(instant, Instant.now()); - logger.log(Level.INFO, MessageFormat.format("parsed delay seconds is {0}", secs)); } - logger.log(Level.INFO, MessageFormat.format("setting delay seconds to {0}", secs)); } } request.setRetry(true); try { if (secs > 0L) { - logger.log(Level.INFO, MessageFormat.format("waiting for {0} seconds (retry-after)", secs)); Thread.sleep(1000 * secs); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - logger.log(Level.SEVERE, "interrupted"); } return; } if (status == 429) { try { - logger.log(Level.WARNING, "received 429 Too many requests, waiting 10 seconds..."); Thread.sleep(10000L); } catch (InterruptedException e) { // ignore @@ -122,7 +111,7 @@ public class ListRecordsResponse extends AbstractOAIResponse { throw new OAIException("status = " + status + " response = " + content); } // activate XSLT only if OAI XML content type is returned - String contentType = message.getHeaders().getHeader(HttpHeaderNames.CONTENT_TYPE); + String contentType = message.getHeaders().getHeader("content-type"); if (contentType != null && !contentType.startsWith("text/xml")) { throw new OAIException("no XML content type in response: " + contentType); } @@ -133,7 +122,6 @@ public class ListRecordsResponse extends AbstractOAIResponse { Transformer transformer = transformerFactory.newTransformer(); Source source = new SAXSource(filterreader, new InputSource(new StringReader(XMLUtil.sanitize(content)))); StreamResult streamResult = new StreamResult(writer); - logger.log(Level.FINE, "transforming"); transformer.transform(source, streamResult); if ("noRecordsMatch".equals(error)) { throw new NoRecordsMatchException("metadataPrefix=" + request.getMetadataPrefix() diff --git a/oai-client/src/test/java/org/xbib/oai/client/BundeskunsthalleTest.java b/oai-client/src/test/java/org/xbib/oai/client/BundeskunsthalleTest.java index 833ca3b..4820d77 100644 --- a/oai-client/src/test/java/org/xbib/oai/client/BundeskunsthalleTest.java +++ b/oai-client/src/test/java/org/xbib/oai/client/BundeskunsthalleTest.java @@ -28,12 +28,12 @@ import java.util.logging.Logger; /** * */ -@Disabled("takes very long time") class BundeskunsthalleTest { private static final Logger logger = Logger.getLogger(BundeskunsthalleTest.class.getName()); @Test + @Disabled("takes long time") void testListRecords() { URL url = URL.create("https://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh"); try (Client httpClient = Client.builder() @@ -63,7 +63,7 @@ class BundeskunsthalleTest { ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest(); listRecordsRequest.setDateTimeFormatter(dateTimeFormatter); listRecordsRequest.setMetadataPrefix("marcxml"); - try (MarcJsonWriter writer = new MarcJsonWriter("bk-bulk%d.jsonl", 1000, + try (MarcJsonWriter writer = new MarcJsonWriter("build/bk-bulk%d.jsonl", 1000, EnumSet.of(MarcJsonWriter.Style.ELASTICSEARCH_BULK), 65536, false) .setIndex("testindex", "testtype")) { writer.startDocument(); diff --git a/oai-client/src/test/java/org/xbib/oai/client/DNBClientTest.java b/oai-client/src/test/java/org/xbib/oai/client/DNBClientTest.java index 30506e9..6226a38 100644 --- a/oai-client/src/test/java/org/xbib/oai/client/DNBClientTest.java +++ b/oai-client/src/test/java/org/xbib/oai/client/DNBClientTest.java @@ -1,6 +1,7 @@ package org.xbib.oai.client; import io.netty.handler.codec.http.HttpHeaderNames; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; import org.xbib.net.URL; import org.xbib.netty.http.client.Client; @@ -18,8 +19,6 @@ import java.io.StringWriter; import java.net.ConnectException; import java.nio.charset.StandardCharsets; import java.time.Instant; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; @@ -52,9 +51,6 @@ class DNBClientTest { httpClient.execute(request).get(); String granularity = identifyResponse.getGranularity(); logger.log(Level.INFO, "granularity = " + granularity); - DateTimeFormatter dateTimeFormatter = "YYYY-MM-DD".equals(granularity) ? - DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")) : - DateTimeFormatter.ISO_DATE_TIME; ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest(); listRecordsRequest.setFrom(Instant.parse("2016-01-01T00:00:00Z")); listRecordsRequest.setUntil(Instant.parse("2016-01-10T00:00:00Z")); @@ -83,6 +79,7 @@ class DNBClientTest { } fileWriter.close(); logger.log(Level.INFO, "count=" + handler.count()); + assertTrue(handler.count() > 0); } catch (Exception e) { logger.log(Level.SEVERE, "skipped, HTTP exception"); } diff --git a/oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java b/oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java index 5b2e7e4..e8ee310 100644 --- a/oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java +++ b/oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java @@ -12,9 +12,10 @@ import org.xbib.oai.client.listrecords.ListRecordsRequest; import org.xbib.oai.client.listrecords.ListRecordsResponse; import org.xbib.oai.xml.SimpleMetadataHandler; -import java.io.File; -import java.io.FileWriter; import java.io.StringWriter; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Paths; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -30,7 +31,7 @@ class DOAJClientTest { private static final Logger logger = Logger.getLogger(DOAJClientTest.class.getName()); @Test - @Disabled // takes too long time + @Disabled("takes long time") void testListRecordsDOAJ() { URL url = URL.create("https://doaj.org/oai"); try (Client httpClient = Client.builder() @@ -60,26 +61,24 @@ class DOAJClientTest { listRecordsRequest.setUntil(Instant.parse("2018-01-01T00:00:00Z")); listRecordsRequest.setMetadataPrefix("oai_dc"); Handler handler = new Handler(); - File file = File.createTempFile("doaj.", ".xml"); - file.deleteOnExit(); - FileWriter fileWriter = new FileWriter(file); - while (listRecordsRequest != null) { - ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest); - listRecordsRequest.addHandler(handler); - logger.log(Level.INFO,"sending " + listRecordsRequest.getURL()); - request = Request.get() - .url(url.resolve(listRecordsRequest.getURL())) - .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") - .setResponseListener(resp -> { - listRecordsResponse.receivedResponse(resp, fileWriter); - logger.log(Level.FINE, "response headers = " + resp.getHeaders() + - " resumption-token = {}" + listRecordsResponse.getResumptionToken()); - }) - .build(); - httpClient.execute(request).get(); - listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); + try (Writer writer = Files.newBufferedWriter(Paths.get("build/doaj.xml"))) { + while (listRecordsRequest != null) { + ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest); + listRecordsRequest.addHandler(handler); + logger.log(Level.INFO, "sending " + listRecordsRequest.getURL()); + request = Request.get() + .url(url.resolve(listRecordsRequest.getURL())) + .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") + .setResponseListener(resp -> { + listRecordsResponse.receivedResponse(resp, writer); + logger.log(Level.FINE, "response headers = " + resp.getHeaders() + + " resumption-token = {}" + listRecordsResponse.getResumptionToken()); + }) + .build(); + httpClient.execute(request).get(); + listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken()); + } } - fileWriter.close(); logger.log(Level.INFO, "count = " + handler.count()); } catch (Exception e) { logger.log(Level.SEVERE, e.getMessage(), e); diff --git a/oai-common/build.gradle b/oai-common/build.gradle index e0d37a4..ba93ff7 100644 --- a/oai-common/build.gradle +++ b/oai-common/build.gradle @@ -1,3 +1,5 @@ dependencies { - compile "org.xbib:content-rdf:${project.property('xbib-content.version')}" + api "org.xbib:content-rdf:${project.property('xbib-content.version')}" + api "org.xbib:content-resource:${project.property('xbib-content.version')}" + api "org.xbib:content-xml:${project.property('xbib-content.version')}" } diff --git a/oai-common/src/main/java/module-info.java b/oai-common/src/main/java/module-info.java new file mode 100644 index 0000000..2a1ea79 --- /dev/null +++ b/oai-common/src/main/java/module-info.java @@ -0,0 +1,11 @@ +module org.xbib.oai { + exports org.xbib.oai; + exports org.xbib.oai.exceptions; + exports org.xbib.oai.rdf; + exports org.xbib.oai.util; + exports org.xbib.oai.xml; + requires org.xbib.content.rdf; + requires org.xbib.content.resource; + requires org.xbib.content.xml; + requires java.xml; +} diff --git a/oai-common/src/main/java/org/xbib/oai/rdf/RdfResourceHandler.java b/oai-common/src/main/java/org/xbib/oai/rdf/RdfResourceHandler.java index e18f953..b5df5bc 100644 --- a/oai-common/src/main/java/org/xbib/oai/rdf/RdfResourceHandler.java +++ b/oai-common/src/main/java/org/xbib/oai/rdf/RdfResourceHandler.java @@ -4,7 +4,7 @@ import org.xbib.content.rdf.RdfContentParams; import org.xbib.content.rdf.io.xml.AbstractXmlResourceHandler; import org.xbib.content.rdf.io.xml.XmlHandler; import org.xbib.content.resource.IRI; -import org.xbib.content.resource.IRINamespaceContext; +import org.xbib.content.resource.NamespaceContext; import org.xbib.oai.OAIConstants; import javax.xml.namespace.QName; @@ -48,12 +48,12 @@ public class RdfResourceHandler extends AbstractXmlResourceHandler setNamespaceContext(IRINamespaceContext namespaceContext) { + public XmlHandler setNamespaceContext(NamespaceContext namespaceContext) { return this; } @Override - public IRINamespaceContext getNamespaceContext() { + public NamespaceContext getNamespaceContext() { return getParams().getNamespaceContext(); } } diff --git a/oai-common/src/main/java/org/xbib/oai/xml/XmlSimpleMetadataHandler.java b/oai-common/src/main/java/org/xbib/oai/xml/XmlSimpleMetadataHandler.java index 9471135..acb9748 100644 --- a/oai-common/src/main/java/org/xbib/oai/xml/XmlSimpleMetadataHandler.java +++ b/oai-common/src/main/java/org/xbib/oai/xml/XmlSimpleMetadataHandler.java @@ -14,8 +14,6 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; import javax.xml.stream.Location; import javax.xml.stream.XMLEventFactory; import javax.xml.stream.XMLEventWriter; @@ -29,15 +27,13 @@ import javax.xml.stream.events.Namespace; */ public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements OAIConstants { - private static final Logger logger = Logger.getLogger(XmlSimpleMetadataHandler.class.getName()); - private static final XMLOutputFactory outputFactory = XMLOutputFactory.newInstance(); private static final XMLEventFactory eventFactory = XMLEventFactory.newInstance(); private List namespaces = new ArrayList<>(); - private Deque> nsStack = new ArrayDeque<>(); + private final Deque> nsStack = new ArrayDeque<>(); private Locator locator; @@ -49,14 +45,10 @@ public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements O private boolean needToCallStartDocument = false; - public XmlSimpleMetadataHandler setWriter(Writer writer) { + public XmlSimpleMetadataHandler setWriter(Writer writer) throws XMLStreamException { this.writer = writer; - try { - outputFactory.setProperty("javax.xml.stream.isRepairingNamespaces", Boolean.TRUE); - this.eventWriter = outputFactory.createXMLEventWriter(writer); - } catch (XMLStreamException e) { - logger.log(Level.FINE, e.getMessage(), e); - } + outputFactory.setProperty("javax.xml.stream.isRepairingNamespaces", Boolean.TRUE); + this.eventWriter = outputFactory.createXMLEventWriter(writer); return this; } @@ -150,7 +142,7 @@ public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements O try { eventWriter.add(eventFactory.createStartDocument()); } catch (XMLStreamException e) { - logger.log(Level.FINE, e.getMessage(), e); + throw new SAXException(e); } needToCallStartDocument = false; } @@ -272,10 +264,14 @@ public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements O } private static final class SAXLocation implements Location { - private int lineNumber; - private int columnNumber; - private String publicId; - private String systemId; + + private final int lineNumber; + + private final int columnNumber; + + private final String publicId; + + private final String systemId; private SAXLocation(Locator locator) { lineNumber = locator.getLineNumber(); diff --git a/oai-server/build.gradle b/oai-server/build.gradle index 336374e..39166e6 100644 --- a/oai-server/build.gradle +++ b/oai-server/build.gradle @@ -1,3 +1,3 @@ dependencies { - compile project(':oai-common') -} \ No newline at end of file + api project(':oai-common') +}