update to Java named modules, Gradle 6.4.1

This commit is contained in:
Jörg Prante 2020-05-26 16:12:26 +02:00
parent a8fe25a21e
commit 276771f96f
24 changed files with 337 additions and 278 deletions

View file

@ -1,179 +1,39 @@
plugins { plugins {
id "com.github.spotbugs" version "2.0.0" id "de.marcphilipp.nexus-publish" version "0.4.0"
id "org.sonarqube" version "2.8"
id "io.codearte.nexus-staging" version "0.21.1" id "io.codearte.nexus-staging" version "0.21.1"
id "org.xbib.gradle.plugin.asciidoctor" version "1.6.0.1"
} }
apply plugin: "io.codearte.nexus-staging" wrapper {
gradleVersion = "${project.property('gradle.wrapper.version')}"
distributionType = Wrapper.DistributionType.ALL
}
ext {
user = 'xbib'
name = 'oai'
description = 'Open Archive Initiative library for Java'
inceptionYear = '2016'
url = 'https://github.com/' + user + '/' + name
scmUrl = 'https://github.com/' + user + '/' + name
scmConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git'
scmDeveloperConnection = 'scm:git:ssh://git@github.com:' + user + '/' + name + '.git'
issueManagementSystem = 'Github'
issueManagementUrl = ext.scmUrl + '/issues'
licenseName = 'The Apache License, Version 2.0'
licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
}
subprojects { subprojects {
apply plugin: 'java-library' apply plugin: 'java-library'
apply plugin: 'maven'
apply plugin: 'pmd'
apply plugin: 'checkstyle'
apply plugin: "com.github.spotbugs"
apply plugin: 'org.xbib.gradle.plugin.asciidoctor'
repositories {
mavenCentral()
}
dependencies { dependencies {
testCompile "org.junit.jupiter:junit-jupiter-api:${project.property('junit.version')}" testImplementation "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
testCompile "org.junit.jupiter:junit-jupiter-params:${project.property('junit.version')}"
testCompile "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}"
testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
} }
compileJava { apply from: rootProject.file('gradle/ide/idea.gradle')
sourceCompatibility = JavaVersion.VERSION_11 apply from: rootProject.file('gradle/compile/java.gradle')
targetCompatibility = JavaVersion.VERSION_11 apply from: rootProject.file('gradle/test/junit5.gradle')
} apply from: rootProject.file('gradle/publishing/publication.gradle')
compileTestJava {
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
}
tasks.withType(JavaCompile) {
options.compilerArgs << "-Xlint:all"
}
clean {
delete 'out'
}
test {
useJUnitPlatform()
failFast = false
testLogging {
events 'STARTED', 'PASSED', 'FAILED', 'SKIPPED'
showStandardStreams = false
}
afterSuite { desc, result ->
if (!desc.parent) {
println "\nTest result: ${result.resultType}"
println "Test summary: ${result.testCount} tests, " +
"${result.successfulTestCount} succeeded, " +
"${result.failedTestCount} failed, " +
"${result.skippedTestCount} skipped"
}
}
}
task sourcesJar(type: Jar, dependsOn: classes) {
classifier 'sources'
from sourceSets.main.allSource
}
task javadocJar(type: Jar, dependsOn: javadoc) {
classifier 'javadoc'
}
artifacts {
archives sourcesJar, javadocJar
}
ext {
user = 'xbib'
projectName = 'oai'
projectDescription = 'Open Archive Initiative library for Java'
scmUrl = 'https://github.com/xbib/oai'
scmConnection = 'scm:git:git://github.com/xbib/oai.git'
scmDeveloperConnection = 'scm:git:git://github.com/xbib/oai.git'
}
task sonatypeUpload(type: Upload, dependsOn: build) {
group = 'publish'
configuration = configurations.archives
uploadDescriptor = true
repositories {
if (project.hasProperty('ossrhUsername')) {
mavenDeployer {
beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) }
repository(url: uri(ossrhReleaseUrl)) {
authentication(userName: ossrhUsername, password: ossrhPassword)
}
snapshotRepository(url: uri(ossrhSnapshotUrl)) {
authentication(userName: ossrhUsername, password: ossrhPassword)
}
pom.project {
groupId project.group
artifactId project.name
version project.version
name project.name
description projectDescription
packaging 'jar'
inceptionYear '2016'
url scmUrl
organization {
name 'xbib'
url 'http://xbib.org'
}
developers {
developer {
id user
name 'Jörg Prante'
email 'joergprante@gmail.com'
url 'https://github.com/jprante'
}
}
scm {
url scmUrl
connection scmConnection
developerConnection scmDeveloperConnection
}
licenses {
license {
name 'The Apache License, Version 2.0'
url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
}
}
}
}
}
}
}
nexusStaging {
packageGroup = "org.xbib"
}
spotbugs {
effort = "max"
reportLevel = "low"
}
tasks.withType(com.github.spotbugs.SpotBugsTask) {
ignoreFailures = true
reports {
xml.enabled = false
html.enabled = true
}
}
pmd {
toolVersion = '6.11.0'
ruleSets = ['category/java/bestpractices.xml']
ignoreFailures = true
}
checkstyle {
configFile = rootProject.file('config/checkstyle/checkstyle.xml')
ignoreFailures = true
showViolations = true
}
sonarqube {
properties {
property "sonar.projectName", "${project.group} ${project.name}"
property "sonar.sourceEncoding", "UTF-8"
property "sonar.tests", "src/test/java"
property "sonar.scm.provider", "git"
property "sonar.junit.reportsPath", "build/test-results/test/"
}
}
} }
apply from: rootProject.file('gradle/publishing/sonatype.gradle')

View file

@ -1,12 +1,10 @@
group = org.xbib group = org.xbib
name = oai name = oai
version = 2.2.2 version = 2.3.0
xbib-content.version = 2.0.5 gradle.wrapper.version = 6.4.1
xbib-netty-http.version = 4.1.48.0 xbib-content.version = 2.2.0
xbib-netty-http.version = 4.1.49.1
tcnative.version = 2.0.28.Final tcnative.version = 2.0.28.Final
xbib-marc.version = 2.3.0
# test only xbib-bibliographic-character-sets.version = 2.0.0
junit.version = 5.5.1
xbib-bibliographic-character-sets.version = 1.0.0
xbib-marc.version = 2.1.0

View file

@ -0,0 +1,35 @@
apply plugin: 'java-library'
java {
modularity.inferModulePath.set(true)
}
compileJava {
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
}
compileTestJava {
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
}
jar {
manifest {
attributes('Implementation-Version': project.version)
}
}
task sourcesJar(type: Jar, dependsOn: classes) {
classifier 'sources'
from sourceSets.main.allSource
}
task javadocJar(type: Jar, dependsOn: javadoc) {
classifier 'javadoc'
}
artifacts {
archives sourcesJar, javadocJar
}

View file

@ -0,0 +1,55 @@
apply plugin: 'org.xbib.gradle.plugin.asciidoctor'
configurations {
asciidoclet
}
dependencies {
asciidoclet "org.asciidoctor:asciidoclet:${project.property('asciidoclet.version')}"
}
asciidoctor {
backends 'html5'
outputDir = file("${rootProject.projectDir}/docs")
separateOutputDirs = false
attributes 'source-highlighter': 'coderay',
idprefix: '',
idseparator: '-',
toc: 'left',
doctype: 'book',
icons: 'font',
encoding: 'utf-8',
sectlink: true,
sectanchors: true,
linkattrs: true,
imagesdir: 'img',
stylesheet: "${projectDir}/src/docs/asciidoc/css/foundation.css"
}
/*javadoc {
options.docletpath = configurations.asciidoclet.files.asType(List)
options.doclet = 'org.asciidoctor.Asciidoclet'
//options.overview = "src/docs/asciidoclet/overview.adoc"
options.addStringOption "-base-dir", "${projectDir}"
options.addStringOption "-attribute",
"name=${project.name},version=${project.version},title-link=https://github.com/xbib/${project.name}"
configure(options) {
noTimestamp = true
}
}*/
/*javadoc {
options.docletpath = configurations.asciidoclet.files.asType(List)
options.doclet = 'org.asciidoctor.Asciidoclet'
options.overview = "${rootProject.projectDir}/src/docs/asciidoclet/overview.adoc"
options.addStringOption "-base-dir", "${projectDir}"
options.addStringOption "-attribute",
"name=${project.name},version=${project.version},title-link=https://github.com/xbib/${project.name}"
options.destinationDirectory(file("${projectDir}/docs/javadoc"))
configure(options) {
noTimestamp = true
}
}*/

13
gradle/ide/idea.gradle Normal file
View file

@ -0,0 +1,13 @@
apply plugin: 'idea'
idea {
module {
outputDir file('build/classes/java/main')
testOutputDir file('build/classes/java/test')
}
}
if (project.convention.findPlugin(JavaPluginConvention)) {
//sourceSets.main.output.classesDirs = file("build/classes/java/main")
//sourceSets.test.output.classesDirs = file("build/classes/java/test")
}

View file

@ -0,0 +1,64 @@
apply plugin: "de.marcphilipp.nexus-publish"
publishing {
publications {
mavenJava(MavenPublication) {
from components.java
artifact sourcesJar
artifact javadocJar
pom {
name = project.name
description = rootProject.ext.description
url = rootProject.ext.url
inceptionYear = rootProject.ext.inceptionYear
packaging = 'jar'
organization {
name = 'xbib'
url = 'https://xbib.org'
}
developers {
developer {
id = 'jprante'
name = 'Jörg Prante'
email = 'joergprante@gmail.com'
url = 'https://github.com/jprante'
}
}
scm {
url = rootProject.ext.scmUrl
connection = rootProject.ext.scmConnection
developerConnection = rootProject.ext.scmDeveloperConnection
}
issueManagement {
system = rootProject.ext.issueManagementSystem
url = rootProject.ext.issueManagementUrl
}
licenses {
license {
name = rootProject.ext.licenseName
url = rootProject.ext.licenseUrl
distribution = 'repo'
}
}
}
}
}
}
if (project.hasProperty("signing.keyId")) {
apply plugin: 'signing'
signing {
sign publishing.publications.mavenJava
}
}
nexusPublishing {
repositories {
sonatype {
username = project.property('ossrhUsername')
password = project.property('ossrhPassword')
packageGroup = "org.xbib"
}
}
}

View file

@ -0,0 +1,11 @@
if (project.hasProperty('ossrhUsername') && project.hasProperty('ossrhPassword')) {
apply plugin: 'io.codearte.nexus-staging'
nexusStaging {
username = project.property('ossrhUsername')
password = project.property('ossrhPassword')
packageGroup = "org.xbib"
}
}

27
gradle/test/junit5.gradle Normal file
View file

@ -0,0 +1,27 @@
def junitVersion = project.hasProperty('junit.version')?project.property('junit.version'):'5.6.2'
def hamcrestVersion = project.hasProperty('hamcrest.version')?project.property('hamcrest.version'):'2.2'
dependencies {
testImplementation "org.junit.jupiter:junit-jupiter-api:${junitVersion}"
testImplementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}"
testImplementation "org.hamcrest:hamcrest-library:${hamcrestVersion}"
testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}"
}
test {
useJUnitPlatform()
failFast = true
testLogging {
events 'STARTED', 'PASSED', 'FAILED', 'SKIPPED'
}
afterSuite { desc, result ->
if (!desc.parent) {
println "\nTest result: ${result.resultType}"
println "Test summary: ${result.testCount} tests, " +
"${result.successfulTestCount} succeeded, " +
"${result.failedTestCount} failed, " +
"${result.skippedTestCount} skipped"
}
}
}

Binary file not shown.

View file

@ -1,6 +1,5 @@
#Mon Jan 13 14:23:25 CET 2020
distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-all.zip
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
zipStorePath=wrapper/dists distributionUrl=https\://services.gradle.org/distributions/gradle-6.4.1-all.zip
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

31
gradlew vendored
View file

@ -82,6 +82,7 @@ esac
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM. # Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
@ -129,6 +130,7 @@ fi
if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"` APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"` JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath # We build the pattern for arguments to be converted via cygpath
@ -154,19 +156,19 @@ if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
else else
eval `echo args$i`="\"$arg\"" eval `echo args$i`="\"$arg\""
fi fi
i=$((i+1)) i=`expr $i + 1`
done done
case $i in case $i in
(0) set -- ;; 0) set -- ;;
(1) set -- "$args0" ;; 1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;; 2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;; 3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;; 4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac esac
fi fi
@ -175,14 +177,9 @@ save () {
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
echo " " echo " "
} }
APP_ARGS=$(save "$@") APP_ARGS=`save "$@"`
# Collect all arguments for the java command, following the shell quoting and substitution rules # Collect all arguments for the java command, following the shell quoting and substitution rules
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
cd "$(dirname "$0")"
fi
exec "$JAVACMD" "$@" exec "$JAVACMD" "$@"

4
gradlew.bat vendored
View file

@ -29,6 +29,9 @@ if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0 set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME% set APP_HOME=%DIRNAME%
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@ -81,6 +84,7 @@ set CMD_LINE_ARGS=%*
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle @rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%

View file

@ -1,8 +1,7 @@
dependencies { dependencies {
compile project(':oai-common') api project(':oai-common')
compile "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}" api "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}"
compile "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}" implementation "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}"
testCompile "org.xbib:marc:${project.property('xbib-marc.version')}" testImplementation "org.xbib:marc:${project.property('xbib-marc.version')}"
testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}" testImplementation "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
} }

View file

@ -0,0 +1,13 @@
module org.xbib.oai.client {
exports org.xbib.oai.client;
exports org.xbib.oai.client.getrecord;
exports org.xbib.oai.client.identify;
exports org.xbib.oai.client.listidentifiers;
exports org.xbib.oai.client.listrecords;
exports org.xbib.oai.client.listsets;
requires org.xbib.oai;
requires org.xbib.net.url;
requires org.xbib.netty.http.common;
requires org.xbib.content.xml;
requires java.xml;
}

View file

@ -7,6 +7,7 @@ import org.xbib.oai.util.ResumptionToken;
import org.xbib.oai.xml.MetadataHandler; import org.xbib.oai.xml.MetadataHandler;
import org.xml.sax.Attributes; import org.xml.sax.Attributes;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import org.xml.sax.ContentHandler;
import java.time.Instant; import java.time.Instant;
import java.time.LocalDate; import java.time.LocalDate;
@ -14,21 +15,17 @@ import java.time.ZoneId;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException; import java.time.format.DateTimeParseException;
import java.util.logging.Level;
import java.util.logging.Logger;
/** /**
* *
*/ */
public class ListRecordsFilterReader extends XMLFilterReader { public class ListRecordsFilterReader extends XMLFilterReader {
private static final Logger logger = Logger.getLogger(ListRecordsFilterReader.class.getName());
private final ListRecordsRequest request; private final ListRecordsRequest request;
private final ListRecordsResponse response; private final ListRecordsResponse response;
private StringBuilder content; private final StringBuilder content;
private RecordHeader header; private RecordHeader header;
@ -54,14 +51,12 @@ public class ListRecordsFilterReader extends XMLFilterReader {
@Override @Override
public void startDocument() throws SAXException { public void startDocument() throws SAXException {
logger.log(Level.FINE, "start of document");
super.startDocument(); super.startDocument();
request.setResumptionToken(null); request.setResumptionToken(null);
} }
@Override @Override
public void endDocument() throws SAXException { public void endDocument() throws SAXException {
logger.log(Level.FINE, "end of document");
super.endDocument(); super.endDocument();
} }
@ -97,7 +92,6 @@ public class ListRecordsFilterReader extends XMLFilterReader {
request.setResumptionToken(token); request.setResumptionToken(token);
} }
} catch (Exception e) { } catch (Exception e) {
logger.log(Level.FINE, e.getMessage(), e);
throw new SAXException(e); throw new SAXException(e);
} }
break; break;
@ -139,7 +133,6 @@ public class ListRecordsFilterReader extends XMLFilterReader {
// feedback to request // feedback to request
request.setResumptionToken(token); request.setResumptionToken(token);
} else { } else {
logger.log(Level.WARNING, "empty resumption token value");
// some servers send a null or an empty token as last token // some servers send a null or an empty token as last token
token = null; token = null;
request.setResumptionToken(null); request.setResumptionToken(null);
@ -167,9 +160,7 @@ public class ListRecordsFilterReader extends XMLFilterReader {
ZonedDateTime zonedDateTime = localDate.atStartOfDay(ZoneId.of("UTC")); ZonedDateTime zonedDateTime = localDate.atStartOfDay(ZoneId.of("UTC"));
header.setDate(Instant.from(zonedDateTime)); header.setDate(Instant.from(zonedDateTime));
} catch (DateTimeParseException e2) { } catch (DateTimeParseException e2) {
if (logger.isLoggable(Level.WARNING)) { throw new IllegalArgumentException("unable to parse: " + e2.getMessage(), e2);
logger.log(Level.WARNING, "unable to parse date: " + s + " reason = " + e2.getMessage());
}
} }
} }
} }

View file

@ -1,6 +1,5 @@
package org.xbib.oai.client.listrecords; package org.xbib.oai.client.listrecords;
import io.netty.handler.codec.http.HttpHeaderNames;
import org.xbib.content.xml.transform.TransformerURIResolver; import org.xbib.content.xml.transform.TransformerURIResolver;
import org.xbib.content.xml.util.XMLUtil; import org.xbib.content.xml.util.XMLUtil;
import org.xbib.netty.http.common.HttpResponse; import org.xbib.netty.http.common.HttpResponse;
@ -16,12 +15,9 @@ import org.xml.sax.InputSource;
import java.io.StringReader; import java.io.StringReader;
import java.io.Writer; import java.io.Writer;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.text.MessageFormat;
import java.time.Instant; import java.time.Instant;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit; import java.time.temporal.ChronoUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.transform.Source; import javax.xml.transform.Source;
import javax.xml.transform.Transformer; import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerException;
@ -34,8 +30,6 @@ import javax.xml.transform.stream.StreamResult;
*/ */
public class ListRecordsResponse extends AbstractOAIResponse { public class ListRecordsResponse extends AbstractOAIResponse {
private static final Logger logger = Logger.getLogger(ListRecordsResponse.class.getName());
private static final String[] RETRY_AFTER_HEADERS = { private static final String[] RETRY_AFTER_HEADERS = {
"retry-after", "Retry-after", "Retry-After" "retry-after", "Retry-after", "Retry-After"
}; };
@ -93,26 +87,21 @@ public class ListRecordsResponse extends AbstractOAIResponse {
// parse RFC date, e.g. Fri, 31 Dec 1999 23:59:59 GMT // parse RFC date, e.g. Fri, 31 Dec 1999 23:59:59 GMT
Instant instant = Instant.from(DateTimeFormatter.RFC_1123_DATE_TIME.parse(retryAfter)); Instant instant = Instant.from(DateTimeFormatter.RFC_1123_DATE_TIME.parse(retryAfter));
secs = ChronoUnit.SECONDS.between(instant, Instant.now()); secs = ChronoUnit.SECONDS.between(instant, Instant.now());
logger.log(Level.INFO, MessageFormat.format("parsed delay seconds is {0}", secs));
} }
logger.log(Level.INFO, MessageFormat.format("setting delay seconds to {0}", secs));
} }
} }
request.setRetry(true); request.setRetry(true);
try { try {
if (secs > 0L) { if (secs > 0L) {
logger.log(Level.INFO, MessageFormat.format("waiting for {0} seconds (retry-after)", secs));
Thread.sleep(1000 * secs); Thread.sleep(1000 * secs);
} }
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
logger.log(Level.SEVERE, "interrupted");
} }
return; return;
} }
if (status == 429) { if (status == 429) {
try { try {
logger.log(Level.WARNING, "received 429 Too many requests, waiting 10 seconds...");
Thread.sleep(10000L); Thread.sleep(10000L);
} catch (InterruptedException e) { } catch (InterruptedException e) {
// ignore // ignore
@ -122,7 +111,7 @@ public class ListRecordsResponse extends AbstractOAIResponse {
throw new OAIException("status = " + status + " response = " + content); throw new OAIException("status = " + status + " response = " + content);
} }
// activate XSLT only if OAI XML content type is returned // activate XSLT only if OAI XML content type is returned
String contentType = message.getHeaders().getHeader(HttpHeaderNames.CONTENT_TYPE); String contentType = message.getHeaders().getHeader("content-type");
if (contentType != null && !contentType.startsWith("text/xml")) { if (contentType != null && !contentType.startsWith("text/xml")) {
throw new OAIException("no XML content type in response: " + contentType); throw new OAIException("no XML content type in response: " + contentType);
} }
@ -133,7 +122,6 @@ public class ListRecordsResponse extends AbstractOAIResponse {
Transformer transformer = transformerFactory.newTransformer(); Transformer transformer = transformerFactory.newTransformer();
Source source = new SAXSource(filterreader, new InputSource(new StringReader(XMLUtil.sanitize(content)))); Source source = new SAXSource(filterreader, new InputSource(new StringReader(XMLUtil.sanitize(content))));
StreamResult streamResult = new StreamResult(writer); StreamResult streamResult = new StreamResult(writer);
logger.log(Level.FINE, "transforming");
transformer.transform(source, streamResult); transformer.transform(source, streamResult);
if ("noRecordsMatch".equals(error)) { if ("noRecordsMatch".equals(error)) {
throw new NoRecordsMatchException("metadataPrefix=" + request.getMetadataPrefix() throw new NoRecordsMatchException("metadataPrefix=" + request.getMetadataPrefix()

View file

@ -28,12 +28,12 @@ import java.util.logging.Logger;
/** /**
* *
*/ */
@Disabled("takes very long time")
class BundeskunsthalleTest { class BundeskunsthalleTest {
private static final Logger logger = Logger.getLogger(BundeskunsthalleTest.class.getName()); private static final Logger logger = Logger.getLogger(BundeskunsthalleTest.class.getName());
@Test @Test
@Disabled("takes long time")
void testListRecords() { void testListRecords() {
URL url = URL.create("https://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh"); URL url = URL.create("https://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh");
try (Client httpClient = Client.builder() try (Client httpClient = Client.builder()
@ -63,7 +63,7 @@ class BundeskunsthalleTest {
ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest(); ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest();
listRecordsRequest.setDateTimeFormatter(dateTimeFormatter); listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
listRecordsRequest.setMetadataPrefix("marcxml"); listRecordsRequest.setMetadataPrefix("marcxml");
try (MarcJsonWriter writer = new MarcJsonWriter("bk-bulk%d.jsonl", 1000, try (MarcJsonWriter writer = new MarcJsonWriter("build/bk-bulk%d.jsonl", 1000,
EnumSet.of(MarcJsonWriter.Style.ELASTICSEARCH_BULK), 65536, false) EnumSet.of(MarcJsonWriter.Style.ELASTICSEARCH_BULK), 65536, false)
.setIndex("testindex", "testtype")) { .setIndex("testindex", "testtype")) {
writer.startDocument(); writer.startDocument();

View file

@ -1,6 +1,7 @@
package org.xbib.oai.client; package org.xbib.oai.client;
import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaderNames;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.xbib.net.URL; import org.xbib.net.URL;
import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Client;
@ -18,8 +19,6 @@ import java.io.StringWriter;
import java.net.ConnectException; import java.net.ConnectException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.time.Instant; import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
@ -52,9 +51,6 @@ class DNBClientTest {
httpClient.execute(request).get(); httpClient.execute(request).get();
String granularity = identifyResponse.getGranularity(); String granularity = identifyResponse.getGranularity();
logger.log(Level.INFO, "granularity = " + granularity); logger.log(Level.INFO, "granularity = " + granularity);
DateTimeFormatter dateTimeFormatter = "YYYY-MM-DD".equals(granularity) ?
DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")) :
DateTimeFormatter.ISO_DATE_TIME;
ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest(); ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest();
listRecordsRequest.setFrom(Instant.parse("2016-01-01T00:00:00Z")); listRecordsRequest.setFrom(Instant.parse("2016-01-01T00:00:00Z"));
listRecordsRequest.setUntil(Instant.parse("2016-01-10T00:00:00Z")); listRecordsRequest.setUntil(Instant.parse("2016-01-10T00:00:00Z"));
@ -83,6 +79,7 @@ class DNBClientTest {
} }
fileWriter.close(); fileWriter.close();
logger.log(Level.INFO, "count=" + handler.count()); logger.log(Level.INFO, "count=" + handler.count());
assertTrue(handler.count() > 0);
} catch (Exception e) { } catch (Exception e) {
logger.log(Level.SEVERE, "skipped, HTTP exception"); logger.log(Level.SEVERE, "skipped, HTTP exception");
} }

View file

@ -12,9 +12,10 @@ import org.xbib.oai.client.listrecords.ListRecordsRequest;
import org.xbib.oai.client.listrecords.ListRecordsResponse; import org.xbib.oai.client.listrecords.ListRecordsResponse;
import org.xbib.oai.xml.SimpleMetadataHandler; import org.xbib.oai.xml.SimpleMetadataHandler;
import java.io.File;
import java.io.FileWriter;
import java.io.StringWriter; import java.io.StringWriter;
import java.io.Writer;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Instant; import java.time.Instant;
import java.time.ZoneId; import java.time.ZoneId;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
@ -30,7 +31,7 @@ class DOAJClientTest {
private static final Logger logger = Logger.getLogger(DOAJClientTest.class.getName()); private static final Logger logger = Logger.getLogger(DOAJClientTest.class.getName());
@Test @Test
@Disabled // takes too long time @Disabled("takes long time")
void testListRecordsDOAJ() { void testListRecordsDOAJ() {
URL url = URL.create("https://doaj.org/oai"); URL url = URL.create("https://doaj.org/oai");
try (Client httpClient = Client.builder() try (Client httpClient = Client.builder()
@ -60,26 +61,24 @@ class DOAJClientTest {
listRecordsRequest.setUntil(Instant.parse("2018-01-01T00:00:00Z")); listRecordsRequest.setUntil(Instant.parse("2018-01-01T00:00:00Z"));
listRecordsRequest.setMetadataPrefix("oai_dc"); listRecordsRequest.setMetadataPrefix("oai_dc");
Handler handler = new Handler(); Handler handler = new Handler();
File file = File.createTempFile("doaj.", ".xml"); try (Writer writer = Files.newBufferedWriter(Paths.get("build/doaj.xml"))) {
file.deleteOnExit(); while (listRecordsRequest != null) {
FileWriter fileWriter = new FileWriter(file); ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
while (listRecordsRequest != null) { listRecordsRequest.addHandler(handler);
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest); logger.log(Level.INFO, "sending " + listRecordsRequest.getURL());
listRecordsRequest.addHandler(handler); request = Request.get()
logger.log(Level.INFO,"sending " + listRecordsRequest.getURL()); .url(url.resolve(listRecordsRequest.getURL()))
request = Request.get() .addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.url(url.resolve(listRecordsRequest.getURL())) .setResponseListener(resp -> {
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8") listRecordsResponse.receivedResponse(resp, writer);
.setResponseListener(resp -> { logger.log(Level.FINE, "response headers = " + resp.getHeaders() +
listRecordsResponse.receivedResponse(resp, fileWriter); " resumption-token = {}" + listRecordsResponse.getResumptionToken());
logger.log(Level.FINE, "response headers = " + resp.getHeaders() + })
" resumption-token = {}" + listRecordsResponse.getResumptionToken()); .build();
}) httpClient.execute(request).get();
.build(); listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
httpClient.execute(request).get(); }
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} }
fileWriter.close();
logger.log(Level.INFO, "count = " + handler.count()); logger.log(Level.INFO, "count = " + handler.count());
} catch (Exception e) { } catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e); logger.log(Level.SEVERE, e.getMessage(), e);

View file

@ -1,3 +1,5 @@
dependencies { dependencies {
compile "org.xbib:content-rdf:${project.property('xbib-content.version')}" api "org.xbib:content-rdf:${project.property('xbib-content.version')}"
api "org.xbib:content-resource:${project.property('xbib-content.version')}"
api "org.xbib:content-xml:${project.property('xbib-content.version')}"
} }

View file

@ -0,0 +1,11 @@
module org.xbib.oai {
exports org.xbib.oai;
exports org.xbib.oai.exceptions;
exports org.xbib.oai.rdf;
exports org.xbib.oai.util;
exports org.xbib.oai.xml;
requires org.xbib.content.rdf;
requires org.xbib.content.resource;
requires org.xbib.content.xml;
requires java.xml;
}

View file

@ -4,7 +4,7 @@ import org.xbib.content.rdf.RdfContentParams;
import org.xbib.content.rdf.io.xml.AbstractXmlResourceHandler; import org.xbib.content.rdf.io.xml.AbstractXmlResourceHandler;
import org.xbib.content.rdf.io.xml.XmlHandler; import org.xbib.content.rdf.io.xml.XmlHandler;
import org.xbib.content.resource.IRI; import org.xbib.content.resource.IRI;
import org.xbib.content.resource.IRINamespaceContext; import org.xbib.content.resource.NamespaceContext;
import org.xbib.oai.OAIConstants; import org.xbib.oai.OAIConstants;
import javax.xml.namespace.QName; import javax.xml.namespace.QName;
@ -48,12 +48,12 @@ public class RdfResourceHandler extends AbstractXmlResourceHandler<RdfContentPar
} }
@Override @Override
public XmlHandler<RdfContentParams> setNamespaceContext(IRINamespaceContext namespaceContext) { public XmlHandler<RdfContentParams> setNamespaceContext(NamespaceContext namespaceContext) {
return this; return this;
} }
@Override @Override
public IRINamespaceContext getNamespaceContext() { public NamespaceContext getNamespaceContext() {
return getParams().getNamespaceContext(); return getParams().getNamespaceContext();
} }
} }

View file

@ -14,8 +14,6 @@ import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.stream.Location; import javax.xml.stream.Location;
import javax.xml.stream.XMLEventFactory; import javax.xml.stream.XMLEventFactory;
import javax.xml.stream.XMLEventWriter; import javax.xml.stream.XMLEventWriter;
@ -29,15 +27,13 @@ import javax.xml.stream.events.Namespace;
*/ */
public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements OAIConstants { public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements OAIConstants {
private static final Logger logger = Logger.getLogger(XmlSimpleMetadataHandler.class.getName());
private static final XMLOutputFactory outputFactory = XMLOutputFactory.newInstance(); private static final XMLOutputFactory outputFactory = XMLOutputFactory.newInstance();
private static final XMLEventFactory eventFactory = XMLEventFactory.newInstance(); private static final XMLEventFactory eventFactory = XMLEventFactory.newInstance();
private List<String> namespaces = new ArrayList<>(); private List<String> namespaces = new ArrayList<>();
private Deque<Collection<Namespace>> nsStack = new ArrayDeque<>(); private final Deque<Collection<Namespace>> nsStack = new ArrayDeque<>();
private Locator locator; private Locator locator;
@ -49,14 +45,10 @@ public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements O
private boolean needToCallStartDocument = false; private boolean needToCallStartDocument = false;
public XmlSimpleMetadataHandler setWriter(Writer writer) { public XmlSimpleMetadataHandler setWriter(Writer writer) throws XMLStreamException {
this.writer = writer; this.writer = writer;
try { outputFactory.setProperty("javax.xml.stream.isRepairingNamespaces", Boolean.TRUE);
outputFactory.setProperty("javax.xml.stream.isRepairingNamespaces", Boolean.TRUE); this.eventWriter = outputFactory.createXMLEventWriter(writer);
this.eventWriter = outputFactory.createXMLEventWriter(writer);
} catch (XMLStreamException e) {
logger.log(Level.FINE, e.getMessage(), e);
}
return this; return this;
} }
@ -150,7 +142,7 @@ public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements O
try { try {
eventWriter.add(eventFactory.createStartDocument()); eventWriter.add(eventFactory.createStartDocument());
} catch (XMLStreamException e) { } catch (XMLStreamException e) {
logger.log(Level.FINE, e.getMessage(), e); throw new SAXException(e);
} }
needToCallStartDocument = false; needToCallStartDocument = false;
} }
@ -272,10 +264,14 @@ public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements O
} }
private static final class SAXLocation implements Location { private static final class SAXLocation implements Location {
private int lineNumber;
private int columnNumber; private final int lineNumber;
private String publicId;
private String systemId; private final int columnNumber;
private final String publicId;
private final String systemId;
private SAXLocation(Locator locator) { private SAXLocation(Locator locator) {
lineNumber = locator.getLineNumber(); lineNumber = locator.getLineNumber();

View file

@ -1,3 +1,3 @@
dependencies { dependencies {
compile project(':oai-common') api project(':oai-common')
} }