update to Java named modules, Gradle 6.4.1

This commit is contained in:
Jörg Prante 2020-05-26 16:12:26 +02:00
parent a8fe25a21e
commit 276771f96f
24 changed files with 337 additions and 278 deletions

View file

@ -1,179 +1,39 @@
plugins {
id "com.github.spotbugs" version "2.0.0"
id "org.sonarqube" version "2.8"
id "de.marcphilipp.nexus-publish" version "0.4.0"
id "io.codearte.nexus-staging" version "0.21.1"
id "org.xbib.gradle.plugin.asciidoctor" version "1.6.0.1"
}
apply plugin: "io.codearte.nexus-staging"
wrapper {
gradleVersion = "${project.property('gradle.wrapper.version')}"
distributionType = Wrapper.DistributionType.ALL
}
ext {
user = 'xbib'
name = 'oai'
description = 'Open Archive Initiative library for Java'
inceptionYear = '2016'
url = 'https://github.com/' + user + '/' + name
scmUrl = 'https://github.com/' + user + '/' + name
scmConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git'
scmDeveloperConnection = 'scm:git:ssh://git@github.com:' + user + '/' + name + '.git'
issueManagementSystem = 'Github'
issueManagementUrl = ext.scmUrl + '/issues'
licenseName = 'The Apache License, Version 2.0'
licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
}
subprojects {
apply plugin: 'java-library'
apply plugin: 'maven'
apply plugin: 'pmd'
apply plugin: 'checkstyle'
apply plugin: "com.github.spotbugs"
apply plugin: 'org.xbib.gradle.plugin.asciidoctor'
repositories {
mavenCentral()
}
dependencies {
testCompile "org.junit.jupiter:junit-jupiter-api:${project.property('junit.version')}"
testCompile "org.junit.jupiter:junit-jupiter-params:${project.property('junit.version')}"
testCompile "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}"
testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
testImplementation "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
}
compileJava {
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
}
compileTestJava {
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
}
tasks.withType(JavaCompile) {
options.compilerArgs << "-Xlint:all"
}
clean {
delete 'out'
}
test {
useJUnitPlatform()
failFast = false
testLogging {
events 'STARTED', 'PASSED', 'FAILED', 'SKIPPED'
showStandardStreams = false
}
afterSuite { desc, result ->
if (!desc.parent) {
println "\nTest result: ${result.resultType}"
println "Test summary: ${result.testCount} tests, " +
"${result.successfulTestCount} succeeded, " +
"${result.failedTestCount} failed, " +
"${result.skippedTestCount} skipped"
}
}
}
task sourcesJar(type: Jar, dependsOn: classes) {
classifier 'sources'
from sourceSets.main.allSource
}
task javadocJar(type: Jar, dependsOn: javadoc) {
classifier 'javadoc'
}
artifacts {
archives sourcesJar, javadocJar
}
ext {
user = 'xbib'
projectName = 'oai'
projectDescription = 'Open Archive Initiative library for Java'
scmUrl = 'https://github.com/xbib/oai'
scmConnection = 'scm:git:git://github.com/xbib/oai.git'
scmDeveloperConnection = 'scm:git:git://github.com/xbib/oai.git'
}
task sonatypeUpload(type: Upload, dependsOn: build) {
group = 'publish'
configuration = configurations.archives
uploadDescriptor = true
repositories {
if (project.hasProperty('ossrhUsername')) {
mavenDeployer {
beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) }
repository(url: uri(ossrhReleaseUrl)) {
authentication(userName: ossrhUsername, password: ossrhPassword)
}
snapshotRepository(url: uri(ossrhSnapshotUrl)) {
authentication(userName: ossrhUsername, password: ossrhPassword)
}
pom.project {
groupId project.group
artifactId project.name
version project.version
name project.name
description projectDescription
packaging 'jar'
inceptionYear '2016'
url scmUrl
organization {
name 'xbib'
url 'http://xbib.org'
}
developers {
developer {
id user
name 'Jörg Prante'
email 'joergprante@gmail.com'
url 'https://github.com/jprante'
}
}
scm {
url scmUrl
connection scmConnection
developerConnection scmDeveloperConnection
}
licenses {
license {
name 'The Apache License, Version 2.0'
url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
}
}
}
}
}
}
}
nexusStaging {
packageGroup = "org.xbib"
}
spotbugs {
effort = "max"
reportLevel = "low"
}
tasks.withType(com.github.spotbugs.SpotBugsTask) {
ignoreFailures = true
reports {
xml.enabled = false
html.enabled = true
}
}
pmd {
toolVersion = '6.11.0'
ruleSets = ['category/java/bestpractices.xml']
ignoreFailures = true
}
checkstyle {
configFile = rootProject.file('config/checkstyle/checkstyle.xml')
ignoreFailures = true
showViolations = true
}
sonarqube {
properties {
property "sonar.projectName", "${project.group} ${project.name}"
property "sonar.sourceEncoding", "UTF-8"
property "sonar.tests", "src/test/java"
property "sonar.scm.provider", "git"
property "sonar.junit.reportsPath", "build/test-results/test/"
}
}
apply from: rootProject.file('gradle/ide/idea.gradle')
apply from: rootProject.file('gradle/compile/java.gradle')
apply from: rootProject.file('gradle/test/junit5.gradle')
apply from: rootProject.file('gradle/publishing/publication.gradle')
}
apply from: rootProject.file('gradle/publishing/sonatype.gradle')

View file

@ -1,12 +1,10 @@
group = org.xbib
name = oai
version = 2.2.2
version = 2.3.0
xbib-content.version = 2.0.5
xbib-netty-http.version = 4.1.48.0
gradle.wrapper.version = 6.4.1
xbib-content.version = 2.2.0
xbib-netty-http.version = 4.1.49.1
tcnative.version = 2.0.28.Final
# test only
junit.version = 5.5.1
xbib-bibliographic-character-sets.version = 1.0.0
xbib-marc.version = 2.1.0
xbib-marc.version = 2.3.0
xbib-bibliographic-character-sets.version = 2.0.0

View file

@ -0,0 +1,35 @@
apply plugin: 'java-library'
java {
modularity.inferModulePath.set(true)
}
compileJava {
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
}
compileTestJava {
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
}
jar {
manifest {
attributes('Implementation-Version': project.version)
}
}
task sourcesJar(type: Jar, dependsOn: classes) {
classifier 'sources'
from sourceSets.main.allSource
}
task javadocJar(type: Jar, dependsOn: javadoc) {
classifier 'javadoc'
}
artifacts {
archives sourcesJar, javadocJar
}

View file

@ -0,0 +1,55 @@
apply plugin: 'org.xbib.gradle.plugin.asciidoctor'
configurations {
asciidoclet
}
dependencies {
asciidoclet "org.asciidoctor:asciidoclet:${project.property('asciidoclet.version')}"
}
asciidoctor {
backends 'html5'
outputDir = file("${rootProject.projectDir}/docs")
separateOutputDirs = false
attributes 'source-highlighter': 'coderay',
idprefix: '',
idseparator: '-',
toc: 'left',
doctype: 'book',
icons: 'font',
encoding: 'utf-8',
sectlink: true,
sectanchors: true,
linkattrs: true,
imagesdir: 'img',
stylesheet: "${projectDir}/src/docs/asciidoc/css/foundation.css"
}
/*javadoc {
options.docletpath = configurations.asciidoclet.files.asType(List)
options.doclet = 'org.asciidoctor.Asciidoclet'
//options.overview = "src/docs/asciidoclet/overview.adoc"
options.addStringOption "-base-dir", "${projectDir}"
options.addStringOption "-attribute",
"name=${project.name},version=${project.version},title-link=https://github.com/xbib/${project.name}"
configure(options) {
noTimestamp = true
}
}*/
/*javadoc {
options.docletpath = configurations.asciidoclet.files.asType(List)
options.doclet = 'org.asciidoctor.Asciidoclet'
options.overview = "${rootProject.projectDir}/src/docs/asciidoclet/overview.adoc"
options.addStringOption "-base-dir", "${projectDir}"
options.addStringOption "-attribute",
"name=${project.name},version=${project.version},title-link=https://github.com/xbib/${project.name}"
options.destinationDirectory(file("${projectDir}/docs/javadoc"))
configure(options) {
noTimestamp = true
}
}*/

13
gradle/ide/idea.gradle Normal file
View file

@ -0,0 +1,13 @@
apply plugin: 'idea'
idea {
module {
outputDir file('build/classes/java/main')
testOutputDir file('build/classes/java/test')
}
}
if (project.convention.findPlugin(JavaPluginConvention)) {
//sourceSets.main.output.classesDirs = file("build/classes/java/main")
//sourceSets.test.output.classesDirs = file("build/classes/java/test")
}

View file

@ -0,0 +1,64 @@
apply plugin: "de.marcphilipp.nexus-publish"
publishing {
publications {
mavenJava(MavenPublication) {
from components.java
artifact sourcesJar
artifact javadocJar
pom {
name = project.name
description = rootProject.ext.description
url = rootProject.ext.url
inceptionYear = rootProject.ext.inceptionYear
packaging = 'jar'
organization {
name = 'xbib'
url = 'https://xbib.org'
}
developers {
developer {
id = 'jprante'
name = 'Jörg Prante'
email = 'joergprante@gmail.com'
url = 'https://github.com/jprante'
}
}
scm {
url = rootProject.ext.scmUrl
connection = rootProject.ext.scmConnection
developerConnection = rootProject.ext.scmDeveloperConnection
}
issueManagement {
system = rootProject.ext.issueManagementSystem
url = rootProject.ext.issueManagementUrl
}
licenses {
license {
name = rootProject.ext.licenseName
url = rootProject.ext.licenseUrl
distribution = 'repo'
}
}
}
}
}
}
if (project.hasProperty("signing.keyId")) {
apply plugin: 'signing'
signing {
sign publishing.publications.mavenJava
}
}
nexusPublishing {
repositories {
sonatype {
username = project.property('ossrhUsername')
password = project.property('ossrhPassword')
packageGroup = "org.xbib"
}
}
}

View file

@ -0,0 +1,11 @@
if (project.hasProperty('ossrhUsername') && project.hasProperty('ossrhPassword')) {
apply plugin: 'io.codearte.nexus-staging'
nexusStaging {
username = project.property('ossrhUsername')
password = project.property('ossrhPassword')
packageGroup = "org.xbib"
}
}

27
gradle/test/junit5.gradle Normal file
View file

@ -0,0 +1,27 @@
def junitVersion = project.hasProperty('junit.version')?project.property('junit.version'):'5.6.2'
def hamcrestVersion = project.hasProperty('hamcrest.version')?project.property('hamcrest.version'):'2.2'
dependencies {
testImplementation "org.junit.jupiter:junit-jupiter-api:${junitVersion}"
testImplementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}"
testImplementation "org.hamcrest:hamcrest-library:${hamcrestVersion}"
testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}"
}
test {
useJUnitPlatform()
failFast = true
testLogging {
events 'STARTED', 'PASSED', 'FAILED', 'SKIPPED'
}
afterSuite { desc, result ->
if (!desc.parent) {
println "\nTest result: ${result.resultType}"
println "Test summary: ${result.testCount} tests, " +
"${result.successfulTestCount} succeeded, " +
"${result.failedTestCount} failed, " +
"${result.skippedTestCount} skipped"
}
}
}

Binary file not shown.

View file

@ -1,6 +1,5 @@
#Mon Jan 13 14:23:25 CET 2020
distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-all.zip
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-6.4.1-all.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

31
gradlew vendored
View file

@ -82,6 +82,7 @@ esac
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
@ -129,6 +130,7 @@ fi
if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
@ -154,19 +156,19 @@ if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
i=`expr $i + 1`
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
0) set -- ;;
1) set -- "$args0" ;;
2) set -- "$args0" "$args1" ;;
3) set -- "$args0" "$args1" "$args2" ;;
4) set -- "$args0" "$args1" "$args2" "$args3" ;;
5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
@ -175,14 +177,9 @@ save () {
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
echo " "
}
APP_ARGS=$(save "$@")
APP_ARGS=`save "$@"`
# Collect all arguments for the java command, following the shell quoting and substitution rules
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
cd "$(dirname "$0")"
fi
exec "$JAVACMD" "$@"

4
gradlew.bat vendored
View file

@ -29,6 +29,9 @@ if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@ -81,6 +84,7 @@ set CMD_LINE_ARGS=%*
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%

View file

@ -1,8 +1,7 @@
dependencies {
compile project(':oai-common')
compile "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}"
compile "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}"
testCompile "org.xbib:marc:${project.property('xbib-marc.version')}"
testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
api project(':oai-common')
api "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}"
implementation "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}"
testImplementation "org.xbib:marc:${project.property('xbib-marc.version')}"
testImplementation "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
}

View file

@ -0,0 +1,13 @@
module org.xbib.oai.client {
exports org.xbib.oai.client;
exports org.xbib.oai.client.getrecord;
exports org.xbib.oai.client.identify;
exports org.xbib.oai.client.listidentifiers;
exports org.xbib.oai.client.listrecords;
exports org.xbib.oai.client.listsets;
requires org.xbib.oai;
requires org.xbib.net.url;
requires org.xbib.netty.http.common;
requires org.xbib.content.xml;
requires java.xml;
}

View file

@ -7,6 +7,7 @@ import org.xbib.oai.util.ResumptionToken;
import org.xbib.oai.xml.MetadataHandler;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.ContentHandler;
import java.time.Instant;
import java.time.LocalDate;
@ -14,21 +15,17 @@ import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
*
*/
public class ListRecordsFilterReader extends XMLFilterReader {
private static final Logger logger = Logger.getLogger(ListRecordsFilterReader.class.getName());
private final ListRecordsRequest request;
private final ListRecordsResponse response;
private StringBuilder content;
private final StringBuilder content;
private RecordHeader header;
@ -54,14 +51,12 @@ public class ListRecordsFilterReader extends XMLFilterReader {
@Override
public void startDocument() throws SAXException {
logger.log(Level.FINE, "start of document");
super.startDocument();
request.setResumptionToken(null);
}
@Override
public void endDocument() throws SAXException {
logger.log(Level.FINE, "end of document");
super.endDocument();
}
@ -97,7 +92,6 @@ public class ListRecordsFilterReader extends XMLFilterReader {
request.setResumptionToken(token);
}
} catch (Exception e) {
logger.log(Level.FINE, e.getMessage(), e);
throw new SAXException(e);
}
break;
@ -139,7 +133,6 @@ public class ListRecordsFilterReader extends XMLFilterReader {
// feedback to request
request.setResumptionToken(token);
} else {
logger.log(Level.WARNING, "empty resumption token value");
// some servers send a null or an empty token as last token
token = null;
request.setResumptionToken(null);
@ -167,9 +160,7 @@ public class ListRecordsFilterReader extends XMLFilterReader {
ZonedDateTime zonedDateTime = localDate.atStartOfDay(ZoneId.of("UTC"));
header.setDate(Instant.from(zonedDateTime));
} catch (DateTimeParseException e2) {
if (logger.isLoggable(Level.WARNING)) {
logger.log(Level.WARNING, "unable to parse date: " + s + " reason = " + e2.getMessage());
}
throw new IllegalArgumentException("unable to parse: " + e2.getMessage(), e2);
}
}
}

View file

@ -1,6 +1,5 @@
package org.xbib.oai.client.listrecords;
import io.netty.handler.codec.http.HttpHeaderNames;
import org.xbib.content.xml.transform.TransformerURIResolver;
import org.xbib.content.xml.util.XMLUtil;
import org.xbib.netty.http.common.HttpResponse;
@ -16,12 +15,9 @@ import org.xml.sax.InputSource;
import java.io.StringReader;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.text.MessageFormat;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
@ -34,8 +30,6 @@ import javax.xml.transform.stream.StreamResult;
*/
public class ListRecordsResponse extends AbstractOAIResponse {
private static final Logger logger = Logger.getLogger(ListRecordsResponse.class.getName());
private static final String[] RETRY_AFTER_HEADERS = {
"retry-after", "Retry-after", "Retry-After"
};
@ -93,26 +87,21 @@ public class ListRecordsResponse extends AbstractOAIResponse {
// parse RFC date, e.g. Fri, 31 Dec 1999 23:59:59 GMT
Instant instant = Instant.from(DateTimeFormatter.RFC_1123_DATE_TIME.parse(retryAfter));
secs = ChronoUnit.SECONDS.between(instant, Instant.now());
logger.log(Level.INFO, MessageFormat.format("parsed delay seconds is {0}", secs));
}
logger.log(Level.INFO, MessageFormat.format("setting delay seconds to {0}", secs));
}
}
request.setRetry(true);
try {
if (secs > 0L) {
logger.log(Level.INFO, MessageFormat.format("waiting for {0} seconds (retry-after)", secs));
Thread.sleep(1000 * secs);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.log(Level.SEVERE, "interrupted");
}
return;
}
if (status == 429) {
try {
logger.log(Level.WARNING, "received 429 Too many requests, waiting 10 seconds...");
Thread.sleep(10000L);
} catch (InterruptedException e) {
// ignore
@ -122,7 +111,7 @@ public class ListRecordsResponse extends AbstractOAIResponse {
throw new OAIException("status = " + status + " response = " + content);
}
// activate XSLT only if OAI XML content type is returned
String contentType = message.getHeaders().getHeader(HttpHeaderNames.CONTENT_TYPE);
String contentType = message.getHeaders().getHeader("content-type");
if (contentType != null && !contentType.startsWith("text/xml")) {
throw new OAIException("no XML content type in response: " + contentType);
}
@ -133,7 +122,6 @@ public class ListRecordsResponse extends AbstractOAIResponse {
Transformer transformer = transformerFactory.newTransformer();
Source source = new SAXSource(filterreader, new InputSource(new StringReader(XMLUtil.sanitize(content))));
StreamResult streamResult = new StreamResult(writer);
logger.log(Level.FINE, "transforming");
transformer.transform(source, streamResult);
if ("noRecordsMatch".equals(error)) {
throw new NoRecordsMatchException("metadataPrefix=" + request.getMetadataPrefix()

View file

@ -28,12 +28,12 @@ import java.util.logging.Logger;
/**
*
*/
@Disabled("takes very long time")
class BundeskunsthalleTest {
private static final Logger logger = Logger.getLogger(BundeskunsthalleTest.class.getName());
@Test
@Disabled("takes long time")
void testListRecords() {
URL url = URL.create("https://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh");
try (Client httpClient = Client.builder()
@ -63,7 +63,7 @@ class BundeskunsthalleTest {
ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest();
listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
listRecordsRequest.setMetadataPrefix("marcxml");
try (MarcJsonWriter writer = new MarcJsonWriter("bk-bulk%d.jsonl", 1000,
try (MarcJsonWriter writer = new MarcJsonWriter("build/bk-bulk%d.jsonl", 1000,
EnumSet.of(MarcJsonWriter.Style.ELASTICSEARCH_BULK), 65536, false)
.setIndex("testindex", "testtype")) {
writer.startDocument();

View file

@ -1,6 +1,7 @@
package org.xbib.oai.client;
import io.netty.handler.codec.http.HttpHeaderNames;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
import org.xbib.net.URL;
import org.xbib.netty.http.client.Client;
@ -18,8 +19,6 @@ import java.io.StringWriter;
import java.net.ConnectException;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
@ -52,9 +51,6 @@ class DNBClientTest {
httpClient.execute(request).get();
String granularity = identifyResponse.getGranularity();
logger.log(Level.INFO, "granularity = " + granularity);
DateTimeFormatter dateTimeFormatter = "YYYY-MM-DD".equals(granularity) ?
DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")) :
DateTimeFormatter.ISO_DATE_TIME;
ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest();
listRecordsRequest.setFrom(Instant.parse("2016-01-01T00:00:00Z"));
listRecordsRequest.setUntil(Instant.parse("2016-01-10T00:00:00Z"));
@ -83,6 +79,7 @@ class DNBClientTest {
}
fileWriter.close();
logger.log(Level.INFO, "count=" + handler.count());
assertTrue(handler.count() > 0);
} catch (Exception e) {
logger.log(Level.SEVERE, "skipped, HTTP exception");
}

View file

@ -12,9 +12,10 @@ import org.xbib.oai.client.listrecords.ListRecordsRequest;
import org.xbib.oai.client.listrecords.ListRecordsResponse;
import org.xbib.oai.xml.SimpleMetadataHandler;
import java.io.File;
import java.io.FileWriter;
import java.io.StringWriter;
import java.io.Writer;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
@ -30,7 +31,7 @@ class DOAJClientTest {
private static final Logger logger = Logger.getLogger(DOAJClientTest.class.getName());
@Test
@Disabled // takes too long time
@Disabled("takes long time")
void testListRecordsDOAJ() {
URL url = URL.create("https://doaj.org/oai");
try (Client httpClient = Client.builder()
@ -60,26 +61,24 @@ class DOAJClientTest {
listRecordsRequest.setUntil(Instant.parse("2018-01-01T00:00:00Z"));
listRecordsRequest.setMetadataPrefix("oai_dc");
Handler handler = new Handler();
File file = File.createTempFile("doaj.", ".xml");
file.deleteOnExit();
FileWriter fileWriter = new FileWriter(file);
while (listRecordsRequest != null) {
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
listRecordsRequest.addHandler(handler);
logger.log(Level.INFO,"sending " + listRecordsRequest.getURL());
request = Request.get()
.url(url.resolve(listRecordsRequest.getURL()))
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.setResponseListener(resp -> {
listRecordsResponse.receivedResponse(resp, fileWriter);
logger.log(Level.FINE, "response headers = " + resp.getHeaders() +
" resumption-token = {}" + listRecordsResponse.getResumptionToken());
})
.build();
httpClient.execute(request).get();
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
try (Writer writer = Files.newBufferedWriter(Paths.get("build/doaj.xml"))) {
while (listRecordsRequest != null) {
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
listRecordsRequest.addHandler(handler);
logger.log(Level.INFO, "sending " + listRecordsRequest.getURL());
request = Request.get()
.url(url.resolve(listRecordsRequest.getURL()))
.addHeader(HttpHeaderNames.ACCEPT.toString(), "utf-8")
.setResponseListener(resp -> {
listRecordsResponse.receivedResponse(resp, writer);
logger.log(Level.FINE, "response headers = " + resp.getHeaders() +
" resumption-token = {}" + listRecordsResponse.getResumptionToken());
})
.build();
httpClient.execute(request).get();
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
}
}
fileWriter.close();
logger.log(Level.INFO, "count = " + handler.count());
} catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage(), e);

View file

@ -1,3 +1,5 @@
dependencies {
compile "org.xbib:content-rdf:${project.property('xbib-content.version')}"
api "org.xbib:content-rdf:${project.property('xbib-content.version')}"
api "org.xbib:content-resource:${project.property('xbib-content.version')}"
api "org.xbib:content-xml:${project.property('xbib-content.version')}"
}

View file

@ -0,0 +1,11 @@
module org.xbib.oai {
exports org.xbib.oai;
exports org.xbib.oai.exceptions;
exports org.xbib.oai.rdf;
exports org.xbib.oai.util;
exports org.xbib.oai.xml;
requires org.xbib.content.rdf;
requires org.xbib.content.resource;
requires org.xbib.content.xml;
requires java.xml;
}

View file

@ -4,7 +4,7 @@ import org.xbib.content.rdf.RdfContentParams;
import org.xbib.content.rdf.io.xml.AbstractXmlResourceHandler;
import org.xbib.content.rdf.io.xml.XmlHandler;
import org.xbib.content.resource.IRI;
import org.xbib.content.resource.IRINamespaceContext;
import org.xbib.content.resource.NamespaceContext;
import org.xbib.oai.OAIConstants;
import javax.xml.namespace.QName;
@ -48,12 +48,12 @@ public class RdfResourceHandler extends AbstractXmlResourceHandler<RdfContentPar
}
@Override
public XmlHandler<RdfContentParams> setNamespaceContext(IRINamespaceContext namespaceContext) {
public XmlHandler<RdfContentParams> setNamespaceContext(NamespaceContext namespaceContext) {
return this;
}
@Override
public IRINamespaceContext getNamespaceContext() {
public NamespaceContext getNamespaceContext() {
return getParams().getNamespaceContext();
}
}

View file

@ -14,8 +14,6 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.stream.Location;
import javax.xml.stream.XMLEventFactory;
import javax.xml.stream.XMLEventWriter;
@ -29,15 +27,13 @@ import javax.xml.stream.events.Namespace;
*/
public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements OAIConstants {
private static final Logger logger = Logger.getLogger(XmlSimpleMetadataHandler.class.getName());
private static final XMLOutputFactory outputFactory = XMLOutputFactory.newInstance();
private static final XMLEventFactory eventFactory = XMLEventFactory.newInstance();
private List<String> namespaces = new ArrayList<>();
private Deque<Collection<Namespace>> nsStack = new ArrayDeque<>();
private final Deque<Collection<Namespace>> nsStack = new ArrayDeque<>();
private Locator locator;
@ -49,14 +45,10 @@ public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements O
private boolean needToCallStartDocument = false;
public XmlSimpleMetadataHandler setWriter(Writer writer) {
public XmlSimpleMetadataHandler setWriter(Writer writer) throws XMLStreamException {
this.writer = writer;
try {
outputFactory.setProperty("javax.xml.stream.isRepairingNamespaces", Boolean.TRUE);
this.eventWriter = outputFactory.createXMLEventWriter(writer);
} catch (XMLStreamException e) {
logger.log(Level.FINE, e.getMessage(), e);
}
outputFactory.setProperty("javax.xml.stream.isRepairingNamespaces", Boolean.TRUE);
this.eventWriter = outputFactory.createXMLEventWriter(writer);
return this;
}
@ -150,7 +142,7 @@ public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements O
try {
eventWriter.add(eventFactory.createStartDocument());
} catch (XMLStreamException e) {
logger.log(Level.FINE, e.getMessage(), e);
throw new SAXException(e);
}
needToCallStartDocument = false;
}
@ -272,10 +264,14 @@ public class XmlSimpleMetadataHandler extends SimpleMetadataHandler implements O
}
private static final class SAXLocation implements Location {
private int lineNumber;
private int columnNumber;
private String publicId;
private String systemId;
private final int lineNumber;
private final int columnNumber;
private final String publicId;
private final String systemId;
private SAXLocation(Locator locator) {
lineNumber = locator.getLineNumber();

View file

@ -1,3 +1,3 @@
dependencies {
compile project(':oai-common')
}
api project(':oai-common')
}