joerg/oai
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

some renamings, clean up, add Bundeskunsthalle harvest, update to gradle 4.3.1

Browse source
This commit is contained in:
Jörg Prante 2017-12-08 11:24:59 +01:00
parent cd5004b292
commit d6591cd68e
50 changed files with 645 additions and 376 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -14,3 +14,4 @@ build
/sessions
*~
*.MARC
out

44
build.gradle
View file

@ -1,13 +1,25 @@
plugins {
id "org.sonarqube" version "2.2"
id "org.ajoberstar.github-pages" version "1.6.0-rc.1"
id "org.xbib.gradle.plugin.jbake" version "1.2.1"
id "org.sonarqube" version "2.6.1"
id "io.codearte.nexus-staging" version "0.11.0"
id "org.xbib.gradle.plugin.asciidoctor" version "1.5.4.1.0"
}
allprojects {
printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGroovy: %s\nGradle: %s\n" +
"Build: group: ${project.group} name: ${project.name} version: ${project.version}\n",
InetAddress.getLocalHost(),
System.getProperty("os.name"),
System.getProperty("os.arch"),
System.getProperty("os.version"),
System.getProperty("java.version"),
System.getProperty("java.vm.version"),
System.getProperty("java.vm.vendor"),
System.getProperty("java.vm.name"),
GroovySystem.getVersion(),
gradle.gradleVersion
group = 'org.xbib'
version = '1.0.2'
apply plugin: "io.codearte.nexus-staging"
allprojects {
apply plugin: 'java'
apply plugin: 'maven'
@ -16,20 +28,25 @@ allprojects {
apply plugin: 'pmd'
apply plugin: 'checkstyle'
apply plugin: "jacoco"
apply plugin: 'org.xbib.gradle.plugin.asciidoctor'
repositories {
mavenCentral()
}
configurations {
alpnagent
asciidoclet
wagon
}
dependencies {
testCompile 'junit:junit:4.12'
testCompile 'org.apache.logging.log4j:log4j-core:2.7'
testCompile 'org.apache.logging.log4j:log4j-jul:2.7'
wagon 'org.apache.maven.wagon:wagon-ssh-external:2.10'
testCompile "junit:junit:${project.property('junit.version')}"
testCompile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}"
testCompile "org.apache.logging.log4j:log4j-jul:${project.property('log4j.version')}"
testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
asciidoclet "org.asciidoctor:asciidoclet:${project.property('asciidoclet.version')}"
wagon "org.apache.maven.wagon:wagon-ssh:${project.property('wagon.version')}"
}
sourceCompatibility = JavaVersion.VERSION_1_8
@ -40,6 +57,10 @@ allprojects {
options.compilerArgs << "-Xlint:all" << "-profile" << "compact2"
}
clean {
delete 'out'
}
test {
testLogging {
showStandardStreams = false
@ -52,12 +73,15 @@ allprojects {
classifier 'sources'
from sourceSets.main.allSource
}
task javadocJar(type: Jar, dependsOn: javadoc) {
classifier 'javadoc'
}
artifacts {
archives sourcesJar, javadocJar
}
if (project.hasProperty('signing.keyId')) {
signing {
sign configurations.archives

14
gradle.properties Normal file
View file

@ -0,0 +1,14 @@
group = org.xbib
name = oai
version = 1.1.0
xbib-content.version = 1.1.0
xbib-bibliographic-character-sets.version = 1.0.0
xbib-marc.version = 1.0.17
helianthus.version = 1.0.10
tcnative.version = 2.0.1.Final
alpnagent.version = 2.0.6
junit.version 4.12
log4j.version = 2.8.2
wagon.version = 2.12
asciidoclet.version = 1.5.4

4
gradle/ext.gradle
View file

@ -5,8 +5,4 @@ ext {
scmUrl = 'https://github.com/xbib/oai'
scmConnection = 'scm:git:git://github.com/xbib/oai.git'
scmDeveloperConnection = 'scm:git:git://github.com/xbib/oai.git'
versions = [
'tcnative': '1.1.33.Fork23',
'alpnboot': '8.1.9.v20160720'
]
}

6
gradle/sonarqube.gradle
View file

@ -22,10 +22,8 @@ tasks.withType(Checkstyle) {
jacocoTestReport {
reports {
xml.enabled true
csv.enabled false
xml.destination "${buildDir}/reports/jacoco-xml"
html.destination "${buildDir}/reports/jacoco-html"
xml.enabled = true
csv.enabled = false
}
}

BIN
gradle/wrapper/gradle-wrapper.jar vendored
View file

Binary file not shown.

4
gradle/wrapper/gradle-wrapper.properties vendored
View file

@ -1,6 +1,6 @@
#Mon Oct 03 00:03:03 CEST 2016
#Wed Dec 06 21:25:32 CET 2017
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-all.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-4.3.1-all.zip

172
gradlew vendored Executable file
View file

@ -0,0 +1,172 @@
#!/usr/bin/env sh
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/" >/dev/null
APP_HOME="`pwd -P`"
cd "$SAVED" >/dev/null
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn () {
echo "$*"
}
die () {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
NONSTOP* )
nonstop=true
;;
esac
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin, switch paths to Windows format before running java
if $cygwin ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Escape application args
save () {
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
echo " "
}
APP_ARGS=$(save "$@")
# Collect all arguments for the java command, following the shell quoting and substitution rules
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
cd "$(dirname "$0")"
fi
exec "$JAVACMD" "$@"

84
gradlew.bat vendored Normal file
View file

@ -0,0 +1,84 @@
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto init
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto init
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:init
@rem Get command-line arguments, handling Windows variants
if not "%OS%" == "Windows_NT" goto win9xME_args
:win9xME_args
@rem Slurp the command line arguments.
set CMD_LINE_ARGS=
set _SKIP=2
:win9xME_args_slurp
if "x%~1" == "x" goto execute
set CMD_LINE_ARGS=%*
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega

24
oai-client/build.gradle
View file

@ -4,30 +4,18 @@ configurations {
dependencies {
compile project(':oai-common')
compile "org.xbib.helianthus:helianthus-client:1.0.3"
if ('os x' == org.gradle.internal.os.OperatingSystem.current().getFamilyName()) {
testCompile "io.netty:netty-tcnative:${versions.tcnative}:osx-x86_64"
compile "org.xbib:marc:${project.property('xbib-marc.version')}"
compile "org.xbib.helianthus:helianthus-client:${project.property('helianthus.version')}"
compile "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}"
testCompile "org.xbib:bibliographic-character-sets:${project.property('xbib-bibliographic-character-sets.version')}"
alpnagent "org.mortbay.jetty.alpn:jetty-alpn-agent:${project.property('alpnagent.version')}"
}
if ('linux' == org.gradle.internal.os.OperatingSystem.current().getFamilyName()) {
if (new File("/etc/redhat-release").exists()) {
// use this for linking to libssl.so.10 (RHEL/Fedora/CentOS)
testCompile "io.netty:netty-tcnative:${versions.tcnative}:linux-x86_64-fedora"
} else {
// use this for linking to libssl.so.1.0.0
testCompile "io.netty:netty-tcnative:${versions.tcnative}:linux-x86_64"
}
}
alpnboot "org.mortbay.jetty.alpn:alpn-boot:${versions.alpnboot}"
}
test {
jvmArgs "-javaagent:" + configurations.alpnagent.asPath
testLogging {
showStandardStreams = true
exceptionFormat = 'full'
}
// note: bootstrapClasspath does not use /p (or /a)
jvmArgs "-Xbootclasspath/p:" + configurations.alpnboot.asPath
systemProperty 'java.util.logging.manager', 'org.apache.logging.log4j.jul.LogManager'
systemProperty 'io.netty.leakDetection.level', 'advanced'
}

18
oai-client/src/main/java/org/xbib/oai/client/ClientOAIRequest.java → oai-client/src/main/java/org/xbib/oai/client/AbstractOAIRequest.java
View file

@ -17,9 +17,9 @@ import java.util.logging.Logger;
/**
* Client OAI request.
*/
public class ClientOAIRequest implements OAIRequest {
public abstract class AbstractOAIRequest implements OAIRequest {
private static final Logger logger = Logger.getLogger(ClientOAIRequest.class.getName());
private static final Logger logger = Logger.getLogger(AbstractOAIRequest.class.getName());
private URIBuilder uriBuilder;
@ -37,7 +37,7 @@ public class ClientOAIRequest implements OAIRequest {
private boolean retry;
protected ClientOAIRequest() {
protected AbstractOAIRequest() {
uriBuilder = new URIBuilder();
}
@ -135,35 +135,35 @@ public class ClientOAIRequest implements OAIRequest {
return retry;
}
class GetRecord extends ClientOAIRequest {
class GetRecord extends AbstractOAIRequest {
public GetRecord() {
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.GET_RECORD);
}
}
class Identify extends ClientOAIRequest {
class Identify extends AbstractOAIRequest {
public Identify() {
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.IDENTIFY);
}
}
class ListIdentifiers extends ClientOAIRequest {
class ListIdentifiers extends AbstractOAIRequest {
public ListIdentifiers() {
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_IDENTIFIERS);
}
}
class ListMetadataFormats extends ClientOAIRequest {
class ListMetadataFormats extends AbstractOAIRequest {
public ListMetadataFormats() {
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_METADATA_FORMATS);
}
}
class ListRecordsRequest extends ClientOAIRequest {
class ListRecordsRequest extends AbstractOAIRequest {
public ListRecordsRequest() {
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_RECORDS);
@ -171,7 +171,7 @@ public class ClientOAIRequest implements OAIRequest {
}
class ListSetsRequest extends ClientOAIRequest {
class ListSetsRequest extends AbstractOAIRequest {
public ListSetsRequest() {
addParameter(OAIConstants.VERB_PARAMETER, OAIConstants.LIST_SETS);

4
oai-client/src/main/java/org/xbib/oai/client/ClientOAIResponse.java → oai-client/src/main/java/org/xbib/oai/client/AbstractOAIResponse.java
View file

@ -9,7 +9,7 @@ import java.io.Writer;
/**
* Default OAI response.
*/
public interface ClientOAIResponse extends OAIResponse {
public abstract class AbstractOAIResponse implements OAIResponse {
void receivedResponse(AggregatedHttpMessage message, Writer writer) throws IOException;
public abstract void receivedResponse(AggregatedHttpMessage message, Writer writer) throws IOException;
}

74
oai-client/src/main/java/org/xbib/oai/client/DefaultOAIClient.java → oai-client/src/main/java/org/xbib/oai/client/OAIClient.java
View file

@ -16,9 +16,9 @@ import java.net.URL;
import java.time.Duration;
/**
* Default OAI client.
* OAI client.
*/
public class DefaultOAIClient implements OAIClientMethods, AutoCloseable {
public class OAIClient implements AutoCloseable {
private HttpClient client;
@ -26,13 +26,11 @@ public class DefaultOAIClient implements OAIClientMethods, AutoCloseable {
private URL url;
@Override
public DefaultOAIClient setURL(URL url) throws URISyntaxException {
public OAIClient setURL(URL url) throws URISyntaxException {
return setURL(url, false);
}
@Override
public DefaultOAIClient setURL(URL url, boolean trustAlways) throws URISyntaxException {
public OAIClient setURL(URL url, boolean trustAlways) throws URISyntaxException {
this.url = url;
this.clientFactory = ClientFactory.DEFAULT;
this.client = new ClientBuilder("none+" + url.toURI())
@ -42,64 +40,103 @@ public class DefaultOAIClient implements OAIClientMethods, AutoCloseable {
return this;
}
@Override
public URL getURL() {
return url;
}
@Override
public HttpClient getHttpClient() {
return client;
}
@Override
public ClientFactory getFactory() {
return clientFactory;
}
@Override
/**
* This verb is used to retrieve information about a repository.
* Some of the information returned is required as part of the OAI-PMH.
* Repositories may also employ the Identify verb to return additional
* descriptive information.
* @return identify request
*/
public IdentifyRequest newIdentifyRequest() {
IdentifyRequest request = new IdentifyRequest();
request.setURL(url);
return request;
}
@Override
/**
* This verb is used to retrieve the metadata formats available
* from a repository. An optional argument restricts the request
* to the formats available for a specific item.
* @return list metadata formats request
*/
public ListMetadataFormatsRequest newListMetadataFormatsRequest() {
ListMetadataFormatsRequest request = new ListMetadataFormatsRequest();
request.setURL(getURL());
return request;
}
@Override
/**
* This verb is used to retrieve the set structure of a repository,
* useful for selective harvesting.
* @return list sets request
*/
public ListSetsRequest newListSetsRequest() {
ListSetsRequest request = new ListSetsRequest();
request.setURL(getURL());
return request;
}
@Override
/**
* This verb is an abbreviated form of ListRecords, retrieving only
* headers rather than records. Optional arguments permit selective
* harvesting of headers based on set membership and/or datestamp.
* Depending on the repository's support for deletions, a returned
* header may have a status attribute of "deleted" if a record
* matching the arguments specified in the request has been deleted.
* @return list identifiers request
*
*/
public ListIdentifiersRequest newListIdentifiersRequest() {
ListIdentifiersRequest request = new ListIdentifiersRequest();
request.setURL(getURL());
return request;
}
@Override
/**
* This verb is used to retrieve an individual metadata record from
* a repository. Required arguments specify the identifier of the item
* from which the record is requested and the format of the metadata
* that should be included in the record. Depending on the level at
* which a repository tracks deletions, a header with a "deleted" value
* for the status attribute may be returned, in case the metadata format
* specified by the metadataPrefix is no longer available from the
* repository or from the specified item.
* @return get record request
*/
public GetRecordRequest newGetRecordRequest() {
GetRecordRequest request = new GetRecordRequest();
request.setURL(getURL());
return request;
}
@Override
/**
* This verb is used to harvest records from a repository.
* Optional arguments permit selective harvesting of records based on
* set membership and/or datestamp. Depending on the repository's
* support for deletions, a returned header may have a status
* attribute of "deleted" if a record matching the arguments
* specified in the request has been deleted. No metadata
* will be present for records with deleted status.
* @return list records request
*/
public ListRecordsRequest newListRecordsRequest() {
ListRecordsRequest request = new ListRecordsRequest();
request.setURL(getURL());
return request;
}
@Override
public IdentifyRequest resume(IdentifyRequest request, ResumptionToken<?> token) {
if (request.isRetry()) {
request.setRetry(false);
@ -113,7 +150,6 @@ public class DefaultOAIClient implements OAIClientMethods, AutoCloseable {
return nextRequest;
}
@Override
public ListRecordsRequest resume(ListRecordsRequest request, ResumptionToken<?> token) {
if (request.isRetry()) {
request.setRetry(false);
@ -127,7 +163,6 @@ public class DefaultOAIClient implements OAIClientMethods, AutoCloseable {
return nextRequest;
}
@Override
public ListIdentifiersRequest resume(ListIdentifiersRequest request, ResumptionToken<?> token) {
if (request.isRetry()) {
request.setRetry(false);
@ -141,7 +176,6 @@ public class DefaultOAIClient implements OAIClientMethods, AutoCloseable {
return nextRequest;
}
@Override
public ListMetadataFormatsRequest resume(ListMetadataFormatsRequest request, ResumptionToken<?> token) {
if (request.isRetry()) {
request.setRetry(false);
@ -155,7 +189,6 @@ public class DefaultOAIClient implements OAIClientMethods, AutoCloseable {
return nextRequest;
}
@Override
public ListSetsRequest resume(ListSetsRequest request, ResumptionToken<?> token) {
if (request.isRetry()) {
request.setRetry(false);
@ -169,7 +202,6 @@ public class DefaultOAIClient implements OAIClientMethods, AutoCloseable {
return nextRequest;
}
@Override
public GetRecordRequest resume(GetRecordRequest request, ResumptionToken<?> token) {
if (request.isRetry()) {
request.setRetry(false);

105
oai-client/src/main/java/org/xbib/oai/client/OAIClientMethods.java
View file

@ -1,105 +0,0 @@
package org.xbib.oai.client;
import org.xbib.helianthus.client.ClientFactory;
import org.xbib.helianthus.client.http.HttpClient;
import org.xbib.oai.client.getrecord.GetRecordRequest;
import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.listidentifiers.ListIdentifiersRequest;
import org.xbib.oai.client.listmetadataformats.ListMetadataFormatsRequest;
import org.xbib.oai.client.listrecords.ListRecordsRequest;
import org.xbib.oai.client.listsets.ListSetsRequest;
import org.xbib.oai.util.ResumptionToken;
import java.net.URISyntaxException;
import java.net.URL;
/**
* OAI client API.
*
*/
public interface OAIClientMethods {
OAIClientMethods setURL(URL uri, boolean trustAlways) throws URISyntaxException;
OAIClientMethods setURL(URL uri) throws URISyntaxException;
URL getURL();
HttpClient getHttpClient();
ClientFactory getFactory();
/**
* This verb is used to retrieve information about a repository.
* Some of the information returned is required as part of the OAI-PMH.
* Repositories may also employ the Identify verb to return additional
* descriptive information.
* @return identify request
*/
IdentifyRequest newIdentifyRequest();
IdentifyRequest resume(IdentifyRequest request, ResumptionToken<?> token);
/**
* This verb is an abbreviated form of ListRecords, retrieving only
* headers rather than records. Optional arguments permit selective
* harvesting of headers based on set membership and/or datestamp.
* Depending on the repository's support for deletions, a returned
* header may have a status attribute of "deleted" if a record
* matching the arguments specified in the request has been deleted.
* @return list identifiers request
*
*/
ListIdentifiersRequest newListIdentifiersRequest();
ListIdentifiersRequest resume(ListIdentifiersRequest request, ResumptionToken<?> token);
/**
* This verb is used to retrieve the metadata formats available
* from a repository. An optional argument restricts the request
* to the formats available for a specific item.
* @return list metadata formats request
*/
ListMetadataFormatsRequest newListMetadataFormatsRequest();
ListMetadataFormatsRequest resume(ListMetadataFormatsRequest request, ResumptionToken<?> token);
/**
* This verb is used to retrieve the set structure of a repository,
* useful for selective harvesting.
* @return list sets request
*/
ListSetsRequest newListSetsRequest();
ListSetsRequest resume(ListSetsRequest request, ResumptionToken<?> token);
/**
* This verb is used to harvest records from a repository.
* Optional arguments permit selective harvesting of records based on
* set membership and/or datestamp. Depending on the repository's
* support for deletions, a returned header may have a status
* attribute of "deleted" if a record matching the arguments
* specified in the request has been deleted. No metadata
* will be present for records with deleted status.
* @return list records request
*/
ListRecordsRequest newListRecordsRequest();
ListRecordsRequest resume(ListRecordsRequest request, ResumptionToken<?> token);
/**
* This verb is used to retrieve an individual metadata record from
* a repository. Required arguments specify the identifier of the item
* from which the record is requested and the format of the metadata
* that should be included in the record. Depending on the level at
* which a repository tracks deletions, a header with a "deleted" value
* for the status attribute may be returned, in case the metadata format
* specified by the metadataPrefix is no longer available from the
* repository or from the specified item.
* @return get record request
*/
GetRecordRequest newGetRecordRequest();
GetRecordRequest resume(GetRecordRequest request, ResumptionToken<?> token);
}

4
oai-client/src/main/java/org/xbib/oai/client/getrecord/GetRecordRequest.java
View file

@ -1,11 +1,11 @@
package org.xbib.oai.client.getrecord;
import org.xbib.oai.client.ClientOAIRequest;
import org.xbib.oai.client.AbstractOAIRequest;
/**
*
*/
public class GetRecordRequest extends ClientOAIRequest {
public class GetRecordRequest extends AbstractOAIRequest {
public GetRecordRequest() {
super();

7
oai-client/src/main/java/org/xbib/oai/client/getrecord/GetRecordResponse.java
View file

@ -1,18 +1,17 @@
package org.xbib.oai.client.getrecord;
import org.xbib.helianthus.common.http.AggregatedHttpMessage;
import org.xbib.oai.client.ClientOAIResponse;
import org.xbib.oai.client.AbstractOAIResponse;
import java.io.IOException;
import java.io.Writer;
/**
*
*/
public class GetRecordResponse implements ClientOAIResponse {
public class GetRecordResponse extends AbstractOAIResponse {
@Override
public void receivedResponse(AggregatedHttpMessage message, Writer writer) throws IOException {
public void receivedResponse(AggregatedHttpMessage message, Writer writer) {
// not implemented yet
}
}

5
oai-client/src/main/java/org/xbib/oai/client/identify/IdentifyRequest.java
View file

@ -1,12 +1,11 @@
package org.xbib.oai.client.identify;
import org.xbib.oai.OAIRequest;
import org.xbib.oai.client.ClientOAIRequest;
import org.xbib.oai.client.AbstractOAIRequest;
/**
*
*/
public class IdentifyRequest extends ClientOAIRequest implements OAIRequest {
public class IdentifyRequest extends AbstractOAIRequest {
public IdentifyRequest() {
super();

4
oai-client/src/main/java/org/xbib/oai/client/identify/IdentifyResponse.java
View file

@ -4,7 +4,7 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xbib.helianthus.common.http.AggregatedHttpMessage;
import org.xbib.oai.client.ClientOAIResponse;
import org.xbib.oai.client.AbstractOAIResponse;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
@ -22,7 +22,7 @@ import javax.xml.parsers.ParserConfigurationException;
/**
*
*/
public class IdentifyResponse implements ClientOAIResponse {
public class IdentifyResponse extends AbstractOAIResponse {
private String repositoryName;

5
oai-client/src/main/java/org/xbib/oai/client/listidentifiers/ListIdentifiersRequest.java
View file

@ -1,12 +1,11 @@
package org.xbib.oai.client.listidentifiers;
import org.xbib.oai.OAIRequest;
import org.xbib.oai.client.ClientOAIRequest;
import org.xbib.oai.client.AbstractOAIRequest;
/**
*
*/
public class ListIdentifiersRequest extends ClientOAIRequest implements OAIRequest {
public class ListIdentifiersRequest extends AbstractOAIRequest {
public ListIdentifiersRequest() {
super();

4
oai-client/src/main/java/org/xbib/oai/client/listidentifiers/ListIdentifiersResponse.java
View file

@ -1,7 +1,7 @@
package org.xbib.oai.client.listidentifiers;
import org.xbib.helianthus.common.http.AggregatedHttpMessage;
import org.xbib.oai.client.ClientOAIResponse;
import org.xbib.oai.client.AbstractOAIResponse;
import java.io.IOException;
import java.io.Writer;
@ -9,7 +9,7 @@ import java.io.Writer;
/**
*
*/
public class ListIdentifiersResponse implements ClientOAIResponse {
public class ListIdentifiersResponse extends AbstractOAIResponse {
@Override
public void receivedResponse(AggregatedHttpMessage message, Writer writer) throws IOException {

6
oai-client/src/main/java/org/xbib/oai/client/listmetadataformats/ListMetadataFormatsRequest.java
View file

@ -1,16 +1,14 @@
package org.xbib.oai.client.listmetadataformats;
import org.xbib.oai.OAIRequest;
import org.xbib.oai.client.ClientOAIRequest;
import org.xbib.oai.client.AbstractOAIRequest;
/**
*
*/
public class ListMetadataFormatsRequest extends ClientOAIRequest implements OAIRequest {
public class ListMetadataFormatsRequest extends AbstractOAIRequest {
public ListMetadataFormatsRequest() {
super();
addParameter(VERB_PARAMETER, LIST_METADATA_FORMATS);
}
}

4
oai-client/src/main/java/org/xbib/oai/client/listmetadataformats/ListMetadataFormatsResponse.java
View file

@ -1,7 +1,7 @@
package org.xbib.oai.client.listmetadataformats;
import org.xbib.helianthus.common.http.AggregatedHttpMessage;
import org.xbib.oai.client.ClientOAIResponse;
import org.xbib.oai.client.AbstractOAIResponse;
import java.io.IOException;
import java.io.Writer;
@ -9,7 +9,7 @@ import java.io.Writer;
/**
*
*/
public class ListMetadataFormatsResponse implements ClientOAIResponse {
public class ListMetadataFormatsResponse extends AbstractOAIResponse {
@Override
public void receivedResponse(AggregatedHttpMessage message, Writer writer) throws IOException {

4
oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsRequest.java
View file

@ -1,7 +1,7 @@
package org.xbib.oai.client.listrecords;
import org.xbib.oai.OAIConstants;
import org.xbib.oai.client.ClientOAIRequest;
import org.xbib.oai.client.AbstractOAIRequest;
import org.xbib.oai.xml.MetadataHandler;
import java.util.LinkedList;
@ -10,7 +10,7 @@ import java.util.List;
/**
*
*/
public class ListRecordsRequest extends ClientOAIRequest {
public class ListRecordsRequest extends AbstractOAIRequest {
private List<MetadataHandler> handlers = new LinkedList<>();

4
oai-client/src/main/java/org/xbib/oai/client/listrecords/ListRecordsResponse.java
View file

@ -5,7 +5,7 @@ import io.netty.util.AsciiString;
import org.xbib.content.xml.transform.TransformerURIResolver;
import org.xbib.content.xml.util.XMLUtil;
import org.xbib.helianthus.common.http.AggregatedHttpMessage;
import org.xbib.oai.client.ClientOAIResponse;
import org.xbib.oai.client.AbstractOAIResponse;
import org.xbib.oai.exceptions.BadArgumentException;
import org.xbib.oai.exceptions.BadResumptionTokenException;
import org.xbib.oai.exceptions.NoRecordsMatchException;
@ -32,7 +32,7 @@ import javax.xml.transform.stream.StreamResult;
/**
*
*/
public class ListRecordsResponse implements ClientOAIResponse {
public class ListRecordsResponse extends AbstractOAIResponse {
private static final Logger logger = Logger.getLogger(ListRecordsResponse.class.getName());
private static final String[] RETRY_AFTER_HEADERS = {

4
oai-client/src/main/java/org/xbib/oai/client/listsets/ListSetsRequest.java
View file

@ -1,11 +1,11 @@
package org.xbib.oai.client.listsets;
import org.xbib.oai.client.ClientOAIRequest;
import org.xbib.oai.client.AbstractOAIRequest;
/**
*
*/
public class ListSetsRequest extends ClientOAIRequest {
public class ListSetsRequest extends AbstractOAIRequest {
public ListSetsRequest() {
super();

4
oai-client/src/main/java/org/xbib/oai/client/listsets/ListSetsResponse.java
View file

@ -1,7 +1,7 @@
package org.xbib.oai.client.listsets;
import org.xbib.helianthus.common.http.AggregatedHttpMessage;
import org.xbib.oai.client.ClientOAIResponse;
import org.xbib.oai.client.AbstractOAIResponse;
import java.io.IOException;
import java.io.Writer;
@ -9,7 +9,7 @@ import java.io.Writer;
/**
*
*/
public class ListSetsResponse implements ClientOAIResponse {
public class ListSetsResponse extends AbstractOAIResponse {
@Override
public void receivedResponse(AggregatedHttpMessage message, Writer writer) throws IOException {

67
oai-client/src/test/java/org/xbib/oai/client/ArxivClientTest.java
View file

@ -4,6 +4,7 @@ import static org.junit.Assert.assertTrue;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.Ignore;
import org.junit.Test;
import org.xbib.helianthus.client.http.HttpClient;
import org.xbib.helianthus.common.http.AggregatedHttpMessage;
@ -15,8 +16,6 @@ import org.xbib.oai.client.identify.IdentifyResponse;
import org.xbib.oai.client.listrecords.ListRecordsRequest;
import org.xbib.oai.client.listrecords.ListRecordsResponse;
import org.xbib.oai.xml.SimpleMetadataHandler;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import java.io.File;
import java.io.FileWriter;
@ -33,13 +32,14 @@ import java.util.concurrent.atomic.AtomicLong;
/**
*
*/
@Ignore
public class ArxivClientTest {
private static final Logger logger = LogManager.getLogger(ArxivClientTest.class.getName());
@Test
public void testListRecordsArxiv() throws Exception {
try (DefaultOAIClient client = new DefaultOAIClient().setURL(new URL("http://export.arxiv.org/oai2"))) {
try (OAIClient client = new OAIClient().setURL(new URL("http://export.arxiv.org/oai2"))) {
IdentifyRequest identifyRequest = client.newIdentifyRequest();
HttpClient httpClient = client.getHttpClient();
AggregatedHttpMessage response = httpClient.execute(HttpHeaders.of(HttpMethod.GET, identifyRequest.getPath())
@ -58,46 +58,13 @@ public class ArxivClientTest {
listRecordsRequest.setFrom(Instant.parse("2016-11-01T00:00:00Z"));
listRecordsRequest.setUntil(Instant.parse("2016-11-02T00:00:00Z"));
listRecordsRequest.setMetadataPrefix("arXiv");
final AtomicLong count = new AtomicLong(0L);
SimpleMetadataHandler simpleMetadataHandler = new SimpleMetadataHandler() {
@Override
public void startDocument() throws SAXException {
logger.debug("start doc");
}
@Override
public void endDocument() throws SAXException {
logger.debug("end doc");
count.incrementAndGet();
}
@Override
public void startPrefixMapping(String prefix, String uri) throws SAXException {
}
@Override
public void endPrefixMapping(String prefix) throws SAXException {
}
@Override
public void startElement(String ns, String localname, String qname, Attributes atrbts) throws SAXException {
}
@Override
public void endElement(String ns, String localname, String qname) throws SAXException {
}
@Override
public void characters(char[] chars, int pos, int len) throws SAXException {
}
};
Handler handler = new Handler();
File file = File.createTempFile("arxiv.", ".xml");
file.deleteOnExit();
FileWriter fileWriter = new FileWriter(file);
while (listRecordsRequest != null) {
try {
listRecordsRequest.addHandler(simpleMetadataHandler);
listRecordsRequest.addHandler(handler);
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
logger.info("sending {}", listRecordsRequest.getPath());
response = httpClient.execute(HttpHeaders.of(HttpMethod.GET, listRecordsRequest.getPath())
@ -112,12 +79,32 @@ public class ArxivClientTest {
}
}
fileWriter.close();
logger.info("count={}", count.get());
assertTrue(count.get() > 0L);
logger.info("count={}", handler.count());
assertTrue(handler.count() > 0L);
} catch (ConnectException | ExecutionException e) {
logger.warn("skipped, can not connect", e);
} catch (InterruptedException | IOException e) {
throw e;
}
}
class Handler extends SimpleMetadataHandler {
final AtomicLong count = new AtomicLong(0L);
@Override
public void startDocument() {
logger.debug("start doc");
}
@Override
public void endDocument() {
logger.debug("end doc");
count.incrementAndGet();
}
long count() {
return count.get();
}
}
}

112
oai-client/src/test/java/org/xbib/oai/client/BundeskunsthalleTest.java Normal file
View file

@ -0,0 +1,112 @@
package org.xbib.oai.client;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.Test;
import org.xbib.helianthus.client.Clients;
import org.xbib.helianthus.client.http.HttpClient;
import org.xbib.helianthus.common.http.AggregatedHttpMessage;
import org.xbib.helianthus.common.http.HttpHeaderNames;
import org.xbib.helianthus.common.http.HttpHeaders;
import org.xbib.helianthus.common.http.HttpMethod;
import org.xbib.marc.Marc;
import org.xbib.marc.json.MarcJsonWriter;
import org.xbib.marc.xml.MarcContentHandler;
import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.identify.IdentifyResponse;
import org.xbib.oai.client.listrecords.ListRecordsRequest;
import org.xbib.oai.client.listrecords.ListRecordsResponse;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.ConnectException;
import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.concurrent.ExecutionException;
/**
*
*/
public class BundeskunsthalleTest {
private static final Logger logger = LogManager.getLogger(BundeskunsthalleTest.class.getName());
@Test
public void testListRecords() throws Exception {
String spec = "http://www.bundeskunsthalle.de/cgi-bin/bib/oai-pmh";
try (OAIClient oaiClient = new OAIClient().setURL(new URL(spec), true)) {
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
HttpClient client = oaiClient.getHttpClient();
AggregatedHttpMessage response = client.execute(HttpHeaders.of(HttpMethod.GET, identifyRequest.getPath())
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
// follow a maximum of 10 HTTP redirects
int max = 10;
while (response.followUrl() != null && max-- > 0) {
URI uri = URI.create(response.followUrl());
client = Clients.newClient(oaiClient.getFactory(), "none+" + uri, HttpClient.class);
response = client.execute(HttpHeaders.of(HttpMethod.GET, response.followUrl())
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
}
IdentifyResponse identifyResponse = new IdentifyResponse();
logger.debug("identifyResponse = {}", response.content().toStringUtf8());
identifyResponse.receivedResponse(response, new StringWriter());
String granularity = identifyResponse.getGranularity();
logger.info("granularity = {}", granularity);
DateTimeFormatter dateTimeFormatter = "YYYY-MM-DD".equals(granularity) ?
DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("GMT")) : null;
ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest();
listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
listRecordsRequest.setMetadataPrefix("marcxml");
try (MarcJsonWriter writer = new MarcJsonWriter("bk-bulk%d.jsonl", 1000,
MarcJsonWriter.Style.ELASTICSEARCH_BULK, 65536, false)
.setIndex("testindex", "testtype")) {
writer.startDocument();
writer.beginCollection();
while (listRecordsRequest != null) {
try {
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
logger.debug("response = {}", response.headers());
client = oaiClient.getHttpClient();
response = client.execute(HttpHeaders.of(HttpMethod.GET, listRecordsRequest.getPath())
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
// follow a maximum of 10 HTTP redirects
max = 10;
while (response.followUrl() != null && max-- > 0) {
URI uri = URI.create(response.followUrl());
client = Clients.newClient(oaiClient.getFactory(), "none+" + uri, HttpClient.class);
response = client.execute(HttpHeaders.of(HttpMethod.GET, response.followUrl())
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
}
InputStream inputStream = new ByteArrayInputStream(response.content().array());
Marc.builder()
.setInputStream(inputStream)
.setCharset(StandardCharsets.UTF_8)
.setContentHandler(new MarcContentHandler()
.setFormat("MarcXML")
.setType("Bibliographic")
.addNamespace("http://www.loc.gov/MARC21/slim")
.setMarcListener(writer))
.build()
.xmlReader().parse();
listRecordsResponse.receivedResponse(response, new StringWriter());
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} catch (IOException e) {
logger.error(e.getMessage(), e);
listRecordsRequest = null;
}
}
writer.endCollection();
writer.endDocument();
}
} catch (ConnectException | ExecutionException e) {
logger.warn("skipped, can not connect, exception is:", e);
} catch (InterruptedException | IOException e) {
throw e;
}
}
}

70
oai-client/src/test/java/org/xbib/oai/client/DNBClientTest.java
View file

@ -4,6 +4,7 @@ import static org.junit.Assert.assertEquals;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.Ignore;
import org.junit.Test;
import org.xbib.helianthus.client.http.HttpClient;
import org.xbib.helianthus.common.http.AggregatedHttpMessage;
@ -14,8 +15,6 @@ import org.xbib.oai.client.identify.IdentifyRequest;
import org.xbib.oai.client.listrecords.ListRecordsRequest;
import org.xbib.oai.client.listrecords.ListRecordsResponse;
import org.xbib.oai.xml.SimpleMetadataHandler;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import java.io.File;
import java.io.FileWriter;
@ -29,13 +28,14 @@ import java.util.concurrent.atomic.AtomicLong;
/**
*
*/
@Ignore
public class DNBClientTest {
private static final Logger logger = LogManager.getLogger(DNBClientTest.class.getName());
@Test
public void testIdentify() throws Exception {
DefaultOAIClient client = new DefaultOAIClient().setURL(new URL("http://services.dnb.de/oai/repository"));
OAIClient client = new OAIClient().setURL(new URL("http://services.dnb.de/oai/repository"));
IdentifyRequest request = client.newIdentifyRequest();
HttpClient httpClient = client.getHttpClient();
assertEquals("/oai/repository?verb=Identify", request.getPath());
@ -45,55 +45,23 @@ public class DNBClientTest {
@Test
public void testListRecordsDNB() throws Exception {
try (DefaultOAIClient client = new DefaultOAIClient().setURL(new URL("http://services.dnb.de/oai/repository"))){
try (OAIClient client = new OAIClient().setURL(new URL("http://services.dnb.de/oai/repository"))){
ListRecordsRequest listRecordsRequest = client.newListRecordsRequest();
listRecordsRequest.setFrom(Instant.parse("2016-01-01T00:00:00Z"));
listRecordsRequest.setUntil(Instant.parse("2016-01-10T00:00:00Z"));
listRecordsRequest.setSet("bib");
listRecordsRequest.setMetadataPrefix("PicaPlus-xml");
final AtomicLong count = new AtomicLong(0L);
SimpleMetadataHandler simpleMetadataHandler = new SimpleMetadataHandler() {
@Override
public void startDocument() throws SAXException {
logger.debug("startDocument");
}
@Override
public void endDocument() throws SAXException {
count.incrementAndGet();
logger.debug("endDocument");
}
@Override
public void startPrefixMapping(String prefix, String uri) throws SAXException {
}
@Override
public void endPrefixMapping(String prefix) throws SAXException {
}
@Override
public void startElement(String ns, String localname, String qname, Attributes atrbts) throws SAXException {
}
@Override
public void endElement(String ns, String localname, String qname) throws SAXException {
}
@Override
public void characters(char[] chars, int pos, int len) throws SAXException {
}
};
Handler handler = new Handler();
File file = File.createTempFile("dnb-bib-pica.", ".xml");
file.deleteOnExit();
FileWriter sw = new FileWriter(file);
while (listRecordsRequest != null) {
try {
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
listRecordsRequest.addHandler(simpleMetadataHandler);
listRecordsRequest.addHandler(handler);
HttpClient httpClient = client.getHttpClient();
AggregatedHttpMessage response = httpClient.execute(HttpHeaders.of(HttpMethod.GET, listRecordsRequest.getPath())
AggregatedHttpMessage response =
httpClient.execute(HttpHeaders.of(HttpMethod.GET, listRecordsRequest.getPath())
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
String content = response.content().toStringUtf8();
listRecordsResponse.receivedResponse(response, sw);
@ -104,11 +72,31 @@ public class DNBClientTest {
}
}
sw.close();
logger.info("count={}", count.get());
logger.info("count={}", handler.count());
} catch (ConnectException | ExecutionException e) {
logger.warn("skipped, can not connect");
} catch (IOException e) {
logger.warn("skipped, HTTP exception");
}
}
class Handler extends SimpleMetadataHandler {
final AtomicLong count = new AtomicLong(0L);
@Override
public void startDocument() {
logger.debug("start doc");
}
@Override
public void endDocument() {
logger.debug("end doc");
count.incrementAndGet();
}
long count() {
return count.get();
}
}
}

74
oai-client/src/test/java/org/xbib/oai/client/DOAJClientTest.java
View file

@ -2,6 +2,7 @@ package org.xbib.oai.client;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.Ignore;
import org.junit.Test;
import org.xbib.helianthus.client.Clients;
import org.xbib.helianthus.client.http.HttpClient;
@ -14,8 +15,6 @@ import org.xbib.oai.client.identify.IdentifyResponse;
import org.xbib.oai.client.listrecords.ListRecordsRequest;
import org.xbib.oai.client.listrecords.ListRecordsResponse;
import org.xbib.oai.xml.SimpleMetadataHandler;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import java.io.File;
import java.io.FileWriter;
@ -30,11 +29,10 @@ import java.time.format.DateTimeFormatter;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.Assert.assertTrue;
/**
*
*/
@Ignore
public class DOAJClientTest {
private static final Logger logger = LogManager.getLogger(DOAJClientTest.class.getName());
@ -42,7 +40,7 @@ public class DOAJClientTest {
@Test
public void testListRecordsDOAJ() throws Exception {
// will redirect to https://doaj.org/oai
try (DefaultOAIClient oaiClient = new DefaultOAIClient().setURL(new URL("http://doaj.org/oai"), true)) {
try (OAIClient oaiClient = new OAIClient().setURL(new URL("http://doaj.org/oai"), true)) {
IdentifyRequest identifyRequest = oaiClient.newIdentifyRequest();
HttpClient client = oaiClient.getHttpClient();
AggregatedHttpMessage response = client.execute(HttpHeaders.of(HttpMethod.GET, identifyRequest.getPath())
@ -65,48 +63,18 @@ public class DOAJClientTest {
DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("GMT")) : null;
ListRecordsRequest listRecordsRequest = oaiClient.newListRecordsRequest();
listRecordsRequest.setDateTimeFormatter(dateTimeFormatter);
listRecordsRequest.setFrom(Instant.parse("2016-01-06T00:00:00Z"));
listRecordsRequest.setUntil(Instant.parse("2016-11-07T00:00:00Z"));
listRecordsRequest.setFrom(Instant.parse("2017-01-01T00:00:00Z"));
listRecordsRequest.setUntil(Instant.parse("2018-01-01T00:00:00Z"));
listRecordsRequest.setMetadataPrefix("oai_dc");
final AtomicLong count = new AtomicLong(0L);
SimpleMetadataHandler simpleMetadataHandler = new SimpleMetadataHandler() {
@Override
public void startDocument() throws SAXException {
logger.debug("start doc");
}
@Override
public void endDocument() throws SAXException {
logger.debug("end doc");
count.incrementAndGet();
}
@Override
public void startPrefixMapping(String prefix, String uri) throws SAXException {
}
@Override
public void endPrefixMapping(String prefix) throws SAXException {
}
@Override
public void startElement(String ns, String localname, String qname, Attributes atrbts) throws SAXException {
}
@Override
public void endElement(String ns, String localname, String qname) throws SAXException {
}
@Override
public void characters(char[] chars, int pos, int len) throws SAXException {
}
};
Handler handler = new Handler();
File file = File.createTempFile("doaj.", ".xml");
file.deleteOnExit();
FileWriter fileWriter = new FileWriter(file);
while (listRecordsRequest != null) {
try {
listRecordsRequest.addHandler(simpleMetadataHandler);
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
logger.debug("response = {}", response.headers());
listRecordsRequest.addHandler(handler);
client = oaiClient.getHttpClient();
response = client.execute(HttpHeaders.of(HttpMethod.GET, listRecordsRequest.getPath())
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
@ -118,8 +86,6 @@ public class DOAJClientTest {
response = client.execute(HttpHeaders.of(HttpMethod.GET, response.followUrl())
.set(HttpHeaderNames.ACCEPT, "utf-8")).aggregate().get();
}
ListRecordsResponse listRecordsResponse = new ListRecordsResponse(listRecordsRequest);
logger.debug("response = {}", response.headers());
listRecordsResponse.receivedResponse(response, fileWriter);
listRecordsRequest = oaiClient.resume(listRecordsRequest, listRecordsResponse.getResumptionToken());
} catch (IOException e) {
@ -128,8 +94,7 @@ public class DOAJClientTest {
}
}
fileWriter.close();
logger.info("count={}", count.get());
assertTrue(count.get() > 0L);
logger.info("count={}", handler.count());
} catch (ConnectException | ExecutionException e) {
logger.warn("skipped, can not connect, exception is:", e);
} catch (InterruptedException | IOException e) {
@ -137,4 +102,23 @@ public class DOAJClientTest {
}
}
class Handler extends SimpleMetadataHandler {
final AtomicLong count = new AtomicLong(0L);
@Override
public void startDocument() {
logger.debug("start doc");
}
@Override
public void endDocument() {
logger.debug("end doc");
count.incrementAndGet();
}
long count() {
return count.get();
}
}
}

2
oai-common/build.gradle
View file

@ -1,3 +1,3 @@
dependencies {
compile "org.xbib:content-rdf:1.0.5"
compile "org.xbib:content-rdf:${project.property('xbib-content.version')}"
}

4
oai-server/src/main/java/org/xbib/oai/server/ServerOAIRequest.java → oai-server/src/main/java/org/xbib/oai/server/AbstractOAIRequest.java
View file

@ -11,7 +11,7 @@ import java.util.Map;
/**
*
*/
public abstract class ServerOAIRequest implements OAIRequest {
public abstract class AbstractOAIRequest implements OAIRequest {
private String path;
@ -29,7 +29,7 @@ public abstract class ServerOAIRequest implements OAIRequest {
private boolean retry;
protected ServerOAIRequest() {
protected AbstractOAIRequest() {
this.parameters = new HashMap<>();
}

4
oai-server/src/main/java/org/xbib/oai/server/ServerOAIResponse.java → oai-server/src/main/java/org/xbib/oai/server/AbstractOAIResponse.java
View file

@ -7,11 +7,11 @@ import javax.xml.stream.util.XMLEventConsumer;
/**
* Default OAI response.
*/
public class ServerOAIResponse implements OAIResponse {
public abstract class AbstractOAIResponse implements OAIResponse {
private XMLEventConsumer consumer;
public ServerOAIResponse setConsumer(XMLEventConsumer consumer) {
public AbstractOAIResponse setConsumer(XMLEventConsumer consumer) {
this.consumer = consumer;
return this;
}

4
oai-server/src/main/java/org/xbib/oai/server/getrecord/GetRecordServerRequest.java
View file

@ -1,10 +1,10 @@
package org.xbib.oai.server.getrecord;
import org.xbib.oai.server.ServerOAIRequest;
import org.xbib.oai.server.AbstractOAIRequest;
/**
*
*/
public class GetRecordServerRequest extends ServerOAIRequest {
public class GetRecordServerRequest extends AbstractOAIRequest {
}

4
oai-server/src/main/java/org/xbib/oai/server/getrecord/GetRecordServerResponse.java
View file

@ -1,9 +1,9 @@
package org.xbib.oai.server.getrecord;
import org.xbib.oai.server.ServerOAIResponse;
import org.xbib.oai.server.AbstractOAIResponse;
/**
*
*/
public class GetRecordServerResponse extends ServerOAIResponse {
public class GetRecordServerResponse extends AbstractOAIResponse {
}

4
oai-server/src/main/java/org/xbib/oai/server/identify/IdentifyServerRequest.java
View file

@ -1,9 +1,9 @@
package org.xbib.oai.server.identify;
import org.xbib.oai.server.ServerOAIRequest;
import org.xbib.oai.server.AbstractOAIRequest;
/**
*
*/
public class IdentifyServerRequest extends ServerOAIRequest {
public class IdentifyServerRequest extends AbstractOAIRequest {
}

4
oai-server/src/main/java/org/xbib/oai/server/identify/IdentifyServerResponse.java
View file

@ -1,6 +1,6 @@
package org.xbib.oai.server.identify;
import org.xbib.oai.server.ServerOAIResponse;
import org.xbib.oai.server.AbstractOAIResponse;
import java.net.URL;
import java.util.ArrayList;
@ -10,7 +10,7 @@ import java.util.List;
/**
*
*/
public class IdentifyServerResponse extends ServerOAIResponse {
public class IdentifyServerResponse extends AbstractOAIResponse {
private String repositoryName;

4
oai-server/src/main/java/org/xbib/oai/server/listidentifiers/ListIdentifiersServerRequest.java
View file

@ -1,10 +1,10 @@
package org.xbib.oai.server.listidentifiers;
import org.xbib.oai.server.ServerOAIRequest;
import org.xbib.oai.server.AbstractOAIRequest;
/**
*
*/
public class ListIdentifiersServerRequest extends ServerOAIRequest {
public class ListIdentifiersServerRequest extends AbstractOAIRequest {
}

4
oai-server/src/main/java/org/xbib/oai/server/listidentifiers/ListIdentifiersServerResponse.java
View file

@ -1,10 +1,10 @@
package org.xbib.oai.server.listidentifiers;
import org.xbib.oai.server.ServerOAIResponse;
import org.xbib.oai.server.AbstractOAIResponse;
/**
*
*/
public class ListIdentifiersServerResponse extends ServerOAIResponse {
public class ListIdentifiersServerResponse extends AbstractOAIResponse {
}

4
oai-server/src/main/java/org/xbib/oai/server/listmetadataformats/ListMetadataFormatsServerRequest.java
View file

@ -1,10 +1,10 @@
package org.xbib.oai.server.listmetadataformats;
import org.xbib.oai.server.ServerOAIRequest;
import org.xbib.oai.server.AbstractOAIRequest;
/**
*
*/
public class ListMetadataFormatsServerRequest extends ServerOAIRequest {
public class ListMetadataFormatsServerRequest extends AbstractOAIRequest {
}

4
oai-server/src/main/java/org/xbib/oai/server/listmetadataformats/ListMetadataFormatsServerResponse.java
View file

@ -1,10 +1,10 @@
package org.xbib.oai.server.listmetadataformats;
import org.xbib.oai.server.ServerOAIResponse;
import org.xbib.oai.server.AbstractOAIResponse;
/**
*
*/
public class ListMetadataFormatsServerResponse extends ServerOAIResponse {
public class ListMetadataFormatsServerResponse extends AbstractOAIResponse {
}

4
oai-server/src/main/java/org/xbib/oai/server/listrecords/ListRecordsServerRequest.java
View file

@ -1,10 +1,10 @@
package org.xbib.oai.server.listrecords;
import org.xbib.oai.server.ServerOAIRequest;
import org.xbib.oai.server.AbstractOAIRequest;
/**
*
*/
public class ListRecordsServerRequest extends ServerOAIRequest {
public class ListRecordsServerRequest extends AbstractOAIRequest {
}

4
oai-server/src/main/java/org/xbib/oai/server/listrecords/ListRecordsServerResponse.java
View file

@ -1,13 +1,13 @@
package org.xbib.oai.server.listrecords;
import org.xbib.oai.server.ServerOAIResponse;
import org.xbib.oai.server.AbstractOAIResponse;
import java.util.Date;
/**
*
*/
public class ListRecordsServerResponse extends ServerOAIResponse {
public class ListRecordsServerResponse extends AbstractOAIResponse {
private String error;

4
oai-server/src/main/java/org/xbib/oai/server/listsets/ListSetsServerRequest.java
View file

@ -1,10 +1,10 @@
package org.xbib.oai.server.listsets;
import org.xbib.oai.server.ServerOAIRequest;
import org.xbib.oai.server.AbstractOAIRequest;
/**
*
*/
public class ListSetsServerRequest extends ServerOAIRequest {
public class ListSetsServerRequest extends AbstractOAIRequest {
}

4
oai-server/src/main/java/org/xbib/oai/server/listsets/ListSetsServerResponse.java
View file

@ -1,9 +1,9 @@
package org.xbib.oai.server.listsets;
import org.xbib.oai.server.ServerOAIResponse;
import org.xbib.oai.server.AbstractOAIResponse;
/**
*
*/
public class ListSetsServerResponse extends ServerOAIResponse {
public class ListSetsServerResponse extends AbstractOAIResponse {
}

15
oai-server/src/main/java/org/xbib/oai/server/verb/AbstractVerb.java
View file

@ -3,10 +3,9 @@ package org.xbib.oai.server.verb;
import org.xbib.oai.OAIConstants;
import org.xbib.oai.exceptions.OAIException;
import org.xbib.oai.server.OAIServer;
import org.xbib.oai.server.ServerOAIRequest;
import org.xbib.oai.server.ServerOAIResponse;
import org.xbib.oai.server.AbstractOAIRequest;
import org.xbib.oai.server.AbstractOAIResponse;
import java.io.IOException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Date;
@ -29,22 +28,22 @@ public abstract class AbstractVerb {
private static final String NS_PREFIX = "xsi";
private final ServerOAIRequest request;
private final AbstractOAIRequest request;
private final ServerOAIResponse response;
private final AbstractOAIResponse response;
public AbstractVerb(ServerOAIRequest request, ServerOAIResponse response) {
public AbstractVerb(AbstractOAIRequest request, AbstractOAIResponse response) {
this.request = request;
this.response = response;
}
public abstract void execute(OAIServer adapter) throws OAIException;
public abstract void execute(OAIServer server) throws OAIException;
protected void beginDocument() throws XMLStreamException {
response.getConsumer().add(eventFactory.createStartDocument());
}
protected void endDocument() throws XMLStreamException, IOException {
protected void endDocument() throws XMLStreamException {
response.getConsumer().add(eventFactory.createEndDocument());
}

4
oai-server/src/main/java/org/xbib/oai/server/verb/Identify.java
View file

@ -15,10 +15,10 @@ public class Identify extends AbstractVerb {
}
@Override
public void execute(OAIServer adapter) throws OAIException {
public void execute(OAIServer server) throws OAIException {
try {
beginDocument();
beginOAIPMH(adapter.getBaseURL());
beginOAIPMH(server.getBaseURL());
beginElement("Identify");
endElement("Identify");
endOAIPMH();

6
oai-server/src/main/java/org/xbib/oai/server/verb/ListMetadataFormats.java
View file

@ -2,15 +2,15 @@ package org.xbib.oai.server.verb;
import org.xbib.oai.exceptions.OAIException;
import org.xbib.oai.server.OAIServer;
import org.xbib.oai.server.ServerOAIRequest;
import org.xbib.oai.server.ServerOAIResponse;
import org.xbib.oai.server.AbstractOAIRequest;
import org.xbib.oai.server.AbstractOAIResponse;
/**
*
*/
public class ListMetadataFormats extends AbstractVerb {
public ListMetadataFormats(ServerOAIRequest request, ServerOAIResponse response) {
public ListMetadataFormats(AbstractOAIRequest request, AbstractOAIResponse response) {
super(request, response);
}

1
oai-server/src/test/java/org/xbib/oai/server/SimpleServer.java
View file

@ -16,7 +16,6 @@ import org.xbib.oai.server.listsets.ListSetsServerResponse;
import org.xbib.oai.server.verb.Identify;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Date;

3
settings.gradle
View file

@ -1,4 +1,5 @@
rootProject.name = 'oai'
rootProject.name = name
include 'oai-common'
include 'oai-client'
include 'oai-server'