fixed validity check for subfield codes, changed MarcGenerator close() to flush() method, fixes for sonarqube, update to gradle 4.4
This commit is contained in:
parent
58c2a49293
commit
7fe42850dc
19 changed files with 225 additions and 194 deletions
16
build.gradle
16
build.gradle
|
@ -64,13 +64,15 @@ test {
|
||||||
}
|
}
|
||||||
|
|
||||||
asciidoctor {
|
asciidoctor {
|
||||||
backends 'html5'
|
attributes toc: 'left',
|
||||||
separateOutputDirs = false
|
doctype: 'book',
|
||||||
attributes 'source-highlighter': 'coderay',
|
icons: 'font',
|
||||||
toc : '',
|
encoding: 'utf-8',
|
||||||
idprefix : '',
|
sectlink: true,
|
||||||
idseparator : '-',
|
sectanchors: true,
|
||||||
stylesheet: "${projectDir}/src/docs/asciidoc/css/foundation.css"
|
linkattrs: true,
|
||||||
|
imagesdir: 'img',
|
||||||
|
'source-highlighter': 'coderay'
|
||||||
}
|
}
|
||||||
|
|
||||||
javadoc {
|
javadoc {
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
group = org.xbib
|
group = org.xbib
|
||||||
name = marc
|
name = marc
|
||||||
version = 1.0.17
|
version = 1.0.18
|
||||||
|
|
||||||
xbib-content.version = 1.0.7
|
xbib-content.version = 1.0.7
|
||||||
xbib-bibliographic-character-sets.version = 1.0.0
|
xbib-bibliographic-character-sets.version = 1.0.0
|
||||||
|
|
BIN
gradle/wrapper/gradle-wrapper.jar
vendored
BIN
gradle/wrapper/gradle-wrapper.jar
vendored
Binary file not shown.
4
gradle/wrapper/gradle-wrapper.properties
vendored
4
gradle/wrapper/gradle-wrapper.properties
vendored
|
@ -1,6 +1,6 @@
|
||||||
#Tue Dec 05 21:14:54 CET 2017
|
#Thu Dec 14 12:16:56 CET 2017
|
||||||
distributionBase=GRADLE_USER_HOME
|
distributionBase=GRADLE_USER_HOME
|
||||||
distributionPath=wrapper/dists
|
distributionPath=wrapper/dists
|
||||||
zipStoreBase=GRADLE_USER_HOME
|
zipStoreBase=GRADLE_USER_HOME
|
||||||
zipStorePath=wrapper/dists
|
zipStorePath=wrapper/dists
|
||||||
distributionUrl=https\://services.gradle.org/distributions/gradle-4.3.1-all.zip
|
distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip
|
||||||
|
|
|
@ -5,10 +5,8 @@ Version 1.0
|
||||||
:toc: preamble
|
:toc: preamble
|
||||||
:toclevels: 4
|
:toclevels: 4
|
||||||
:!toc-title: Content
|
:!toc-title: Content
|
||||||
:experimental:
|
|
||||||
:description: MARC bibliographic data processing
|
:description: MARC bibliographic data processing
|
||||||
:keywords: MARC, Java, bibliographic data processing
|
:keywords: MARC, Java, bibliographic data processing
|
||||||
:icons: font
|
|
||||||
|
|
||||||
== Introduction
|
== Introduction
|
||||||
|
|
||||||
|
|
|
@ -314,7 +314,7 @@ public final class Marc {
|
||||||
count++;
|
count++;
|
||||||
}
|
}
|
||||||
stream.close();
|
stream.close();
|
||||||
builder.marcGenerator.close();
|
builder.marcGenerator.flush();
|
||||||
if (withCollection) {
|
if (withCollection) {
|
||||||
marcListener.endCollection();
|
marcListener.endCollection();
|
||||||
if (marcListener instanceof ContentHandler) {
|
if (marcListener instanceof ContentHandler) {
|
||||||
|
@ -384,7 +384,7 @@ public final class Marc {
|
||||||
l.incrementAndGet();
|
l.incrementAndGet();
|
||||||
});
|
});
|
||||||
stream.close();
|
stream.close();
|
||||||
builder.marcGenerator.close();
|
builder.marcGenerator.flush();
|
||||||
if (withCollection) {
|
if (withCollection) {
|
||||||
marcRecordListener.endCollection();
|
marcRecordListener.endCollection();
|
||||||
if (marcRecordListener instanceof ContentHandler) {
|
if (marcRecordListener instanceof ContentHandler) {
|
||||||
|
@ -567,7 +567,7 @@ public final class Marc {
|
||||||
while ((chunk = stream.readChunk()) != null) {
|
while ((chunk = stream.readChunk()) != null) {
|
||||||
marcGenerator.chunk(chunk);
|
marcGenerator.chunk(chunk);
|
||||||
}
|
}
|
||||||
marcGenerator.close();
|
marcGenerator.flush();
|
||||||
} finally {
|
} finally {
|
||||||
builder.getInputStream().close();
|
builder.getInputStream().close();
|
||||||
}
|
}
|
||||||
|
@ -1100,18 +1100,19 @@ public final class Marc {
|
||||||
@Override
|
@Override
|
||||||
public boolean hasNext() {
|
public boolean hasNext() {
|
||||||
try {
|
try {
|
||||||
setMarcRecord(null);
|
MarcRecord record;
|
||||||
|
record(null);
|
||||||
Chunk<byte[], BytesReference> chunk;
|
Chunk<byte[], BytesReference> chunk;
|
||||||
while ((chunk = stream.readChunk()) != null) {
|
while ((chunk = stream.readChunk()) != null) {
|
||||||
marcGenerator.chunk(chunk);
|
marcGenerator.chunk(chunk);
|
||||||
MarcRecord marcRecord = getMarcRecord();
|
record = getMarcRecord();
|
||||||
if (marcRecord != null) {
|
if (record != null) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
marcGenerator.close();
|
marcGenerator.flush();
|
||||||
MarcRecord marcRecord = getMarcRecord();
|
record = getMarcRecord();
|
||||||
if (marcRecord != null) {
|
if (record != null) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
|
@ -1122,11 +1123,11 @@ public final class Marc {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public MarcRecord next() {
|
public MarcRecord next() {
|
||||||
MarcRecord marcRecord = getMarcRecord();
|
MarcRecord record = getMarcRecord();
|
||||||
if (marcRecord == null) {
|
if (record == null) {
|
||||||
throw new NoSuchElementException();
|
throw new NoSuchElementException();
|
||||||
}
|
}
|
||||||
return marcRecord;
|
return record;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
@ -1158,10 +1159,6 @@ public final class Marc {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void setMarcRecord(MarcRecord marcRecord) {
|
|
||||||
this.marcRecord = marcRecord;
|
|
||||||
}
|
|
||||||
|
|
||||||
private MarcRecord getMarcRecord() {
|
private MarcRecord getMarcRecord() {
|
||||||
return marcRecord;
|
return marcRecord;
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,8 +19,12 @@ package org.xbib.marc;
|
||||||
import org.xbib.marc.dialects.mab.MabSubfieldControl;
|
import org.xbib.marc.dialects.mab.MabSubfieldControl;
|
||||||
import org.xbib.marc.label.RecordLabel;
|
import org.xbib.marc.label.RecordLabel;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Deque;
|
||||||
|
import java.util.HashSet;
|
||||||
import java.util.LinkedList;
|
import java.util.LinkedList;
|
||||||
import java.util.ListIterator;
|
import java.util.ListIterator;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
@ -39,6 +43,21 @@ public class MarcField implements Comparable<MarcField> {
|
||||||
|
|
||||||
private static final String BLANK_STRING = " ";
|
private static final String BLANK_STRING = " ";
|
||||||
|
|
||||||
|
private static final Set<Character> ASCII_GRAPHICS = new HashSet<>(Arrays.asList(
|
||||||
|
'\u0020', '\u0021', '\u0022', '\u0023', '\u0024', '\u0025', '\u0026', '\'',
|
||||||
|
'\u0028', '\u0029', '\u002A', '\u002B', '\u002C', '\u002D', '\u002E', '\u002F',
|
||||||
|
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
|
||||||
|
'\u003A', '\u003B', '\u003C', '\u003D', '\u003E', '\u003F', '\u0040',
|
||||||
|
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
|
||||||
|
'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
|
||||||
|
'U', 'V', 'W', 'X', 'Y', 'Z',
|
||||||
|
'\u005B', '\\', '\u005D', '\u005E', '\u005F', '\u0060',
|
||||||
|
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j',
|
||||||
|
'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't',
|
||||||
|
'u', 'v', 'w', 'x', 'y', 'z',
|
||||||
|
'\u007B', '\u007C', '\u007D', '\u007E'
|
||||||
|
));
|
||||||
|
|
||||||
private final String tag;
|
private final String tag;
|
||||||
|
|
||||||
private final String indicator;
|
private final String indicator;
|
||||||
|
@ -51,12 +70,12 @@ public class MarcField implements Comparable<MarcField> {
|
||||||
|
|
||||||
private final String subfieldIds;
|
private final String subfieldIds;
|
||||||
|
|
||||||
private final LinkedList<Subfield> subfields;
|
private final Deque<Subfield> subfields;
|
||||||
|
|
||||||
private final boolean iscontrol;
|
private final boolean iscontrol;
|
||||||
|
|
||||||
private MarcField(String tag, String indicator, int position, int length,
|
private MarcField(String tag, String indicator, int position, int length,
|
||||||
String value, LinkedList<Subfield> subfields, String subfieldIds,
|
String value, Deque<Subfield> subfields, String subfieldIds,
|
||||||
boolean iscontrol) {
|
boolean iscontrol) {
|
||||||
this.tag = tag;
|
this.tag = tag;
|
||||||
this.indicator = indicator;
|
this.indicator = indicator;
|
||||||
|
@ -120,7 +139,7 @@ public class MarcField implements Comparable<MarcField> {
|
||||||
* Return the subfields associated with this MARC field.
|
* Return the subfields associated with this MARC field.
|
||||||
* @return a list of MARC subfields
|
* @return a list of MARC subfields
|
||||||
*/
|
*/
|
||||||
public LinkedList<Subfield> getSubfields() {
|
public Deque<Subfield> getSubfields() {
|
||||||
return subfields;
|
return subfields;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -129,7 +148,7 @@ public class MarcField implements Comparable<MarcField> {
|
||||||
* @param subfieldId subfield ID
|
* @param subfieldId subfield ID
|
||||||
* @return list of subfields
|
* @return list of subfields
|
||||||
*/
|
*/
|
||||||
public LinkedList<Subfield> getSubfield(String subfieldId) {
|
public Deque<Subfield> getSubfield(String subfieldId) {
|
||||||
return subfields.stream()
|
return subfields.stream()
|
||||||
.filter(subfield -> subfield.getId().equals(subfieldId))
|
.filter(subfield -> subfield.getId().equals(subfieldId))
|
||||||
.collect(Collectors.toCollection(LinkedList::new));
|
.collect(Collectors.toCollection(LinkedList::new));
|
||||||
|
@ -144,8 +163,8 @@ public class MarcField implements Comparable<MarcField> {
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getFirstSubfieldValue(String subfieldId) {
|
public String getFirstSubfieldValue(String subfieldId) {
|
||||||
LinkedList<Subfield> list = getSubfield(subfieldId);
|
Deque<Subfield> deque = getSubfield(subfieldId);
|
||||||
return list.isEmpty() ? null : list.getFirst().getValue();
|
return deque.isEmpty() ? null : deque.getFirst().getValue();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -157,8 +176,8 @@ public class MarcField implements Comparable<MarcField> {
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getLastSubfieldValue(String subfieldId) {
|
public String getLastSubfieldValue(String subfieldId) {
|
||||||
LinkedList<Subfield> list = getSubfield(subfieldId);
|
Deque<Subfield> deque = getSubfield(subfieldId);
|
||||||
return list.isEmpty() ? null : list.getLast().getValue();
|
return deque.isEmpty() ? null : deque.getLast().getValue();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -229,13 +248,7 @@ public class MarcField implements Comparable<MarcField> {
|
||||||
}
|
}
|
||||||
boolean b = true;
|
boolean b = true;
|
||||||
for (int i = 0; i < subfieldIds.length(); i++) {
|
for (int i = 0; i < subfieldIds.length(); i++) {
|
||||||
b = subfieldIds.charAt(i) == ' '
|
b = ASCII_GRAPHICS.contains(subfieldIds.charAt(i));
|
||||||
|| (subfieldIds.charAt(i) >= '0' && subfieldIds.charAt(i) <= '9')
|
|
||||||
|| (subfieldIds.charAt(i) >= 'a' && subfieldIds.charAt(i) <= 'z')
|
|
||||||
|| (subfieldIds.charAt(i) >= 'A' && subfieldIds.charAt(i) <= 'Z') // can appear in german MARC
|
|
||||||
|| subfieldIds.charAt(i) == '$' // can appear in german MARC
|
|
||||||
|| subfieldIds.charAt(i) == '=' // can appear in german MARC
|
|
||||||
;
|
|
||||||
if (!b) {
|
if (!b) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,25 +22,31 @@ package org.xbib.marc;
|
||||||
public class MarcFieldAdapter implements MarcListener {
|
public class MarcFieldAdapter implements MarcListener {
|
||||||
@Override
|
@Override
|
||||||
public void beginCollection() {
|
public void beginCollection() {
|
||||||
|
// empty by design
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void beginRecord(String format, String type) {
|
public void beginRecord(String format, String type) {
|
||||||
|
// empty by design
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void leader(String label) {
|
public void leader(String label) {
|
||||||
|
// empty by design
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void field(MarcField field) {
|
public void field(MarcField field) {
|
||||||
|
// empty by design
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void endRecord() {
|
public void endRecord() {
|
||||||
|
// empty by design
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void endCollection() {
|
public void endCollection() {
|
||||||
|
// empty by design
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,7 +29,6 @@ import org.xbib.marc.transformer.MarcTransformer;
|
||||||
import org.xbib.marc.transformer.field.MarcFieldTransformers;
|
import org.xbib.marc.transformer.field.MarcFieldTransformers;
|
||||||
import org.xbib.marc.transformer.value.MarcValueTransformers;
|
import org.xbib.marc.transformer.value.MarcValueTransformers;
|
||||||
|
|
||||||
import java.io.Closeable;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.charset.Charset;
|
import java.nio.charset.Charset;
|
||||||
import java.util.LinkedList;
|
import java.util.LinkedList;
|
||||||
|
@ -38,7 +37,7 @@ import java.util.List;
|
||||||
/**
|
/**
|
||||||
* This chunk listener interprets the chunks from a stream and generates MARC events to a given MARC listener.
|
* This chunk listener interprets the chunks from a stream and generates MARC events to a given MARC listener.
|
||||||
*/
|
*/
|
||||||
public class MarcGenerator implements ChunkListener<byte[], BytesReference>, Closeable {
|
public class MarcGenerator implements ChunkListener<byte[], BytesReference> {
|
||||||
|
|
||||||
private String format;
|
private String format;
|
||||||
|
|
||||||
|
@ -180,7 +179,8 @@ public class MarcGenerator implements ChunkListener<byte[], BytesReference>, Clo
|
||||||
builder.indicator(data.substring(0, pos));
|
builder.indicator(data.substring(0, pos));
|
||||||
if (pos < data.length()) {
|
if (pos < data.length()) {
|
||||||
builder.value(this.data.substring(pos));
|
builder.value(this.data.substring(pos));
|
||||||
} }
|
}
|
||||||
|
}
|
||||||
found = true;
|
found = true;
|
||||||
break;
|
break;
|
||||||
} else if (directory.containsKey(position - offset)) {
|
} else if (directory.containsKey(position - offset)) {
|
||||||
|
@ -221,7 +221,7 @@ public class MarcGenerator implements ChunkListener<byte[], BytesReference>, Clo
|
||||||
* This method will emit the last record, if not emitted already.
|
* This method will emit the last record, if not emitted already.
|
||||||
* Useful if chunk streams have no closing record separator.
|
* Useful if chunk streams have no closing record separator.
|
||||||
*/
|
*/
|
||||||
public void close() throws IOException {
|
public void flush() {
|
||||||
if (position > 0) {
|
if (position > 0) {
|
||||||
emitMarcRecord();
|
emitMarcRecord();
|
||||||
}
|
}
|
||||||
|
|
|
@ -57,7 +57,7 @@ public class MarcWriter extends MarcContentHandler implements Flushable, Closeab
|
||||||
* @param charset the character set
|
* @param charset the character set
|
||||||
* @throws IOException if writer can not be created
|
* @throws IOException if writer can not be created
|
||||||
*/
|
*/
|
||||||
public MarcWriter(OutputStream out, Charset charset) throws IOException {
|
public MarcWriter(OutputStream out, Charset charset) {
|
||||||
this(out, charset, DEFAULT_BUFFER_SIZE);
|
this(out, charset, DEFAULT_BUFFER_SIZE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -68,7 +68,7 @@ public class MarcWriter extends MarcContentHandler implements Flushable, Closeab
|
||||||
* @param buffersize the buffer size writing to the underlying output stream
|
* @param buffersize the buffer size writing to the underlying output stream
|
||||||
* @throws IOException if writer can not be created
|
* @throws IOException if writer can not be created
|
||||||
*/
|
*/
|
||||||
public MarcWriter(OutputStream out, Charset charset, int buffersize) throws IOException {
|
public MarcWriter(OutputStream out, Charset charset, int buffersize) {
|
||||||
this.out = new SeparatorOutputStream(out, buffersize);
|
this.out = new SeparatorOutputStream(out, buffersize);
|
||||||
this.charset = charset;
|
this.charset = charset;
|
||||||
this.bytesStreamOutput = new BytesStreamOutput();
|
this.bytesStreamOutput = new BytesStreamOutput();
|
||||||
|
|
|
@ -16,6 +16,9 @@ public class MabSubfieldControl {
|
||||||
FIELDS.put("856", 2);
|
FIELDS.put("856", 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private MabSubfieldControl() {
|
||||||
|
}
|
||||||
|
|
||||||
public static Integer getSubfieldIdLen(String tag) {
|
public static Integer getSubfieldIdLen(String tag) {
|
||||||
return FIELDS.getOrDefault(tag, 0);
|
return FIELDS.getOrDefault(tag, 0);
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,7 +24,15 @@ import org.xbib.marc.label.RecordLabel;
|
||||||
import org.xbib.marc.transformer.value.MarcValueTransformers;
|
import org.xbib.marc.transformer.value.MarcValueTransformers;
|
||||||
import org.xbib.marc.xml.MarcContentHandler;
|
import org.xbib.marc.xml.MarcContentHandler;
|
||||||
|
|
||||||
import java.io.*;
|
import java.io.BufferedOutputStream;
|
||||||
|
import java.io.BufferedWriter;
|
||||||
|
import java.io.Closeable;
|
||||||
|
import java.io.Flushable;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.io.OutputStreamWriter;
|
||||||
|
import java.io.UncheckedIOException;
|
||||||
|
import java.io.Writer;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Paths;
|
import java.nio.file.Paths;
|
||||||
|
@ -97,23 +105,23 @@ public class MarcJsonWriter extends MarcContentHandler implements Flushable, Clo
|
||||||
*/
|
*/
|
||||||
private boolean top;
|
private boolean top;
|
||||||
|
|
||||||
public MarcJsonWriter(OutputStream out) throws IOException {
|
public MarcJsonWriter(OutputStream out) {
|
||||||
this(out, Style.ARRAY);
|
this(out, Style.ARRAY);
|
||||||
}
|
}
|
||||||
|
|
||||||
public MarcJsonWriter(OutputStream out, Style style) throws IOException {
|
public MarcJsonWriter(OutputStream out, Style style) {
|
||||||
this(out, DEFAULT_BUFFER_SIZE, style);
|
this(out, DEFAULT_BUFFER_SIZE, style);
|
||||||
}
|
}
|
||||||
|
|
||||||
public MarcJsonWriter(OutputStream out, int bufferSize, Style style) throws IOException {
|
public MarcJsonWriter(OutputStream out, int bufferSize, Style style) {
|
||||||
this(new OutputStreamWriter(out, StandardCharsets.UTF_8), style, bufferSize);
|
this(new OutputStreamWriter(out, StandardCharsets.UTF_8), style, bufferSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
public MarcJsonWriter(Writer writer) throws IOException {
|
public MarcJsonWriter(Writer writer) {
|
||||||
this(writer, Style.ARRAY, DEFAULT_BUFFER_SIZE);
|
this(writer, Style.ARRAY, DEFAULT_BUFFER_SIZE);
|
||||||
}
|
}
|
||||||
|
|
||||||
public MarcJsonWriter(Writer writer, Style style, int bufferSize) throws IOException {
|
public MarcJsonWriter(Writer writer, Style style, int bufferSize) {
|
||||||
this.writer = new BufferedWriter(writer, bufferSize);
|
this.writer = new BufferedWriter(writer, bufferSize);
|
||||||
this.bufferSize = bufferSize;
|
this.bufferSize = bufferSize;
|
||||||
this.style = style;
|
this.style = style;
|
||||||
|
|
|
@ -45,7 +45,7 @@ public class MarcTool {
|
||||||
private String stylesheet = null;
|
private String stylesheet = null;
|
||||||
private String result = null;
|
private String result = null;
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
public static void main(String[] args) {
|
||||||
MarcTool marcTool = new MarcTool();
|
MarcTool marcTool = new MarcTool();
|
||||||
marcTool.parse(args);
|
marcTool.parse(args);
|
||||||
System.exit(marcTool.run());
|
System.exit(marcTool.run());
|
||||||
|
@ -99,39 +99,36 @@ public class MarcTool {
|
||||||
if (mode == null) {
|
if (mode == null) {
|
||||||
mode = "marc2xml";
|
mode = "marc2xml";
|
||||||
}
|
}
|
||||||
switch (mode) {
|
if ("marc2xml".equals(mode)) {
|
||||||
case "marc2xml": {
|
try (InputStream in = Files.newInputStream(Paths.get(input));
|
||||||
try (InputStream in = Files.newInputStream(Paths.get(input));
|
MarcXchangeWriter writer = new MarcXchangeWriter(Files.newBufferedWriter(Paths.get(output)), true)) {
|
||||||
MarcXchangeWriter writer = new MarcXchangeWriter(Files.newBufferedWriter(Paths.get(output)), true)) {
|
Marc.Builder builder = Marc.builder()
|
||||||
Marc.Builder builder = Marc.builder()
|
.setInputStream(in)
|
||||||
.setInputStream(in)
|
.setCharset(Charset.forName(charset))
|
||||||
.setCharset(Charset.forName(charset))
|
.setMarcListener(writer);
|
||||||
.setMarcListener(writer);
|
if (schema != null && stylesheet != null && result != null) {
|
||||||
if (schema != null && stylesheet != null && result != null) {
|
System.setProperty("http.agent", "Java Agent");
|
||||||
System.setProperty("http.agent", "Java Agent");
|
builder.setSchema(schema).build().transform(new URL(stylesheet),
|
||||||
builder.setSchema(schema).build().transform(new URL(stylesheet),
|
new StreamResult(Files.newBufferedWriter(Paths.get(result))));
|
||||||
new StreamResult(Files.newBufferedWriter(Paths.get(result))));
|
} else {
|
||||||
} else {
|
builder.build().writeCollection();
|
||||||
builder.build().writeCollection();
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.log(Level.SEVERE, e.getMessage(), e);
|
|
||||||
return 1;
|
|
||||||
}
|
}
|
||||||
return 0;
|
} catch (Exception e) {
|
||||||
}
|
logger.log(Level.SEVERE, e.getMessage(), e);
|
||||||
default: {
|
return 1;
|
||||||
String help = "Usage: " + getClass().getName()
|
|
||||||
+ " --mode [marc2xml] set operation mode\n"
|
|
||||||
+ " --input <path> \n"
|
|
||||||
+ " --output <path> \n"
|
|
||||||
+ " --charset <name> \n"
|
|
||||||
+ " --schema [MARC21|MarcXchange] \n"
|
|
||||||
+ " --stylesheet <URL> \n"
|
|
||||||
+ " --result <path> \n";
|
|
||||||
logger.log(Level.INFO, help);
|
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
return 0;
|
||||||
|
} else {
|
||||||
|
String help = "Usage: " + getClass().getName()
|
||||||
|
+ " --mode [marc2xml] set operation mode\n"
|
||||||
|
+ " --input <path> \n"
|
||||||
|
+ " --output <path> \n"
|
||||||
|
+ " --charset <name> \n"
|
||||||
|
+ " --schema [MARC21|MarcXchange] \n"
|
||||||
|
+ " --stylesheet <URL> \n"
|
||||||
|
+ " --result <path> \n";
|
||||||
|
logger.log(Level.INFO, help);
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,6 +21,7 @@ import static org.xbib.marc.transformer.field.MarcFieldTransformer.Operator.HEAD
|
||||||
import org.xbib.marc.MarcField;
|
import org.xbib.marc.MarcField;
|
||||||
|
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
|
import java.util.Iterator;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.logging.Level;
|
import java.util.logging.Level;
|
||||||
|
@ -131,12 +132,13 @@ public class MarcFieldTransformer extends LinkedHashMap<String, MarcField> {
|
||||||
builder.subfield(subfield.getId(), subfield.getValue());
|
builder.subfield(subfield.getId(), subfield.getValue());
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// map subfields
|
// transform subfields
|
||||||
for (int i = 0; i < marcField.getSubfields().size(); i++) {
|
Iterator<MarcField.Subfield> subfields = marcField.getSubfields().iterator();
|
||||||
if (i < newMarcField.getSubfields().size()) {
|
Iterator<MarcField.Subfield> newSubfields = newMarcField.getSubfields().iterator();
|
||||||
builder.subfield(newMarcField.getSubfields().get(i).getId(),
|
while (subfields.hasNext() && newSubfields.hasNext()) {
|
||||||
marcField.getSubfields().get(i).getValue());
|
MarcField.Subfield subfield = subfields.next();
|
||||||
}
|
MarcField.Subfield newSubfield = newSubfields.next();
|
||||||
|
builder.subfield(newSubfield.getId(), subfield.getValue());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -192,11 +194,12 @@ public class MarcFieldTransformer extends LinkedHashMap<String, MarcField> {
|
||||||
} else {
|
} else {
|
||||||
// get the correct MARC field to map subfield IDs
|
// get the correct MARC field to map subfield IDs
|
||||||
MarcField marcField1 = get(key);
|
MarcField marcField1 = get(key);
|
||||||
for (int i = 0; i < marcField.getSubfields().size(); i++) {
|
Iterator<MarcField.Subfield> subfields = marcField.getSubfields().iterator();
|
||||||
if (i < marcField1.getSubfields().size()) {
|
Iterator<MarcField.Subfield> newSubfields = marcField1.getSubfields().iterator();
|
||||||
builder.subfield(marcField1.getSubfields().get(i).getId(),
|
while (subfields.hasNext() && newSubfields.hasNext()) {
|
||||||
marcField.getSubfields().get(i).getValue());
|
MarcField.Subfield subfield = subfields.next();
|
||||||
}
|
MarcField.Subfield newSubfield = newSubfields.next();
|
||||||
|
builder.subfield(newSubfield.getId(), subfield.getValue());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
lastBuilt = builder.build();
|
lastBuilt = builder.build();
|
||||||
|
|
|
@ -25,12 +25,14 @@ import java.util.Map;
|
||||||
*/
|
*/
|
||||||
public class MarcValueTransformers {
|
public class MarcValueTransformers {
|
||||||
|
|
||||||
|
private static final String DEFAULT = "_default";
|
||||||
|
|
||||||
private final Map<String, MarcValueTransformer> marcValueTransformerMap = new HashMap<>();
|
private final Map<String, MarcValueTransformer> marcValueTransformerMap = new HashMap<>();
|
||||||
|
|
||||||
private final Map<String, String> subfieldMap = new HashMap<>();
|
private final Map<String, String> subfieldMap = new HashMap<>();
|
||||||
|
|
||||||
public MarcValueTransformers setMarcValueTransformer(MarcValueTransformer transformer) {
|
public MarcValueTransformers setMarcValueTransformer(MarcValueTransformer transformer) {
|
||||||
this.marcValueTransformerMap.put("_default", transformer);
|
this.marcValueTransformerMap.put(DEFAULT, transformer);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -62,7 +64,7 @@ public class MarcValueTransformers {
|
||||||
return field;
|
return field;
|
||||||
}
|
}
|
||||||
final MarcValueTransformer transformer = marcValueTransformerMap.containsKey(key) ?
|
final MarcValueTransformer transformer = marcValueTransformerMap.containsKey(key) ?
|
||||||
marcValueTransformerMap.get(key) : marcValueTransformerMap.get("_default");
|
marcValueTransformerMap.get(key) : marcValueTransformerMap.get(DEFAULT);
|
||||||
if (transformer != null) {
|
if (transformer != null) {
|
||||||
MarcField.Builder builder = MarcField.builder();
|
MarcField.Builder builder = MarcField.builder();
|
||||||
builder.tag(field.getTag()).indicator(field.getIndicator());
|
builder.tag(field.getTag()).indicator(field.getIndicator());
|
||||||
|
@ -80,7 +82,7 @@ public class MarcValueTransformers {
|
||||||
}
|
}
|
||||||
|
|
||||||
public String transform(String value) {
|
public String transform(String value) {
|
||||||
MarcValueTransformer marcValueTransformer = marcValueTransformerMap.get("_default");
|
MarcValueTransformer marcValueTransformer = marcValueTransformerMap.get(DEFAULT);
|
||||||
return marcValueTransformer != null ? marcValueTransformer.transform(value) : value;
|
return marcValueTransformer != null ? marcValueTransformer.transform(value) : value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,7 +29,7 @@ public class MarcFieldTest extends Assert {
|
||||||
@Test
|
@Test
|
||||||
public void testFieldData() {
|
public void testFieldData() {
|
||||||
MarcField marcField = MarcField.builder().tag("100").indicator("").value("Hello World").build();
|
MarcField marcField = MarcField.builder().tag("100").indicator("").value("Hello World").build();
|
||||||
assertEquals(marcField.getValue(), "Hello World");
|
assertEquals("Hello World", marcField.getValue());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -159,7 +159,7 @@ public class MarcFieldTest extends Assert {
|
||||||
MarcField marcField = MarcField.builder()
|
MarcField marcField = MarcField.builder()
|
||||||
.tag("100")
|
.tag("100")
|
||||||
.indicator("0")
|
.indicator("0")
|
||||||
.subfield("-", null)
|
.subfield("\u007f", null)
|
||||||
.build();
|
.build();
|
||||||
assertFalse(marcField.isSubfieldValid());
|
assertFalse(marcField.isSubfieldValid());
|
||||||
}
|
}
|
||||||
|
|
|
@ -92,85 +92,85 @@ public class MarcRecordTest extends Assert {
|
||||||
@Test
|
@Test
|
||||||
public void testFilterKeyIterable() throws Exception {
|
public void testFilterKeyIterable() throws Exception {
|
||||||
String s = "summerland.mrc";
|
String s = "summerland.mrc";
|
||||||
InputStream in = getClass().getResource(s).openStream();
|
try (InputStream in = getClass().getResource(s).openStream()) {
|
||||||
Marc.Builder builder = Marc.builder()
|
Marc.Builder builder = Marc.builder()
|
||||||
.setInputStream(in)
|
.setInputStream(in)
|
||||||
.setCharset(Charset.forName("ANSEL"));
|
.setCharset(Charset.forName("ANSEL"));
|
||||||
// only single record
|
// only single record
|
||||||
for (MarcRecord marcRecord : builder.iterable()) {
|
for (MarcRecord marcRecord : builder.iterable()) {
|
||||||
// single 245 field
|
// single 245 field
|
||||||
assertEquals(1, marcRecord.filterKey(Pattern.compile("^245.*")).size());
|
assertEquals(1, marcRecord.filterKey(Pattern.compile("^245.*")).size());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
in.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testFilterKey() throws Exception {
|
public void testFilterKey() throws Exception {
|
||||||
String s = "summerland.mrc";
|
String s = "summerland.mrc";
|
||||||
InputStream in = getClass().getResource(s).openStream();
|
try (InputStream in = getClass().getResource(s).openStream()) {
|
||||||
Marc.Builder builder = Marc.builder()
|
Marc.Builder builder = Marc.builder()
|
||||||
.setInputStream(in)
|
.setInputStream(in)
|
||||||
.setCharset(Charset.forName("ANSEL"))
|
.setCharset(Charset.forName("ANSEL"))
|
||||||
.setKeyPattern(Pattern.compile("^245.*"));
|
.setKeyPattern(Pattern.compile("^245.*"));
|
||||||
// record with single field
|
// record with single field
|
||||||
for (MarcRecord marcRecord : builder.iterable()) {
|
for (MarcRecord marcRecord : builder.iterable()) {
|
||||||
assertEquals(1, marcRecord.getFields().size());
|
assertEquals(1, marcRecord.getFields().size());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
in.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testFilterValueIterable() throws Exception {
|
public void testFilterValueIterable() throws Exception {
|
||||||
String s = "summerland.mrc";
|
String s = "summerland.mrc";
|
||||||
InputStream in = getClass().getResource(s).openStream();
|
try (InputStream in = getClass().getResource(s).openStream()) {
|
||||||
Marc.Builder builder = Marc.builder()
|
Marc.Builder builder = Marc.builder()
|
||||||
.setInputStream(in)
|
.setInputStream(in)
|
||||||
.setCharset(Charset.forName("ANSEL"));
|
.setCharset(Charset.forName("ANSEL"));
|
||||||
for (MarcRecord marcRecord : builder.iterable()) {
|
for (MarcRecord marcRecord : builder.iterable()) {
|
||||||
assertEquals(2, marcRecord.filterValue(Pattern.compile(".*?Chabon.*")).size());
|
assertEquals(2, marcRecord.filterValue(Pattern.compile(".*?Chabon.*")).size());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
in.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testFilterValue() throws Exception {
|
public void testFilterValue() throws Exception {
|
||||||
String s = "summerland.mrc";
|
String s = "summerland.mrc";
|
||||||
InputStream in = getClass().getResource(s).openStream();
|
try (InputStream in = getClass().getResource(s).openStream()) {
|
||||||
Marc.Builder builder = Marc.builder()
|
Marc.Builder builder = Marc.builder()
|
||||||
.setInputStream(in)
|
.setInputStream(in)
|
||||||
.setCharset(Charset.forName("ANSEL"))
|
.setCharset(Charset.forName("ANSEL"))
|
||||||
.setValuePattern(Pattern.compile(".*?Chabon.*"));
|
.setValuePattern(Pattern.compile(".*?Chabon.*"));
|
||||||
for (MarcRecord marcRecord : builder.iterable()) {
|
for (MarcRecord marcRecord : builder.iterable()) {
|
||||||
assertEquals(2, marcRecord.getFields().size());
|
assertEquals(2, marcRecord.getFields().size());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
in.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSequentialIteration() throws Exception {
|
public void testSequentialIteration() throws Exception {
|
||||||
String s = "dialects/unimarc/periouni.mrc";
|
String s = "dialects/unimarc/periouni.mrc";
|
||||||
InputStream in = getClass().getResource(s).openStream();
|
try (InputStream in = getClass().getResource(s).openStream()) {
|
||||||
Marc.Builder builder = Marc.builder()
|
Marc.Builder builder = Marc.builder()
|
||||||
.setInputStream(in).setCharset(StandardCharsets.UTF_8);
|
.setInputStream(in).setCharset(StandardCharsets.UTF_8);
|
||||||
final AtomicInteger count = new AtomicInteger();
|
final AtomicInteger count = new AtomicInteger();
|
||||||
// test for loop
|
// test for loop
|
||||||
for (MarcRecord marcRecord : builder.iterable()) {
|
for (MarcRecord marcRecord : builder.iterable()) {
|
||||||
count.incrementAndGet();
|
count.incrementAndGet();
|
||||||
|
}
|
||||||
|
assertEquals(3064, count.get());
|
||||||
}
|
}
|
||||||
in.close();
|
|
||||||
assertEquals(3064, count.get());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRecordStream() throws Exception {
|
public void testRecordStream() throws Exception {
|
||||||
String s = "dialects/unimarc/periouni.mrc";
|
String s = "dialects/unimarc/periouni.mrc";
|
||||||
InputStream in = getClass().getResource(s).openStream();
|
try (InputStream in = getClass().getResource(s).openStream()) {
|
||||||
Marc.Builder builder = Marc.builder()
|
Marc.Builder builder = Marc.builder()
|
||||||
.setInputStream(in)
|
.setInputStream(in)
|
||||||
.setCharset(StandardCharsets.UTF_8);
|
.setCharset(StandardCharsets.UTF_8);
|
||||||
long count = builder.recordStream().map(r -> r.get("001")).count();
|
long count = builder.recordStream().map(r -> r.get("001")).count();
|
||||||
in.close();
|
assertEquals(3064, count);
|
||||||
assertEquals(3064, count);
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -179,47 +179,49 @@ public class MarcRecordTest extends Assert {
|
||||||
@Test
|
@Test
|
||||||
public void testIRMARC8AsRecordStream() throws Exception {
|
public void testIRMARC8AsRecordStream() throws Exception {
|
||||||
String s = "IRMARC8.bin";
|
String s = "IRMARC8.bin";
|
||||||
InputStream in = getClass().getResource(s).openStream();
|
try (InputStream in = getClass().getResource(s).openStream()) {
|
||||||
File file = File.createTempFile(s + ".", ".xml");
|
File file = File.createTempFile(s + ".", ".xml");
|
||||||
file.deleteOnExit();
|
file.deleteOnExit();
|
||||||
FileOutputStream out = new FileOutputStream(file);
|
FileOutputStream out = new FileOutputStream(file);
|
||||||
MarcValueTransformers marcValueTransformers = new MarcValueTransformers();
|
MarcValueTransformers marcValueTransformers = new MarcValueTransformers();
|
||||||
marcValueTransformers.setMarcValueTransformer(value -> Normalizer.normalize(value, Normalizer.Form.NFC));
|
marcValueTransformers.setMarcValueTransformer(value -> Normalizer.normalize(value, Normalizer.Form.NFC));
|
||||||
try (MarcXchangeWriter writer = new MarcXchangeWriter(out)
|
try (MarcXchangeWriter writer = new MarcXchangeWriter(out)
|
||||||
.setMarcValueTransformers(marcValueTransformers)) {
|
.setMarcValueTransformers(marcValueTransformers)) {
|
||||||
Marc.builder()
|
Marc.builder()
|
||||||
.setInputStream(in)
|
.setInputStream(in)
|
||||||
.setCharset(Charset.forName("ANSEL"))
|
.setCharset(Charset.forName("ANSEL"))
|
||||||
.setMarcRecordListener(writer)
|
.setMarcRecordListener(writer)
|
||||||
.build()
|
.build()
|
||||||
.writeRecordCollection();
|
.writeRecordCollection();
|
||||||
assertNull(writer.getException());
|
assertNull(writer.getException());
|
||||||
|
}
|
||||||
|
assertThat(file, CompareMatcher.isIdenticalTo(getClass().getResource(s + ".xml").openStream()));
|
||||||
}
|
}
|
||||||
assertThat(file, CompareMatcher.isIdenticalTo(getClass().getResource(s + ".xml").openStream()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testIRMARC8AsLightweightRecordAdapter() throws Exception {
|
public void testIRMARC8AsLightweightRecordAdapter() throws Exception {
|
||||||
String s = "IRMARC8.bin";
|
String s = "IRMARC8.bin";
|
||||||
InputStream in = getClass().getResource(s).openStream();
|
try (InputStream in = getClass().getResource(s).openStream()) {
|
||||||
File file = File.createTempFile(s + ".", ".xml");
|
File file = File.createTempFile(s + ".", ".xml");
|
||||||
file.deleteOnExit();
|
file.deleteOnExit();
|
||||||
FileOutputStream out = new FileOutputStream(file);
|
FileOutputStream out = new FileOutputStream(file);
|
||||||
MarcValueTransformers marcValueTransformers = new MarcValueTransformers();
|
MarcValueTransformers marcValueTransformers = new MarcValueTransformers();
|
||||||
marcValueTransformers.setMarcValueTransformer(value -> Normalizer.normalize(value, Normalizer.Form.NFC));
|
marcValueTransformers.setMarcValueTransformer(value -> Normalizer.normalize(value, Normalizer.Form.NFC));
|
||||||
try (MarcXchangeWriter writer = new MarcXchangeWriter(out)
|
try (MarcXchangeWriter writer = new MarcXchangeWriter(out)
|
||||||
.setMarcValueTransformers(marcValueTransformers)) {
|
.setMarcValueTransformers(marcValueTransformers)) {
|
||||||
writer.startDocument(); // just write XML processing instruction
|
writer.startDocument(); // just write XML processing instruction
|
||||||
Marc.builder()
|
Marc.builder()
|
||||||
.setInputStream(in)
|
.setInputStream(in)
|
||||||
.setCharset(Charset.forName("ANSEL"))
|
.setCharset(Charset.forName("ANSEL"))
|
||||||
.setMarcListener(new LightweightMarcRecordAdapter(writer))
|
.setMarcListener(new LightweightMarcRecordAdapter(writer))
|
||||||
.build()
|
.build()
|
||||||
.writeCollection();
|
.writeCollection();
|
||||||
assertNull(writer.getException());
|
assertNull(writer.getException());
|
||||||
writer.endDocument();
|
writer.endDocument();
|
||||||
|
}
|
||||||
|
assertThat(file, CompareMatcher.isIdenticalTo(getClass().getResource(s + ".xml").openStream()));
|
||||||
}
|
}
|
||||||
assertThat(file, CompareMatcher.isIdenticalTo(getClass().getResource(s + ".xml").openStream()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
package org.xbib.marc.dialects.mab;
|
package org.xbib.marc.dialects.mab;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.xbib.marc.Marc;
|
import org.xbib.marc.Marc;
|
||||||
import org.xbib.marc.MarcRecord;
|
import org.xbib.marc.MarcRecord;
|
||||||
|
@ -9,9 +12,6 @@ import java.nio.charset.Charset;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
import java.util.logging.Logger;
|
import java.util.logging.Logger;
|
||||||
|
|
||||||
import static org.junit.Assert.assertEquals;
|
|
||||||
import static org.junit.Assert.assertTrue;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -89,11 +89,11 @@ public class BufferedSeparatorInputStreamTest {
|
||||||
listener.chunk(chunk);
|
listener.chunk(chunk);
|
||||||
}
|
}
|
||||||
in.close();
|
in.close();
|
||||||
assertEquals(unitCount, 23);
|
assertEquals(23, unitCount);
|
||||||
assertEquals(groupCount, 9);
|
assertEquals(9, groupCount);
|
||||||
assertEquals(dataCount, 389);
|
assertEquals(389, dataCount);
|
||||||
assertEquals(recordCount, 356);
|
assertEquals(356, recordCount);
|
||||||
assertEquals(fileCount, 1);
|
assertEquals(1, fileCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
Loading…
Reference in a new issue