joerg/z3950
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

enable UTF-8 for PQF queries by default

Browse source
This commit is contained in:
Jörg Prante 2023-03-22 22:32:26 +01:00
parent 758ab83b3c
commit 6f53533c63
11 changed files with 99 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
gradle.properties
View file

@ -1,5 +1,5 @@
group = org.xbib
name = z3950
version = 5.0.3
version = 5.0.4
org.gradle.warning.mode = ALL

3
z3950-asn1/src/main/java/org/xbib/asn1/ASN1Any.java
View file

@ -45,8 +45,7 @@ public class ASN1Any {
* @throws ASN1Exception If the BER encoding is incorrect.
* Never occurs for ASN1Any.
*/
public void berDecode(BEREncoding berEncoding, boolean checkTag)
throws ASN1Exception {
public void berDecode(BEREncoding berEncoding, boolean checkTag) throws ASN1Exception {
asn1anyBer = berEncoding;
}

20
z3950-asn1/src/main/java/org/xbib/asn1/ASN1OctetString.java
View file

@ -1,5 +1,6 @@
package org.xbib.asn1;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
/**
@ -27,6 +28,8 @@ public class ASN1OctetString extends ASN1Any {
private byte[] octets;
private final Charset charset;
/**
* Constructor for an OCTET STRING object. The tag is set to the
* default of UNIVERSAL 4, and its value to the given bytes.
@ -34,6 +37,7 @@ public class ASN1OctetString extends ASN1Any {
*/
public ASN1OctetString(byte[] data) {
octets = new byte[data.length];
this.charset = StandardCharsets.ISO_8859_1;
System.arraycopy(data, 0, octets, 0, data.length);
}
@ -44,7 +48,12 @@ public class ASN1OctetString extends ASN1Any {
* @param str string
*/
public ASN1OctetString(String str) {
octets = str.getBytes(StandardCharsets.ISO_8859_1);
this(str, StandardCharsets.ISO_8859_1);
}
public ASN1OctetString(String str, Charset charset) {
this.octets = str.getBytes(charset);
this.charset = charset;
}
/**
@ -55,7 +64,8 @@ public class ASN1OctetString extends ASN1Any {
* @throws ASN1Exception If the BER encoding is incorrect.
*/
public ASN1OctetString(BEREncoding ber, boolean checkTag) throws ASN1Exception {
super(ber, checkTag);
this.charset = StandardCharsets.ISO_8859_1;
berDecode(ber, checkTag);
}
/**
@ -80,7 +90,7 @@ public class ASN1OctetString extends ASN1Any {
for (int anEncoding : encoding) {
buf.append((char) (anEncoding & 0x00ff));
}
octets = buf.toString().getBytes(StandardCharsets.ISO_8859_1);
octets = buf.toString().getBytes(charset);
} else {
throw new ASN1EncodingException("decode from constructed NOT IMPLEMENTED YET");
}
@ -140,7 +150,7 @@ public class ASN1OctetString extends ASN1Any {
* @return the object.
*/
public ASN1OctetString set(String str) {
octets = str.getBytes(StandardCharsets.ISO_8859_1);
octets = str.getBytes(charset);
return this;
}
@ -150,7 +160,7 @@ public class ASN1OctetString extends ASN1Any {
* @return the OCTET STRING's current value.
*/
public String get() {
return new String(octets, StandardCharsets.ISO_8859_1);
return new String(octets, charset);
}
/**

3
z3950-client-jdk/src/main/java/org/xbib/z3950/client/jdk/JDKZClient.java
View file

@ -1,5 +1,6 @@
package org.xbib.z3950.client.jdk;
import java.nio.charset.StandardCharsets;
import org.xbib.asn1.io.InputStreamBERReader;
import org.xbib.asn1.io.OutputStreamBERWriter;
import org.xbib.z3950.api.TimeoutListener;
@ -164,7 +165,7 @@ public class JDKZClient implements Client, Closeable {
try {
lock.lock();
SearchOperation search = new SearchOperation(berReader, berWriter, resultSetName, databases, host);
search.executePQF(query);
search.executePQF(query, StandardCharsets.UTF_8);
if (!search.isSuccess()) {
logger.log(Level.WARNING, MessageFormat.format("search was not a success [{0}]", query));
} else {

6
z3950-client-jdk/src/test/java/org/xbib/z3950/client/jdk/test/LVIZClientTest.java
View file

@ -14,7 +14,7 @@ class LVIZClientTest {
@Test
void testLVI() {
String query = "@attr 1=4 linux";
String query = "@attr 1=4 Köln";
int from = 1;
int size = 10;
try (JDKZClient client = newZClient()) {
@ -32,8 +32,8 @@ class LVIZClientTest {
private JDKZClient newZClient() {
JDKZClient.Builder builder = JDKZClient.builder();
builder.setHost("sru.hbz-nrw.de");
builder.setPort(210);
builder.setHost("localhost");
builder.setPort(1210);
builder.setDatabases(Collections.singletonList("LVI"));
builder.setElementSetName(null);
builder.setPreferredRecordSyntax("xml");

36
z3950-common/src/main/java/org/xbib/z3950/common/operations/AbstractOperation.java
View file

@ -1,5 +1,7 @@
package org.xbib.z3950.common.operations;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.xbib.asn1.ASN1Any;
import org.xbib.asn1.ASN1Exception;
import org.xbib.asn1.BEREncoding;
@ -53,6 +55,8 @@ import java.io.IOException;
*/
public class AbstractOperation<IN extends ASN1Any, OUT extends ASN1Any> {
private static final Logger logger = Logger.getLogger(AbstractOperation.class.getName());
protected final BERReader reader;
protected final BERWriter writer;
@ -122,18 +126,24 @@ public class AbstractOperation<IN extends ASN1Any, OUT extends ASN1Any> {
}
try {
switch (ber.getTag()) {
case 20:
case 20 -> {
return (IN) new InitializeRequest(ber, false);
case 21:
}
case 21 -> {
return (IN) new InitializeResponse(ber, false);
case 22:
}
case 22 -> {
return (IN) new SearchRequest(ber, false);
case 23:
}
case 23 -> {
return (IN) new SearchResponse(ber, false);
case 24:
}
case 24 -> {
return (IN) new PresentRequest(ber, false);
case 25:
}
case 25 -> {
return (IN) new PresentResponse(ber, false);
}
// 26 new DeleteResultSetRequest(ber, false);
// 27 new DeleteResultSetResponse(ber, false);
// 28 new AccessControlRequest(ber, false);
@ -143,23 +153,29 @@ public class AbstractOperation<IN extends ASN1Any, OUT extends ASN1Any> {
// 32 new TriggerResourceControlRequest(ber, false);
// 33 new ResourceReportRequest(ber, false);
// 34 new ResourceReportResponse(ber, false);
case 35:
case 35 -> {
return (IN) new ScanRequest(ber, false);
case 36:
}
case 36 -> {
return (IN) new ScanResponse(ber, false);
}
// 43 new SortRequest(ber, false);
// 44 new SortResponse(ber, false);
// 45 new Segment(ber, false);
// 46 new ExtendedServicesRequest(ber, false);
// 47 new ExtendedServicesResponse(ber, false);
case 48:
case 48 -> {
return (IN) new Close(ber, false);
}
default -> {
throw new ASN1Exception("bad BER encoding: " + ber.getTag() + " not matched");
}
}
} catch (Exception e) {
// class cast exception if Close is returned, ignore
logger.log(Level.SEVERE, e.getMessage(), e);
return null;
}
throw new ASN1Exception("bad BER encoding: choice not matched");
}
}

50
z3950-common/src/main/java/org/xbib/z3950/common/operations/InitOperation.java
View file

@ -86,38 +86,40 @@ public class InitOperation extends AbstractOperation<InitializeResponse, Initial
}
write(init);
InitializeResponse initResp = read();
if (initResp.implementationName != null) {
targetInfo = initResp.implementationName.toString();
if (initResp.implementationVersion != null) {
targetInfo += " - " + initResp.implementationVersion;
}
} else {
targetInfo = "server";
}
int targetVersion = 0;
if (initResp.protocolVersion != null) {
for (int n = 0; n < initResp.protocolVersion.value.get().length; n++) {
if (initResp.protocolVersion.value.get()[n]) {
targetVersion = n + 1;
if (initResp != null) {
if (initResp.implementationName != null) {
targetInfo = initResp.implementationName.toString();
if (initResp.implementationVersion != null) {
targetInfo += " - " + initResp.implementationVersion;
}
} else {
targetInfo = "server";
}
if (targetVersion > 0) {
targetInfo += " (Version " + targetVersion + ")";
if (initResp.protocolVersion != null) {
for (int n = 0; n < initResp.protocolVersion.value.get().length; n++) {
if (initResp.protocolVersion.value.get()[n]) {
targetVersion = n + 1;
}
}
if (targetVersion > 0) {
targetInfo += " (Version " + targetVersion + ")";
}
} else {
targetInfo += " (Version unknown)";
}
} else {
targetInfo += " (Version unknown)";
if (initResp.userInformationField != null && initResp.userInformationField.getSingleASN1Type() != null) {
targetInfo += "\n" + initResp.userInformationField.getSingleASN1Type().toString();
}
if (initResp.otherInfo != null) {
targetInfo += "\n" + initResp.otherInfo.toString();
}
targetInfo = targetInfo.replaceAll("\"", "");
}
if (initResp.userInformationField != null && initResp.userInformationField.getSingleASN1Type() != null) {
targetInfo += "\n" + initResp.userInformationField.getSingleASN1Type().toString();
}
if (initResp.otherInfo != null) {
targetInfo += "\n" + initResp.otherInfo.toString();
}
targetInfo = targetInfo.replaceAll("\"", "");
if (initListener != null) {
initListener.onInit(targetVersion, targetInfo);
}
return !initResp.result.get();
return initResp != null && !initResp.result.get();
}
public String getTargetInfo() {

9
z3950-common/src/main/java/org/xbib/z3950/common/operations/SearchOperation.java
View file

@ -1,5 +1,6 @@
package org.xbib.z3950.common.operations;
import java.nio.charset.Charset;
import org.xbib.asn1.ASN1Boolean;
import org.xbib.asn1.ASN1GeneralString;
import org.xbib.asn1.ASN1Integer;
@ -53,8 +54,8 @@ public class SearchOperation extends AbstractOperation<SearchResponse, SearchReq
this.status = false;
}
public boolean executePQF(String pqf) throws IOException {
return execute(createRPNQueryFromPQF(pqf));
public boolean executePQF(String pqf, Charset charset) throws IOException {
return execute(createRPNQueryFromPQF(pqf, charset));
}
public boolean executeCQL(String cql) throws IOException {
@ -125,8 +126,8 @@ public class SearchOperation extends AbstractOperation<SearchResponse, SearchReq
return generator.getQueryResult();
}
private RPNQuery createRPNQueryFromPQF(String query) {
PQFRPNGenerator generator = new PQFRPNGenerator();
private RPNQuery createRPNQueryFromPQF(String query, Charset charset) {
PQFRPNGenerator generator = new PQFRPNGenerator(charset);
PQFParser parser = new PQFParser(new StringReader(query));
parser.parse();
parser.getResult().accept(generator);

14
z3950-common/src/main/java/org/xbib/z3950/common/pqf/PQFRPNGenerator.java
View file

@ -1,5 +1,7 @@
package org.xbib.z3950.common.pqf;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import org.xbib.asn1.ASN1Any;
import org.xbib.asn1.ASN1Integer;
import org.xbib.asn1.ASN1Null;
@ -27,12 +29,19 @@ public class PQFRPNGenerator implements Visitor {
private static final Logger logger = Logger.getLogger(PQFRPNGenerator.class.getName());
private final Charset charset;
private final Stack<ASN1Any> result;
private RPNQuery rpnQuery;
public PQFRPNGenerator() {
this(StandardCharsets.ISO_8859_1);
}
public PQFRPNGenerator(Charset charset) {
this.result = new Stack<>();
this.charset = charset;
}
public RPNQuery getResult() {
@ -46,9 +55,6 @@ public class PQFRPNGenerator implements Visitor {
ASN1Any any = result.pop();
if (any instanceof RPNStructure) {
rpnQuery.rpn = (RPNStructure) any;
} else if (any instanceof ASN1OctetString) {
// TODO
logger.log(Level.INFO, "found ASN1OctetString = " + ((ASN1OctetString) any).get());
}
if (pqf.getAttrSet() == null) {
// Z39.50 BIB-1: urn:oid:1.2.840.10003.3.1
@ -131,7 +137,7 @@ public class PQFRPNGenerator implements Visitor {
@Override
public void visit(Term term) {
result.push(new ASN1OctetString(term.getValue()));
result.push(new ASN1OctetString(term.getValue(), charset));
}
@Override

3
z3950-common/src/main/java/org/xbib/z3950/common/pqf/Term.java
View file

@ -2,9 +2,6 @@ package org.xbib.z3950.common.pqf;
import java.math.BigDecimal;
/**
*
*/
public class Term extends Node {
private final String value;

11
z3950-common/src/test/java/org/xbib/z3950/common/pqf/PQFTest.java
View file

@ -1,5 +1,6 @@
package org.xbib.z3950.common.pqf;
import java.nio.charset.StandardCharsets;
import org.junit.jupiter.api.Test;
import org.xbib.z3950.common.v3.RPNQuery;
@ -38,8 +39,16 @@ class PQFTest {
createRPNQueryFromPQF(q).toString());
}
@Test
void testRPN4() {
// test UTF-8
String q = "@attr 1=3 Jörg";
assertEquals("{attributeSetId 1.2.840.10003.3.1, rpn {op {attrTerm {attributes {{attributeType 1, attributeValue {numeric 3}}}, term {general \"J\\703\\666rg\"}}}}}",
createRPNQueryFromPQF(q).toString());
}
private RPNQuery createRPNQueryFromPQF(String query) {
PQFRPNGenerator generator = new PQFRPNGenerator();
PQFRPNGenerator generator = new PQFRPNGenerator(StandardCharsets.UTF_8);
org.xbib.z3950.common.pqf.PQFParser parser = new org.xbib.z3950.common.pqf.PQFParser(new StringReader(query));
parser.parse();
parser.getResult().accept(generator);