enable UTF-8 for PQF queries by default

gradle.properties

@@ -1,5 +1,5 @@
 group = org.xbib
 name = z3950
-version = 5.0.3
+version = 5.0.4
 
 org.gradle.warning.mode = ALL

z3950-asn1/src/main/java/org/xbib/asn1/ASN1Any.java

@@ -45,8 +45,7 @@ public class ASN1Any {
      * @throws ASN1Exception If the BER encoding is incorrect.
      *                       Never occurs for ASN1Any.
      */
-    public void berDecode(BEREncoding berEncoding, boolean checkTag)
+    public void berDecode(BEREncoding berEncoding, boolean checkTag) throws ASN1Exception {
-            throws ASN1Exception {
         asn1anyBer = berEncoding;
     }
 

z3950-asn1/src/main/java/org/xbib/asn1/ASN1OctetString.java

@@ -1,5 +1,6 @@
 package org.xbib.asn1;
 
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 
 /**
@@ -27,6 +28,8 @@ public class ASN1OctetString extends ASN1Any {
 
     private byte[] octets;
 
+    private final Charset charset;
+
     /**
      * Constructor for an OCTET STRING object. The tag is set to the
      * default of UNIVERSAL 4, and its value to the given bytes.
@@ -34,6 +37,7 @@ public class ASN1OctetString extends ASN1Any {
      */
     public ASN1OctetString(byte[] data) {
         octets = new byte[data.length];
+        this.charset = StandardCharsets.ISO_8859_1;
         System.arraycopy(data, 0, octets, 0, data.length);
     }
 
@@ -44,7 +48,12 @@ public class ASN1OctetString extends ASN1Any {
      * @param str string
      */
     public ASN1OctetString(String str) {
-        octets = str.getBytes(StandardCharsets.ISO_8859_1);
+        this(str, StandardCharsets.ISO_8859_1);
+    }
+
+    public ASN1OctetString(String str, Charset charset) {
+        this.octets = str.getBytes(charset);
+        this.charset = charset;
     }
 
     /**
@@ -55,7 +64,8 @@ public class ASN1OctetString extends ASN1Any {
      * @throws ASN1Exception If the BER encoding is incorrect.
      */
     public ASN1OctetString(BEREncoding ber, boolean checkTag) throws ASN1Exception {
-        super(ber, checkTag);
+        this.charset = StandardCharsets.ISO_8859_1;
+        berDecode(ber, checkTag);
     }
 
     /**
@@ -80,7 +90,7 @@ public class ASN1OctetString extends ASN1Any {
             for (int anEncoding : encoding) {
                 buf.append((char) (anEncoding & 0x00ff));
             }
-            octets = buf.toString().getBytes(StandardCharsets.ISO_8859_1);
+            octets = buf.toString().getBytes(charset);
         } else {
             throw new ASN1EncodingException("decode from constructed NOT IMPLEMENTED YET");
         }
@@ -140,7 +150,7 @@ public class ASN1OctetString extends ASN1Any {
      * @return the object.
      */
     public ASN1OctetString set(String str) {
-        octets = str.getBytes(StandardCharsets.ISO_8859_1);
+        octets = str.getBytes(charset);
         return this;
     }
 
@@ -150,7 +160,7 @@ public class ASN1OctetString extends ASN1Any {
      * @return the OCTET STRING's current value.
      */
     public String get() {
-        return new String(octets, StandardCharsets.ISO_8859_1);
+        return new String(octets, charset);
     }
 
     /**

z3950-client-jdk/src/main/java/org/xbib/z3950/client/jdk/JDKZClient.java

@@ -1,5 +1,6 @@
 package org.xbib.z3950.client.jdk;
 
+import java.nio.charset.StandardCharsets;
 import org.xbib.asn1.io.InputStreamBERReader;
 import org.xbib.asn1.io.OutputStreamBERWriter;
 import org.xbib.z3950.api.TimeoutListener;
@@ -164,7 +165,7 @@ public class JDKZClient implements Client, Closeable {
         try {
             lock.lock();
             SearchOperation search = new SearchOperation(berReader, berWriter, resultSetName, databases, host);
-            search.executePQF(query);
+            search.executePQF(query, StandardCharsets.UTF_8);
             if (!search.isSuccess()) {
                 logger.log(Level.WARNING, MessageFormat.format("search was not a success [{0}]", query));
             } else {

z3950-client-jdk/src/test/java/org/xbib/z3950/client/jdk/test/LVIZClientTest.java

@@ -14,7 +14,7 @@ class LVIZClientTest {
 
     @Test
     void testLVI() {
-        String query = "@attr 1=4 linux";
+        String query = "@attr 1=4 Köln";
         int from = 1;
         int size = 10;
         try (JDKZClient client = newZClient()) {
@@ -32,8 +32,8 @@ class LVIZClientTest {
 
     private JDKZClient newZClient() {
         JDKZClient.Builder builder = JDKZClient.builder();
-        builder.setHost("sru.hbz-nrw.de");
+        builder.setHost("localhost");
-        builder.setPort(210);
+        builder.setPort(1210);
         builder.setDatabases(Collections.singletonList("LVI"));
         builder.setElementSetName(null);
         builder.setPreferredRecordSyntax("xml");

z3950-common/src/main/java/org/xbib/z3950/common/operations/AbstractOperation.java

@@ -1,5 +1,7 @@
 package org.xbib.z3950.common.operations;
 
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import org.xbib.asn1.ASN1Any;
 import org.xbib.asn1.ASN1Exception;
 import org.xbib.asn1.BEREncoding;
@@ -53,6 +55,8 @@ import java.io.IOException;
  */
 public class AbstractOperation<IN extends ASN1Any, OUT extends ASN1Any> {
 
+    private static final Logger logger = Logger.getLogger(AbstractOperation.class.getName());
+
     protected final BERReader reader;
 
     protected final BERWriter writer;
@@ -122,18 +126,24 @@ public class AbstractOperation<IN extends ASN1Any, OUT extends ASN1Any> {
         }
         try {
             switch (ber.getTag()) {
-                case 20:
+                case 20 -> {
                     return (IN) new InitializeRequest(ber, false);
-                case 21:
+                }
+                case 21 -> {
                     return (IN) new InitializeResponse(ber, false);
-                case 22:
+                }
+                case 22 -> {
                     return (IN) new SearchRequest(ber, false);
-                case 23:
+                }
+                case 23 -> {
                     return (IN) new SearchResponse(ber, false);
-                case 24:
+                }
+                case 24 -> {
                     return (IN) new PresentRequest(ber, false);
-                case 25:
+                }
+                case 25 -> {
                     return (IN) new PresentResponse(ber, false);
+                }
                 // 26 new DeleteResultSetRequest(ber, false);
                 // 27 new DeleteResultSetResponse(ber, false);
                 // 28 new AccessControlRequest(ber, false);
@@ -143,23 +153,29 @@ public class AbstractOperation<IN extends ASN1Any, OUT extends ASN1Any> {
                 // 32 new TriggerResourceControlRequest(ber, false);
                 // 33 new ResourceReportRequest(ber, false);
                 // 34 new ResourceReportResponse(ber, false);
-                case 35:
+                case 35 -> {
                     return (IN) new ScanRequest(ber, false);
-                case 36:
+                }
+                case 36 -> {
                     return (IN) new ScanResponse(ber, false);
+                }
                 // 43 new SortRequest(ber, false);
                 // 44  new SortResponse(ber, false);
                 // 45  new Segment(ber, false);
                 // 46  new ExtendedServicesRequest(ber, false);
                 // 47  new ExtendedServicesResponse(ber, false);
-                case 48:
+                case 48 -> {
                     return (IN) new Close(ber, false);
+                }
+                default -> {
+                    throw new ASN1Exception("bad BER encoding: " + ber.getTag() + " not matched");
+                }
             }
         } catch (Exception e) {
             // class cast exception if Close is returned, ignore
+            logger.log(Level.SEVERE, e.getMessage(), e);
             return null;
         }
-        throw new ASN1Exception("bad BER encoding: choice not matched");
     }
 
 }

z3950-common/src/main/java/org/xbib/z3950/common/operations/InitOperation.java

@@ -86,38 +86,40 @@ public class InitOperation extends AbstractOperation<InitializeResponse, Initial
         }
         write(init);
         InitializeResponse initResp = read();
-        if (initResp.implementationName != null) {
-            targetInfo = initResp.implementationName.toString();
-            if (initResp.implementationVersion != null) {
-                targetInfo += " - " + initResp.implementationVersion;
-            }
-        } else {
-            targetInfo = "server";
-        }
         int targetVersion = 0;
-        if (initResp.protocolVersion != null) {
+        if (initResp != null) {
-            for (int n = 0; n < initResp.protocolVersion.value.get().length; n++) {
+            if (initResp.implementationName != null) {
-                if (initResp.protocolVersion.value.get()[n]) {
+                targetInfo = initResp.implementationName.toString();
-                    targetVersion = n + 1;
+                if (initResp.implementationVersion != null) {
+                    targetInfo += " - " + initResp.implementationVersion;
                 }
+            } else {
+                targetInfo = "server";
             }
-            if (targetVersion > 0) {
+            if (initResp.protocolVersion != null) {
-                targetInfo += " (Version " + targetVersion + ")";
+                for (int n = 0; n < initResp.protocolVersion.value.get().length; n++) {
+                    if (initResp.protocolVersion.value.get()[n]) {
+                        targetVersion = n + 1;
+                    }
+                }
+                if (targetVersion > 0) {
+                    targetInfo += " (Version " + targetVersion + ")";
+                }
+            } else {
+                targetInfo += " (Version unknown)";
             }
-        } else {
+            if (initResp.userInformationField != null && initResp.userInformationField.getSingleASN1Type() != null) {
-            targetInfo += " (Version unknown)";
+                targetInfo += "\n" + initResp.userInformationField.getSingleASN1Type().toString();
+            }
+            if (initResp.otherInfo != null) {
+                targetInfo += "\n" + initResp.otherInfo.toString();
+            }
+            targetInfo = targetInfo.replaceAll("\"", "");
         }
-        if (initResp.userInformationField != null && initResp.userInformationField.getSingleASN1Type() != null) {
-            targetInfo += "\n" + initResp.userInformationField.getSingleASN1Type().toString();
-        }
-        if (initResp.otherInfo != null) {
-            targetInfo += "\n" + initResp.otherInfo.toString();
-        }
-        targetInfo = targetInfo.replaceAll("\"", "");
         if (initListener != null) {
             initListener.onInit(targetVersion, targetInfo);
         }
-        return !initResp.result.get();
+        return initResp != null && !initResp.result.get();
     }
 
     public String getTargetInfo() {

z3950-common/src/main/java/org/xbib/z3950/common/operations/SearchOperation.java

@@ -1,5 +1,6 @@
 package org.xbib.z3950.common.operations;
 
+import java.nio.charset.Charset;
 import org.xbib.asn1.ASN1Boolean;
 import org.xbib.asn1.ASN1GeneralString;
 import org.xbib.asn1.ASN1Integer;
@@ -53,8 +54,8 @@ public class SearchOperation extends AbstractOperation<SearchResponse, SearchReq
         this.status = false;
     }
 
-    public boolean executePQF(String pqf) throws IOException {
+    public boolean executePQF(String pqf, Charset charset) throws IOException {
-        return execute(createRPNQueryFromPQF(pqf));
+        return execute(createRPNQueryFromPQF(pqf, charset));
     }
 
     public boolean executeCQL(String cql) throws IOException {
@@ -125,8 +126,8 @@ public class SearchOperation extends AbstractOperation<SearchResponse, SearchReq
         return generator.getQueryResult();
     }
 
-    private RPNQuery createRPNQueryFromPQF(String query) {
+    private RPNQuery createRPNQueryFromPQF(String query, Charset charset) {
-        PQFRPNGenerator generator = new PQFRPNGenerator();
+        PQFRPNGenerator generator = new PQFRPNGenerator(charset);
         PQFParser parser = new PQFParser(new StringReader(query));
         parser.parse();
         parser.getResult().accept(generator);

z3950-common/src/main/java/org/xbib/z3950/common/pqf/PQFRPNGenerator.java

@@ -1,5 +1,7 @@
 package org.xbib.z3950.common.pqf;
 
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import org.xbib.asn1.ASN1Any;
 import org.xbib.asn1.ASN1Integer;
 import org.xbib.asn1.ASN1Null;
@@ -27,12 +29,19 @@ public class PQFRPNGenerator implements Visitor {
 
     private static final Logger logger = Logger.getLogger(PQFRPNGenerator.class.getName());
 
+    private final Charset charset;
+
     private final Stack<ASN1Any> result;
 
     private RPNQuery rpnQuery;
 
     public PQFRPNGenerator() {
+        this(StandardCharsets.ISO_8859_1);
+    }
+
+    public PQFRPNGenerator(Charset charset) {
         this.result = new Stack<>();
+        this.charset = charset;
     }
 
     public RPNQuery getResult() {
@@ -46,9 +55,6 @@ public class PQFRPNGenerator implements Visitor {
             ASN1Any any = result.pop();
             if (any instanceof RPNStructure) {
                 rpnQuery.rpn = (RPNStructure) any;
-            } else if (any instanceof ASN1OctetString) {
-                // TODO
-                logger.log(Level.INFO, "found ASN1OctetString = " + ((ASN1OctetString) any).get());
             }
             if (pqf.getAttrSet() == null) {
                 // Z39.50 BIB-1: urn:oid:1.2.840.10003.3.1
@@ -131,7 +137,7 @@ public class PQFRPNGenerator implements Visitor {
 
     @Override
     public void visit(Term term) {
-        result.push(new ASN1OctetString(term.getValue()));
+        result.push(new ASN1OctetString(term.getValue(), charset));
     }
 
     @Override

z3950-common/src/main/java/org/xbib/z3950/common/pqf/Term.java

@@ -2,9 +2,6 @@ package org.xbib.z3950.common.pqf;
 
 import java.math.BigDecimal;
 
-/**
- *
- */
 public class Term extends Node {
 
     private final String value;

z3950-common/src/test/java/org/xbib/z3950/common/pqf/PQFTest.java

@@ -1,5 +1,6 @@
 package org.xbib.z3950.common.pqf;
 
+import java.nio.charset.StandardCharsets;
 import org.junit.jupiter.api.Test;
 import org.xbib.z3950.common.v3.RPNQuery;
 
@@ -38,8 +39,16 @@ class PQFTest {
                 createRPNQueryFromPQF(q).toString());
     }
 
+    @Test
+    void testRPN4() {
+        // test UTF-8
+        String q = "@attr 1=3 Jörg";
+        assertEquals("{attributeSetId 1.2.840.10003.3.1, rpn {op {attrTerm {attributes {{attributeType 1, attributeValue {numeric 3}}}, term {general \"J\\703\\666rg\"}}}}}",
+                createRPNQueryFromPQF(q).toString());
+    }
+
     private RPNQuery createRPNQueryFromPQF(String query) {
-        PQFRPNGenerator generator = new PQFRPNGenerator();
+        PQFRPNGenerator generator = new PQFRPNGenerator(StandardCharsets.UTF_8);
         org.xbib.z3950.common.pqf.PQFParser parser = new org.xbib.z3950.common.pqf.PQFParser(new StringReader(query));
         parser.parse();
         parser.getResult().accept(generator);