fix UTF-8 in CQL->RPN

gradle.properties

@@ -1,5 +1,5 @@
 group = org.xbib
 name = z3950
-version = 5.1.6
+version = 5.1.7
 
 org.gradle.warning.mode = ALL

z3950-client-jdk/src/main/java/org/xbib/z3950/client/jdk/JDKZClient.java

@@ -72,7 +72,7 @@ public class JDKZClient implements Client, Closeable {
             lock.lock();
             SearchOperation searchOperation = new SearchOperation(berReader, berWriter,
                     builder.resultSetName, builder.databases, builder.host);
-            boolean success = searchOperation.executeCQL(query, builder.wordListSupported);
+            boolean success = searchOperation.executeCQL(StandardCharsets.UTF_8, query, builder.wordListSupported);
             if (!success) {
                 logger.log(Level.WARNING, MessageFormat.format("search was not a success [{0}]", query));
             } else {
@@ -130,7 +130,7 @@ public class JDKZClient implements Client, Closeable {
             lock.lock();
             SearchOperation searchOperation = new SearchOperation(berReader, berWriter,
                     builder.resultSetName, builder.databases, builder.host);
-            searchOperation.executePQF(query, StandardCharsets.UTF_8);
+            searchOperation.executePQF(StandardCharsets.UTF_8, query);
             if (!searchOperation.isSuccess()) {
                 logger.log(Level.WARNING, MessageFormat.format("search was not a success [{0}]", query));
             } else {

z3950-client-jdk/src/test/java/org/xbib/z3950/client/jdk/test/LVIZClientTest.java

@@ -17,7 +17,7 @@ class LVIZClientTest {
 
     @Disabled
     @Test
-    void testLVI() {
+    void testPQF() {
         String query = "@attr 1=4 @attr 4=6 \"Köln strafrecht\"";
         int offset = 1;
         int size = 10;

z3950-common/src/main/java/org/xbib/z3950/common/cql/CQLRPNGenerator.java

@@ -32,6 +32,9 @@ import org.xbib.z3950.common.v3.Operator;
 import org.xbib.z3950.common.v3.RPNQuery;
 import org.xbib.z3950.common.v3.RPNStructure;
 import org.xbib.z3950.common.v3.RPNStructureRpnRpnOp;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
@@ -46,9 +49,8 @@ import java.util.stream.Collectors;
  */
 public final class CQLRPNGenerator implements Visitor {
 
-    /**
+    private final Charset charset;
-     * Context map.
+
-     */
     private final Map<String, Map<String, String>> contexts;
 
     private final Stack<AttributeElement> attributeElements;
@@ -60,10 +62,15 @@ public final class CQLRPNGenerator implements Visitor {
     private final boolean wordListSupported;
 
     public CQLRPNGenerator() {
-        this(null, true);
+        this(StandardCharsets.ISO_8859_1, null, true);
     }
 
-    public CQLRPNGenerator(Collection<AttributeElement> attributeElements, boolean wordListSupported) {
+    public CQLRPNGenerator(Charset charset) {
+        this(charset, null, true);
+    }
+
+    public CQLRPNGenerator(Charset charset, Collection<AttributeElement> attributeElements, boolean wordListSupported) {
+        this.charset = charset;
         this.attributeElements = new Stack<>();
         if (attributeElements != null) {
             this.attributeElements.addAll(attributeElements);
@@ -266,7 +273,6 @@ public final class CQLRPNGenerator implements Visitor {
 
     @Override
     public void visit(Term term) {
-        // the value
         ASN1OctetString s = transformTerm(term);
         result.push(s);
     }
@@ -358,7 +364,7 @@ public final class CQLRPNGenerator implements Visitor {
         }
         push(attributeElements, createAttributeElement(attributeType, attributeValue));
 
-        return new ASN1OctetString(v);
+        return new ASN1OctetString(v, charset);
     }
 
     private static AttributeElement createAttributeElement(Integer attributeType, Integer attributeValue) {

z3950-common/src/main/java/org/xbib/z3950/common/operations/SearchOperation.java

@@ -54,12 +54,12 @@ public class SearchOperation extends AbstractOperation<SearchResponse, SearchReq
         this.status = false;
     }
 
-    public boolean executePQF(String pqf, Charset charset) throws IOException {
+    public boolean executePQF(Charset charset, String pqf) throws IOException {
-        return execute(createRPNQueryFromPQF(pqf, charset));
+        return execute(createRPNQueryFromPQF(charset, pqf));
     }
 
-    public boolean executeCQL(String cql, boolean wordListSupported) throws IOException {
+    public boolean executeCQL(Charset charset, String cql, boolean wordListSupported) throws IOException {
-        return execute(createRPNQueryFromCQL(cql, wordListSupported));
+        return execute(createRPNQueryFromCQL(charset, cql, wordListSupported));
     }
 
     public boolean execute(RPNQuery rpn) throws IOException {
@@ -118,15 +118,15 @@ public class SearchOperation extends AbstractOperation<SearchResponse, SearchReq
         return status;
     }
 
-    private RPNQuery createRPNQueryFromCQL(String query, boolean wordListSupported) {
+    private RPNQuery createRPNQueryFromCQL(Charset charset, String query, boolean wordListSupported) {
-        CQLRPNGenerator generator = new CQLRPNGenerator(null, wordListSupported);
+        CQLRPNGenerator generator = new CQLRPNGenerator(charset, null, wordListSupported);
         CQLParser parser = new CQLParser(query);
         parser.parse();
         parser.getCQLQuery().accept(generator);
         return generator.getQueryResult();
     }
 
-    private RPNQuery createRPNQueryFromPQF(String query, Charset charset) {
+    private RPNQuery createRPNQueryFromPQF(Charset charset, String query) {
         PQFRPNGenerator generator = new PQFRPNGenerator(charset);
         PQFParser parser = new PQFParser(new StringReader(query));
         parser.parse();

z3950-common/src/test/java/org/xbib/z3950/common/cql/CQL2RPNTest.java

@@ -6,6 +6,8 @@ import org.xbib.asn1.ASN1Integer;
 import org.xbib.cql.CQLParser;
 import org.xbib.z3950.common.v3.AttributeElement;
 import org.xbib.z3950.common.v3.AttributeElementAttributeValue;
+
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 
 class CQL2RPNTest {
@@ -41,7 +43,7 @@ class CQL2RPNTest {
         String cql = "dc.title = \"a phrase\"";
         CQLParser parser = new CQLParser(cql);
         parser.parse();
-        CQLRPNGenerator generator = new CQLRPNGenerator(Collections.singleton(ae), true);
+        CQLRPNGenerator generator = new CQLRPNGenerator(StandardCharsets.UTF_8, Collections.singleton(ae), true);
         parser.getCQLQuery().accept(generator);
         String q = generator.getQueryResult().toString();
         assertEquals("{attributeSetId 1.2.840.10003.3.1, rpn {op {attrTerm {attributes {{attributeType 7, attributeValue {numeric 4}}{attributeType 3, attributeValue {numeric 3}}{attributeType 4, attributeValue {numeric 1}}{attributeType 5, attributeValue {numeric 100}}{attributeType 6, attributeValue {numeric 1}}{attributeType 1, attributeValue {numeric 4}}{attributeType 2, attributeValue {numeric 3}}}, term {general \"a phrase\"}}}}}", q);
@@ -79,4 +81,15 @@ class CQL2RPNTest {
         String q = generator.getQueryResult().toString();
         assertEquals("{attributeSetId 1.2.840.10003.3.1, rpn {op {attrTerm {attributes {{attributeType 3, attributeValue {numeric 3}}{attributeType 5, attributeValue {numeric 100}}{attributeType 6, attributeValue {numeric 1}}{attributeType 4, attributeValue {numeric 5}}{attributeType 1, attributeValue {numeric 31}}{attributeType 2, attributeValue {numeric 3}}}, term {general \"2023\"}}}}}", q);
     }
+
+    @Test
+    void testUTF8() {
+        String cql = "bib.title = Köln";
+        CQLParser parser = new CQLParser(cql);
+        parser.parse();
+        CQLRPNGenerator generator = new CQLRPNGenerator(StandardCharsets.UTF_8);
+        parser.getCQLQuery().accept(generator);
+        String q = generator.getQueryResult().toString();
+        assertEquals("{attributeSetId 1.2.840.10003.3.1, rpn {op {attrTerm {attributes {{attributeType 3, attributeValue {numeric 3}}{attributeType 4, attributeValue {numeric 2}}{attributeType 5, attributeValue {numeric 100}}{attributeType 6, attributeValue {numeric 1}}{attributeType 1, attributeValue {numeric 4}}{attributeType 2, attributeValue {numeric 3}}}, term {general \"K\\703\\666ln\"}}}}}", q);
+    }
 }