From 2bd1d28c21f9d4c911fa5e488155281ea0db8914 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Prante?= <joergprante@gmail.com>
Date: Tue, 18 Apr 2023 15:02:49 +0200
Subject: [PATCH] fix UTF-8 in CQL->RPN

---
 gradle.properties                             |  2 +-
 .../org/xbib/z3950/client/jdk/JDKZClient.java |  4 ++--
 .../z3950/client/jdk/test/LVIZClientTest.java |  2 +-
 .../z3950/common/cql/CQLRPNGenerator.java     | 20 ++++++++++++-------
 .../common/operations/SearchOperation.java    | 14 ++++++-------
 .../xbib/z3950/common/cql/CQL2RPNTest.java    | 15 +++++++++++++-
 6 files changed, 38 insertions(+), 19 deletions(-)

diff --git a/gradle.properties b/gradle.properties
index fecf301..a154171 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,5 +1,5 @@
 group = org.xbib
 name = z3950
-version = 5.1.6
+version = 5.1.7
 
 org.gradle.warning.mode = ALL
diff --git a/z3950-client-jdk/src/main/java/org/xbib/z3950/client/jdk/JDKZClient.java b/z3950-client-jdk/src/main/java/org/xbib/z3950/client/jdk/JDKZClient.java
index 0e20176..f02f37c 100644
--- a/z3950-client-jdk/src/main/java/org/xbib/z3950/client/jdk/JDKZClient.java
+++ b/z3950-client-jdk/src/main/java/org/xbib/z3950/client/jdk/JDKZClient.java
@@ -72,7 +72,7 @@ public class JDKZClient implements Client, Closeable {
             lock.lock();
             SearchOperation searchOperation = new SearchOperation(berReader, berWriter,
                     builder.resultSetName, builder.databases, builder.host);
-            boolean success = searchOperation.executeCQL(query, builder.wordListSupported);
+            boolean success = searchOperation.executeCQL(StandardCharsets.UTF_8, query, builder.wordListSupported);
             if (!success) {
                 logger.log(Level.WARNING, MessageFormat.format("search was not a success [{0}]", query));
             } else {
@@ -130,7 +130,7 @@ public class JDKZClient implements Client, Closeable {
             lock.lock();
             SearchOperation searchOperation = new SearchOperation(berReader, berWriter,
                     builder.resultSetName, builder.databases, builder.host);
-            searchOperation.executePQF(query, StandardCharsets.UTF_8);
+            searchOperation.executePQF(StandardCharsets.UTF_8, query);
             if (!searchOperation.isSuccess()) {
                 logger.log(Level.WARNING, MessageFormat.format("search was not a success [{0}]", query));
             } else {
diff --git a/z3950-client-jdk/src/test/java/org/xbib/z3950/client/jdk/test/LVIZClientTest.java b/z3950-client-jdk/src/test/java/org/xbib/z3950/client/jdk/test/LVIZClientTest.java
index f427c68..2f81738 100644
--- a/z3950-client-jdk/src/test/java/org/xbib/z3950/client/jdk/test/LVIZClientTest.java
+++ b/z3950-client-jdk/src/test/java/org/xbib/z3950/client/jdk/test/LVIZClientTest.java
@@ -17,7 +17,7 @@ class LVIZClientTest {
 
     @Disabled
     @Test
-    void testLVI() {
+    void testPQF() {
         String query = "@attr 1=4 @attr 4=6 \"Köln strafrecht\"";
         int offset = 1;
         int size = 10;
diff --git a/z3950-common/src/main/java/org/xbib/z3950/common/cql/CQLRPNGenerator.java b/z3950-common/src/main/java/org/xbib/z3950/common/cql/CQLRPNGenerator.java
index 0176c7c..023125a 100644
--- a/z3950-common/src/main/java/org/xbib/z3950/common/cql/CQLRPNGenerator.java
+++ b/z3950-common/src/main/java/org/xbib/z3950/common/cql/CQLRPNGenerator.java
@@ -32,6 +32,9 @@ import org.xbib.z3950.common.v3.Operator;
 import org.xbib.z3950.common.v3.RPNQuery;
 import org.xbib.z3950.common.v3.RPNStructure;
 import org.xbib.z3950.common.v3.RPNStructureRpnRpnOp;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
@@ -46,9 +49,8 @@ import java.util.stream.Collectors;
  */
 public final class CQLRPNGenerator implements Visitor {
 
-    /**
-     * Context map.
-     */
+    private final Charset charset;
+
     private final Map<String, Map<String, String>> contexts;
 
     private final Stack<AttributeElement> attributeElements;
@@ -60,10 +62,15 @@ public final class CQLRPNGenerator implements Visitor {
     private final boolean wordListSupported;
 
     public CQLRPNGenerator() {
-        this(null, true);
+        this(StandardCharsets.ISO_8859_1, null, true);
     }
 
-    public CQLRPNGenerator(Collection<AttributeElement> attributeElements, boolean wordListSupported) {
+    public CQLRPNGenerator(Charset charset) {
+        this(charset, null, true);
+    }
+
+    public CQLRPNGenerator(Charset charset, Collection<AttributeElement> attributeElements, boolean wordListSupported) {
+        this.charset = charset;
         this.attributeElements = new Stack<>();
         if (attributeElements != null) {
             this.attributeElements.addAll(attributeElements);
@@ -266,7 +273,6 @@ public final class CQLRPNGenerator implements Visitor {
 
     @Override
     public void visit(Term term) {
-        // the value
         ASN1OctetString s = transformTerm(term);
         result.push(s);
     }
@@ -358,7 +364,7 @@ public final class CQLRPNGenerator implements Visitor {
         }
         push(attributeElements, createAttributeElement(attributeType, attributeValue));
 
-        return new ASN1OctetString(v);
+        return new ASN1OctetString(v, charset);
     }
 
     private static AttributeElement createAttributeElement(Integer attributeType, Integer attributeValue) {
diff --git a/z3950-common/src/main/java/org/xbib/z3950/common/operations/SearchOperation.java b/z3950-common/src/main/java/org/xbib/z3950/common/operations/SearchOperation.java
index ed6b2cd..9ae83ce 100644
--- a/z3950-common/src/main/java/org/xbib/z3950/common/operations/SearchOperation.java
+++ b/z3950-common/src/main/java/org/xbib/z3950/common/operations/SearchOperation.java
@@ -54,12 +54,12 @@ public class SearchOperation extends AbstractOperation<SearchResponse, SearchReq
         this.status = false;
     }
 
-    public boolean executePQF(String pqf, Charset charset) throws IOException {
-        return execute(createRPNQueryFromPQF(pqf, charset));
+    public boolean executePQF(Charset charset, String pqf) throws IOException {
+        return execute(createRPNQueryFromPQF(charset, pqf));
     }
 
-    public boolean executeCQL(String cql, boolean wordListSupported) throws IOException {
-        return execute(createRPNQueryFromCQL(cql, wordListSupported));
+    public boolean executeCQL(Charset charset, String cql, boolean wordListSupported) throws IOException {
+        return execute(createRPNQueryFromCQL(charset, cql, wordListSupported));
     }
 
     public boolean execute(RPNQuery rpn) throws IOException {
@@ -118,15 +118,15 @@ public class SearchOperation extends AbstractOperation<SearchResponse, SearchReq
         return status;
     }
 
-    private RPNQuery createRPNQueryFromCQL(String query, boolean wordListSupported) {
-        CQLRPNGenerator generator = new CQLRPNGenerator(null, wordListSupported);
+    private RPNQuery createRPNQueryFromCQL(Charset charset, String query, boolean wordListSupported) {
+        CQLRPNGenerator generator = new CQLRPNGenerator(charset, null, wordListSupported);
         CQLParser parser = new CQLParser(query);
         parser.parse();
         parser.getCQLQuery().accept(generator);
         return generator.getQueryResult();
     }
 
-    private RPNQuery createRPNQueryFromPQF(String query, Charset charset) {
+    private RPNQuery createRPNQueryFromPQF(Charset charset, String query) {
         PQFRPNGenerator generator = new PQFRPNGenerator(charset);
         PQFParser parser = new PQFParser(new StringReader(query));
         parser.parse();
diff --git a/z3950-common/src/test/java/org/xbib/z3950/common/cql/CQL2RPNTest.java b/z3950-common/src/test/java/org/xbib/z3950/common/cql/CQL2RPNTest.java
index 489703d..1767551 100644
--- a/z3950-common/src/test/java/org/xbib/z3950/common/cql/CQL2RPNTest.java
+++ b/z3950-common/src/test/java/org/xbib/z3950/common/cql/CQL2RPNTest.java
@@ -6,6 +6,8 @@ import org.xbib.asn1.ASN1Integer;
 import org.xbib.cql.CQLParser;
 import org.xbib.z3950.common.v3.AttributeElement;
 import org.xbib.z3950.common.v3.AttributeElementAttributeValue;
+
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 
 class CQL2RPNTest {
@@ -41,7 +43,7 @@ class CQL2RPNTest {
         String cql = "dc.title = \"a phrase\"";
         CQLParser parser = new CQLParser(cql);
         parser.parse();
-        CQLRPNGenerator generator = new CQLRPNGenerator(Collections.singleton(ae), true);
+        CQLRPNGenerator generator = new CQLRPNGenerator(StandardCharsets.UTF_8, Collections.singleton(ae), true);
         parser.getCQLQuery().accept(generator);
         String q = generator.getQueryResult().toString();
         assertEquals("{attributeSetId 1.2.840.10003.3.1, rpn {op {attrTerm {attributes {{attributeType 7, attributeValue {numeric 4}}{attributeType 3, attributeValue {numeric 3}}{attributeType 4, attributeValue {numeric 1}}{attributeType 5, attributeValue {numeric 100}}{attributeType 6, attributeValue {numeric 1}}{attributeType 1, attributeValue {numeric 4}}{attributeType 2, attributeValue {numeric 3}}}, term {general \"a phrase\"}}}}}", q);
@@ -79,4 +81,15 @@ class CQL2RPNTest {
         String q = generator.getQueryResult().toString();
         assertEquals("{attributeSetId 1.2.840.10003.3.1, rpn {op {attrTerm {attributes {{attributeType 3, attributeValue {numeric 3}}{attributeType 5, attributeValue {numeric 100}}{attributeType 6, attributeValue {numeric 1}}{attributeType 4, attributeValue {numeric 5}}{attributeType 1, attributeValue {numeric 31}}{attributeType 2, attributeValue {numeric 3}}}, term {general \"2023\"}}}}}", q);
     }
+
+    @Test
+    void testUTF8() {
+        String cql = "bib.title = Köln";
+        CQLParser parser = new CQLParser(cql);
+        parser.parse();
+        CQLRPNGenerator generator = new CQLRPNGenerator(StandardCharsets.UTF_8);
+        parser.getCQLQuery().accept(generator);
+        String q = generator.getQueryResult().toString();
+        assertEquals("{attributeSetId 1.2.840.10003.3.1, rpn {op {attrTerm {attributes {{attributeType 3, attributeValue {numeric 3}}{attributeType 4, attributeValue {numeric 2}}{attributeType 5, attributeValue {numeric 100}}{attributeType 6, attributeValue {numeric 1}}{attributeType 1, attributeValue {numeric 4}}{attributeType 2, attributeValue {numeric 3}}}, term {general \"K\\703\\666ln\"}}}}}", q);
+    }
 }