joerg/z3950
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix UTF-8 in CQL->RPN

Browse source
This commit is contained in:
Jörg Prante 2023-04-18 15:02:49 +02:00
parent 5a35d608ea
commit 2bd1d28c21
6 changed files with 38 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
gradle.properties
View file

@ -1,5 +1,5 @@
group = org.xbib
name = z3950
version = 5.1.6
version = 5.1.7
org.gradle.warning.mode = ALL

4
z3950-client-jdk/src/main/java/org/xbib/z3950/client/jdk/JDKZClient.java
View file

@ -72,7 +72,7 @@ public class JDKZClient implements Client, Closeable {
lock.lock();
SearchOperation searchOperation = new SearchOperation(berReader, berWriter,
builder.resultSetName, builder.databases, builder.host);
boolean success = searchOperation.executeCQL(query, builder.wordListSupported);
boolean success = searchOperation.executeCQL(StandardCharsets.UTF_8, query, builder.wordListSupported);
if (!success) {
logger.log(Level.WARNING, MessageFormat.format("search was not a success [{0}]", query));
} else {
@ -130,7 +130,7 @@ public class JDKZClient implements Client, Closeable {
lock.lock();
SearchOperation searchOperation = new SearchOperation(berReader, berWriter,
builder.resultSetName, builder.databases, builder.host);
searchOperation.executePQF(query, StandardCharsets.UTF_8);
searchOperation.executePQF(StandardCharsets.UTF_8, query);
if (!searchOperation.isSuccess()) {
logger.log(Level.WARNING, MessageFormat.format("search was not a success [{0}]", query));
} else {

2
z3950-client-jdk/src/test/java/org/xbib/z3950/client/jdk/test/LVIZClientTest.java
View file

@ -17,7 +17,7 @@ class LVIZClientTest {
@Disabled
@Test
void testLVI() {
void testPQF() {
String query = "@attr 1=4 @attr 4=6 \"Köln strafrecht\"";
int offset = 1;
int size = 10;

20
z3950-common/src/main/java/org/xbib/z3950/common/cql/CQLRPNGenerator.java
View file

@ -32,6 +32,9 @@ import org.xbib.z3950.common.v3.Operator;
import org.xbib.z3950.common.v3.RPNQuery;
import org.xbib.z3950.common.v3.RPNStructure;
import org.xbib.z3950.common.v3.RPNStructureRpnRpnOp;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@ -46,9 +49,8 @@ import java.util.stream.Collectors;
*/
public final class CQLRPNGenerator implements Visitor {
/**
* Context map.
*/
private final Charset charset;
private final Map<String, Map<String, String>> contexts;
private final Stack<AttributeElement> attributeElements;
@ -60,10 +62,15 @@ public final class CQLRPNGenerator implements Visitor {
private final boolean wordListSupported;
public CQLRPNGenerator() {
this(null, true);
this(StandardCharsets.ISO_8859_1, null, true);
}
public CQLRPNGenerator(Collection<AttributeElement> attributeElements, boolean wordListSupported) {
public CQLRPNGenerator(Charset charset) {
this(charset, null, true);
}
public CQLRPNGenerator(Charset charset, Collection<AttributeElement> attributeElements, boolean wordListSupported) {
this.charset = charset;
this.attributeElements = new Stack<>();
if (attributeElements != null) {
this.attributeElements.addAll(attributeElements);
@ -266,7 +273,6 @@ public final class CQLRPNGenerator implements Visitor {
@Override
public void visit(Term term) {
// the value
ASN1OctetString s = transformTerm(term);
result.push(s);
}
@ -358,7 +364,7 @@ public final class CQLRPNGenerator implements Visitor {
}
push(attributeElements, createAttributeElement(attributeType, attributeValue));
return new ASN1OctetString(v);
return new ASN1OctetString(v, charset);
}
private static AttributeElement createAttributeElement(Integer attributeType, Integer attributeValue) {

14
z3950-common/src/main/java/org/xbib/z3950/common/operations/SearchOperation.java
View file

@ -54,12 +54,12 @@ public class SearchOperation extends AbstractOperation<SearchResponse, SearchReq
this.status = false;
}
public boolean executePQF(String pqf, Charset charset) throws IOException {
return execute(createRPNQueryFromPQF(pqf, charset));
public boolean executePQF(Charset charset, String pqf) throws IOException {
return execute(createRPNQueryFromPQF(charset, pqf));
}
public boolean executeCQL(String cql, boolean wordListSupported) throws IOException {
return execute(createRPNQueryFromCQL(cql, wordListSupported));
public boolean executeCQL(Charset charset, String cql, boolean wordListSupported) throws IOException {
return execute(createRPNQueryFromCQL(charset, cql, wordListSupported));
}
public boolean execute(RPNQuery rpn) throws IOException {
@ -118,15 +118,15 @@ public class SearchOperation extends AbstractOperation<SearchResponse, SearchReq
return status;
}
private RPNQuery createRPNQueryFromCQL(String query, boolean wordListSupported) {
CQLRPNGenerator generator = new CQLRPNGenerator(null, wordListSupported);
private RPNQuery createRPNQueryFromCQL(Charset charset, String query, boolean wordListSupported) {
CQLRPNGenerator generator = new CQLRPNGenerator(charset, null, wordListSupported);
CQLParser parser = new CQLParser(query);
parser.parse();
parser.getCQLQuery().accept(generator);
return generator.getQueryResult();
}
private RPNQuery createRPNQueryFromPQF(String query, Charset charset) {
private RPNQuery createRPNQueryFromPQF(Charset charset, String query) {
PQFRPNGenerator generator = new PQFRPNGenerator(charset);
PQFParser parser = new PQFParser(new StringReader(query));
parser.parse();

15
z3950-common/src/test/java/org/xbib/z3950/common/cql/CQL2RPNTest.java
View file

@ -6,6 +6,8 @@ import org.xbib.asn1.ASN1Integer;
import org.xbib.cql.CQLParser;
import org.xbib.z3950.common.v3.AttributeElement;
import org.xbib.z3950.common.v3.AttributeElementAttributeValue;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
class CQL2RPNTest {
@ -41,7 +43,7 @@ class CQL2RPNTest {
String cql = "dc.title = \"a phrase\"";
CQLParser parser = new CQLParser(cql);
parser.parse();
CQLRPNGenerator generator = new CQLRPNGenerator(Collections.singleton(ae), true);
CQLRPNGenerator generator = new CQLRPNGenerator(StandardCharsets.UTF_8, Collections.singleton(ae), true);
parser.getCQLQuery().accept(generator);
String q = generator.getQueryResult().toString();
assertEquals("{attributeSetId 1.2.840.10003.3.1, rpn {op {attrTerm {attributes {{attributeType 7, attributeValue {numeric 4}}{attributeType 3, attributeValue {numeric 3}}{attributeType 4, attributeValue {numeric 1}}{attributeType 5, attributeValue {numeric 100}}{attributeType 6, attributeValue {numeric 1}}{attributeType 1, attributeValue {numeric 4}}{attributeType 2, attributeValue {numeric 3}}}, term {general \"a phrase\"}}}}}", q);
@ -79,4 +81,15 @@ class CQL2RPNTest {
String q = generator.getQueryResult().toString();
assertEquals("{attributeSetId 1.2.840.10003.3.1, rpn {op {attrTerm {attributes {{attributeType 3, attributeValue {numeric 3}}{attributeType 5, attributeValue {numeric 100}}{attributeType 6, attributeValue {numeric 1}}{attributeType 4, attributeValue {numeric 5}}{attributeType 1, attributeValue {numeric 31}}{attributeType 2, attributeValue {numeric 3}}}, term {general \"2023\"}}}}}", q);
}
@Test
void testUTF8() {
String cql = "bib.title = Köln";
CQLParser parser = new CQLParser(cql);
parser.parse();
CQLRPNGenerator generator = new CQLRPNGenerator(StandardCharsets.UTF_8);
parser.getCQLQuery().accept(generator);
String q = generator.getQueryResult().toString();
assertEquals("{attributeSetId 1.2.840.10003.3.1, rpn {op {attrTerm {attributes {{attributeType 3, attributeValue {numeric 3}}{attributeType 4, attributeValue {numeric 2}}{attributeType 5, attributeValue {numeric 100}}{attributeType 6, attributeValue {numeric 1}}{attributeType 1, attributeValue {numeric 4}}{attributeType 2, attributeValue {numeric 3}}}, term {general \"K\\703\\666ln\"}}}}}", q);
}
}