update to gradle 5.6, more resilient percent decoding

This commit is contained in:
Jörg Prante 2019-09-23 17:04:03 +02:00
parent f3173094e5
commit 339327ee81
11 changed files with 59 additions and 55 deletions

View file

@ -1,6 +1,6 @@
group = org.xbib group = org.xbib
name = net name = net
version = 2.0.2 version = 2.0.3
# test # test
junit.version = 5.5.1 junit.version = 5.5.1

View file

@ -1,6 +1,5 @@
#Wed Aug 07 23:28:18 CEST 2019
distributionUrl=https\://services.gradle.org/distributions/gradle-5.3.1-all.zip
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
zipStorePath=wrapper/dists distributionUrl=https\://services.gradle.org/distributions/gradle-5.6-bin.zip
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

6
gradlew vendored
View file

@ -7,7 +7,7 @@
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # https://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
@ -125,8 +125,8 @@ if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi fi
# For Cygwin, switch paths to Windows format before running java # For Cygwin or MSYS, switch paths to Windows format before running java
if $cygwin ; then if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"` APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"` JAVACMD=`cygpath --unix "$JAVACMD"`

2
gradlew.bat vendored
View file

@ -5,7 +5,7 @@
@rem you may not use this file except in compliance with the License. @rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at @rem You may obtain a copy of the License at
@rem @rem
@rem http://www.apache.org/licenses/LICENSE-2.0 @rem https://www.apache.org/licenses/LICENSE-2.0
@rem @rem
@rem Unless required by applicable law or agreed to in writing, software @rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS, @rem distributed under the License is distributed on an "AS IS" BASIS,

View file

@ -34,7 +34,9 @@ public class PercentDecoder {
private ByteBuffer encodedBuf; private ByteBuffer encodedBuf;
public PercentDecoder() { public PercentDecoder() {
this(StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT)); this(StandardCharsets.UTF_8.newDecoder()
.onUnmappableCharacter(CodingErrorAction.REPORT)
.onMalformedInput(CodingErrorAction.REPORT));
} }
/** /**
@ -52,8 +54,7 @@ public class PercentDecoder {
* @param initialEncodedByteBufSize Initial size of buffer that holds encoded bytes * @param initialEncodedByteBufSize Initial size of buffer that holds encoded bytes
* @param decodedCharBufSize Size of buffer that encoded bytes are decoded into * @param decodedCharBufSize Size of buffer that encoded bytes are decoded into
*/ */
public PercentDecoder(CharsetDecoder charsetDecoder, int initialEncodedByteBufSize, public PercentDecoder(CharsetDecoder charsetDecoder, int initialEncodedByteBufSize, int decodedCharBufSize) {
int decodedCharBufSize) {
this.outputBuf = new StringBuilder(); this.outputBuf = new StringBuilder();
this.encodedBuf = ByteBuffer.allocate(initialEncodedByteBufSize); this.encodedBuf = ByteBuffer.allocate(initialEncodedByteBufSize);
this.decodedCharBuf = CharBuffer.allocate(decodedCharBufSize); this.decodedCharBuf = CharBuffer.allocate(decodedCharBufSize);
@ -85,9 +86,7 @@ public class PercentDecoder {
continue; continue;
} }
if (i + 2 >= input.length()) { if (i + 2 >= input.length()) {
throw new IllegalArgumentException("could not percent decode <" throw new MalformedInputException(i);
+ input
+ ">: incomplete %-pair at position " + i);
} }
if (encodedBuf.remaining() == 0) { if (encodedBuf.remaining() == 0) {
ByteBuffer largerBuf = ByteBuffer.allocate(encodedBuf.capacity() * 2); ByteBuffer largerBuf = ByteBuffer.allocate(encodedBuf.capacity() * 2);
@ -97,21 +96,18 @@ public class PercentDecoder {
} }
int c1 = input.charAt(++i); int c1 = input.charAt(++i);
int c2 = input.charAt(++i); int c2 = input.charAt(++i);
encodedBuf.put(decode((char) c1, (char) c2)); byte b1 = (byte) decode((char) c1);
byte b2 = (byte) decode((char) c2);
/*if (b1 == -1 || b2 == -1) {
throw new MalformedInputException(i);
}*/
byte b = (byte) ((b1 & 0xf) << 4 | (b2 & 0xf));
encodedBuf.put(b);
} }
handleEncodedBytes(); handleEncodedBytes();
return outputBuf.toString(); return outputBuf.toString();
} }
private static byte decode(char c1, char c2) {
byte b1 = (byte) decode(c1);
byte b2 = (byte) decode(c2);
if (b1 == -1 || b2 == -1) {
throw new IllegalArgumentException("invalid %-tuple <%" + c1 + c2 + ">");
}
return (byte) ((b1 & 0xf) << 4 | (b2 & 0xf));
}
private static int decode(char c) { private static int decode(char c) {
return (c >= '0' && c <= '9') ? c - '0' : return (c >= '0' && c <= '9') ? c - '0' :
(c >= 'A' && c <= 'F') ? c - 'A' + 10 : (c >= 'A' && c <= 'F') ? c - 'A' + 10 :

View file

@ -11,6 +11,8 @@ import java.util.stream.Collectors;
/** /**
* Query parameter list, of limited size. Default is 1024 pairs. * Query parameter list, of limited size. Default is 1024 pairs.
*
* This class is not thread-safe.
*/ */
@SuppressWarnings("serial") @SuppressWarnings("serial")
public class QueryParameters extends ArrayList<Pair<String, String>> { public class QueryParameters extends ArrayList<Pair<String, String>> {
@ -41,7 +43,6 @@ public class QueryParameters extends ArrayList<Pair<String, String>> {
.onMalformedInput(CodingErrorAction.REPLACE)), max); .onMalformedInput(CodingErrorAction.REPLACE)), max);
} }
public QueryParameters(PercentDecoder percentDecoder) { public QueryParameters(PercentDecoder percentDecoder) {
this(percentDecoder, 1024); this(percentDecoder, 1024);
} }
@ -82,10 +83,8 @@ public class QueryParameters extends ArrayList<Pair<String, String>> {
try { try {
add(percentDecoder.decode(pair.getFirst()), add(percentDecoder.decode(pair.getFirst()),
percentDecoder.decode(pair.getSecond())); percentDecoder.decode(pair.getSecond()));
} catch (MalformedInputException e) { } catch (MalformedInputException | UnmappableCharacterException e) {
// never thrown throw new IllegalArgumentException(e);
} catch (UnmappableCharacterException e) {
// never thrown
} }
} }
s = pairs.getSecond(); s = pairs.getSecond();

View file

@ -870,6 +870,7 @@ public class URL implements Comparable<URL> {
this.codingErrorAction = codingErrorAction; this.codingErrorAction = codingErrorAction;
this.regNameEncoder = PercentEncoders.getRegNameEncoder(charset); this.regNameEncoder = PercentEncoders.getRegNameEncoder(charset);
CharsetDecoder charsetDecoder = charset.newDecoder() CharsetDecoder charsetDecoder = charset.newDecoder()
.onMalformedInput(codingErrorAction)
.onUnmappableCharacter(codingErrorAction); .onUnmappableCharacter(codingErrorAction);
this.percentDecoder = new PercentDecoder(charsetDecoder); this.percentDecoder = new PercentDecoder(charsetDecoder);
this.queryParams = new QueryParameters(percentDecoder); this.queryParams = new QueryParameters(percentDecoder);

View file

@ -1,5 +1,6 @@
package org.xbib.net; package org.xbib.net;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.nio.charset.MalformedInputException; import java.nio.charset.MalformedInputException;
@ -22,7 +23,7 @@ class PercentDecoderTest {
private static final int CODE_POINT_IN_BMP = 1; private static final int CODE_POINT_IN_BMP = 1;
private PercentDecoder decoder = new PercentDecoder(StandardCharsets.UTF_8.newDecoder()); private PercentDecoder decoder = new PercentDecoder();
@Test @Test
void testDecodesWithoutPercents() throws Exception { void testDecodesWithoutPercents() throws Exception {
@ -35,33 +36,24 @@ class PercentDecoderTest {
} }
@Test @Test
void testIncompletePercentPairNoNumbers() throws Exception { void testIncompletePercentPairNoNumbers() {
try { Assertions.assertThrows(MalformedInputException.class, () ->{
decoder.decode("%"); decoder.decode("%");
fail(); });
} catch (IllegalArgumentException e) {
assertEquals("could not percent decode <%>: incomplete %-pair at position 0", e.getMessage());
}
} }
@Test @Test
void testIncompletePercentPairOneNumber() throws Exception { void testIncompletePercentPairOneNumber() {
try { Assertions.assertThrows(MalformedInputException.class, () ->{
decoder.decode("%2"); decoder.decode("%2");
fail(); });
} catch (IllegalArgumentException e) {
assertEquals("could not percent decode <%2>: incomplete %-pair at position 0", e.getMessage());
}
} }
@Test @Test
void testInvalidHex() throws Exception { void testInvalidHex() {
try { Assertions.assertThrows(MalformedInputException.class, () ->{
decoder.decode("%xz"); decoder.decode("%xz");
fail(); });
} catch (IllegalArgumentException e) {
assertEquals("invalid %-tuple <%xz>", e.getMessage());
}
} }
@Test @Test

View file

@ -3,6 +3,7 @@ package org.xbib.net;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.nio.charset.Charset;
import java.nio.charset.CodingErrorAction; import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Iterator; import java.util.Iterator;
@ -444,6 +445,25 @@ class URLParserTest {
assertEquals("e", url.getFragment()); assertEquals("e", url.getFragment());
} }
@Test
void testUrlCharsetReplacementAndReport() {
Charset charset = StandardCharsets.UTF_8;
URL url = URL.builder()
.charset(charset, CodingErrorAction.REPLACE)
.path("/bla%PDblabla?a=b")
.build();
QueryParameters queryParameters = url.getQueryParams();
// %EF%BF%B = 0xFFFD UNICODE REPLACEMENT CHARACTER
assertEquals("/bla%EF%BF%BDblabla", url.getPath());
assertEquals("[a=b]", queryParameters.toString());
Assertions.assertThrows(IllegalArgumentException.class, () -> {
URL.builder()
.charset(charset, CodingErrorAction.REPORT)
.path("/bla%PDblabla?a=b")
.build();
});
}
private void assertUrlCompatibility(String url) throws Exception { private void assertUrlCompatibility(String url) throws Exception {
String s = URL.from(url).toExternalForm(); String s = URL.from(url).toExternalForm();
assertEquals(s, URL.from(s).toExternalForm()); assertEquals(s, URL.from(s).toExternalForm());

View file

@ -26,7 +26,7 @@ class URLTest {
if (test.failure) { if (test.failure) {
try { try {
URL.base(base).resolve(input); URL.base(base).resolve(input);
fail(); fail("base = " + base + " input = " + input);
} catch (Exception e) { } catch (Exception e) {
// pass // pass
} }

View file

@ -2064,8 +2064,7 @@
"port": "", "port": "",
"pathname": "/foo%2zbar", "pathname": "/foo%2zbar",
"search": "", "search": "",
"hash": "", "hash": ""
"failure": true
}, },
{ {
"input": "http://example.com/foo%2©zbar", "input": "http://example.com/foo%2©zbar",
@ -2080,8 +2079,7 @@
"port": "", "port": "",
"pathname": "/foo%2%C3%82%C2%A9zbar", "pathname": "/foo%2%C3%82%C2%A9zbar",
"search": "", "search": "",
"hash": "", "hash": ""
"failure": true
}, },
{ {
"input": "http://example.com/foo%41%7a", "input": "http://example.com/foo%41%7a",
@ -4607,8 +4605,7 @@
"port": "", "port": "",
"pathname": "%1G", "pathname": "%1G",
"search": "", "search": "",
"hash": "", "hash": ""
"failure": true
}, },
"# Hosts and percent-encoding", "# Hosts and percent-encoding",
{ {