From 60e8143da7846cd1ee7a7e0d8266084fdce34e3c Mon Sep 17 00:00:00 2001 From: Pierre Salagnac Date: Thu, 23 Jul 2026 11:06:56 +0200 Subject: [PATCH] SOLR-18315: Optimize UTF conversions by using standard Java APIs --- .../SOLR-18315-optimize-utf-conversion.yml | 7 + .../solr/bench/search/RequestWriters.java | 2 +- .../response/TestJavaBinResponseWriter.java | 8 +- .../util/ByteArrayUtf8CharSequence.java | 25 +- .../apache/solr/common/util/ByteUtils.java | 230 ------------------ .../apache/solr/common/util/JavaBinCodec.java | 92 +++---- .../org/apache/solr/common/util/Utils.java | 9 +- .../solr/common/util/TestJavaBinCodec.java | 45 +++- 8 files changed, 105 insertions(+), 313 deletions(-) create mode 100644 changelog/unreleased/SOLR-18315-optimize-utf-conversion.yml delete mode 100644 solr/solrj/src/java/org/apache/solr/common/util/ByteUtils.java diff --git a/changelog/unreleased/SOLR-18315-optimize-utf-conversion.yml b/changelog/unreleased/SOLR-18315-optimize-utf-conversion.yml new file mode 100644 index 000000000000..fece89082142 --- /dev/null +++ b/changelog/unreleased/SOLR-18315-optimize-utf-conversion.yml @@ -0,0 +1,7 @@ +title: Replace custom code for UTF8-UTF16 conversions by standard Java API for better performances. +type: changed +authors: + - name: Pierre Salagnac +links: + - name: SOLR-18315 + url: https://issues.apache.org/jira/browse/SOLR-18315 diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/RequestWriters.java b/solr/benchmark/src/java/org/apache/solr/bench/search/RequestWriters.java index d55800ad06e4..258b53a5a81f 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/RequestWriters.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/RequestWriters.java @@ -60,7 +60,7 @@ public class RequestWriters { @State(Scope.Benchmark) public static class BenchState { - @Param({"xml", "binary"}) + @Param({"xml", "javabin"}) String type; @Param({"10", "100", "1000", "10000"}) diff --git a/solr/core/src/test/org/apache/solr/response/TestJavaBinResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestJavaBinResponseWriter.java index 48d990562923..63eb167618c8 100644 --- a/solr/core/src/test/org/apache/solr/response/TestJavaBinResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestJavaBinResponseWriter.java @@ -18,6 +18,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; import java.util.Locale; import java.util.Map; import java.util.UUID; @@ -26,7 +27,6 @@ import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.ByteUtils; import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.common.util.NamedList; import org.apache.solr.request.SolrQueryRequest; @@ -57,10 +57,10 @@ public void testBytesRefWriting() { } private void compareStringFormat(String input) { - byte[] bytes1 = new byte[1024]; - int len1 = ByteUtils.UTF16toUTF8(input, 0, input.length(), bytes1, 0); + byte[] bytes1 = input.getBytes(StandardCharsets.UTF_8); + int len1 = bytes1.length; BytesRef bytesref = new BytesRef(input); - System.out.println(); + assertEquals(len1, bytesref.length); for (int i = 0; i < len1; i++) { assertEquals(input + " not matching char at :" + i, bytesref.bytes[i], bytes1[i]); diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ByteArrayUtf8CharSequence.java b/solr/solrj/src/java/org/apache/solr/common/util/ByteArrayUtf8CharSequence.java index 052514e75c9c..16bf68fe5f7b 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/ByteArrayUtf8CharSequence.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/ByteArrayUtf8CharSequence.java @@ -17,13 +17,12 @@ package org.apache.solr.common.util; +import java.nio.charset.StandardCharsets; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collection; import java.util.Map; import java.util.Set; -import java.util.function.Function; -import org.noggit.CharArr; /** * A mutable byte[] backed Utf8CharSequence. This is quite similar to the BytesRef of Lucene Do not @@ -36,18 +35,13 @@ public class ByteArrayUtf8CharSequence implements Utf8CharSequence { protected int offset; protected int hashCode = Integer.MIN_VALUE; protected int length; - protected volatile String utf16; - public Function stringProvider; + protected String utf16; public ByteArrayUtf8CharSequence(String utf16) { - buf = new byte[Math.multiplyExact(utf16.length(), 3)]; + this.utf16 = utf16; + buf = utf16.getBytes(StandardCharsets.UTF_8); offset = 0; - length = ByteUtils.UTF16toUTF8(utf16, 0, utf16.length(), buf, 0); - if (buf.length > length) { - byte[] copy = new byte[length]; - System.arraycopy(buf, 0, copy, 0, length); - buf = copy; - } + length = buf.length; assert isValid(); } @@ -154,15 +148,8 @@ public char charAt(int index) { } private String _getStr() { - String utf16 = this.utf16; if (utf16 == null) { - if (stringProvider != null) { - this.utf16 = utf16 = stringProvider.apply(this); - } else { - CharArr arr = new CharArr(); - ByteUtils.UTF8toUTF16(buf, offset, length, arr); - this.utf16 = utf16 = arr.toString(); - } + utf16 = new String(buf, offset, length, StandardCharsets.UTF_8); } return utf16; } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ByteUtils.java b/solr/solrj/src/java/org/apache/solr/common/util/ByteUtils.java deleted file mode 100644 index b7a13dca9746..000000000000 --- a/solr/solrj/src/java/org/apache/solr/common/util/ByteUtils.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.common.util; - -import java.io.IOException; -import java.io.OutputStream; -import org.noggit.CharArr; - -public class ByteUtils { - - /** Maximum number of UTF8 bytes per UTF16 character. */ - public static final int MAX_UTF8_BYTES_PER_CHAR = 3; - - /** - * Converts utf8 to utf16 and returns the number of 16-bit Java chars written. Full characters are - * read, even if this reads past the length passed (and can result in an ArrayOutOfBoundsException - * if invalid UTF8 is passed). Explicit checks for valid UTF8 are not performed. The char[] out - * should probably have enough room to hold the worst case of each byte becoming a Java char. - */ - public static int UTF8toUTF16(byte[] utf8, int offset, int len, char[] out, int out_offset) { - int out_start = out_offset; - final int limit = offset + len; - while (offset < limit) { - int b = utf8[offset++] & 0xff; - - if (b < 0xc0) { - assert b < 0x80; - out[out_offset++] = (char) b; - } else if (b < 0xe0) { - out[out_offset++] = (char) (((b & 0x1f) << 6) + (utf8[offset++] & 0x3f)); - } else if (b < 0xf0) { - out[out_offset++] = - (char) (((b & 0xf) << 12) + ((utf8[offset] & 0x3f) << 6) + (utf8[offset + 1] & 0x3f)); - offset += 2; - } else { - assert b < 0xf8; - int ch = - ((b & 0x7) << 18) - + ((utf8[offset] & 0x3f) << 12) - + ((utf8[offset + 1] & 0x3f) << 6) - + (utf8[offset + 2] & 0x3f); - offset += 3; - if (ch < 0xffff) { - out[out_offset++] = (char) ch; - } else { - int chHalf = ch - 0x0010000; - out[out_offset++] = (char) ((chHalf >> 10) + 0xD800); - out[out_offset++] = (char) ((chHalf & 0x3FFL) + 0xDC00); - } - } - } - - return out_offset - out_start; - } - - /** Convert UTF8 bytes into UTF16 characters. */ - public static void UTF8toUTF16(byte[] utf8, int offset, int len, CharArr out) { - // TODO: do in chunks if the input is large - out.reserve(len); - int n = UTF8toUTF16(utf8, offset, len, out.getArray(), out.getEnd()); - out.setEnd(out.getEnd() + n); - } - - /** Convert UTF8 bytes into a String */ - public static String UTF8toUTF16(byte[] utf8, int offset, int len) { - char[] out = new char[len]; - int n = UTF8toUTF16(utf8, offset, len, out, 0); - return new String(out, 0, n); - } - - /** - * Writes UTF8 into the byte array, starting at offset. The caller should ensure that there is - * enough space for the worst-case scenario. - * - * @return the number of bytes written - */ - public static int UTF16toUTF8( - CharSequence s, int offset, int len, byte[] result, int resultOffset) { - final int end = offset + len; - - int upto = resultOffset; - for (int i = offset; i < end; i++) { - final int code = (int) s.charAt(i); - - if (code < 0x80) result[upto++] = (byte) code; - else if (code < 0x800) { - result[upto++] = (byte) (0xC0 | (code >> 6)); - result[upto++] = (byte) (0x80 | (code & 0x3F)); - } else if (code < 0xD800 || code > 0xDFFF) { - result[upto++] = (byte) (0xE0 | (code >> 12)); - result[upto++] = (byte) (0x80 | ((code >> 6) & 0x3F)); - result[upto++] = (byte) (0x80 | (code & 0x3F)); - } else { - // surrogate pair - // confirm valid high surrogate - if (code < 0xDC00 && (i < end - 1)) { - int utf32 = (int) s.charAt(i + 1); - // confirm valid low surrogate and write pair - if (utf32 >= 0xDC00 && utf32 <= 0xDFFF) { - utf32 = ((code - 0xD7C0) << 10) + (utf32 & 0x3FF); - i++; - result[upto++] = (byte) (0xF0 | (utf32 >> 18)); - result[upto++] = (byte) (0x80 | ((utf32 >> 12) & 0x3F)); - result[upto++] = (byte) (0x80 | ((utf32 >> 6) & 0x3F)); - result[upto++] = (byte) (0x80 | (utf32 & 0x3F)); - continue; - } - } - // replace unpaired surrogate or out-of-order low surrogate - // with substitution character - result[upto++] = (byte) 0xEF; - result[upto++] = (byte) 0xBF; - result[upto++] = (byte) 0xBD; - } - } - - return upto - resultOffset; - } - - /** - * Writes UTF8 into the given OutputStream by first writing to the given scratch array and then - * writing the contents of the scratch array to the OutputStream. The given scratch byte array is - * used to buffer intermediate data before it is written to the output stream. - * - * @return the number of bytes written - */ - public static int writeUTF16toUTF8( - CharSequence s, int offset, int len, OutputStream fos, byte[] scratch) throws IOException { - final int end = offset + len; - - int upto = 0, totalBytes = 0; - for (int i = offset; i < end; i++) { - final int code = (int) s.charAt(i); - - if (upto > scratch.length - 4) { - // a code point may take up to 4 bytes, and we don't have enough space, so reset - totalBytes += upto; - if (fos == null) throw new IOException("buffer over flow"); - fos.write(scratch, 0, upto); - upto = 0; - } - - if (code < 0x80) scratch[upto++] = (byte) code; - else if (code < 0x800) { - scratch[upto++] = (byte) (0xC0 | (code >> 6)); - scratch[upto++] = (byte) (0x80 | (code & 0x3F)); - } else if (code < 0xD800 || code > 0xDFFF) { - scratch[upto++] = (byte) (0xE0 | (code >> 12)); - scratch[upto++] = (byte) (0x80 | ((code >> 6) & 0x3F)); - scratch[upto++] = (byte) (0x80 | (code & 0x3F)); - } else { - // surrogate pair - // confirm valid high surrogate - if (code < 0xDC00 && (i < end - 1)) { - int utf32 = (int) s.charAt(i + 1); - // confirm valid low surrogate and write pair - if (utf32 >= 0xDC00 && utf32 <= 0xDFFF) { - utf32 = ((code - 0xD7C0) << 10) + (utf32 & 0x3FF); - i++; - scratch[upto++] = (byte) (0xF0 | (utf32 >> 18)); - scratch[upto++] = (byte) (0x80 | ((utf32 >> 12) & 0x3F)); - scratch[upto++] = (byte) (0x80 | ((utf32 >> 6) & 0x3F)); - scratch[upto++] = (byte) (0x80 | (utf32 & 0x3F)); - continue; - } - } - // replace unpaired surrogate or out-of-order low surrogate - // with substitution character - scratch[upto++] = (byte) 0xEF; - scratch[upto++] = (byte) 0xBF; - scratch[upto++] = (byte) 0xBD; - } - } - - totalBytes += upto; - if (fos != null) fos.write(scratch, 0, upto); - - return totalBytes; - } - - /** - * Calculates the number of UTF8 bytes necessary to write a UTF16 string. - * - * @return the number of bytes written - */ - public static int calcUTF16toUTF8Length(CharSequence s, int offset, int len) { - final int end = offset + len; - - int res = 0; - for (int i = offset; i < end; i++) { - final int code = (int) s.charAt(i); - - if (code < 0x80) res++; - else if (code < 0x800) { - res += 2; - } else if (code < 0xD800 || code > 0xDFFF) { - res += 3; - } else { - // surrogate pair - // confirm valid high surrogate - if (code < 0xDC00 && (i < end - 1)) { - int utf32 = (int) s.charAt(i + 1); - // confirm valid low surrogate and write pair - if (utf32 >= 0xDC00 && utf32 <= 0xDFFF) { - i++; - res += 4; - continue; - } - } - res += 3; - } - } - - return res; - } -} diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java index f47ff63b9805..4e4770fbcaf1 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java @@ -23,6 +23,9 @@ import java.io.OutputStream; import java.lang.invoke.MethodHandles; import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -40,7 +43,6 @@ import java.util.concurrent.atomic.LongAccumulator; import java.util.concurrent.atomic.LongAdder; import java.util.function.BiConsumer; -import java.util.function.Function; import java.util.function.Predicate; import org.apache.solr.client.api.util.ReflectWritable; import org.apache.solr.common.ConditionalKeyMapWriter; @@ -54,7 +56,6 @@ import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; import org.apache.solr.common.params.CommonParams; -import org.noggit.CharArr; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,6 +116,9 @@ public class JavaBinCodec implements PushWriter { private static final int MIN_UTF8_SIZE_FOR_ARRAY_GROW_STRATEGY = 512; private static final int MAX_UTF8_SIZE_FOR_ARRAY_GROW_STRATEGY = 65536; + /** Maximum number of UTF8 bytes per UTF16 character. */ + private static final int MAX_UTF8_BYTES_PER_CHAR = 3; + private static final byte VERSION = 2; private final ObjectResolver resolver; protected FastOutputStream daos; @@ -1064,28 +1068,56 @@ public void writeStr(CharSequence s) throws IOException { return; } int end = s.length(); - int maxSize = end * ByteUtils.MAX_UTF8_BYTES_PER_CHAR; + int maxSize = end * MAX_UTF8_BYTES_PER_CHAR; if (maxSize <= MAX_UTF8_SIZE_FOR_ARRAY_GROW_STRATEGY) { - if (bytes == null || bytes.length < maxSize) { - int bufferSize = getBufferSize(maxSize); - bytes = new byte[bufferSize]; - } - - int sz = ByteUtils.UTF16toUTF8(s, 0, end, bytes, 0); - writeTag(STR, sz); - daos.write(bytes, 0, sz); + byte[] utf8 = s.toString().getBytes(StandardCharsets.UTF_8); + writeTag(STR, utf8.length); + daos.write(utf8); } else { // double pass logic for large strings, see SOLR-7971 - int sz = ByteUtils.calcUTF16toUTF8Length(s, 0, end); - writeTag(STR, sz); + CharBuffer cb = CharBuffer.wrap(s); + CharsetEncoder encoder = StandardCharsets.UTF_8.newEncoder(); if (bytes == null || bytes.length < 8192) bytes = new byte[8192]; - ByteUtils.writeUTF16toUTF8(s, 0, end, daos, bytes); + ByteBuffer scratch = ByteBuffer.wrap(bytes); + + // first pass: count UTF-8 bytes without retaining output + int sz = 0; + cb.rewind(); + encoder.reset(); + boolean endOfInput = false; + while (!endOfInput) { + endOfInput = !cb.hasRemaining(); + scratch.clear(); + encoder.encode(cb, scratch, endOfInput); + sz += scratch.position(); + } + scratch.clear(); + encoder.flush(scratch); + sz += scratch.position(); + + // second pass: stream-encode into daos + writeTag(STR, sz); + cb.rewind(); + encoder.reset(); + endOfInput = false; + while (!endOfInput) { + endOfInput = !cb.hasRemaining(); + scratch.clear(); + encoder.encode(cb, scratch, endOfInput); + if (scratch.position() > 0) { + daos.write(bytes, 0, scratch.position()); + } + } + scratch.clear(); + encoder.flush(scratch); + if (scratch.position() > 0) { + daos.write(bytes, 0, scratch.position()); + } } } byte[] bytes; - CharArr arr = new CharArr(); private StringBytes bytesRef = new StringBytes(bytes, 0, 0); public CharSequence readStr(DataInputInputStream dis) throws IOException { @@ -1112,9 +1144,7 @@ private CharSequence _readStr(DataInputInputStream dis, StringCache stringCache, if (stringCache != null) { return stringCache.get(bytesRef.reset(bytes, 0, sz)); } else { - arr.reset(); - ByteUtils.UTF8toUTF16(bytes, 0, sz, arr); - return arr.toString(); + return new String(bytes, 0, sz, StandardCharsets.UTF_8); } } @@ -1142,7 +1172,6 @@ static int getBufferSize(int required) { /////////// code to optimize reading UTF8 static final int MAX_UTF8_SZ = 1024 * 64; // too big strings can cause too much memory allocation - private Function stringProvider; private BytesBlock bytesBlock; protected CharSequence readUtf8(DataInputInputStream dis) throws IOException { @@ -1153,7 +1182,6 @@ protected CharSequence readUtf8(DataInputInputStream dis) throws IOException { protected CharSequence readUtf8(DataInputInputStream dis, int sz) throws IOException { ByteArrayUtf8CharSequence result = new ByteArrayUtf8CharSequence(null, 0, 0); if (dis.readDirectUtf8(result, sz)) { - result.stringProvider = getStringProvider(); return result; } @@ -1163,29 +1191,9 @@ protected CharSequence readUtf8(DataInputInputStream dis, int sz) throws IOExcep dis.readFully(block.getBuf(), block.getStartPos(), sz); result.reset(block.getBuf(), block.getStartPos(), sz, null); - result.stringProvider = getStringProvider(); return result; } - private Function getStringProvider() { - if (stringProvider == null) { - stringProvider = - new Function<>() { - final CharArr charArr = new CharArr(8); - - @Override - public String apply(ByteArrayUtf8CharSequence butf8cs) { - synchronized (charArr) { - charArr.reset(); - ByteUtils.UTF8toUTF16(butf8cs.buf, butf8cs.offset(), butf8cs.size(), charArr); - return charArr.toString(); - } - } - }; - } - return this.stringProvider; - } - public void writeInt(int val) throws IOException { if (val > 0) { int b = SINT | (val & 0x0f); @@ -1461,9 +1469,7 @@ public String get(StringBytes b) { StringBytes copy = new StringBytes( Arrays.copyOfRange(b.bytes, b.offset, b.offset + b.length), 0, b.length); - CharArr arr = new CharArr(); - ByteUtils.UTF8toUTF16(b.bytes, b.offset, b.length, arr); - result = arr.toString(); + result = new String(b.bytes, b.offset, b.length, StandardCharsets.UTF_8); putIntoCache(copy, result); } return result; diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java index acbbdac41d72..dd72d9066bd9 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java @@ -43,12 +43,12 @@ import java.net.URL; import java.nio.BufferOverflowException; import java.nio.ByteBuffer; +import java.nio.CharBuffer; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.AbstractMap; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -271,9 +271,10 @@ public static String toJSONString(Object o) { } public static byte[] toUTF8(CharArr out) { - byte[] arr = new byte[out.size() * 3]; - int nBytes = ByteUtils.UTF16toUTF8(out, 0, out.size(), arr, 0); - return Arrays.copyOf(arr, nBytes); + ByteBuffer bb = UTF_8.encode(CharBuffer.wrap(out.getArray(), out.getStart(), out.size())); + byte[] arr = new byte[bb.remaining()]; + bb.get(arr); + return arr; } public static Object fromJSON(byte[] utf8) { diff --git a/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java b/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java index 07c4cb64cd36..fb2406e7699c 100644 --- a/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java @@ -23,6 +23,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -42,7 +43,6 @@ import org.apache.solr.common.SolrInputField; import org.apache.solr.util.RTimer; import org.junit.Test; -import org.noggit.CharArr; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,22 +61,45 @@ public class TestJavaBinCodec extends SolrTestCaseJ4 { private static final String SOLRJ_DOCS_1 = "/solrj/docs1.xml"; private static final String SOLRJ_DOCS_2 = "/solrj/sampleClusteringResponse.xml"; + @Test public void testStrings() throws Exception { for (int i = 0; i < 10000 * RANDOM_MULTIPLIER; i++) { - String s = TestUtil.randomUnicodeString(random()); + String s1 = TestUtil.randomUnicodeString(random()); + String s2 = TestUtil.randomUnicodeString(random()); try (JavaBinCodec jbcO = new JavaBinCodec(); ByteArrayOutputStream os = new ByteArrayOutputStream()) { - jbcO.marshal(s, os); - try (JavaBinCodec jbcI = new JavaBinCodec(); + jbcO.initWrite(os); + jbcO.writeVal(s1); + jbcO.writeVal(s2); + jbcO.daos.flush(); + try (JavaBinCodec jbc1 = new JavaBinCodec(); ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray())) { - jbcI.readMapAsNamedList(false); - Object o = jbcI.unmarshal(is); - assertEquals(s, o); + FastInputStream fis = jbc1.initRead(is); + Object o1 = jbc1.readVal(fis); + Object o2 = jbc1.readVal(fis); + assertEquals(s1, o1); + assertEquals(s2, o2); } } } } + @Test + public void testLongString() throws Exception { + String s = TestUtil.randomUnicodeString(random(), 65536 * 10); + + try (JavaBinCodec jbcO = new JavaBinCodec(); + ByteArrayOutputStream os = new ByteArrayOutputStream()) { + jbcO.marshal(s, os); + try (JavaBinCodec jbc1 = new JavaBinCodec(); + ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray())) { + Object o = jbc1.unmarshal(is); + assertEquals(s, o); + } + } + } + + @Test public void testReadAsCharSeq() throws Exception { List types = new ArrayList<>(); SolrInputDocument idoc = new SolrInputDocument(); @@ -87,6 +110,7 @@ public void testReadAsCharSeq() throws Exception { compareObjects((List) getObject(getBytes(types, true)), (List) types); } + @Test public void testReadMap() throws Exception { Map types = new HashMap<>(); types.put("1", "one"); @@ -563,7 +587,7 @@ protected void putIntoCache(StringBytes b, String val) { int end = s.length(); int maxSize = end * 4; if (bytes == null || bytes.length < maxSize) bytes = new byte[maxSize]; - int sz = ByteUtils.UTF16toUTF8(s, 0, end, bytes, 0); + int sz = s.getBytes(StandardCharsets.UTF_8).length; STRING_CACHE.get(stringBytes.reset(bytes, 0, sz)); } printMem("after cache init"); @@ -591,12 +615,9 @@ protected void putIntoCache(StringBytes b, String val) { THREADS, () -> { String a = null; - CharArr arr = new CharArr(); for (int i = 0; i < ITERS; i++) { StringBytes sb = l.get(i % l.size()); - arr.reset(); - ByteUtils.UTF8toUTF16(sb.bytes, 0, sb.bytes.length, arr); - a = arr.toString(); + a = new String(sb.bytes, StandardCharsets.UTF_8); } });