--- /dev/null
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.nio.CharBuffer;
+import java.nio.ByteBuffer;
+
+/**
+ * Provides support for converting byte sequences to Strings and back again.
+ * The resulting Strings preserve the original byte sequences' sort order.
+ * <p/>
+ * The Strings are constructed using a Base 8000h encoding of the original
+ * binary data - each char of an encoded String represents a 15-bit chunk
+ * from the byte sequence. Base 8000h was chosen because it allows for all
+ * lower 15 bits of char to be used without restriction; the surrogate range
+ * [U+D8000-U+DFFF] does not represent valid chars, and would require
+ * complicated handling to avoid them and allow use of char's high bit.
+ * <p/>
+ * Although unset bits are used as padding in the final char, the original
+ * byte sequence could contain trailing bytes with no set bits (null bytes):
+ * padding is indistinguishable from valid information. To overcome this
+ * problem, a char is appended, indicating the number of encoded bytes in the
+ * final content char.
+ * <p/>
+ * Some methods in this class are defined over CharBuffers and ByteBuffers, but
+ * these are deprecated in favor of methods that operate directly on byte[] and
+ * char[] arrays. Note that this class calls array() and arrayOffset()
+ * on the CharBuffers and ByteBuffers it uses, so only wrapped arrays may be
+ * used. This class interprets the arrayOffset() and limit() values returned
+ * by its input buffers as beginning and end+1 positions on the wrapped array,
+ * respectively; similarly, on the output buffer, arrayOffset() is the first
+ * position written to, and limit() is set to one past the final output array
+ * position.
+ * <p/>
+ * WARNING: This means that the deprecated Buffer-based methods
+ * only work correctly with buffers that have an offset of 0. For example, they
+ * will not correctly interpret buffers returned by {@link ByteBuffer#slice}.
+ *
+ * @lucene.experimental
+ */
+public final class IndexableBinaryStringTools {
+
+ private static final CodingCase[] CODING_CASES = {
+ // CodingCase(int initialShift, int finalShift)
+ new CodingCase( 7, 1 ),
+ // CodingCase(int initialShift, int middleShift, int finalShift)
+ new CodingCase(14, 6, 2),
+ new CodingCase(13, 5, 3),
+ new CodingCase(12, 4, 4),
+ new CodingCase(11, 3, 5),
+ new CodingCase(10, 2, 6),
+ new CodingCase( 9, 1, 7),
+ new CodingCase( 8, 0 )
+ };
+
+ // Export only static methods
+ private IndexableBinaryStringTools() {}
+
+ /**
+ * Returns the number of chars required to encode the given byte sequence.
+ *
+ * @param original The byte sequence to be encoded. Must be backed by an
+ * array.
+ * @return The number of chars required to encode the given byte sequence
+ * @throws IllegalArgumentException If the given ByteBuffer is not backed by
+ * an array
+ * @deprecated Use {@link #getEncodedLength(byte[], int, int)} instead. This
+ * method will be removed in Lucene 4.0
+ */
+ @Deprecated
+ public static int getEncodedLength(ByteBuffer original)
+ throws IllegalArgumentException {
+ if (original.hasArray()) {
+ return getEncodedLength(original.array(), original.arrayOffset(),
+ original.limit() - original.arrayOffset());
+ } else {
+ throw new IllegalArgumentException("original argument must have a backing array");
+ }
+ }
+
+ /**
+ * Returns the number of chars required to encode the given bytes.
+ *
+ * @param inputArray byte sequence to be encoded
+ * @param inputOffset initial offset into inputArray
+ * @param inputLength number of bytes in inputArray
+ * @return The number of chars required to encode the number of bytes.
+ */
+ public static int getEncodedLength(byte[] inputArray, int inputOffset,
+ int inputLength) {
+ // Use long for intermediaries to protect against overflow
+ return (int)((8L * inputLength + 14L) / 15L) + 1;
+ }
+
+
+ /**
+ * Returns the number of bytes required to decode the given char sequence.
+ *
+ * @param encoded The char sequence to be decoded. Must be backed by an array.
+ * @return The number of bytes required to decode the given char sequence
+ * @throws IllegalArgumentException If the given CharBuffer is not backed by
+ * an array
+ * @deprecated Use {@link #getDecodedLength(char[], int, int)} instead. This
+ * method will be removed in Lucene 4.0
+ */
+ @Deprecated
+ public static int getDecodedLength(CharBuffer encoded)
+ throws IllegalArgumentException {
+ if (encoded.hasArray()) {
+ return getDecodedLength(encoded.array(), encoded.arrayOffset(),
+ encoded.limit() - encoded.arrayOffset());
+ } else {
+ throw new IllegalArgumentException("encoded argument must have a backing array");
+ }
+ }
+
+ /**
+ * Returns the number of bytes required to decode the given char sequence.
+ *
+ * @param encoded char sequence to be decoded
+ * @param offset initial offset
+ * @param length number of characters
+ * @return The number of bytes required to decode the given char sequence
+ */
+ public static int getDecodedLength(char[] encoded, int offset, int length) {
+ final int numChars = length - 1;
+ if (numChars <= 0) {
+ return 0;
+ } else {
+ // Use long for intermediaries to protect against overflow
+ final long numFullBytesInFinalChar = encoded[offset + length - 1];
+ final long numEncodedChars = numChars - 1;
+ return (int)((numEncodedChars * 15L + 7L) / 8L + numFullBytesInFinalChar);
+ }
+ }
+
+ /**
+ * Encodes the input byte sequence into the output char sequence. Before
+ * calling this method, ensure that the output CharBuffer has sufficient
+ * capacity by calling {@link #getEncodedLength(java.nio.ByteBuffer)}.
+ *
+ * @param input The byte sequence to encode
+ * @param output Where the char sequence encoding result will go. The limit is
+ * set to one past the position of the final char.
+ * @throws IllegalArgumentException If either the input or the output buffer
+ * is not backed by an array
+ * @deprecated Use {@link #encode(byte[], int, int, char[], int, int)}
+ * instead. This method will be removed in Lucene 4.0
+ */
+ @Deprecated
+ public static void encode(ByteBuffer input, CharBuffer output) {
+ if (input.hasArray() && output.hasArray()) {
+ final int inputOffset = input.arrayOffset();
+ final int inputLength = input.limit() - inputOffset;
+ final int outputOffset = output.arrayOffset();
+ final int outputLength = getEncodedLength(input.array(), inputOffset,
+ inputLength);
+ output.limit(outputLength + outputOffset);
+ output.position(0);
+ encode(input.array(), inputOffset, inputLength, output.array(),
+ outputOffset, outputLength);
+ } else {
+ throw new IllegalArgumentException("Arguments must have backing arrays");
+ }
+ }
+
+ /**
+ * Encodes the input byte sequence into the output char sequence. Before
+ * calling this method, ensure that the output array has sufficient
+ * capacity by calling {@link #getEncodedLength(byte[], int, int)}.
+ *
+ * @param inputArray byte sequence to be encoded
+ * @param inputOffset initial offset into inputArray
+ * @param inputLength number of bytes in inputArray
+ * @param outputArray char sequence to store encoded result
+ * @param outputOffset initial offset into outputArray
+ * @param outputLength length of output, must be getEncodedLength
+ */
+ public static void encode(byte[] inputArray, int inputOffset,
+ int inputLength, char[] outputArray, int outputOffset, int outputLength) {
+ assert (outputLength == getEncodedLength(inputArray, inputOffset,
+ inputLength));
+ if (inputLength > 0) {
+ int inputByteNum = inputOffset;
+ int caseNum = 0;
+ int outputCharNum = outputOffset;
+ CodingCase codingCase;
+ for (; inputByteNum + CODING_CASES[caseNum].numBytes <= inputLength; ++outputCharNum) {
+ codingCase = CODING_CASES[caseNum];
+ if (2 == codingCase.numBytes) {
+ outputArray[outputCharNum] = (char) (((inputArray[inputByteNum] & 0xFF) << codingCase.initialShift)
+ + (((inputArray[inputByteNum + 1] & 0xFF) >>> codingCase.finalShift) & codingCase.finalMask) & (short) 0x7FFF);
+ } else { // numBytes is 3
+ outputArray[outputCharNum] = (char) (((inputArray[inputByteNum] & 0xFF) << codingCase.initialShift)
+ + ((inputArray[inputByteNum + 1] & 0xFF) << codingCase.middleShift)
+ + (((inputArray[inputByteNum + 2] & 0xFF) >>> codingCase.finalShift) & codingCase.finalMask) & (short) 0x7FFF);
+ }
+ inputByteNum += codingCase.advanceBytes;
+ if (++caseNum == CODING_CASES.length) {
+ caseNum = 0;
+ }
+ }
+ // Produce final char (if any) and trailing count chars.
+ codingCase = CODING_CASES[caseNum];
+
+ if (inputByteNum + 1 < inputLength) { // codingCase.numBytes must be 3
+ outputArray[outputCharNum++] = (char) ((((inputArray[inputByteNum] & 0xFF) << codingCase.initialShift) + ((inputArray[inputByteNum + 1] & 0xFF) << codingCase.middleShift)) & (short) 0x7FFF);
+ // Add trailing char containing the number of full bytes in final char
+ outputArray[outputCharNum++] = (char) 1;
+ } else if (inputByteNum < inputLength) {
+ outputArray[outputCharNum++] = (char) (((inputArray[inputByteNum] & 0xFF) << codingCase.initialShift) & (short) 0x7FFF);
+ // Add trailing char containing the number of full bytes in final char
+ outputArray[outputCharNum++] = caseNum == 0 ? (char) 1 : (char) 0;
+ } else { // No left over bits - last char is completely filled.
+ // Add trailing char containing the number of full bytes in final char
+ outputArray[outputCharNum++] = (char) 1;
+ }
+ }
+ }
+
+ /**
+ * Decodes the input char sequence into the output byte sequence. Before
+ * calling this method, ensure that the output ByteBuffer has sufficient
+ * capacity by calling {@link #getDecodedLength(java.nio.CharBuffer)}.
+ *
+ * @param input The char sequence to decode
+ * @param output Where the byte sequence decoding result will go. The limit is
+ * set to one past the position of the final char.
+ * @throws IllegalArgumentException If either the input or the output buffer
+ * is not backed by an array
+ * @deprecated Use {@link #decode(char[], int, int, byte[], int, int)}
+ * instead. This method will be removed in Lucene 4.0
+ */
+ @Deprecated
+ public static void decode(CharBuffer input, ByteBuffer output) {
+ if (input.hasArray() && output.hasArray()) {
+ final int inputOffset = input.arrayOffset();
+ final int inputLength = input.limit() - inputOffset;
+ final int outputOffset = output.arrayOffset();
+ final int outputLength = getDecodedLength(input.array(), inputOffset,
+ inputLength);
+ output.limit(outputLength + outputOffset);
+ output.position(0);
+ decode(input.array(), inputOffset, inputLength, output.array(),
+ outputOffset, outputLength);
+ } else {
+ throw new IllegalArgumentException("Arguments must have backing arrays");
+ }
+ }
+
+ /**
+ * Decodes the input char sequence into the output byte sequence. Before
+ * calling this method, ensure that the output array has sufficient capacity
+ * by calling {@link #getDecodedLength(char[], int, int)}.
+ *
+ * @param inputArray char sequence to be decoded
+ * @param inputOffset initial offset into inputArray
+ * @param inputLength number of chars in inputArray
+ * @param outputArray byte sequence to store encoded result
+ * @param outputOffset initial offset into outputArray
+ * @param outputLength length of output, must be
+ * getDecodedLength(inputArray, inputOffset, inputLength)
+ */
+ public static void decode(char[] inputArray, int inputOffset,
+ int inputLength, byte[] outputArray, int outputOffset, int outputLength) {
+ assert (outputLength == getDecodedLength(inputArray, inputOffset,
+ inputLength));
+ final int numInputChars = inputLength - 1;
+ final int numOutputBytes = outputLength;
+
+ if (numOutputBytes > 0) {
+ int caseNum = 0;
+ int outputByteNum = outputOffset;
+ int inputCharNum = inputOffset;
+ short inputChar;
+ CodingCase codingCase;
+ for (; inputCharNum < numInputChars - 1; ++inputCharNum) {
+ codingCase = CODING_CASES[caseNum];
+ inputChar = (short) inputArray[inputCharNum];
+ if (2 == codingCase.numBytes) {
+ if (0 == caseNum) {
+ outputArray[outputByteNum] = (byte) (inputChar >>> codingCase.initialShift);
+ } else {
+ outputArray[outputByteNum] += (byte) (inputChar >>> codingCase.initialShift);
+ }
+ outputArray[outputByteNum + 1] = (byte) ((inputChar & codingCase.finalMask) << codingCase.finalShift);
+ } else { // numBytes is 3
+ outputArray[outputByteNum] += (byte) (inputChar >>> codingCase.initialShift);
+ outputArray[outputByteNum + 1] = (byte) ((inputChar & codingCase.middleMask) >>> codingCase.middleShift);
+ outputArray[outputByteNum + 2] = (byte) ((inputChar & codingCase.finalMask) << codingCase.finalShift);
+ }
+ outputByteNum += codingCase.advanceBytes;
+ if (++caseNum == CODING_CASES.length) {
+ caseNum = 0;
+ }
+ }
+ // Handle final char
+ inputChar = (short) inputArray[inputCharNum];
+ codingCase = CODING_CASES[caseNum];
+ if (0 == caseNum) {
+ outputArray[outputByteNum] = 0;
+ }
+ outputArray[outputByteNum] += (byte) (inputChar >>> codingCase.initialShift);
+ final int bytesLeft = numOutputBytes - outputByteNum;
+ if (bytesLeft > 1) {
+ if (2 == codingCase.numBytes) {
+ outputArray[outputByteNum + 1] = (byte) ((inputChar & codingCase.finalMask) >>> codingCase.finalShift);
+ } else { // numBytes is 3
+ outputArray[outputByteNum + 1] = (byte) ((inputChar & codingCase.middleMask) >>> codingCase.middleShift);
+ if (bytesLeft > 2) {
+ outputArray[outputByteNum + 2] = (byte) ((inputChar & codingCase.finalMask) << codingCase.finalShift);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Decodes the given char sequence, which must have been encoded by
+ * {@link #encode(java.nio.ByteBuffer)} or
+ * {@link #encode(java.nio.ByteBuffer, java.nio.CharBuffer)}.
+ *
+ * @param input The char sequence to decode
+ * @return A byte sequence containing the decoding result. The limit is set to
+ * one past the position of the final char.
+ * @throws IllegalArgumentException If the input buffer is not backed by an
+ * array
+ * @deprecated Use {@link #decode(char[], int, int, byte[], int, int)}
+ * instead. This method will be removed in Lucene 4.0
+ */
+ @Deprecated
+ public static ByteBuffer decode(CharBuffer input) {
+ byte[] outputArray = new byte[getDecodedLength(input)];
+ ByteBuffer output = ByteBuffer.wrap(outputArray);
+ decode(input, output);
+ return output;
+ }
+
+ /**
+ * Encodes the input byte sequence.
+ *
+ * @param input The byte sequence to encode
+ * @return A char sequence containing the encoding result. The limit is set to
+ * one past the position of the final char.
+ * @throws IllegalArgumentException If the input buffer is not backed by an
+ * array
+ * @deprecated Use {@link #encode(byte[], int, int, char[], int, int)}
+ * instead. This method will be removed in Lucene 4.0
+ */
+ @Deprecated
+ public static CharBuffer encode(ByteBuffer input) {
+ char[] outputArray = new char[getEncodedLength(input)];
+ CharBuffer output = CharBuffer.wrap(outputArray);
+ encode(input, output);
+ return output;
+ }
+
+ static class CodingCase {
+ int numBytes, initialShift, middleShift, finalShift, advanceBytes = 2;
+ short middleMask, finalMask;
+
+ CodingCase(int initialShift, int middleShift, int finalShift) {
+ this.numBytes = 3;
+ this.initialShift = initialShift;
+ this.middleShift = middleShift;
+ this.finalShift = finalShift;
+ this.finalMask = (short)((short)0xFF >>> finalShift);
+ this.middleMask = (short)((short)0xFF << middleShift);
+ }
+
+ CodingCase(int initialShift, int finalShift) {
+ this.numBytes = 2;
+ this.initialShift = initialShift;
+ this.finalShift = finalShift;
+ this.finalMask = (short)((short)0xFF >>> finalShift);
+ if (finalShift != 0) {
+ advanceBytes = 1;
+ }
+ }
+ }
+}
fnp / pylucene.git / blobdiff