--- /dev/null
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.lucene.store.DataOutput;
+
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+
+/**
+ * Class that Posting and PostingVector use to write byte
+ * streams into shared fixed-size byte[] arrays. The idea
+ * is to allocate slices of increasing lengths For
+ * example, the first slice is 5 bytes, the next slice is
+ * 14, etc. We start by writing our bytes into the first
+ * 5 bytes. When we hit the end of the slice, we allocate
+ * the next slice and then write the address of the new
+ * slice into the last 4 bytes of the previous slice (the
+ * "forwarding address").
+ *
+ * Each slice is filled with 0's initially, and we mark
+ * the end with a non-zero byte. This way the methods
+ * that are writing into the slice don't need to record
+ * its length and instead allocate a new slice once they
+ * hit a non-zero byte.
+ *
+ * @lucene.internal
+ **/
+public final class ByteBlockPool {
+ public final static int BYTE_BLOCK_SHIFT = 15;
+ public final static int BYTE_BLOCK_SIZE = 1 << BYTE_BLOCK_SHIFT;
+ public final static int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1;
+
+ public abstract static class Allocator {
+ protected final int blockSize;
+
+ public Allocator(int blockSize) {
+ this.blockSize = blockSize;
+ }
+
+ public abstract void recycleByteBlocks(byte[][] blocks, int start, int end);
+
+ public void recycleByteBlocks(List<byte[]> blocks) {
+ final byte[][] b = blocks.toArray(new byte[blocks.size()][]);
+ recycleByteBlocks(b, 0, b.length);
+ }
+
+ public byte[] getByteBlock() {
+ return new byte[blockSize];
+ }
+ }
+
+ public static final class DirectAllocator extends Allocator {
+
+ public DirectAllocator() {
+ this(BYTE_BLOCK_SIZE);
+ }
+
+ public DirectAllocator(int blockSize) {
+ super(blockSize);
+ }
+
+ @Override
+ public void recycleByteBlocks(byte[][] blocks, int start, int end) {
+ }
+
+ }
+
+ public static class DirectTrackingAllocator extends Allocator {
+ private final AtomicLong bytesUsed;
+
+ public DirectTrackingAllocator(AtomicLong bytesUsed) {
+ this(BYTE_BLOCK_SIZE, bytesUsed);
+ }
+
+ public DirectTrackingAllocator(int blockSize, AtomicLong bytesUsed) {
+ super(blockSize);
+ this.bytesUsed = bytesUsed;
+ }
+
+ public byte[] getByteBlock() {
+ bytesUsed.addAndGet(blockSize);
+ return new byte[blockSize];
+ }
+ @Override
+ public void recycleByteBlocks(byte[][] blocks, int start, int end) {
+ bytesUsed.addAndGet(-((end-start)* blockSize));
+ for (int i = start; i < end; i++) {
+ blocks[i] = null;
+ }
+ }
+
+ };
+
+
+ public byte[][] buffers = new byte[10][];
+
+ int bufferUpto = -1; // Which buffer we are upto
+ public int byteUpto = BYTE_BLOCK_SIZE; // Where we are in head buffer
+
+ public byte[] buffer; // Current head buffer
+ public int byteOffset = -BYTE_BLOCK_SIZE; // Current head offset
+
+ private final Allocator allocator;
+
+ public ByteBlockPool(Allocator allocator) {
+ this.allocator = allocator;
+ }
+
+ public void dropBuffersAndReset() {
+ if (bufferUpto != -1) {
+ // Recycle all but the first buffer
+ allocator.recycleByteBlocks(buffers, 0, 1+bufferUpto);
+
+ // Re-use the first buffer
+ bufferUpto = -1;
+ byteUpto = BYTE_BLOCK_SIZE;
+ byteOffset = -BYTE_BLOCK_SIZE;
+ buffers = new byte[10][];
+ buffer = null;
+ }
+ }
+
+ public void reset() {
+ if (bufferUpto != -1) {
+ // We allocated at least one buffer
+
+ for(int i=0;i<bufferUpto;i++)
+ // Fully zero fill buffers that we fully used
+ Arrays.fill(buffers[i], (byte) 0);
+
+ // Partial zero fill the final buffer
+ Arrays.fill(buffers[bufferUpto], 0, byteUpto, (byte) 0);
+
+ if (bufferUpto > 0)
+ // Recycle all but the first buffer
+ allocator.recycleByteBlocks(buffers, 1, 1+bufferUpto);
+
+ // Re-use the first buffer
+ bufferUpto = 0;
+ byteUpto = 0;
+ byteOffset = 0;
+ buffer = buffers[0];
+ }
+ }
+
+ public void nextBuffer() {
+ if (1+bufferUpto == buffers.length) {
+ byte[][] newBuffers = new byte[ArrayUtil.oversize(buffers.length+1,
+ NUM_BYTES_OBJECT_REF)][];
+ System.arraycopy(buffers, 0, newBuffers, 0, buffers.length);
+ buffers = newBuffers;
+ }
+ buffer = buffers[1+bufferUpto] = allocator.getByteBlock();
+ bufferUpto++;
+
+ byteUpto = 0;
+ byteOffset += BYTE_BLOCK_SIZE;
+ }
+
+ public int newSlice(final int size) {
+ if (byteUpto > BYTE_BLOCK_SIZE-size)
+ nextBuffer();
+ final int upto = byteUpto;
+ byteUpto += size;
+ buffer[byteUpto-1] = 16;
+ return upto;
+ }
+
+ // Size of each slice. These arrays should be at most 16
+ // elements (index is encoded with 4 bits). First array
+ // is just a compact way to encode X+1 with a max. Second
+ // array is the length of each slice, ie first slice is 5
+ // bytes, next slice is 14 bytes, etc.
+
+ public final static int[] nextLevelArray = {1, 2, 3, 4, 5, 6, 7, 8, 9, 9};
+ public final static int[] levelSizeArray = {5, 14, 20, 30, 40, 40, 80, 80, 120, 200};
+ public final static int FIRST_LEVEL_SIZE = levelSizeArray[0];
+
+ public int allocSlice(final byte[] slice, final int upto) {
+
+ final int level = slice[upto] & 15;
+ final int newLevel = nextLevelArray[level];
+ final int newSize = levelSizeArray[newLevel];
+
+ // Maybe allocate another block
+ if (byteUpto > BYTE_BLOCK_SIZE-newSize)
+ nextBuffer();
+
+ final int newUpto = byteUpto;
+ final int offset = newUpto + byteOffset;
+ byteUpto += newSize;
+
+ // Copy forward the past 3 bytes (which we are about
+ // to overwrite with the forwarding address):
+ buffer[newUpto] = slice[upto-3];
+ buffer[newUpto+1] = slice[upto-2];
+ buffer[newUpto+2] = slice[upto-1];
+
+ // Write forwarding address at end of last slice:
+ slice[upto-3] = (byte) (offset >>> 24);
+ slice[upto-2] = (byte) (offset >>> 16);
+ slice[upto-1] = (byte) (offset >>> 8);
+ slice[upto] = (byte) offset;
+
+ // Write new level:
+ buffer[byteUpto-1] = (byte) (16|newLevel);
+
+ return newUpto+3;
+ }
+
+ // Fill in a BytesRef from term's length & bytes encoded in
+ // byte block
+ public final BytesRef setBytesRef(BytesRef term, int textStart) {
+ final byte[] bytes = term.bytes = buffers[textStart >> BYTE_BLOCK_SHIFT];
+ int pos = textStart & BYTE_BLOCK_MASK;
+ if ((bytes[pos] & 0x80) == 0) {
+ // length is 1 byte
+ term.length = bytes[pos];
+ term.offset = pos+1;
+ } else {
+ // length is 2 bytes
+ term.length = (bytes[pos]&0x7f) + ((bytes[pos+1]&0xff)<<7);
+ term.offset = pos+2;
+ }
+ assert term.length >= 0;
+ return term;
+ }
+
+ /**
+ * Copies the given {@link BytesRef} at the current positions (
+ * {@link #byteUpto} across buffer boundaries
+ */
+ public final void copy(final BytesRef bytes) {
+ int length = bytes.length;
+ int offset = bytes.offset;
+ int overflow = (length + byteUpto) - BYTE_BLOCK_SIZE;
+ do {
+ if (overflow <= 0) {
+ System.arraycopy(bytes.bytes, offset, buffer, byteUpto, length);
+ byteUpto += length;
+ break;
+ } else {
+ final int bytesToCopy = length-overflow;
+ System.arraycopy(bytes.bytes, offset, buffer, byteUpto, bytesToCopy);
+ offset += bytesToCopy;
+ length -= bytesToCopy;
+ nextBuffer();
+ overflow = overflow - BYTE_BLOCK_SIZE;
+ }
+ } while(true);
+ }
+
+ /**
+ * Writes the pools content to the given {@link DataOutput}
+ */
+ public final void writePool(final DataOutput out) throws IOException {
+ int bytesOffset = byteOffset;
+ int block = 0;
+ while (bytesOffset > 0) {
+ out.writeBytes(buffers[block++], BYTE_BLOCK_SIZE);
+ bytesOffset -= BYTE_BLOCK_SIZE;
+ }
+ out.writeBytes(buffers[block], byteUpto);
+ }
+}
+
fnp / pylucene.git / blobdiff