+++ /dev/null
-package org.apache.lucene.util;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.io.InputStream;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.zip.GZIPInputStream;
-import java.util.Random;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-
-/** Minimal port of contrib/benchmark's LneDocSource +
- * DocMaker, so tests can enum docs from a line file created
- * by contrib/benchmark's WriteLineDoc task */
-public class LineFileDocs implements Closeable {
-
- private BufferedReader reader;
- private final static int BUFFER_SIZE = 1 << 16; // 64K
- private final AtomicInteger id = new AtomicInteger();
- private final String path;
-
- /** If forever is true, we rewind the file at EOF (repeat
- * the docs over and over) */
- public LineFileDocs(Random random, String path) throws IOException {
- this.path = path;
- open(random);
- }
-
- public LineFileDocs(Random random) throws IOException {
- this(random, LuceneTestCase.TEST_LINE_DOCS_FILE);
- }
-
- public synchronized void close() throws IOException {
- if (reader != null) {
- reader.close();
- reader = null;
- }
- }
-
- private synchronized void open(Random random) throws IOException {
- InputStream is = getClass().getResourceAsStream(path);
- if (is == null) {
- // if its not in classpath, we load it as absolute filesystem path (e.g. Hudson's home dir)
- is = new FileInputStream(path);
- }
- File file = new File(path);
- long size;
- if (file.exists()) {
- size = file.length();
- } else {
- size = is.available();
- }
- if (path.endsWith(".gz")) {
- is = new GZIPInputStream(is);
- // guestimate:
- size *= 2.8;
- }
-
- reader = new BufferedReader(new InputStreamReader(is, "UTF-8"), BUFFER_SIZE);
-
- // Override sizes for currently "known" line files:
- if (path.equals("europarl.lines.txt.gz")) {
- size = 15129506L;
- } else if (path.equals("/home/hudson/lucene-data/enwiki.random.lines.txt.gz")) {
- size = 3038178822L;
- }
-
- // Randomly seek to starting point:
- if (random != null && size > 3) {
- final long seekTo = (random.nextLong()&Long.MAX_VALUE) % (size/3);
- if (LuceneTestCase.VERBOSE) {
- System.out.println("TEST: LineFileDocs: seek to fp=" + seekTo + " on open");
- }
- reader.skip(seekTo);
- reader.readLine();
- }
- }
-
- public synchronized void reset(Random random) throws IOException {
- close();
- open(random);
- id.set(0);
- }
-
- private final static char SEP = '\t';
-
- private static final class DocState {
- final Document doc;
- final Field titleTokenized;
- final Field title;
- final Field body;
- final Field id;
- final Field date;
-
- public DocState() {
- doc = new Document();
-
- title = new Field("title", "", Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS);
- doc.add(title);
-
- titleTokenized = new Field("titleTokenized", "", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(titleTokenized);
-
- body = new Field("body", "", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(body);
-
- id = new Field("docid", "", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS);
- doc.add(id);
-
- date = new Field("date", "", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS);
- doc.add(date);
- }
- }
-
- private final ThreadLocal<DocState> threadDocs = new ThreadLocal<DocState>();
-
- /** Note: Document instance is re-used per-thread */
- public Document nextDoc() throws IOException {
- String line;
- synchronized(this) {
- line = reader.readLine();
- if (line == null) {
- // Always rewind at end:
- if (LuceneTestCase.VERBOSE) {
- System.out.println("TEST: LineFileDocs: now rewind file...");
- }
- close();
- open(null);
- line = reader.readLine();
- }
- }
-
- DocState docState = threadDocs.get();
- if (docState == null) {
- docState = new DocState();
- threadDocs.set(docState);
- }
-
- int spot = line.indexOf(SEP);
- if (spot == -1) {
- throw new RuntimeException("line: [" + line + "] is in an invalid format !");
- }
- int spot2 = line.indexOf(SEP, 1 + spot);
- if (spot2 == -1) {
- throw new RuntimeException("line: [" + line + "] is in an invalid format !");
- }
-
- docState.body.setValue(line.substring(1+spot2, line.length()));
- final String title = line.substring(0, spot);
- docState.title.setValue(title);
- docState.titleTokenized.setValue(title);
- docState.date.setValue(line.substring(1+spot, spot2));
- docState.id.setValue(Integer.toString(id.getAndIncrement()));
- return docState.doc;
- }
-}
fnp / pylucene.git / blobdiff