--- /dev/null
+package org.apache.lucene.search;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.util.FixedBitSet;
+
+public class DuplicateFilter extends Filter
+{
+
+ String fieldName;
+
+ /**
+ * KeepMode determines which document id to consider as the master, all others being
+ * identified as duplicates. Selecting the "first occurrence" can potentially save on IO.
+ */
+ int keepMode=KM_USE_FIRST_OCCURRENCE;
+ public static final int KM_USE_FIRST_OCCURRENCE=1;
+ public static final int KM_USE_LAST_OCCURRENCE=2;
+
+ /**
+ * "Full" processing mode starts by setting all bits to false and only setting bits
+ * for documents that contain the given field and are identified as none-duplicates.
+
+ * "Fast" processing sets all bits to true then unsets all duplicate docs found for the
+ * given field. This approach avoids the need to read TermDocs for terms that are seen
+ * to have a document frequency of exactly "1" (i.e. no duplicates). While a potentially
+ * faster approach , the downside is that bitsets produced will include bits set for
+ * documents that do not actually contain the field given.
+ *
+ */
+ int processingMode=PM_FULL_VALIDATION;
+ public static final int PM_FULL_VALIDATION=1;
+ public static final int PM_FAST_INVALIDATION=2;
+
+
+
+ public DuplicateFilter(String fieldName)
+ {
+ this(fieldName, KM_USE_LAST_OCCURRENCE,PM_FULL_VALIDATION);
+ }
+
+
+ public DuplicateFilter(String fieldName, int keepMode, int processingMode)
+ {
+ this.fieldName = fieldName;
+ this.keepMode = keepMode;
+ this.processingMode = processingMode;
+ }
+
+ @Override
+ public DocIdSet getDocIdSet(IndexReader reader) throws IOException
+ {
+ if(processingMode==PM_FAST_INVALIDATION)
+ {
+ return fastBits(reader);
+ }
+ else
+ {
+ return correctBits(reader);
+ }
+ }
+
+ private FixedBitSet correctBits(IndexReader reader) throws IOException
+ {
+
+ FixedBitSet bits=new FixedBitSet(reader.maxDoc()); //assume all are INvalid
+ Term startTerm=new Term(fieldName);
+ TermEnum te = reader.terms(startTerm);
+ if(te!=null)
+ {
+ Term currTerm=te.term();
+ while((currTerm!=null)&&(currTerm.field()==startTerm.field())) //term fieldnames are interned
+ {
+ int lastDoc=-1;
+ //set non duplicates
+ TermDocs td = reader.termDocs(currTerm);
+ if(td.next())
+ {
+ if(keepMode==KM_USE_FIRST_OCCURRENCE)
+ {
+ bits.set(td.doc());
+ }
+ else
+ {
+ do
+ {
+ lastDoc=td.doc();
+ }while(td.next());
+ bits.set(lastDoc);
+ }
+ }
+ if(!te.next())
+ {
+ break;
+ }
+ currTerm=te.term();
+ }
+ }
+ return bits;
+ }
+
+ private FixedBitSet fastBits(IndexReader reader) throws IOException
+ {
+
+ FixedBitSet bits=new FixedBitSet(reader.maxDoc());
+ bits.set(0,reader.maxDoc()); //assume all are valid
+ Term startTerm=new Term(fieldName);
+ TermEnum te = reader.terms(startTerm);
+ if(te!=null)
+ {
+ Term currTerm=te.term();
+
+ while((currTerm!=null)&&(currTerm.field()==startTerm.field())) //term fieldnames are interned
+ {
+ if(te.docFreq()>1)
+ {
+ int lastDoc=-1;
+ //unset potential duplicates
+ TermDocs td = reader.termDocs(currTerm);
+ td.next();
+ if(keepMode==KM_USE_FIRST_OCCURRENCE)
+ {
+ td.next();
+ }
+ do
+ {
+ lastDoc=td.doc();
+ bits.clear(lastDoc);
+ }while(td.next());
+ if(keepMode==KM_USE_LAST_OCCURRENCE)
+ {
+ //restore the last bit
+ bits.set(lastDoc);
+ }
+ }
+ if(!te.next())
+ {
+ break;
+ }
+ currTerm=te.term();
+ }
+ }
+ return bits;
+ }
+
+ public String getFieldName()
+ {
+ return fieldName;
+ }
+
+
+ public void setFieldName(String fieldName)
+ {
+ this.fieldName = fieldName;
+ }
+
+
+ public int getKeepMode()
+ {
+ return keepMode;
+ }
+
+
+ public void setKeepMode(int keepMode)
+ {
+ this.keepMode = keepMode;
+ }
+
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if(this == obj)
+ return true;
+ if((obj == null) || (obj.getClass() != this.getClass()))
+ return false;
+ DuplicateFilter other = (DuplicateFilter)obj;
+ return keepMode == other.keepMode &&
+ processingMode == other.processingMode &&
+ (fieldName == other.fieldName || (fieldName != null && fieldName.equals(other.fieldName)));
+ }
+
+
+
+ @Override
+ public int hashCode()
+ {
+ int hash = 217;
+ hash = 31 * hash + keepMode;
+ hash = 31 * hash + processingMode;
+ hash = 31 * hash + fieldName.hashCode();
+ return hash;
+ }
+
+
+ public int getProcessingMode()
+ {
+ return processingMode;
+ }
+
+
+ public void setProcessingMode(int processingMode)
+ {
+ this.processingMode = processingMode;
+ }
+
+
+
+}
fnp / pylucene.git / blobdiff