X-Git-Url: https://git.mdrn.pl/wolnelektury.git/blobdiff_plain/55bc643c16dc56cf2377bd5c4d6e924d5fc62698..276656dcc680e1a35103d310dea046152ddc3785:/apps/search/index.py?ds=sidebyside diff --git a/apps/search/index.py b/apps/search/index.py index e59c38b17..307376de9 100644 --- a/apps/search/index.py +++ b/apps/search/index.py @@ -25,6 +25,7 @@ import re import errno from librarian import dcparser from librarian.parser import WLDocument +from lxml import etree import catalogue.models from multiprocessing.pool import ThreadPool from threading import current_thread @@ -212,7 +213,7 @@ class Index(BaseIndex): for tag in catalogue.models.Tag.objects.all(): doc = Document() - doc.add(NumericField("tag_id", Field.Store.YES, True).setIntValue(tag.id)) + doc.add(NumericField("tag_id", Field.Store.YES, True).setIntValue(int(tag.id))) doc.add(Field("tag_name", tag.name, Field.Store.NO, Field.Index.ANALYZED)) doc.add(Field("tag_name_pl", tag.name, Field.Store.NO, Field.Index.ANALYZED)) doc.add(Field("tag_category", tag.category, Field.Store.NO, Field.Index.NOT_ANALYZED)) @@ -223,9 +224,9 @@ class Index(BaseIndex): Create a lucene document referring book id. """ doc = Document() - doc.add(NumericField("book_id", Field.Store.YES, True).setIntValue(book.id)) + doc.add(NumericField("book_id", Field.Store.YES, True).setIntValue(int(book.id))) if book.parent is not None: - doc.add(NumericField("parent_id", Field.Store.YES, True).setIntValue(book.parent.id)) + doc.add(NumericField("parent_id", Field.Store.YES, True).setIntValue(int(book.parent.id))) return doc def remove_book(self, book): @@ -401,28 +402,28 @@ class Index(BaseIndex): if header.tag in self.skip_header_tags: continue + if header.tag is etree.Comment: + continue - content = u' '.join([t for t in header.itertext()]) - content = fix_format(content) - - doc = add_part(snippets, header_index=position, header_type=header.tag, content=content) + # section content + content = [] - self.index.addDocument(doc) - for start, end in walker(header): + # handle fragments and themes. if start is not None and start.tag == 'begin': fid = start.attrib['id'][1:] fragments[fid] = {'content': [], 'themes': [], 'start_section': position, 'start_header': header.tag} - fragments[fid]['content'].append(start.tail) + elif start is not None and start.tag == 'motyw': fid = start.attrib['id'][1:] if start.text is not None: fragments[fid]['themes'] += map(str.strip, map(give_me_utf8, start.text.split(','))) - fragments[fid]['content'].append(start.tail) + elif start is not None and start.tag == 'end': fid = start.attrib['id'][1:] if fid not in fragments: continue # a broken node, skip it + # import pdb; pdb.set_trace() frag = fragments[fid] if frag['themes'] == []: continue # empty themes list. @@ -442,12 +443,23 @@ class Index(BaseIndex): themes=frag['themes']) self.index.addDocument(doc) + + # Collect content. elif start is not None: for frag in fragments.values(): frag['content'].append(start.text) + content.append(start.text) elif end is not None: for frag in fragments.values(): frag['content'].append(end.tail) + content.append(end.tail) + + # in the end, add a section text. + doc = add_part(snippets, header_index=position, header_type=header.tag, + content=fix_format(u' '.join(filter(lambda s: s is not None, content)))) + + self.index.addDocument(doc) + finally: snippets.close()