- for start, end in walker(master):
- print("%s %s" % (start, end))
- if start is not None and start.tag == 'begin':
- fid = start.attrib['id'][1:]
- fragments[fid] = {'content': [], 'themes': []}
- fragments[fid]['content'].append(start.tail)
- elif start is not None and start.tag == 'motyw':
- fid = start.attrib['id'][1:]
- fragments[fid]['themes'].append(start.text)
- fragments[fid]['content'].append(start.tail)
- elif start is not None and start.tag == 'end':
- fid = start.attrib['id'][1:]
- frag = fragments[fid]
- del fragments[fid]
- print("Fragment %s complete, themes: %s contents: %s" % (fid, frag['themes'], frag['content']))
-
- doc = self.create_book_doc(book)
- doc.add(Field("fragment_anchor", fid, Field.Store.YES, Field.Index.NOT_ANALYZED))
- doc.add(Field("content", u' '.join(filter(lambda s: s is not None, frag['content'])), Field.Store.NO, Field.Index.ANALYZED))
- doc.add(Field("themes", u' '.join(frag['themes']), Field.Store.NO, Field.Index.ANALYZED))
- fragment_docs.append(doc)
- elif start is not None:
- for frag in fragments.values():
- frag['content'].append(start.text)
- elif end is not None:
- for frag in fragments.values():
- frag['content'].append(end.tail)
-
- return header_docs + fragment_docs
-
- def __enter__(self):
- self.open()
+ snippets = Snippets(book.id).open('w')
+ try:
+ for header, position in zip(list(master), range(len(master))):
+
+ if header.tag in self.skip_header_tags:
+ continue
+ if header.tag is etree.Comment:
+ continue
+
+ # section content
+ content = []
+ footnote = []
+
+ def all_content(text):
+ for frag in fragments.values():
+ frag['text'].append(text)
+ content.append(text)
+ handle_text = [all_content]
+
+ for start, text, end in walker(header, ignore_tags=self.ignore_content_tags):
+ # handle footnotes
+ if start is not None and start.tag in self.footnote_tags:
+ footnote = []
+
+ def collect_footnote(t):
+ footnote.append(t)
+
+ handle_text.append(collect_footnote)
+ elif end is not None and footnote is not [] and end.tag in self.footnote_tags:
+ handle_text.pop()
+ doc = add_part(snippets, header_index=position, header_type=header.tag,
+ text=u''.join(footnote),
+ is_footnote=True)
+ self.index.add(doc)
+ #print "@ footnote text: %s" % footnote
+ footnote = []
+
+ # handle fragments and themes.
+ if start is not None and start.tag == 'begin':
+ fid = start.attrib['id'][1:]
+ fragments[fid] = {'text': [], 'themes': [], 'start_section': position, 'start_header': header.tag}
+
+ # themes for this fragment
+ elif start is not None and start.tag == 'motyw':
+ fid = start.attrib['id'][1:]
+ handle_text.append(None)
+ if start.text is not None:
+ fragments[fid]['themes'] += map(unicode.strip, map(unicode, (start.text.split(','))))
+ elif end is not None and end.tag == 'motyw':
+ handle_text.pop()
+
+ elif start is not None and start.tag == 'end':
+ fid = start.attrib['id'][1:]
+ if fid not in fragments:
+ continue # a broken <end> node, skip it
+ frag = fragments[fid]
+ if frag['themes'] == []:
+ continue # empty themes list.
+ del fragments[fid]
+
+ doc = add_part(snippets,
+ header_type=frag['start_header'],
+ header_index=frag['start_section'],
+ header_span=position - frag['start_section'] + 1,
+ fragment_anchor=fid,
+ text=fix_format(frag['text']),
+ themes=frag['themes'])
+ #print '@ FRAG %s' % frag['content']
+ self.index.add(doc)
+
+ # Collect content.
+
+ if text is not None and handle_text is not []:
+ hdl = handle_text[-1]
+ if hdl is not None:
+ hdl(text)
+
+ # in the end, add a section text.
+ doc = add_part(snippets, header_index=position,
+ header_type=header.tag, text=fix_format(content))
+ #print '@ CONTENT: %s' % fix_format(content)
+
+ self.index.add(doc)
+
+ finally:
+ snippets.close()
+
+
+class SearchResult(object):
+ def __init__(self, doc, how_found=None, query=None, query_terms=None):
+ # self.search = search
+ self.boost = 1.0
+ self._hits = []
+ self._processed_hits = None # processed hits
+ self.snippets = []
+ self.query_terms = query_terms
+
+ if 'score' in doc:
+ self._score = doc['score']
+ else:
+ self._score = 0
+
+ self.book_id = int(doc["book_id"])
+
+ try:
+ self.published_date = int(doc.get("published_date"))
+ except ValueError:
+ self.published_date = 0
+
+ # content hits
+ header_type = doc.get("header_type", None)
+ # we have a content hit in some header of fragment
+ if header_type is not None:
+ sec = (header_type, int(doc["header_index"]))
+ header_span = doc['header_span']
+ header_span = header_span is not None and int(header_span) or 1
+ fragment = doc.get("fragment_anchor", None)
+ snippets_pos = (doc['snippets_position'], doc['snippets_length'])
+ snippets_rev = doc.get('snippets_revision', None)
+
+ hit = (sec + (header_span,), fragment, self._score, {
+ 'how_found': how_found,
+ 'snippets_pos': snippets_pos,
+ 'snippets_revision': snippets_rev,
+ 'themes': doc.get('themes', []),
+ 'themes_pl': doc.get('themes_pl', [])
+ })
+
+ self._hits.append(hit)
+
+ def __unicode__(self):
+ return u"<SR id=%d %d(%d) hits score=%f %d snippets>" % \
+ (self.book_id, len(self._hits), self._processed_hits and len(self._processed_hits) or -1, self._score, len(self.snippets))
+
+ def __str__(self):
+ return unicode(self).encode('utf-8')
+
+ @property
+ def score(self):
+ return self._score * self.boost
+
+ def merge(self, other):
+ if self.book_id != other.book_id:
+ raise ValueError("this search result is or book %d; tried to merge with %d" % (self.book_id, other.book_id))
+ self._hits += other._hits
+ if other.score > self.score:
+ self._score = other._score
fnp / wolnelektury.git / blobdiff