+ themes = self.add_gaps(themes, 'themes')
+ themes_pl = self.add_gaps(themes_pl, 'themes_pl')
+
+ for t in themes:
+ doc.add(t)
+ for t in themes_pl:
+ doc.add(t)
+
+ return doc
+
+ def give_me_utf8(s):
+ if isinstance(s, unicode):
+ return s.encode('utf-8')
+ else:
+ return s
+
+ fragments = {}
+ snippets = Snippets(book.id).open('w')
+ try:
+ for header, position in zip(list(master), range(len(master))):
+
+ if header.tag in self.skip_header_tags:
+ continue
+ if header.tag is etree.Comment:
+ continue
+
+ # section content
+ content = []
+ footnote = []
+
+ def all_content(text):
+ for frag in fragments.values():
+ frag['content'].append(text)
+ content.append(text)
+ handle_text = [all_content]
+
+
+ for start, text, end in walker(header, ignore_tags=self.ignore_content_tags):
+ # handle footnotes
+ if start is not None and start.tag in self.footnote_tags:
+ footnote = []
+ def collect_footnote(t):
+ footnote.append(t)
+ handle_text.append(collect_footnote)
+ elif end is not None and footnote is not [] and end.tag in self.footnote_tags:
+ handle_text.pop()
+ doc = add_part(snippets, header_index=position, header_type=header.tag,
+ content=u''.join(footnote),
+ is_footnote=Field("is_footnote", 'true', Field.Store.NO, Field.Index.NOT_ANALYZED))
+
+ self.index.addDocument(doc)
+ #print "@ footnote text: %s" % footnote
+ footnote = []
+
+ # handle fragments and themes.
+ if start is not None and start.tag == 'begin':
+ fid = start.attrib['id'][1:]
+ fragments[fid] = {'content': [], 'themes': [], 'start_section': position, 'start_header': header.tag}
+
+ # themes for this fragment
+ elif start is not None and start.tag == 'motyw':
+ fid = start.attrib['id'][1:]
+ handle_text.append(None)
+ if start.text is not None:
+ fragments[fid]['themes'] += map(str.strip, map(give_me_utf8, start.text.split(',')))
+ elif end is not None and end.tag == 'motyw':
+ handle_text.pop()
+
+ elif start is not None and start.tag == 'end':
+ fid = start.attrib['id'][1:]
+ if fid not in fragments:
+ continue # a broken <end> node, skip it
+ frag = fragments[fid]
+ if frag['themes'] == []:
+ continue # empty themes list.
+ del fragments[fid]
+
+ doc = add_part(snippets,
+ header_type=frag['start_header'],
+ header_index=frag['start_section'],
+ header_span=position - frag['start_section'] + 1,
+ fragment_anchor=fid,
+ content=fix_format(frag['content']),
+ themes=frag['themes'])
+ #print '@ FRAG %s' % frag['content']
+ self.index.addDocument(doc)
+
+ # Collect content.
+
+ if text is not None and handle_text is not []:
+ hdl = handle_text[-1]
+ if hdl is not None:
+ hdl(text)
+
+ # in the end, add a section text.
+ doc = add_part(snippets, header_index=position, header_type=header.tag,
+ content=fix_format(content))
+ #print '@ CONTENT: %s' % fix_format(content)
+
+ self.index.addDocument(doc)
+
+ finally:
+ snippets.close()
+
+
+def log_exception_wrapper(f):
+ def _wrap(*a):
+ try:
+ f(*a)
+ except Exception, e:
+ print("Error in indexing thread: %s" % e)
+ traceback.print_exc()
+ raise e
+ return _wrap