From: Marcin Koziej Date: Wed, 21 Dec 2011 09:03:56 +0000 (+0100) Subject: merge search into pretty branch X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/f59e7c3de6bd0f85a61a4d9481db60cd7369ae92 merge search into pretty branch --- f59e7c3de6bd0f85a61a4d9481db60cd7369ae92 diff --cc apps/catalogue/management/commands/importbooks.py index d097ddd1a,4ea0fd359..b6ddc5540 --- a/apps/catalogue/management/commands/importbooks.py +++ b/apps/catalogue/management/commands/importbooks.py @@@ -29,38 -28,14 +29,40 @@@ class Command(BaseCommand) help='Don\'t build TXT file'), make_option('-P', '--no-build-pdf', action='store_false', dest='build_pdf', default=True, help='Don\'t build PDF file'), + make_option('-S', '--no-search-index', action='store_false', dest='search_index', default=True, + help='Don\'t build PDF file'), make_option('-w', '--wait-until', dest='wait_until', metavar='TIME', help='Wait until specified time (Y-M-D h:m:s)'), + make_option('-p', '--picture', action='store_true', dest='import_picture', default=False, + help='Import pictures'), - ) help = 'Imports books from the specified directories.' args = 'directory [directory ...]' + def import_book(self, file_path, options): + verbose = options.get('verbose') + file_base, ext = os.path.splitext(file_path) + book = Book.from_xml_file(file_path, overwrite=options.get('force'), - build_epub=options.get('build_epub'), - build_txt=options.get('build_txt'), - build_pdf=options.get('build_pdf'), - build_mobi=options.get('build_mobi')) ++ build_epub=options.get('build_epub'), ++ build_txt=options.get('build_txt'), ++ build_pdf=options.get('build_pdf'), ++ build_mobi=options.get('build_mobi'), ++ search_index=options.get('search_index')) + fileid = book.fileid() + for ebook_format in Book.ebook_formats: + if os.path.isfile(file_base + '.' + ebook_format): + getattr(book, '%s_file' % ebook_format).save( + '%s.%s' % (fileid, ebook_format), + File(file(file_base + '.' + ebook_format))) + if verbose: + print "Importing %s.%s" % (file_base, ebook_format) + + book.save() + + def import_picture(self, file_path, options): + picture = Picture.from_xml_file(file_path, overwrite=options.get('force')) + return picture + def handle(self, *directories, **options): from django.db import transaction diff --cc apps/catalogue/models.py index d615fb65b,1a2e8f86f..8fb210935 --- a/apps/catalogue/models.py +++ b/apps/catalogue/models.py @@@ -23,14 -22,17 +23,16 @@@ from django.conf import setting from newtagging.models import TagBase, tags_updated from newtagging import managers from catalogue.fields import JSONField, OverwritingFileField -from catalogue.utils import ExistingFile, ORMDocProvider, create_zip, remove_zip +from catalogue.utils import create_zip, split_tags +from catalogue.tasks import touch_tag +from shutil import copy +from glob import glob +import re +from os import path -from librarian import dcparser, html, epub, NoDublinCore -import mutagen -from mutagen import id3 -from slughifi import slughifi -from sortify import sortify -from os import unlink + import search + TAG_CATEGORIES = ( ('author', _('author')), ('epoch', _('epoch')), @@@ -726,14 -622,26 +728,29 @@@ class Book(models.Model) def zip_audiobooks(self): bm = BookMedia.objects.filter(book=self, type='mp3') paths = map(lambda bm: (None, bm.file.path), bm) - result = create_zip.delay(paths, self.slug) + result = create_zip.delay(paths, self.fileid()) return result.wait() + def search_index(self): + if settings.SEARCH_INDEX_PARALLEL: + if instance(settings.SEARCH_INDEX_PARALLEL, int): + idx = search.ReusableIndex(threads=4) + else: + idx = search.ReusableIndex() + else: + idx = search.Index() + + idx.open() + try: + idx.index_book(self) + finally: + idx.close() + @classmethod def from_xml_file(cls, xml_file, **kwargs): + from django.core.files import File + from librarian import dcparser + # use librarian to parse meta-data book_info = dcparser.parse(xml_file) @@@ -747,9 -655,9 +764,10 @@@ @classmethod def from_text_and_meta(cls, raw_file, book_info, overwrite=False, - build_epub=True, build_txt=True, build_pdf=True, build_mobi=True): + build_epub=True, build_txt=True, build_pdf=True, build_mobi=True, + search_index=True): import re + from sortify import sortify # check for parts before we do anything children = [] @@@ -815,12 -735,14 +833,15 @@@ if not settings.NO_BUILD_MOBI and build_mobi: book.build_mobi() + if not settings.NO_SEARCH_INDEX and search_index: + book.search_index() + book_descendants = list(book.children.all()) + descendants_tags = set() # add l-tag to descendants and their fragments - # delete unnecessary EPUB files while len(book_descendants) > 0: child_book = book_descendants.pop(0) + descendants_tags.update(child_book.tags) child_book.tags = list(child_book.tags) + [book_tag] child_book.save() for fragment in child_book.fragments.all(): diff --cc lib/librarian index 0718c9d23,d7ba2c607..5b407667c --- a/lib/librarian +++ b/lib/librarian @@@ -1,1 -1,1 +1,1 @@@ - Subproject commit 0718c9d23a5791aa51bc11bba6a011fe9a8a616d -Subproject commit d7ba2c607dacf7a6136b83a1588b5adf2278ad46 ++Subproject commit 5b407667ca47cf4d9752821fd49e5611737146d2 diff --cc wolnelektury/settings.py index b597e3347,826abb2ca..4184c4e66 --- a/wolnelektury/settings.py +++ b/wolnelektury/settings.py @@@ -152,7 -151,10 +153,8 @@@ INSTALLED_APPS = 'sponsors', 'stats', 'suggest', + 'picture', + 'search', - - # - 'django_nose', ] #CACHE_BACKEND = 'locmem:///?max_entries=3000' diff --cc wolnelektury/templates/catalogue/search_multiple_hits.html index 1eea6fc3b,dc3696ece..15c5d71d9 --- a/wolnelektury/templates/catalogue/search_multiple_hits.html +++ b/wolnelektury/templates/catalogue/search_multiple_hits.html @@@ -7,8 -7,37 +7,35 @@@ {% block bodyid %}tagged-object-list{% endblock %} {% block body %} -
-

{{ form.q }} {% trans "or" %} {% trans "return to main page" %}

-
+

{% trans "Search" %}

+
+
    + {% for result in results %} +
  1. +

    {{result.book.pretty_title}} (id: {{result.book_id}}, score: {{result.score}})

    +
      + {% for hit in result.process_hits %} +
    • + {% if hit.fragment %} +
      Tagi/Motywy: {% for tag in hit.themes %}{{tag.name}} {% endfor %}
      + {% endif %} + {% for snip in hit.snippets %} + {{snip|safe}}
      + {% endfor %} +
      • + {% endfor %} + +
    +
    2. + {% empty %} +

    No results.

    + {% endfor %} +
+
+ + + {% comment %}

{% trans "More than one result matching the criteria found." %}

+ {% endcomment %} - {% endblock %} -
-
{% trans "Close" %}
-
-

* {% trans "Loading" %}

-
-
+ {% endblock %}