From: Marcin Koziej <marcin.koziej@nowoczesnapolska.org.pl>
Date: Wed, 21 Dec 2011 09:03:56 +0000 (+0100)
Subject: merge search into pretty branch
X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/f59e7c3de6bd0f85a61a4d9481db60cd7369ae92?hp=--cc
merge search into pretty branch
---
f59e7c3de6bd0f85a61a4d9481db60cd7369ae92
diff --cc apps/catalogue/management/commands/importbooks.py
index d097ddd1a,4ea0fd359..b6ddc5540
--- a/apps/catalogue/management/commands/importbooks.py
+++ b/apps/catalogue/management/commands/importbooks.py
@@@ -29,38 -28,14 +29,40 @@@ class Command(BaseCommand)
help='Don\'t build TXT file'),
make_option('-P', '--no-build-pdf', action='store_false', dest='build_pdf', default=True,
help='Don\'t build PDF file'),
+ make_option('-S', '--no-search-index', action='store_false', dest='search_index', default=True,
+ help='Don\'t build PDF file'),
make_option('-w', '--wait-until', dest='wait_until', metavar='TIME',
help='Wait until specified time (Y-M-D h:m:s)'),
+ make_option('-p', '--picture', action='store_true', dest='import_picture', default=False,
+ help='Import pictures'),
-
)
help = 'Imports books from the specified directories.'
args = 'directory [directory ...]'
+ def import_book(self, file_path, options):
+ verbose = options.get('verbose')
+ file_base, ext = os.path.splitext(file_path)
+ book = Book.from_xml_file(file_path, overwrite=options.get('force'),
- build_epub=options.get('build_epub'),
- build_txt=options.get('build_txt'),
- build_pdf=options.get('build_pdf'),
- build_mobi=options.get('build_mobi'))
++ build_epub=options.get('build_epub'),
++ build_txt=options.get('build_txt'),
++ build_pdf=options.get('build_pdf'),
++ build_mobi=options.get('build_mobi'),
++ search_index=options.get('search_index'))
+ fileid = book.fileid()
+ for ebook_format in Book.ebook_formats:
+ if os.path.isfile(file_base + '.' + ebook_format):
+ getattr(book, '%s_file' % ebook_format).save(
+ '%s.%s' % (fileid, ebook_format),
+ File(file(file_base + '.' + ebook_format)))
+ if verbose:
+ print "Importing %s.%s" % (file_base, ebook_format)
+
+ book.save()
+
+ def import_picture(self, file_path, options):
+ picture = Picture.from_xml_file(file_path, overwrite=options.get('force'))
+ return picture
+
def handle(self, *directories, **options):
from django.db import transaction
diff --cc apps/catalogue/models.py
index d615fb65b,1a2e8f86f..8fb210935
--- a/apps/catalogue/models.py
+++ b/apps/catalogue/models.py
@@@ -23,14 -22,17 +23,16 @@@ from django.conf import setting
from newtagging.models import TagBase, tags_updated
from newtagging import managers
from catalogue.fields import JSONField, OverwritingFileField
-from catalogue.utils import ExistingFile, ORMDocProvider, create_zip, remove_zip
+from catalogue.utils import create_zip, split_tags
+from catalogue.tasks import touch_tag
+from shutil import copy
+from glob import glob
+import re
+from os import path
-from librarian import dcparser, html, epub, NoDublinCore
-import mutagen
-from mutagen import id3
-from slughifi import slughifi
-from sortify import sortify
-from os import unlink
+ import search
+
TAG_CATEGORIES = (
('author', _('author')),
('epoch', _('epoch')),
@@@ -726,14 -622,26 +728,29 @@@ class Book(models.Model)
def zip_audiobooks(self):
bm = BookMedia.objects.filter(book=self, type='mp3')
paths = map(lambda bm: (None, bm.file.path), bm)
- result = create_zip.delay(paths, self.slug)
+ result = create_zip.delay(paths, self.fileid())
return result.wait()
+ def search_index(self):
+ if settings.SEARCH_INDEX_PARALLEL:
+ if instance(settings.SEARCH_INDEX_PARALLEL, int):
+ idx = search.ReusableIndex(threads=4)
+ else:
+ idx = search.ReusableIndex()
+ else:
+ idx = search.Index()
+
+ idx.open()
+ try:
+ idx.index_book(self)
+ finally:
+ idx.close()
+
@classmethod
def from_xml_file(cls, xml_file, **kwargs):
+ from django.core.files import File
+ from librarian import dcparser
+
# use librarian to parse meta-data
book_info = dcparser.parse(xml_file)
@@@ -747,9 -655,9 +764,10 @@@
@classmethod
def from_text_and_meta(cls, raw_file, book_info, overwrite=False,
- build_epub=True, build_txt=True, build_pdf=True, build_mobi=True):
+ build_epub=True, build_txt=True, build_pdf=True, build_mobi=True,
+ search_index=True):
import re
+ from sortify import sortify
# check for parts before we do anything
children = []
@@@ -815,12 -735,14 +833,15 @@@
if not settings.NO_BUILD_MOBI and build_mobi:
book.build_mobi()
+ if not settings.NO_SEARCH_INDEX and search_index:
+ book.search_index()
+
book_descendants = list(book.children.all())
+ descendants_tags = set()
# add l-tag to descendants and their fragments
- # delete unnecessary EPUB files
while len(book_descendants) > 0:
child_book = book_descendants.pop(0)
+ descendants_tags.update(child_book.tags)
child_book.tags = list(child_book.tags) + [book_tag]
child_book.save()
for fragment in child_book.fragments.all():
diff --cc lib/librarian
index 0718c9d23,d7ba2c607..5b407667c
--- a/lib/librarian
+++ b/lib/librarian
@@@ -1,1 -1,1 +1,1 @@@
- Subproject commit 0718c9d23a5791aa51bc11bba6a011fe9a8a616d
-Subproject commit d7ba2c607dacf7a6136b83a1588b5adf2278ad46
++Subproject commit 5b407667ca47cf4d9752821fd49e5611737146d2
diff --cc wolnelektury/settings.py
index b597e3347,826abb2ca..4184c4e66
--- a/wolnelektury/settings.py
+++ b/wolnelektury/settings.py
@@@ -152,7 -151,10 +153,8 @@@ INSTALLED_APPS =
'sponsors',
'stats',
'suggest',
+ 'picture',
+ 'search',
-
- #
- 'django_nose',
]
#CACHE_BACKEND = 'locmem:///?max_entries=3000'
diff --cc wolnelektury/templates/catalogue/search_multiple_hits.html
index 1eea6fc3b,dc3696ece..15c5d71d9
--- a/wolnelektury/templates/catalogue/search_multiple_hits.html
+++ b/wolnelektury/templates/catalogue/search_multiple_hits.html
@@@ -7,8 -7,37 +7,35 @@@
{% block bodyid %}tagged-object-list{% endblock %}
{% block body %}
- <form action="{% url search %}" method="get" accept-charset="utf-8" id="search-form">
- <p>{{ form.q }} <input type="submit" value="{% trans "Search" %}" /> <strong>{% trans "or" %}</strong> <a href="{% url main_page %}">{% trans "return to main page" %}</a></p>
- </form>
+ <h1>{% trans "Search" %}</h1>
+ <div id="results">
+ <ol>
+ {% for result in results %}
+ <li>
+ <p><a href="{{result.book.get_absolute_url}}">{{result.book.pretty_title}}</a> (id: {{result.book_id}}, score: {{result.score}})</p>
+ <ul>
+ {% for hit in result.process_hits %}
+ <li>
+ {% if hit.fragment %}
+ <div style="">Tagi/Motywy: {% for tag in hit.themes %}{{tag.name}} {% endfor %}</div>
+ {% endif %}
+ {% for snip in hit.snippets %}
+ {{snip|safe}}<br/>
+ {% endfor %}
+ </li>
+ {% endfor %}
+
+ </ul>
+ </li>
+ {% empty %}
+ <p>No results.</p>
+ {% endfor %}
+ </ol>
+ </div>
+
+
+ {% comment %}
<div id="books-list">
<p>{% trans "More than one result matching the criteria found." %}</p>
<ul class='matches'>
@@@ -23,5 -52,12 +50,6 @@@
{% endfor %}
</ul>
</div>
+ {% endcomment %}
- {% endblock %}
- <div id="set-window">
- <div class="header"><a href="#" class="jqmClose">{% trans "Close" %}</a></div>
- <div class="target">
- <p><img src="{{ STATIC_URL }}img/indicator.gif" alt="*"/> {% trans "Loading" %}</p>
- </div>
- </div>
+ {% endblock %}