From: Marcin Koziej Date: Mon, 30 Jan 2012 16:44:44 +0000 (+0100) Subject: Merge branch 'pretty' of github.com:fnp/wolnelektury into pretty X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/cb40f164b028883a2fc061307383d0aa98f8b609?hp=e353262951cc8b8d3f1a52b896a825bad743a5a7 Merge branch 'pretty' of github.com:fnp/wolnelektury into pretty Conflicts: wolnelektury/templates/catalogue/book_searched.html --- diff --git a/apps/catalogue/templatetags/catalogue_tags.py b/apps/catalogue/templatetags/catalogue_tags.py index cf80bebb5..e8bc13899 100644 --- a/apps/catalogue/templatetags/catalogue_tags.py +++ b/apps/catalogue/templatetags/catalogue_tags.py @@ -7,6 +7,7 @@ import feedparser from django import template from django.template import Node, Variable, Template, Context +from django.core.cache import cache from django.core.urlresolvers import reverse from django.contrib.auth.forms import UserCreationForm, AuthenticationForm from django.utils.translation import ugettext as _ @@ -266,17 +267,14 @@ def latest_blog_posts(feed_url, posts_to_show=5): def tag_list(tags, choices=None): if choices is None: choices = [] - if len(tags) == 1: + if len(tags) == 1 and tags[0].category not in [t.category for t in choices]: one_tag = tags[0] return locals() + @register.inclusion_tag('catalogue/inline_tag_list.html') def inline_tag_list(tags, choices=None): - if choices is None: - choices = [] - if len(tags) == 1: - one_tag = tags[0] - return locals() + return tag_list(tags, choices) @register.inclusion_tag('catalogue/book_info.html') @@ -348,14 +346,23 @@ def fragment_promo(arg=None): @register.inclusion_tag('catalogue/related_books.html') -def related_books(book, limit=6): - related = list(Book.objects.filter( - common_slug=book.common_slug).exclude(pk=book.pk)[:limit]) - limit -= len(related) - if limit: - related += Book.tagged.related_to(book, - Book.objects.exclude(common_slug=book.common_slug), - ignore_by_tag=book.book_tag())[:limit] +def related_books(book, limit=6, random=1): + cache_key = "catalogue.related_books.%d.%d" % (book.id, limit - random) + related = cache.get(cache_key) + if related is None: + print 'not in cache' + related = list(Book.objects.filter( + common_slug=book.common_slug).exclude(pk=book.pk)[:limit]) + limit -= len(related) + if limit > random: + related += Book.tagged.related_to(book, + Book.objects.exclude(common_slug=book.common_slug), + ignore_by_tag=book.book_tag())[:limit-random] + cache.set(cache_key, related, 1800) + if random: + related += list(Book.objects.exclude( + pk__in=[b.pk for b in related] + [book.pk] + ).order_by('?')[:random]) return { 'books': related, } diff --git a/apps/search/forms.py b/apps/search/forms.py index 60521f96a..9e0a07880 100755 --- a/apps/search/forms.py +++ b/apps/search/forms.py @@ -9,7 +9,7 @@ from search.fields import JQueryAutoCompleteSearchField class SearchForm(forms.Form): - q = JQueryAutoCompleteSearchField() # {'minChars': 2, 'selectFirst': True, 'cacheLength': 50, 'matchContains': "word"}) + q = JQueryAutoCompleteSearchField(label=_('Search')) # {'minChars': 2, 'selectFirst': True, 'cacheLength': 50, 'matchContains': "word"}) def __init__(self, source, *args, **kwargs): kwargs['auto_id'] = False @@ -18,4 +18,4 @@ class SearchForm(forms.Form): self.fields['q'].widget.attrs['autocomplete'] = 'off' self.fields['q'].widget.attrs['data-source'] = source if not 'q' in self.data: - self.fields['q'].widget.attrs['title'] = _('title, author, theme/topic, epoch, kind, genre, phrase') + self.fields['q'].widget.attrs['placeholder'] = _('title, author, theme/topic, epoch, kind, genre, phrase') diff --git a/apps/search/index.py b/apps/search/index.py index 6068fa2fe..71c0ed236 100644 --- a/apps/search/index.py +++ b/apps/search/index.py @@ -27,6 +27,7 @@ from librarian import dcparser from librarian.parser import WLDocument from lxml import etree import catalogue.models +from pdcounter.models import Author as PDCounterAuthor from multiprocessing.pool import ThreadPool from threading import current_thread import atexit @@ -219,6 +220,15 @@ class Index(BaseIndex): doc.add(Field("tag_category", tag.category, Field.Store.NO, Field.Index.NOT_ANALYZED)) self.index.addDocument(doc) + for pdtag in PDCounterAuthor.objects.all(): + doc = Document() + doc.add(NumericField("tag_id", Field.Store.YES, True).setIntValue(int(pdtag.id))) + doc.add(Field("tag_name", pdtag.name, Field.Store.NO, Field.Index.ANALYZED)) + doc.add(Field("tag_name_pl", pdtag.name, Field.Store.NO, Field.Index.ANALYZED)) + doc.add(Field("tag_category", 'pdcounter', Field.Store.NO, Field.Index.NOT_ANALYZED)) + doc.add(Field("is_pdcounter", 'true', Field.Store.YES, Field.Index.NOT_ANALYZED)) + self.index.addDocument(doc) + def create_book_doc(self, book): """ Create a lucene document referring book id. @@ -322,9 +332,10 @@ class Index(BaseIndex): # get published date source = book_info.source_name - match = self.published_date_re.search(source) - if match is not None: - fields["published_date"] = Field("published_date", str(match.groups()[0]), Field.Store.YES, Field.Index.NOT_ANALYZED) + if hasattr(book_info, 'source_name'): + match = self.published_date_re.search(source) + if match is not None: + fields["published_date"] = Field("published_date", str(match.groups()[0]), Field.Store.YES, Field.Index.NOT_ANALYZED) return fields @@ -624,26 +635,25 @@ class SearchResult(object): stored = search.searcher.doc(scoreDocs.doc) self.book_id = int(stored.get("book_id")) - header_type = stored.get("header_type") - if not header_type: - return - - sec = (header_type, int(stored.get("header_index"))) - header_span = stored.get('header_span') - header_span = header_span is not None and int(header_span) or 1 - - fragment = stored.get("fragment_anchor") - pd = stored.get("published_date") if pd is None: pd = 0 self.published_date = int(pd) - if snippets: - snippets = snippets.replace("/\n", "\n") - hit = (sec + (header_span,), fragment, scoreDocs.score, {'how_found': how_found, 'snippets': snippets and [snippets] or []}) + header_type = stored.get("header_type") + # we have a content hit in some header of fragment + if header_type is not None: + sec = (header_type, int(stored.get("header_index"))) + header_span = stored.get('header_span') + header_span = header_span is not None and int(header_span) or 1 + + fragment = stored.get("fragment_anchor") - self._hits.append(hit) + if snippets: + snippets = snippets.replace("/\n", "\n") + hit = (sec + (header_span,), fragment, scoreDocs.score, {'how_found': how_found, 'snippets': snippets and [snippets] or []}) + + self._hits.append(hit) self.search = search self.searched = searched @@ -1228,19 +1238,27 @@ class Search(IndexStore): if terms: return JArray('object')(terms, Term) - def search_tags(self, query, filter=None, max_results=40): + def search_tags(self, query, filters=None, max_results=40, pdcounter=False): """ Search for Tag objects using query. """ - tops = self.searcher.search(query, filter, max_results) + if not pdcounter: + filters = self.chain_filters([filter, self.term_filter(Term('is_pdcounter', 'true'), inverse=True)]) + tops = self.searcher.search(query, filters, max_results) tags = [] for found in tops.scoreDocs: doc = self.searcher.doc(found.doc) - tag = catalogue.models.Tag.objects.get(id=doc.get("tag_id")) - tags.append(tag) - print "%s (%d) -> %f" % (tag, tag.id, found.score) - + is_pdcounter = doc.get('is_pdcounter') + if is_pdcounter: + tag = PDCounterAuthor.objects.get(id=doc.get('tag_id')) + else: + tag = catalogue.models.Tag.objects.get(id=doc.get("tag_id")) + # don't add the pdcounter tag if same tag already exists + if not (is_pdcounter and filter(lambda t: tag.slug == t.slug, tags)): + tags.append(tag) + # print "%s (%d) -> %f" % (tag, tag.id, found.score) + print 'returning %s' % tags return tags def search_books(self, query, filter=None, max_results=10): @@ -1254,7 +1272,7 @@ class Search(IndexStore): bks.append(catalogue.models.Book.objects.get(id=doc.get("book_id"))) return bks - def create_prefix_phrase(self, toks, field): + def make_prefix_phrase(self, toks, field): q = MultiPhraseQuery() for i in range(len(toks)): t = Term(field, toks[i]) @@ -1280,7 +1298,7 @@ class Search(IndexStore): return only_term - def hint_tags(self, string, max_results=50): + def hint_tags(self, string, max_results=50, pdcounter=True, prefix=True): """ Return auto-complete hints for tags using prefix search. @@ -1289,14 +1307,17 @@ class Search(IndexStore): top = BooleanQuery() for field in ['tag_name', 'tag_name_pl']: - q = self.create_prefix_phrase(toks, field) + if prefix: + q = self.make_prefix_phrase(toks, field) + else: + q = self.make_term_query(toks, field) top.add(BooleanClause(q, BooleanClause.Occur.SHOULD)) no_book_cat = self.term_filter(Term("tag_category", "book"), inverse=True) - return self.search_tags(top, no_book_cat, max_results=max_results) + return self.search_tags(top, no_book_cat, max_results=max_results, pdcounter=pdcounter) - def hint_books(self, string, max_results=50): + def hint_books(self, string, max_results=50, prefix=True): """ Returns auto-complete hints for book titles Because we do not index 'pseudo' title-tags. @@ -1304,7 +1325,10 @@ class Search(IndexStore): """ toks = self.get_tokens(string, field='SIMPLE') - q = self.create_prefix_phrase(toks, 'title') + if prefix: + q = self.make_prefix_phrase(toks, 'title') + else: + q = self.make_term_query(toks, 'title') return self.search_books(q, self.term_filter(Term("is_book", "true")), max_results=max_results) diff --git a/apps/search/views.py b/apps/search/views.py index cf008705d..623b311fb 100644 --- a/apps/search/views.py +++ b/apps/search/views.py @@ -8,7 +8,7 @@ from django.views.decorators import cache from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponsePermanentRedirect from django.utils.translation import ugettext as _ -from catalogue.utils import get_random_hash +from catalogue.utils import split_tags from catalogue.models import Book, Tag, Fragment from catalogue.fields import dumps from catalogue.views import JSONResponse @@ -34,7 +34,7 @@ def did_you_mean(query, tokens): authors = Tag.objects.filter(category='author', name__iregex=match_word_re(t)) if len(authors) > 0: continue - + if not dictionary.check(t): try: change[t] = dictionary.suggest(t)[0] @@ -69,7 +69,7 @@ def hint(request): # jezeli tagi dot tylko ksiazki, to wazne zeby te nowe byly w tej samej ksiazce # jesli zas dotycza themes, to wazne, zeby byly w tym samym fragmencie. - tags = s.hint_tags(prefix) + tags = s.hint_tags(prefix, pdcounter=True) books = s.hint_books(prefix) # TODO DODAC TU HINTY @@ -97,26 +97,28 @@ def main(request): fuzzy = False if 'q' in request.GET: - tags = request.GET.get('tags', '') + # tags = request.GET.get('tags', '') query = request.GET['q'] - book_id = request.GET.get('book', None) - book = None - if book_id is not None: - book = get_object_or_404(Book, id=book_id) + # book_id = request.GET.get('book', None) + # book = None + # if book_id is not None: + # book = get_object_or_404(Book, id=book_id) - hint = srch.hint() - try: - tag_list = Tag.get_tag_list(tags) - except: - tag_list = [] + # hint = srch.hint() + # try: + # tag_list = Tag.get_tag_list(tags) + # except: + # tag_list = [] if len(query) < 2: - return render_to_response('catalogue/search_too_short.html', {'tags': tag_list, 'prefix': query}, + return render_to_response('catalogue/search_too_short.html', {'prefix': query}, context_instance=RequestContext(request)) - hint.tags(tag_list) - if book: - hint.books(book) + # hint.tags(tag_list) + # if book: + # hint.books(book) + tags = srch.hint_tags(query, pdcounter=True, prefix=False) + tags = split_tags(tags) toks = StringReader(query) tokens_cache = {} @@ -160,7 +162,7 @@ def main(request): author_results = SearchResult.aggregate(author_results) title_results = SearchResult.aggregate(title_results) - + everywhere = SearchResult.aggregate(everywhere, author_title_rest) for res in [author_results, title_results, text_phrase, everywhere]: @@ -168,15 +170,15 @@ def main(request): for r in res: for h in r.hits: h['snippets'] = map(lambda s: - re.subn(r"(^[ \t\n]+|[ \t\n]+$)", u"", + re.subn(r"(^[ \t\n]+|[ \t\n]+$)", u"", re.subn(r"[ \t\n]*\n[ \t\n]*", u"\n", s)[0])[0], h['snippets']) - + suggestion = did_you_mean(query, srch.get_tokens(toks, field="SIMPLE")) print "dym? %s" % repr(suggestion).encode('utf-8') - + results = author_results + title_results + text_phrase + everywhere results.sort(reverse=True) - + if len(results) == 1: fragment_hits = filter(lambda h: 'fragment' in h, results[0].hits) if len(fragment_hits) == 1: @@ -187,14 +189,15 @@ def main(request): elif len(results) == 0: form = PublishingSuggestForm(initial={"books": query + ", "}) return render_to_response('catalogue/search_no_hits.html', - {'tags': tag_list, + {'tags': tags, 'prefix': query, "form": form, 'did_you_mean': suggestion}, context_instance=RequestContext(request)) + print "TAGS: %s" % tags return render_to_response('catalogue/search_multiple_hits.html', - {'tags': tag_list, + {'tags': tags, 'prefix': query, 'results': { 'author': author_results, 'title': title_results, diff --git a/apps/social/forms.py b/apps/social/forms.py index bbdc43ce5..cfa963177 100755 --- a/apps/social/forms.py +++ b/apps/social/forms.py @@ -19,7 +19,8 @@ class UserSetsForm(forms.Form): class ObjectSetsForm(forms.Form): - tags = forms.CharField(label=_('Tags (comma-separated)'), required=False) + tags = forms.CharField(label=_('Tags (comma-separated)'), required=False, + widget=forms.Textarea()) def __init__(self, obj, user, *args, **kwargs): self._obj = obj diff --git a/apps/social/templates/social/sets_form.html b/apps/social/templates/social/sets_form.html index ab982fe42..2ea1a867d 100755 --- a/apps/social/templates/social/sets_form.html +++ b/apps/social/templates/social/sets_form.html @@ -6,7 +6,8 @@ -
+
    {{ form.as_ul }} diff --git a/apps/social/templates/social/shelf_tags.html b/apps/social/templates/social/shelf_tags.html index 70108eb93..eabc961aa 100755 --- a/apps/social/templates/social/shelf_tags.html +++ b/apps/social/templates/social/shelf_tags.html @@ -1,4 +1,4 @@ -{% if tags %} +{% if tags.exists %}