From a73444333e9557df72656fd1f49c01933ff21348 Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Fri, 15 Dec 2017 10:43:22 +0100 Subject: [PATCH 01/16] better default limit in search hint --- src/search/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/search/views.py b/src/search/views.py index b0f064119..8542e512f 100644 --- a/src/search/views.py +++ b/src/search/views.py @@ -71,10 +71,10 @@ def hint(request): try: limit = int(request.GET.get('max', '')) except ValueError: - limit = -1 + limit = 20 else: if limit < 1: - limit = -1 + limit = 20 data = [ { @@ -83,7 +83,7 @@ def hint(request): 'id': author.id, 'url': author.get_absolute_url(), } - for author in Tag.objects.filter(category='author', name__iregex=u'\m' + prefix)[:10] + for author in Tag.objects.filter(category='author', name__iregex=u'\m' + prefix)[:limit] ] if len(data) < limit: data += [ -- 2.20.1 From 3ba4d1e44766528f0f5b99a825243dd976a7f03c Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Fri, 15 Dec 2017 10:43:43 +0100 Subject: [PATCH 02/16] outstanding migration --- .../migrations/0017_auto_20171214_1746.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/catalogue/migrations/0017_auto_20171214_1746.py diff --git a/src/catalogue/migrations/0017_auto_20171214_1746.py b/src/catalogue/migrations/0017_auto_20171214_1746.py new file mode 100644 index 000000000..9095cb0ec --- /dev/null +++ b/src/catalogue/migrations/0017_auto_20171214_1746.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalogue', '0016_auto_20171031_1232'), + ] + + operations = [ + migrations.AlterField( + model_name='book', + name='changed_at', + field=models.DateTimeField(auto_now=True, verbose_name='change date', db_index=True), + ), + ] -- 2.20.1 From 33f38948f6f42804f6baf7e431bb9a143daf0884 Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Fri, 15 Dec 2017 13:14:19 +0100 Subject: [PATCH 03/16] remove unused code --- src/catalogue/tests/__init__.py | 1 - src/catalogue/tests/search.py | 75 ---------- src/catalogue/urls.py | 2 - src/catalogue/views.py | 237 +------------------------------- 4 files changed, 1 insertion(+), 314 deletions(-) delete mode 100644 src/catalogue/tests/search.py diff --git a/src/catalogue/tests/__init__.py b/src/catalogue/tests/__init__.py index 9c41af207..4eb8e8dd6 100644 --- a/src/catalogue/tests/__init__.py +++ b/src/catalogue/tests/__init__.py @@ -5,7 +5,6 @@ from catalogue.tests.book_import import * from catalogue.tests.bookmedia import * from catalogue.tests.cover import * -from catalogue.tests.search import * from catalogue.tests.tags import * from catalogue.tests.templatetags import * from .visit import * diff --git a/src/catalogue/tests/search.py b/src/catalogue/tests/search.py deleted file mode 100644 index b6b03d828..000000000 --- a/src/catalogue/tests/search.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. -# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. -# -from catalogue import models, views -from catalogue.test_utils import * - -from nose.tools import raises - - -class BasicSearchLogicTests(WLTestCase): - - def setUp(self): - WLTestCase.setUp(self) - self.author_tag = models.Tag.objects.create( - name=u'Adam Mickiewicz [SubWord]', - category=u'author', slug="one") - - self.unicode_tag = models.Tag.objects.create( - name=u'Tadeusz Żeleński (Boy)', - category=u'author', slug="two") - - self.polish_tag = models.Tag.objects.create( - name=u'ĘÓĄŚŁŻŹĆŃęóąśłżźćń', - category=u'author', slug="three") - - @raises(ValueError) - def test_empty_query(self): - """ Check that empty queries raise an error. """ - views.find_best_matches(u'') - - @raises(ValueError) - def test_one_letter_query(self): - """ Check that one letter queries aren't permitted. """ - views.find_best_matches(u't') - - def test_match_by_prefix(self): - """ Tags should be matched by prefix of words within it's name. """ - self.assertEqual(views.find_best_matches(u'Ada'), (self.author_tag,)) - self.assertEqual(views.find_best_matches(u'Mic'), (self.author_tag,)) - self.assertEqual(views.find_best_matches(u'Mickiewicz'), (self.author_tag,)) - - def test_match_case_insensitive(self): - """ Tag names should match case insensitive. """ - self.assertEqual(views.find_best_matches(u'adam mickiewicz'), (self.author_tag,)) - - def test_match_case_insensitive_unicode(self): - """ Tag names should match case insensitive (unicode). """ - self.assertEqual(views.find_best_matches(u'tadeusz żeleński (boy)'), (self.unicode_tag,)) - - def test_word_boundary(self): - self.assertEqual(views.find_best_matches(u'SubWord'), (self.author_tag,)) - self.assertEqual(views.find_best_matches(u'[SubWord'), (self.author_tag,)) - - def test_unrelated_search(self): - self.assertEqual(views.find_best_matches(u'alamakota'), tuple()) - self.assertEqual(views.find_best_matches(u'Adama'), ()) - - def test_infix_doesnt_match(self): - """ Searching for middle of a word shouldn't match. """ - self.assertEqual(views.find_best_matches(u'deusz'), tuple()) - - def test_diactricts_removal_pl(self): - """ Tags should match both with and without national characters. """ - self.assertEqual(views.find_best_matches(u'ĘÓĄŚŁŻŹĆŃęóąśłżźćń'), (self.polish_tag,)) - self.assertEqual(views.find_best_matches(u'EOASLZZCNeoaslzzcn'), (self.polish_tag,)) - self.assertEqual(views.find_best_matches(u'eoaslzzcneoaslzzcn'), (self.polish_tag,)) - - def test_diactricts_query_removal_pl(self): - """ Tags without national characters shouldn't be matched by queries with them. """ - self.assertEqual(views.find_best_matches(u'Adąm'), ()) - - def test_sloppy(self): - self.assertEqual(views.find_best_matches(u'Żelenski'), (self.unicode_tag,)) - self.assertEqual(views.find_best_matches(u'zelenski'), (self.unicode_tag,)) diff --git a/src/catalogue/urls.py b/src/catalogue/urls.py index fb6a7b465..addb0cba0 100644 --- a/src/catalogue/urls.py +++ b/src/catalogue/urls.py @@ -50,8 +50,6 @@ urlpatterns += patterns( url(r'^lektury/(?P[a-zA-Z0-9-]+)/$', 'collection', name='collection'), url(r'^audiobooki/$', 'audiobooks', name='audiobook_list'), url(r'^daisy/$', 'daisy_list', name='daisy_list'), - url(r'^tags/$', 'tags_starting_with', name='hint'), - url(r'^jtags/?$', 'json_tags_starting_with', name='jhint'), url(r'^nowe/$', ListView.as_view( queryset=Book.objects.filter(parent=None).order_by('-created_at'), template_name='catalogue/recent_list.html'), name='recent_list'), diff --git a/src/catalogue/views.py b/src/catalogue/views.py index a247746d0..bba773d32 100644 --- a/src/catalogue/views.py +++ b/src/catalogue/views.py @@ -3,14 +3,13 @@ # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # from collections import OrderedDict -import re import random from django.conf import settings from django.template import RequestContext from django.template.loader import render_to_string from django.shortcuts import render_to_response, get_object_or_404, render, redirect -from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponsePermanentRedirect, JsonResponse +from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponsePermanentRedirect from django.core.urlresolvers import reverse from django.db.models import Q, QuerySet from django.contrib.auth.decorators import login_required, user_passes_test @@ -19,11 +18,9 @@ from django.utils import translation from django.utils.translation import ugettext as _, ugettext_lazy from ajaxable.utils import AjaxableFormView -from pdcounter.models import BookStub, Author from pdcounter import views as pdcounter_views from picture.models import Picture, PictureArea from ssify import ssi_included, ssi_expect, SsiVariable as Var -from suggest.forms import PublishingSuggestForm from catalogue import constants from catalogue import forms from catalogue.helpers import get_top_level_related_tags @@ -310,238 +307,6 @@ def book_text(request, slug): return render_to_response('catalogue/book_text.html', {'book': book}, context_instance=RequestContext(request)) -# ========== -# = Search = -# ========== - -def _no_diacritics_regexp(query): - """ returns a regexp for searching for a query without diacritics - - should be locale-aware """ - names = { - u'a': u'aąĄ', u'c': u'cćĆ', u'e': u'eęĘ', u'l': u'lłŁ', u'n': u'nńŃ', u'o': u'oóÓ', u's': u'sśŚ', - u'z': u'zźżŹŻ', - u'ą': u'ąĄ', u'ć': u'ćĆ', u'ę': u'ęĘ', u'ł': u'łŁ', u'ń': u'ńŃ', u'ó': u'óÓ', u'ś': u'śŚ', u'ź': u'źŹ', - u'ż': u'żŻ' - } - - def repl(m): - l = m.group() - return u"(?:%s)" % '|'.join(names[l]) - - return re.sub(u'[%s]' % (u''.join(names.keys())), repl, query) - - -def unicode_re_escape(query): - """ Unicode-friendly version of re.escape """ - s = list(query) - for i, c in enumerate(query): - if re.match(r'(?u)(\W)', c) and re.match(r'[\x00-\x7e]', c): - if c == "\000": - s[i] = "\\000" - else: - s[i] = "\\" + c - return query[:0].join(s) - - -def _word_starts_with(name, prefix): - """returns a Q object getting models having `name` contain a word - starting with `prefix` - - We define word characters as alphanumeric and underscore, like in JS. - - Works for MySQL, PostgreSQL, Oracle. - For SQLite, _sqlite* version is substituted for this. - """ - kwargs = {} - - prefix = _no_diacritics_regexp(unicode_re_escape(prefix)) - # can't use [[:<:]] (word start), - # but we want both `xy` and `(xy` to catch `(xyz)` - kwargs['%s__iregex' % name] = u"(^|[^[:alnum:]_])%s" % prefix - - return Q(**kwargs) - - -def _word_starts_with_regexp(prefix): - prefix = _no_diacritics_regexp(unicode_re_escape(prefix)) - return ur"(^|(?<=[^\wąćęłńóśźżĄĆĘŁŃÓŚŹŻ]))%s" % prefix - - -def _sqlite_word_starts_with(name, prefix): - """ version of _word_starts_with for SQLite - - SQLite in Django uses Python re module - """ - kwargs = {'%s__iregex' % name: _word_starts_with_regexp(prefix)} - return Q(**kwargs) - - -if hasattr(settings, 'DATABASES'): - if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': - _word_starts_with = _sqlite_word_starts_with -elif settings.DATABASE_ENGINE == 'sqlite3': - _word_starts_with = _sqlite_word_starts_with - - -class App: - def __init__(self, name, view): - self.name = name - self._view = view - self.lower = name.lower() - self.category = 'application' - - def view(self): - return reverse(*self._view) - -_apps = ( - App(u'Leśmianator', (u'lesmianator', )), - ) - - -def _tags_starting_with(prefix, user=None): - prefix = prefix.lower() - # PD counter - book_stubs = BookStub.objects.filter(_word_starts_with('title', prefix)) - authors = Author.objects.filter(_word_starts_with('name', prefix)) - - books = Book.objects.filter(_word_starts_with('title', prefix)) - tags = Tag.objects.filter(_word_starts_with('name', prefix)) - if user and user.is_authenticated(): - tags = tags.filter(~Q(category='set') | Q(user=user)) - else: - tags = tags.exclude(category='set') - - prefix_regexp = re.compile(_word_starts_with_regexp(prefix)) - return list(books) + list(tags) + [app for app in _apps if prefix_regexp.search(app.lower)] + list(book_stubs) + \ - list(authors) - - -def _get_result_link(match, tag_list): - if isinstance(match, Tag): - return reverse('catalogue.views.tagged_object_list', - kwargs={'tags': '/'.join(tag.url_chunk for tag in tag_list + [match])}) - elif isinstance(match, App): - return match.view() - else: - return match.get_absolute_url() - - -def _get_result_type(match): - if isinstance(match, Book) or isinstance(match, BookStub): - match_type = 'book' - else: - match_type = match.category - return match_type - - -def books_starting_with(prefix): - prefix = prefix.lower() - return Book.objects.filter(_word_starts_with('title', prefix)) - - -def find_best_matches(query, user=None): - """ Finds a Book, Tag, BookStub or Author best matching a query. - - Returns a with: - - zero elements when nothing is found, - - one element when a best result is found, - - more then one element on multiple exact matches - - Raises a ValueError on too short a query. - """ - - query = query.lower() - if len(query) < 2: - raise ValueError("query must have at least two characters") - - result = tuple(_tags_starting_with(query, user)) - # remove pdcounter stuff - book_titles = set(match.pretty_title().lower() for match in result - if isinstance(match, Book)) - authors = set(match.name.lower() for match in result - if isinstance(match, Tag) and match.category == 'author') - result = tuple(res for res in result if not ( - (isinstance(res, BookStub) and res.pretty_title().lower() in book_titles) or - (isinstance(res, Author) and res.name.lower() in authors) - )) - - exact_matches = tuple(res for res in result if res.name.lower() == query) - if exact_matches: - return exact_matches - else: - return tuple(result)[:1] - - -def search(request): - tags = request.GET.get('tags', '') - prefix = request.GET.get('q', '') - - try: - tag_list = Tag.get_tag_list(tags) - except (Tag.DoesNotExist, Tag.MultipleObjectsReturned, Tag.UrlDeprecationWarning): - tag_list = [] - - try: - result = find_best_matches(prefix, request.user) - except ValueError: - return render_to_response( - 'catalogue/search_too_short.html', {'tags': tag_list, 'prefix': prefix}, - context_instance=RequestContext(request)) - - if len(result) == 1: - return HttpResponseRedirect(_get_result_link(result[0], tag_list)) - elif len(result) > 1: - return render_to_response( - 'catalogue/search_multiple_hits.html', - { - 'tags': tag_list, 'prefix': prefix, - 'results': ((x, _get_result_link(x, tag_list), _get_result_type(x)) for x in result) - }, - context_instance=RequestContext(request)) - else: - form = PublishingSuggestForm(initial={"books": prefix + ", "}) - return render_to_response( - 'catalogue/search_no_hits.html', - {'tags': tag_list, 'prefix': prefix, "pubsuggest_form": form}, - context_instance=RequestContext(request)) - - -def tags_starting_with(request): - prefix = request.GET.get('q', '') - # Prefix must have at least 2 characters - if len(prefix) < 2: - return HttpResponse('') - tags_list = [] - result = "" - for tag in _tags_starting_with(prefix, request.user): - if tag.name not in tags_list: - result += "\n" + tag.name - tags_list.append(tag.name) - return HttpResponse(result) - - -def json_tags_starting_with(request, callback=None): - # Callback for JSONP - prefix = request.GET.get('q', '') - callback = request.GET.get('callback', '') - # Prefix must have at least 2 characters - if len(prefix) < 2: - return HttpResponse('') - tags_list = [] - for tag in _tags_starting_with(prefix, request.user): - if tag.name not in tags_list: - tags_list.append(tag.name) - if request.GET.get('mozhint', ''): - result = [prefix, tags_list] - else: - result = {"matches": tags_list} - response = JsonResponse(result, safe=False) - if callback: - response.content = callback + "(" + response.content + ");" - return response - - # ========= # = Admin = # ========= -- 2.20.1 From 1801c65004af14a8ed50d60e47d75a415753bbf7 Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Fri, 15 Dec 2017 17:44:05 +0100 Subject: [PATCH 04/16] minor changes in search --- src/search/context_processors.py | 2 +- src/search/views.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/search/context_processors.py b/src/search/context_processors.py index 6ad2fe9e2..63e328583 100644 --- a/src/search/context_processors.py +++ b/src/search/context_processors.py @@ -7,4 +7,4 @@ from search.forms import SearchForm def search_form(request): - return {'search_form': SearchForm(reverse('search.views.hint')+'?max=10', request.GET)} + return {'search_form': SearchForm(reverse('search_hint') + '?max=10', request.GET)} diff --git a/src/search/views.py b/src/search/views.py index 8542e512f..64382e24b 100644 --- a/src/search/views.py +++ b/src/search/views.py @@ -7,11 +7,9 @@ from django.shortcuts import render_to_response from django.template import RequestContext from django.views.decorators import cache from django.http import HttpResponse, JsonResponse -from django.utils.translation import ugettext as _ from catalogue.utils import split_tags from catalogue.models import Book, Tag -from pdcounter.models import Author as PDCounterAuthor, BookStub as PDCounterBook from search.index import Search, SearchResult from suggest.forms import PublishingSuggestForm import re @@ -76,20 +74,20 @@ def hint(request): if limit < 1: limit = 20 + authors = Tag.objects.filter( + category='author', name__iregex='\m' + prefix).only('name', 'id', 'slug', 'category') data = [ { 'label': author.name, - 'category': _('author'), 'id': author.id, 'url': author.get_absolute_url(), } - for author in Tag.objects.filter(category='author', name__iregex=u'\m' + prefix)[:limit] + for author in authors[:limit] ] if len(data) < limit: data += [ { 'label': '%s, %s' % (b.title, b.author_unicode()), - 'category': _('book'), 'id': b.id, 'url': b.get_absolute_url() } -- 2.20.1 From 05e5b36046d980e84776978660f420b008db4879 Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Mon, 18 Dec 2017 10:41:11 +0100 Subject: [PATCH 05/16] skip fragments with no theme hit --- src/search/index.py | 2 +- src/search/templatetags/search_tags.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/search/index.py b/src/search/index.py index 70214c554..0bd5c0530 100644 --- a/src/search/index.py +++ b/src/search/index.py @@ -670,7 +670,7 @@ class SearchResult(object): m.update(f[self.OTHER]) hits.append(m) - hits.sort(lambda a, b: cmp(a['score'], b['score']), reverse=True) + hits.sort(key=lambda h: h['score'], reverse=True) self._processed_hits = hits diff --git a/src/search/templatetags/search_tags.py b/src/search/templatetags/search_tags.py index f03aa6762..da5a85bc7 100644 --- a/src/search/templatetags/search_tags.py +++ b/src/search/templatetags/search_tags.py @@ -25,7 +25,7 @@ def book_searched(context, result): # We don't need hits which lead to sections but do not have # snippets. hits = filter(lambda (idx, h): - result.snippets[idx] is not None or 'fragment' in h, + result.snippets[idx] is not None or ('fragment' in h and h['themes_hit']), enumerate(result.hits)) # print "[tmpl: from %d hits selected %d]" % (len(result.hits), len(hits)) -- 2.20.1 From fe14330cee64ab56c479ff83a4eb29a214c68d8f Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Mon, 18 Dec 2017 15:05:20 +0100 Subject: [PATCH 06/16] fix weird bug with author hints --- src/search/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/search/views.py b/src/search/views.py index 64382e24b..309db73f4 100644 --- a/src/search/views.py +++ b/src/search/views.py @@ -75,7 +75,7 @@ def hint(request): limit = 20 authors = Tag.objects.filter( - category='author', name__iregex='\m' + prefix).only('name', 'id', 'slug', 'category') + category='author', name_pl__iregex='\m' + prefix).only('name', 'id', 'slug', 'category') data = [ { 'label': author.name, -- 2.20.1 From 2395dcee4ca15b655b7500d6105176d7635fff1f Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Mon, 18 Dec 2017 15:38:36 +0100 Subject: [PATCH 07/16] fix pipeline css in audiobook --- src/catalogue/templates/catalogue/player.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalogue/templates/catalogue/player.html b/src/catalogue/templates/catalogue/player.html index 2e32f711f..8412a3a35 100755 --- a/src/catalogue/templates/catalogue/player.html +++ b/src/catalogue/templates/catalogue/player.html @@ -12,7 +12,7 @@ {% trans "Wolne Lektury" %} :: {{ book.title }} - {{ audiobook }} - {% stylesheet "all" %} + {% stylesheet 'main' %} {% stylesheet "player" %} -- 2.20.1 From c0da122946a4af5c4f4e39a7cf17c7a1443b07cb Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Wed, 20 Dec 2017 12:41:08 +0100 Subject: [PATCH 08/16] opowiada/czyta --- src/wolnelektury/templates/bajki.html | 3 +++ src/wolnelektury/templates/podcast.html | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 src/wolnelektury/templates/bajki.html diff --git a/src/wolnelektury/templates/bajki.html b/src/wolnelektury/templates/bajki.html new file mode 100644 index 000000000..7d65c0a87 --- /dev/null +++ b/src/wolnelektury/templates/bajki.html @@ -0,0 +1,3 @@ +{% extends 'podcast.html' %} + +{% block artist_label %}Opowiada{% endblock %} \ No newline at end of file diff --git a/src/wolnelektury/templates/podcast.html b/src/wolnelektury/templates/podcast.html index 17c1d82fa..0fdb07193 100644 --- a/src/wolnelektury/templates/podcast.html +++ b/src/wolnelektury/templates/podcast.html @@ -28,7 +28,7 @@
  • {{ title }}
    - {% if artist %}Czyta {{ artist }},{% endif %} + {% if artist %}{% block artist_label %}Czyta{% endblock %} {{ artist }},{% endif %} {% if director %}reż. {{ director }}{% endif %}
    -- 2.20.1 From c34d6e5cea353a47408db1f7e78574b4f1e9c7e7 Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Thu, 21 Dec 2017 10:40:23 +0100 Subject: [PATCH 09/16] add slug back to filter-books api --- src/api/handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/handlers.py b/src/api/handlers.py index 88106260a..159e4f235 100644 --- a/src/api/handlers.py +++ b/src/api/handlers.py @@ -301,7 +301,7 @@ class QuerySetProxy(models.QuerySet): class FilterBooksHandler(AnonymousBooksHandler): fields = book_tag_categories + [ - 'href', 'title', 'url', 'cover', 'cover_thumb', 'key', 'cover_source_image'] + 'href', 'title', 'url', 'cover', 'cover_thumb', 'slug', 'key', 'cover_source_image'] def read(self, request): key_sep = '$' -- 2.20.1 From 4424d39a5d12ff6e16853b1aca23def0df33cc80 Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Thu, 21 Dec 2017 10:58:28 +0100 Subject: [PATCH 10/16] allow empty part name in bookmedia --- .../migrations/0018_auto_20171221_1106.py | 19 +++++++++++++++++++ src/catalogue/models/bookmedia.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 src/catalogue/migrations/0018_auto_20171221_1106.py diff --git a/src/catalogue/migrations/0018_auto_20171221_1106.py b/src/catalogue/migrations/0018_auto_20171221_1106.py new file mode 100644 index 000000000..e13685d7a --- /dev/null +++ b/src/catalogue/migrations/0018_auto_20171221_1106.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalogue', '0017_auto_20171214_1746'), + ] + + operations = [ + migrations.AlterField( + model_name='bookmedia', + name='part_name', + field=models.CharField(default=b'', max_length=512, verbose_name='part name', blank=True), + ), + ] diff --git a/src/catalogue/models/bookmedia.py b/src/catalogue/models/bookmedia.py index 67d02790c..0bf92db90 100644 --- a/src/catalogue/models/bookmedia.py +++ b/src/catalogue/models/bookmedia.py @@ -30,7 +30,7 @@ class BookMedia(models.Model): type = models.CharField(_('type'), db_index=True, choices=format_choices, max_length=20) name = models.CharField(_('name'), max_length=512) - part_name = models.CharField(_('part name'), default='', max_length=512) + part_name = models.CharField(_('part name'), default='', blank=True, max_length=512) index = models.IntegerField(_('index'), default=0) file = models.FileField(_('file'), max_length=600, upload_to=_file_upload_to, storage=OverwriteStorage()) uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False, db_index=True) -- 2.20.1 From 3f7ea05878c9fa9ac9ff13211e48773ab6cf64eb Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Thu, 21 Dec 2017 11:58:41 +0100 Subject: [PATCH 11/16] minor fix for django 1.9 --- src/contact/widgets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/contact/widgets.py b/src/contact/widgets.py index 785e019fd..62a4976d5 100644 --- a/src/contact/widgets.py +++ b/src/contact/widgets.py @@ -3,7 +3,7 @@ # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # from django import forms -from django.forms.util import flatatt +from django.forms.utils import flatatt from django.utils.html import format_html -- 2.20.1 From 5d48d148b34690b425b1f7ff5a6d38c157fd4362 Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Thu, 21 Dec 2017 11:59:38 +0100 Subject: [PATCH 12/16] ignore abstrakt tags when indexing book --- src/search/index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/search/index.py b/src/search/index.py index 0bd5c0530..bfb1739b1 100644 --- a/src/search/index.py +++ b/src/search/index.py @@ -271,14 +271,14 @@ class Index(SolrIndex): 'dramat_wierszowany_lp', 'dramat_wspolczesny', 'liryka_l', 'liryka_lp', 'wywiad', - ] + ] ignore_content_tags = [ - 'uwaga', 'extra', 'nota_red', + 'uwaga', 'extra', 'nota_red', 'abstrakt', 'zastepnik_tekstu', 'sekcja_asterysk', 'separator_linia', 'zastepnik_wersu', 'didaskalia', 'naglowek_aktu', 'naglowek_sceny', 'naglowek_czesc', - ] + ] footnote_tags = ['pa', 'pt', 'pr', 'pe'] -- 2.20.1 From 09f311cf21633def6e6eda0549633cd07b363cf2 Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Thu, 21 Dec 2017 12:05:47 +0100 Subject: [PATCH 13/16] book/picture-only tags --- src/catalogue/fields.py | 5 +++ .../migrations/0019_auto_20171221_1107.py | 35 +++++++++++++++++++ src/catalogue/models/book.py | 5 +++ src/catalogue/models/tag.py | 3 ++ src/picture/models.py | 14 +++++++- 5 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 src/catalogue/migrations/0019_auto_20171221_1107.py diff --git a/src/catalogue/fields.py b/src/catalogue/fields.py index 21d2bcf72..372cf2735 100644 --- a/src/catalogue/fields.py +++ b/src/catalogue/fields.py @@ -193,6 +193,7 @@ class BuildHtml(BuildEbook): tag.name = theme_name setattr(tag, "name_%s" % lang, theme_name) tag.sort_key = sortify(theme_name.lower()) + tag.for_books = True tag.save() themes.append(tag) elif lang is not None: @@ -214,6 +215,10 @@ class BuildHtml(BuildEbook): new_fragment.save() new_fragment.tags = set(meta_tags + themes) + for theme in themes: + if not theme.for_books: + theme.for_books = True + theme.save() book.html_built.send(sender=type(self), instance=book) return True return False diff --git a/src/catalogue/migrations/0019_auto_20171221_1107.py b/src/catalogue/migrations/0019_auto_20171221_1107.py new file mode 100644 index 000000000..751e707d8 --- /dev/null +++ b/src/catalogue/migrations/0019_auto_20171221_1107.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +def init_tag_for_books_pictures(apps, schema_editor): + Tag = apps.get_model('catalogue', 'Tag') + db_alias = schema_editor.connection.alias + tag_objects = Tag.objects.using(db_alias).exclude(category='set') + tags_for_books = tag_objects.filter(items__content_type__model__in=('book', 'fragment')).distinct() + tags_for_books.update(for_books=True) + tags_for_pictures = tag_objects.filter(items__content_type__model__in=('picture', 'picturearea')).distinct() + tags_for_pictures.update(for_pictures=True) + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalogue', '0018_auto_20171221_1106'), + ] + + operations = [ + migrations.AddField( + model_name='tag', + name='for_books', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='tag', + name='for_pictures', + field=models.BooleanField(default=False), + ), + migrations.RunPython(init_tag_for_books_pictures, migrations.RunPython.noop) + ] diff --git a/src/catalogue/models/book.py b/src/catalogue/models/book.py index 12fdb8d3d..44ee02ecc 100644 --- a/src/catalogue/models/book.py +++ b/src/catalogue/models/book.py @@ -391,6 +391,11 @@ class Book(models.Model): meta_tags = Tag.tags_from_info(book_info) + for tag in meta_tags: + if not tag.for_books: + tag.for_books = True + tag.save() + book.tags = set(meta_tags + book_shelves) cover_changed = old_cover != book.cover_info() diff --git a/src/catalogue/models/tag.py b/src/catalogue/models/tag.py index 828a8b191..830f29f69 100644 --- a/src/catalogue/models/tag.py +++ b/src/catalogue/models/tag.py @@ -60,6 +60,9 @@ class Tag(TagBase): _('category'), max_length=50, blank=False, null=False, db_index=True, choices=TAG_CATEGORIES) description = models.TextField(_('description'), blank=True) + for_books = models.BooleanField(default=False) + for_pictures = models.BooleanField(default=False) + user = models.ForeignKey(User, blank=True, null=True) gazeta_link = models.CharField(blank=True, max_length=240) culturepl_link = models.CharField(blank=True, max_length=240) diff --git a/src/picture/models.py b/src/picture/models.py index 8f913c356..7732e2042 100644 --- a/src/picture/models.py +++ b/src/picture/models.py @@ -198,6 +198,10 @@ class Picture(models.Model): picture.extra_info = picture_xml.picture_info.to_dict() picture_tags = set(catalogue.models.Tag.tags_from_info(picture_xml.picture_info)) + for tag in picture_tags: + if not tag.for_pictures: + tag.for_pictures = True + tag.save() area_data = {'themes': {}, 'things': {}} @@ -221,16 +225,20 @@ class Picture(models.Model): tag.name = objname setattr(tag, 'name_%s' % lang, tag.name) tag.sort_key = sortify(tag.name) + tag.for_pictures = True tag.save() - # thing_tags.add(tag) area_data['things'][tag.slug] = { 'object': objname, 'coords': part['coords'], } _tags.add(tag) + if not tag.for_pictures: + tag.for_pictures = True + tag.save() area = PictureArea.rectangle(picture, 'thing', part['coords']) area.save() + # WTF thing area does not inherit tags from picture and theme area does, is it intentional? area.tags = _tags else: _tags = set() @@ -241,9 +249,13 @@ class Picture(models.Model): if created: tag.name = motif tag.sort_key = sortify(tag.name) + tag.for_pictures = True tag.save() # motif_tags.add(tag) _tags.add(tag) + if not tag.for_pictures: + tag.for_pictures = True + tag.save() area_data['themes'][tag.slug] = { 'theme': motif, 'coords': part['coords'] -- 2.20.1 From 87b22d351ce199326613811ad544e24e0fa53d28 Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Thu, 21 Dec 2017 12:05:59 +0100 Subject: [PATCH 14/16] book/picture-only tags in api --- src/api/handlers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/api/handlers.py b/src/api/handlers.py index 159e4f235..a1e9f6a23 100644 --- a/src/api/handlers.py +++ b/src/api/handlers.py @@ -523,6 +523,13 @@ class TagsHandler(BaseHandler, TagDetails): tags = Tag.objects.filter(category=category_sng).exclude(items=None).order_by('slug') + book_only = request.GET.get('book_only') == 'true' + picture_only = request.GET.get('picture_only') == 'true' + if book_only: + tags = tags.filter(for_books=True) + if picture_only: + tags = tags.filter(for_pictures=True) + if after: tags = tags.filter(slug__gt=after) if before: -- 2.20.1 From 7255df395ced6145a654fb6cfc1e673000008a25 Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Thu, 21 Dec 2017 14:23:54 +0100 Subject: [PATCH 15/16] cover thumb with no box for api --- lib/librarian | 2 +- src/api/handlers.py | 10 ++++--- src/catalogue/fields.py | 9 +++++++ .../migrations/0020_book_cover_api_thumb.py | 21 +++++++++++++++ src/catalogue/models/book.py | 26 ++++++++++++++----- 5 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 src/catalogue/migrations/0020_book_cover_api_thumb.py diff --git a/lib/librarian b/lib/librarian index fa42894ec..f4fdb08e0 160000 --- a/lib/librarian +++ b/lib/librarian @@ -1 +1 @@ -Subproject commit fa42894ec06a78183d736608613c9a96fec2c400 +Subproject commit f4fdb08e0b0ff0da9faed1ac5d89a1523cad7271 diff --git a/src/api/handlers.py b/src/api/handlers.py index a1e9f6a23..f7b30864d 100644 --- a/src/api/handlers.py +++ b/src/api/handlers.py @@ -145,6 +145,10 @@ class BookDetails(object): return MEDIA_BASE + default.backend.get_thumbnail( book.cover, "139x193").url if book.cover else '' + @classmethod + def simple_thumb(cls, book): + return MEDIA_BASE + book.cover_api_thumb.url if book.cover_api_thumb else '' + @classmethod def cover_source_image(cls, book): url = book.cover_source() @@ -158,7 +162,7 @@ class BookDetailHandler(BaseHandler, BookDetails): """ allowed_methods = ['GET'] fields = ['title', 'parent', 'children'] + Book.formats + [ - 'media', 'url', 'cover', 'cover_thumb', 'fragment_data'] + [ + 'media', 'url', 'cover', 'cover_thumb', 'simple_thumb', 'fragment_data'] + [ category_plural[c] for c in book_tag_categories] @piwik_track @@ -177,7 +181,7 @@ class AnonymousBooksHandler(AnonymousBaseHandler, BookDetails): """ allowed_methods = ('GET',) model = Book - fields = book_tag_categories + ['href', 'title', 'url', 'cover', 'cover_thumb', 'slug'] + fields = book_tag_categories + ['href', 'title', 'url', 'cover', 'cover_thumb', 'slug', 'simple_thumb'] @classmethod def genres(cls, book): @@ -301,7 +305,7 @@ class QuerySetProxy(models.QuerySet): class FilterBooksHandler(AnonymousBooksHandler): fields = book_tag_categories + [ - 'href', 'title', 'url', 'cover', 'cover_thumb', 'slug', 'key', 'cover_source_image'] + 'href', 'title', 'url', 'cover', 'cover_thumb', 'simple_thumb', 'slug', 'key', 'cover_source_image'] def read(self, request): key_sep = '$' diff --git a/src/catalogue/fields.py b/src/catalogue/fields.py index 372cf2735..8b28f1c43 100644 --- a/src/catalogue/fields.py +++ b/src/catalogue/fields.py @@ -244,6 +244,15 @@ class BuildCoverThumb(BuildEbook): return WLCover(wldoc.book_info, height=193).output_file() +@BuildEbook.register('cover_api_thumb') +@task(ignore_result=True) +class BuildCoverApiThumb(BuildEbook): + @classmethod + def transform(cls, wldoc, fieldfile): + from librarian.cover import WLNoBoxCover + return WLNoBoxCover(wldoc.book_info, height=500).output_file() + + # not used, but needed for migrations class OverwritingFieldFile(FieldFile): """ diff --git a/src/catalogue/migrations/0020_book_cover_api_thumb.py b/src/catalogue/migrations/0020_book_cover_api_thumb.py new file mode 100644 index 000000000..45db58d13 --- /dev/null +++ b/src/catalogue/migrations/0020_book_cover_api_thumb.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import catalogue.fields +import catalogue.models.book + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalogue', '0019_auto_20171221_1107'), + ] + + operations = [ + migrations.AddField( + model_name='book', + name='cover_api_thumb', + field=catalogue.fields.EbookField(b'cover_api_thumb', max_length=255, upload_to=catalogue.models.book.UploadToPath(b'book/cover_api_thumb/%s.jpg'), null=True, verbose_name='cover thumbnail for API', blank=True), + ), + ] diff --git a/src/catalogue/models/book.py b/src/catalogue/models/book.py index 44ee02ecc..cdb110045 100644 --- a/src/catalogue/models/book.py +++ b/src/catalogue/models/book.py @@ -14,6 +14,7 @@ import django.dispatch from django.contrib.contenttypes.fields import GenericRelation from django.core.urlresolvers import reverse from django.utils.translation import ugettext_lazy as _, get_language +from django.utils.deconstruct import deconstructible import jsonfield from fnpdjango.storage import BofhFileSystemStorage from ssify import flush_ssi_includes @@ -30,18 +31,22 @@ from wolnelektury.utils import makedirs bofh_storage = BofhFileSystemStorage() -def _make_upload_to(path): - def _upload_to(i, n): - return path % i.slug - return _upload_to +@deconstructible +class UploadToPath(object): + def __init__(self, path): + self.path = path + def __call__(self, instance, filename): + return self.path % instance.slug -_cover_upload_to = _make_upload_to('book/cover/%s.jpg') -_cover_thumb_upload_to = _make_upload_to('book/cover_thumb/%s.jpg') + +_cover_upload_to = UploadToPath('book/cover/%s.jpg') +_cover_thumb_upload_to = UploadToPath('book/cover_thumb/%s.jpg') +_cover_api_thumb_opload_to = UploadToPath('book/cover_api_thumb/%s.jpg') def _ebook_upload_to(upload_path): - return _make_upload_to(upload_path) + return UploadToPath(upload_path) class Book(models.Model): @@ -75,6 +80,11 @@ class Book(models.Model): null=True, blank=True, upload_to=_cover_thumb_upload_to, max_length=255) + cover_api_thumb = EbookField( + 'cover_api_thumb', _('cover thumbnail for API'), + null=True, blank=True, + upload_to=_cover_api_thumb_opload_to, + max_length=255) ebook_formats = constants.EBOOK_FORMATS formats = ebook_formats + ['html', 'xml'] @@ -429,6 +439,7 @@ class Book(models.Model): if 'cover' not in dont_build: book.cover.build_delay() book.cover_thumb.build_delay() + book.cover_api_thumb.build_delay() # Build HTML and ebooks. book.html_file.build_delay() @@ -531,6 +542,7 @@ class Book(models.Model): if 'cover' not in app_settings.DONT_BUILD: self.cover.build_delay() self.cover_thumb.build_delay() + self.cover_api_thumb.build_delay() for format_ in constants.EBOOK_FORMATS_WITH_COVERS: if format_ not in app_settings.DONT_BUILD: getattr(self, '%s_file' % format_).build_delay() -- 2.20.1 From d850c26d30dc10b20278e537198e698b84c64e7b Mon Sep 17 00:00:00 2001 From: Jan Szejko Date: Fri, 22 Dec 2017 11:22:51 +0100 Subject: [PATCH 16/16] fix in autocomplete --- src/search/views.py | 3 +- src/wolnelektury/static/js/search.js | 72 ++++++++++++++-------------- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/src/search/views.py b/src/search/views.py index 309db73f4..7e512be14 100644 --- a/src/search/views.py +++ b/src/search/views.py @@ -87,7 +87,8 @@ def hint(request): if len(data) < limit: data += [ { - 'label': '%s, %s' % (b.title, b.author_unicode()), + 'label': b.title, + 'author': b.author_unicode(), 'id': b.id, 'url': b.get_absolute_url() } diff --git a/src/wolnelektury/static/js/search.js b/src/wolnelektury/static/js/search.js index 786a05c79..34c63f3c9 100644 --- a/src/wolnelektury/static/js/search.js +++ b/src/wolnelektury/static/js/search.js @@ -5,44 +5,46 @@ var __bind = function (self, fn) { (function($){ $.widget("wl.search", { - options: { - minLength: 0, - dataType: "json", - host: '', + options: { + minLength: 0, + dataType: "json", + host: '' + }, + + _create: function() { + var opts = { + minLength: this.options.minLength, + select: __bind(this, this.enter), + focus: function() { return false; }, + source: this.element.data('source') + }; + + this.element.autocomplete($.extend(opts, this.options)) + .data("autocomplete")._renderItem = __bind(this, this.render_item); }, - _create: function() { - var opts = { - minLength: this.options.minLength, - select: __bind(this, this.enter), - focus: function() { return false; }, - source: this.element.data('source'), - }; - - this.element.autocomplete($.extend(opts, this.options)) - .data("autocomplete")._renderItem = __bind(this, this.render_item); - }, - - enter: function(event, ui) { - if (ui.item.url != undefined) { - location.href = this.options.host+ui.item.url; - } else { - this.element.closest('form').submit(); - } - }, - - render_item: function (ul, item) { - return $("
  • ").data('item.autocomplete', item) - .append(''+item.label+'') - .appendTo(ul); - }, - - destroy: function() { - - }, - + enter: function(event, ui) { + if (ui.item.url !== undefined) { + location.href = this.options.host+ui.item.url; + } else { + this.element.closest('form').submit(); + } + }, - }); + render_item: function (ul, item) { + var label; + if (item['author']) { + label = '' + item.label + ', ' + item['author']; + } else { + label = item.label; + } + return $("
  • ").data('item.autocomplete', item) + .append('    '+label+'') + .appendTo(ul); + }, + destroy: function() { + } + }); })(jQuery); -- 2.20.1