From 527c5dba85fcdfd70755b6c1f899dbb21f5ec40a Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Thu, 16 Oct 2014 13:45:51 +0200 Subject: [PATCH 1/1] Some dictionary filtering features. --- apps/catalogue/fields.py | 4 ++ apps/catalogue/tasks.py | 5 +- .../migrations/0002_auto_20141006_1422.py | 54 ++++++++++++++ apps/dictionary/models.py | 55 +++++++++++--- .../templates/dictionary/note_list.html | 71 +++++++++++++++++-- apps/dictionary/tests.py | 15 ++-- apps/dictionary/urls.py | 1 - apps/dictionary/views.py | 37 +++++++++- .../static/scss/dictionary/dictionary.scss | 9 +++ apps/wolnelektury_core/static/scss/main.scss | 2 + lib/librarian | 2 +- 11 files changed, 230 insertions(+), 25 deletions(-) create mode 100644 apps/dictionary/migrations/0002_auto_20141006_1422.py create mode 100644 apps/wolnelektury_core/static/scss/dictionary/dictionary.scss diff --git a/apps/catalogue/fields.py b/apps/catalogue/fields.py index 590f2657d..0ff2ca9ef 100644 --- a/apps/catalogue/fields.py +++ b/apps/catalogue/fields.py @@ -10,8 +10,11 @@ from catalogue import app_settings from catalogue.constants import LANGUAGES_3TO2 from catalogue.utils import remove_zip, truncate_html_words from celery.task import Task, task +from celery.utils.log import get_task_logger from waiter.utils import clear_cache +task_logger = get_task_logger(__name__) + class EbookFieldFile(FieldFile): """Represents contents of an ebook file field.""" @@ -82,6 +85,7 @@ class BuildEbook(Task): def run(self, obj, field_name): """Just run `build` on FieldFile, can't pass it directly to Celery.""" + task_logger.info("%s -> %s" % (obj.slug, field_name)) ret = self.build(getattr(obj, field_name)) obj.flush_includes() return ret diff --git a/apps/catalogue/tasks.py b/apps/catalogue/tasks.py index e392ae2da..a2b8be088 100644 --- a/apps/catalogue/tasks.py +++ b/apps/catalogue/tasks.py @@ -5,10 +5,13 @@ from datetime import datetime from traceback import print_exc from celery.task import task +from celery.utils.log import get_task_logger from django.conf import settings from wolnelektury.utils import localtime_to_utc from waiter.models import WaitedFile +task_logger = get_task_logger(__name__) + # TODO: move to model? def touch_tag(tag): @@ -38,7 +41,7 @@ def build_custom_pdf(book_id, customizations, file_name, waiter_id=None): from django.core.files.storage import DefaultStorage from catalogue.models import Book - print "will gen %s" % DefaultStorage().path(file_name) + task_logger.info(DefaultStorage().path(file_name)) if not DefaultStorage().exists(file_name): kwargs = { 'cover': True, diff --git a/apps/dictionary/migrations/0002_auto_20141006_1422.py b/apps/dictionary/migrations/0002_auto_20141006_1422.py new file mode 100644 index 000000000..a9bad32b9 --- /dev/null +++ b/apps/dictionary/migrations/0002_auto_20141006_1422.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalogue', '0004_remove_booktags_count_related_info'), + ('dictionary', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='NoteSource', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('anchor', models.CharField(max_length=64)), + ('book', models.ForeignKey(to='catalogue.Book')), + ('note', models.ForeignKey(to='dictionary.Note')), + ], + options={ + 'ordering': ['book'], + }, + bases=(models.Model,), + ), + migrations.RemoveField( + model_name='note', + name='anchor', + ), + migrations.RemoveField( + model_name='note', + name='book', + ), + migrations.AddField( + model_name='note', + name='fn_type', + field=models.CharField(default='', max_length=10, db_index=True), + preserve_default=False, + ), + migrations.AddField( + model_name='note', + name='language', + field=models.CharField(default='', max_length=10, db_index=True), + preserve_default=False, + ), + migrations.AddField( + model_name='note', + name='qualifier', + field=models.CharField(default='', max_length=128, db_index=True, blank=True), + preserve_default=False, + ), + ] diff --git a/apps/dictionary/models.py b/apps/dictionary/models.py index 2256acf71..73fbb0d47 100644 --- a/apps/dictionary/models.py +++ b/apps/dictionary/models.py @@ -2,33 +2,68 @@ # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # -from django.db import models +from django.db import models, transaction from celery.task import task from sortify import sortify +from celery.utils.log import get_task_logger + +task_logger = get_task_logger(__name__) from catalogue.models import Book class Note(models.Model): """Represents a single annotation from a book.""" - book = models.ForeignKey(Book) - anchor = models.CharField(max_length=64) html = models.TextField() sort_key = models.CharField(max_length=128, db_index=True) + fn_type = models.CharField(max_length=10, db_index=True) + qualifier = models.CharField(max_length=128, db_index=True, blank=True) + language = models.CharField(max_length=10, db_index=True) class Meta: ordering = ['sort_key'] +class NoteSource(models.Model): + """Represents a single annotation from a book.""" + note = models.ForeignKey(Note) + book = models.ForeignKey(Book) + anchor = models.CharField(max_length=64) + + class Meta: + ordering = ['book'] + + @task(ignore_result=True) def build_notes(book): - Note.objects.filter(book=book).delete() - if book.html_file: - from librarian import html - for anchor, text_str, html_str in html.extract_annotations(book.html_file.path): - Note.objects.create(book=book, anchor=anchor, - html=html_str, - sort_key=sortify(text_str).strip()[:128]) + task_logger.info(book.slug) + with transaction.atomic(): + book.notesource_set.all().delete() + if book.html_file: + from librarian import html + for anchor, fn_type, qualifier, text_str, html_str in \ + html.extract_annotations(book.html_file.path): + sort_key = sortify(text_str).strip()[:128] + qualifier = (qualifier or '')[:128] + language = book.language + note = None + notes = Note.objects.filter(sort_key=sort_key, + qualifier=qualifier, fn_type=fn_type, + language=language, html=html_str) + if notes: + note = notes[0] + else: + note = Note.objects.create( + sort_key=sort_key, + qualifier=qualifier, + html=html_str, + fn_type=fn_type, + language=language + ) + note.notesource_set.create(book=book, anchor=anchor) + + Note.objects.filter(notesource=None).delete() + def notes_from_book(sender, instance, **kwargs): build_notes.delay(instance) diff --git a/apps/dictionary/templates/dictionary/note_list.html b/apps/dictionary/templates/dictionary/note_list.html index 5bf9e3f4c..5109454b8 100755 --- a/apps/dictionary/templates/dictionary/note_list.html +++ b/apps/dictionary/templates/dictionary/note_list.html @@ -15,7 +15,7 @@

{% trans "By first letter" %}: {% if letter %} - {% trans "all" %} + {% trans "all" %} {% else %} {% trans "all" %} {% endif %} @@ -25,10 +25,66 @@ {% if let == letter %} {{ let|upper }} {% else %} - {{ let|upper }} + {{ let|upper }} {% endif %} {% endfor %}

+ +

+{% trans "By type" %}: +{% if fn_type %} + {% trans "all" %} +{% else %} + {% trans "all" %} +{% endif %} + +{% for fnt in fn_types %} + | + {% if fnt == fn_type %} + {{ fnt }} + {% else %} + {{ fnt }} + {% endif %} +{% endfor %} +

+ + +

+{% trans "By qualifier" %}: +{% if qualifier %} + {% trans "all" %} +{% else %} + {% trans "all" %} +{% endif %} + +{% for qual in qualifiers %} + | + {% if qual == qualifier %} + {{ qual }} + {% else %} + {{ qual }} + {% endif %} +{% endfor %} +

+ +

+{% trans "By language" %}: +{% if language %} + {% trans "all" %} +{% else %} + {% trans "all" %} +{% endif %} + +{% for lang in languages %} + | + {% if lang == language %} + {{ lang }} + {% else %} + {{ lang }} + {% endif %} +{% endfor %} +

+

@@ -41,10 +97,13 @@ {% paginate %} {% for obj in object_list %}

- {{ obj.html|safe }} -
- ({{ obj.book.pretty_title }}) -
+ {{ obj.html|safe }} + {% for note_source in obj.notesource_set.all %} +
+ {{ note_source.book.pretty_title }} + (źródło na Platformie Redakcyjnej) +
+ {% endfor %}
{% endfor %} {% paginate %} diff --git a/apps/dictionary/tests.py b/apps/dictionary/tests.py index e88fe507d..526819a01 100755 --- a/apps/dictionary/tests.py +++ b/apps/dictionary/tests.py @@ -23,6 +23,8 @@ class DictionaryTests(WLTestCase): BOOK_TEXT = """ rose --- kind of a flower. + rose --- kind of a flower. + rose (color) --- #FF007F. """ @@ -30,16 +32,21 @@ class DictionaryTests(WLTestCase): self.assertEqual( len(self.client.get('/przypisy/').context['object_list']), - 1, + 2, 'There should be a note on the note list.') self.assertEqual( - len(self.client.get('/przypisy/a/').context['object_list']), + len(self.client.get('/przypisy/?ltr=r').context['object_list']), 0, 'There should not be a note for the letter A.') self.assertEqual( - len(self.client.get('/przypisy/r/').context['object_list']), - 1, + len(self.client.get('/przypisy/?ltr=r').context['object_list']), + 2, + 'There should be a note for the letter R.') + + self.assertEqual( + len(self.client.get('/przypisy/?qual=color').context['object_list']), + 2, 'There should be a note for the letter R.') diff --git a/apps/dictionary/urls.py b/apps/dictionary/urls.py index 63e4bbd06..2320f4773 100755 --- a/apps/dictionary/urls.py +++ b/apps/dictionary/urls.py @@ -7,6 +7,5 @@ from dictionary.views import NotesView urlpatterns = patterns('dictionary.views', url(r'^$', NotesView.as_view(), name='dictionary_notes'), - url(r'(?P[a-z]|0-9)/$', NotesView.as_view(), name='dictionary_notes'), ) diff --git a/apps/dictionary/views.py b/apps/dictionary/views.py index 76c89cd85..fa9712dda 100755 --- a/apps/dictionary/views.py +++ b/apps/dictionary/views.py @@ -4,22 +4,55 @@ # from dictionary.models import Note from django.views.generic.list import ListView +from django.db.models import Count class NotesView(ListView): def get_queryset(self): self.letters = ["0-9"] + [chr(a) for a in range(ord('a'), ord('z')+1)] - self.letter = self.kwargs.get('letter') + self.letter = self.request.GET.get('ltr') + + self.qualifiers = Note.objects.order_by('qualifier').filter(qualifier__startswith='f').values_list( + 'qualifier', flat=True).distinct() + self.qualifier = self.request.GET.get('qual') + + self.languages = Note.objects.order_by('language').values_list( + 'language', flat=True).distinct() + self.language = self.request.GET.get('lang') + + self.fn_types = Note.objects.order_by('fn_type').values_list( + 'fn_type', flat=True).distinct() + self.fn_type = self.request.GET.get('type') objects = Note.objects.select_related('book').all() + if self.letter == "0-9": objects = objects.filter(sort_key__regex=r"^[0-9]") elif self.letter: objects = objects.filter(sort_key__startswith=self.letter) + + if self.qualifier: + objects = objects.filter(qualifier=self.qualifier) + + if self.language: + objects = objects.filter(language=self.language) + + if self.fn_type: + objects = objects.filter(fn_type=self.fn_type) + return objects + # TODO: wewn. wyszukiwarka, czy wg definiendum? + # TODO: filtr języka + def get_context_data(self, **kwargs): context = super(NotesView, self).get_context_data(**kwargs) - context['letter'] = self.letter context['letters'] = self.letters + context['letter'] = self.letter + context['qualifiers'] = self.qualifiers + context['qualifier'] = self.qualifier + context['languages'] = self.languages + context['language'] = self.language + context['fn_types'] = self.fn_types + context['fn_type'] = self.fn_type return context diff --git a/apps/wolnelektury_core/static/scss/dictionary/dictionary.scss b/apps/wolnelektury_core/static/scss/dictionary/dictionary.scss new file mode 100644 index 000000000..a90059924 --- /dev/null +++ b/apps/wolnelektury_core/static/scss/dictionary/dictionary.scss @@ -0,0 +1,9 @@ +.dictionary-note { + @include white-box; + @include size(padding, 8px); + @include size(margin, 10px 0); +} +.dictionary-note-source { + @include white-box; + @include size(padding, 8px); +} diff --git a/apps/wolnelektury_core/static/scss/main.scss b/apps/wolnelektury_core/static/scss/main.scss index 467e607bb..bab1fb46f 100644 --- a/apps/wolnelektury_core/static/scss/main.scss +++ b/apps/wolnelektury_core/static/scss/main.scss @@ -20,6 +20,8 @@ @import "main/search"; @import "main/tag"; +@import "dictionary/dictionary"; + @import "funding/funding"; @import "polls/polls"; diff --git a/lib/librarian b/lib/librarian index a91e41e48..a3b684052 160000 --- a/lib/librarian +++ b/lib/librarian @@ -1 +1 @@ -Subproject commit a91e41e489588ebad550cd9e22d157062effa2ff +Subproject commit a3b6840527ec52ce8b6d74819633d8c85e3973ba -- 2.20.1