From: Jan Szejko Date: Wed, 13 Apr 2016 08:17:15 +0000 (+0200) Subject: Merge branch 'dev' X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/9a938c8b406ce05e3bca4a5a483d473ece9e17b0?hp=ed79e41001ebd01207fcea50e09aea5ebcae8a0d Merge branch 'dev' Conflicts: src/api/management/commands/mobileinit.py src/catalogue/templates/catalogue/inline_tag_list.html src/catalogue/templates/catalogue/plain_list.html src/catalogue/views.py src/wolnelektury/static/scss/main/book_box.scss src/wolnelektury/static/scss/main/header.scss src/wolnelektury/templates/main_page.html src/wolnelektury/templates/superbase.html --- diff --git a/fabfile.py b/fabfile.py index ff9a4b4f8..b6f3bbb33 100644 --- a/fabfile.py +++ b/fabfile.py @@ -10,18 +10,26 @@ except ImportError: env.project_name = 'wolnelektury' -def update_counters(): - print '>>> updating counters' - require('app_path', 'project_name') - with cd(get_django_root_path('current')): - run('%(ve)s/bin/python manage.py update_counters' % env, pty=True) +class ManageTask(Task): + def __init__(self, name, params='', **kwargs): + super(ManageTask, self).__init__(**kwargs) + self.name = name + self.params = params + def run(self): + require('app_path', 'project_name') + with cd(get_django_root_path('current')): + run('source %(ve)s/bin/activate && python manage.py %(task)s %(params)s' % { + 've': env.ve, + 'task': self.name, + 'params': self.params, + }, pty=True) -def compile_messages(): - print '>>> compiling messages' - require('app_path', 'project_name') - with cd(get_django_root_path('current')): - run('source %(ve)s/bin/activate && python manage.py localepack -c' % env, pty=True) + +class Memcached(Service): + def run(self): + print '>>> memcached: restart' + sudo('service memcached restart', shell=False) @task @@ -32,12 +40,13 @@ def production(): env.django_root_path = 'src' env.requirements_file = 'requirements/requirements.txt' env.pre_collectstatic = [ - update_counters, - compile_messages, + ManageTask('update_counters'), + ManageTask('localepack', '-c'), ] env.services = [ Supervisord('wolnelektury'), Supervisord('wolnelektury.celery'), + Memcached(), ] @@ -50,8 +59,8 @@ def beta(): env.django_root_path = 'src' env.requirements_file = 'requirements/requirements.txt' env.pre_collectstatic = [ - update_counters, - compile_messages, + ManageTask('update_counters'), + ManageTask('localepack', '-c'), ] env.services = [ Supervisord('beta'), diff --git a/src/api/management/commands/mobileinit.py b/src/api/management/commands/mobileinit.py index ccbff3e21..ccf27bc76 100755 --- a/src/api/management/commands/mobileinit.py +++ b/src/api/management/commands/mobileinit.py @@ -97,7 +97,7 @@ CREATE TABLE state (last_checked INTEGER); """ db.executescript(schema) - db.execute("INSERT INTO state VALUES (:last_checked)", locals()) + db.execute("INSERT INTO state VALUES (:last_checked)", {'last_checked': last_checked}) return db @@ -133,29 +133,30 @@ categories = {'author': 'autor', def add_book(db, book): - title = book.title if book.html_file: html_file = book.html_file.url html_file_size = book.html_file.size else: html_file = html_file_size = None - if book.cover: - cover = book.cover.url - else: - cover = None - parent = book.parent_id - parent_number = book.parent_number - sort_key = book.sort_key - size_str = pretty_size(html_file_size) - authors = book.author_unicode() - db.execute(book_sql, locals()) + db.execute(book_sql, { + 'title': book.title, + 'cover': book.cover.url if book.cover else None, + 'html_file': html_file, + 'html_file_size': html_file_size, + 'parent': book.parent_id, + 'parent_number': book.parent_number, + 'sort_key': book.sort_key, + 'size_str': pretty_size(html_file_size), + 'authors': book.author_unicode(), + }) def add_tag(db, tag): - category = categories[tag.category] - name = tag.name - sort_key = tag.sort_key - books = Book.tagged_top_level([tag]) - book_ids = ','.join(str(b.id) for b in books) - db.execute(tag_sql, locals()) + book_ids = ','.join(str(book_id) for book_id in books.values_list('id', flat=True)) + db.execute(tag_sql, { + 'category': categories[tag.category], + 'name': tag.name, + 'sort_key': tag.sort_key, + 'book_ids': book_ids, + }) diff --git a/src/catalogue/helpers.py b/src/catalogue/helpers.py index e47607928..97c63d08c 100644 --- a/src/catalogue/helpers.py +++ b/src/catalogue/helpers.py @@ -3,6 +3,9 @@ # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.core.cache import cache + from .models import Tag, Book from os.path import getmtime import cPickle @@ -36,7 +39,6 @@ def get_top_level_related_tags(tags, categories=None): related = Tag.objects.filter(pk__in=related_ids) - # TODO: do we really need that? if categories is not None: related = related.filter(category__in=categories) @@ -86,3 +88,16 @@ def update_counters(): with open(settings.CATALOGUE_COUNTERS_FILE, 'w') as f: cPickle.dump(counters, f) + + +def get_audiobook_tags(): + audiobook_tag_ids = cache.get('audiobook_tags') + if audiobook_tag_ids is None: + books_with_audiobook = Book.objects.filter(media__type__in=('mp3', 'ogg'))\ + .distinct().values_list('pk', flat=True) + audiobook_tag_ids = Tag.objects.filter( + items__content_type=ContentType.objects.get_for_model(Book), + items__object_id__in=list(books_with_audiobook)).distinct().values_list('pk', flat=True) + audiobook_tag_ids = list(audiobook_tag_ids) + cache.set('audiobook_tags', audiobook_tag_ids) + return audiobook_tag_ids diff --git a/src/catalogue/management/commands/update_popularity.py b/src/catalogue/management/commands/update_popularity.py new file mode 100644 index 000000000..df9dedf49 --- /dev/null +++ b/src/catalogue/management/commands/update_popularity.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. +# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# +from __future__ import print_function, unicode_literals + +from django.core.management.base import BaseCommand +from django.db.models import Count + +from catalogue.models import Book, BookPopularity + + +class Command(BaseCommand): + help = 'Update popularity counters.' + + def handle(self, **options): + BookPopularity.objects.all().delete() + books_with_popularity = Book.objects.filter(tag_relations__tag__category='set').only('id').distinct()\ + .annotate(pop=Count('tag_relations__tag__user', distinct=True)) + pop_list = [] + for book in books_with_popularity: + pop_list.append(BookPopularity(book=book, count=book.pop)) + books_without_popularity = Book.objects.exclude(tag_relations__tag__category='set') + for book in books_without_popularity: + pop_list.append(BookPopularity(book=book, count=0)) + BookPopularity.objects.bulk_create(pop_list, batch_size=512) diff --git a/src/catalogue/migrations/0010_bookpopularity.py b/src/catalogue/migrations/0010_bookpopularity.py new file mode 100644 index 000000000..42fd6cb7a --- /dev/null +++ b/src/catalogue/migrations/0010_bookpopularity.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalogue', '0009_auto_20160127_1019'), + ] + + operations = [ + migrations.CreateModel( + name='BookPopularity', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('count', models.IntegerField(default=0)), + ('book', models.OneToOneField(related_name='popularity', to='catalogue.Book')), + ], + ), + ] diff --git a/src/catalogue/models/__init__.py b/src/catalogue/models/__init__.py index 73b51090c..7aebc3175 100644 --- a/src/catalogue/models/__init__.py +++ b/src/catalogue/models/__init__.py @@ -5,6 +5,6 @@ from catalogue.models.tag import Tag from catalogue.models.bookmedia import BookMedia from catalogue.models.fragment import Fragment -from catalogue.models.book import Book +from catalogue.models.book import Book, BookPopularity from catalogue.models.collection import Collection from catalogue.models.source import Source diff --git a/src/catalogue/models/book.py b/src/catalogue/models/book.py index d3e2a7756..0ed97161b 100644 --- a/src/catalogue/models/book.py +++ b/src/catalogue/models/book.py @@ -92,7 +92,7 @@ class Book(models.Model): pass class Meta: - ordering = ('sort_key',) + ordering = ('sort_key_author', 'sort_key') verbose_name = _('book') verbose_name_plural = _('books') app_label = 'catalogue' @@ -576,6 +576,15 @@ class Book(models.Model): else: return None + def update_popularity(self): + count = self.tags.filter(category='set').values('user').order_by('user').distinct().count() + try: + pop = self.popularity + pop.count = count + pop.save() + except BookPopularity.DoesNotExist: + BookPopularity.objects.create(book=self, count=count) + def add_file_fields(): for format_ in Book.formats: @@ -595,3 +604,8 @@ def add_file_fields(): ).contribute_to_class(Book, field_name) add_file_fields() + + +class BookPopularity(models.Model): + book = models.OneToOneField(Book, related_name='popularity') + count = models.IntegerField(default=0) diff --git a/src/catalogue/models/collection.py b/src/catalogue/models/collection.py index 0150055e0..15a4e2adc 100644 --- a/src/catalogue/models/collection.py +++ b/src/catalogue/models/collection.py @@ -49,7 +49,7 @@ class Collection(models.Model): def get_books(self): from catalogue.models import Book - return Book.objects.filter(self.get_query()).order_by('sort_key_author', 'sort_key') + return Book.objects.filter(self.get_query()) def flush_includes(self, languages=True): if not languages: diff --git a/src/catalogue/models/tag.py b/src/catalogue/models/tag.py index 57935f8fb..153197307 100644 --- a/src/catalogue/models/tag.py +++ b/src/catalogue/models/tag.py @@ -9,6 +9,7 @@ from django.db import models from django.db.models import permalink from django.dispatch import Signal from django.utils.translation import ugettext_lazy as _ + from newtagging.models import TagBase from ssify import flush_ssi_includes @@ -48,7 +49,9 @@ class Tag(TagBase): after_change = Signal(providing_args=['instance', 'languages']) class UrlDeprecationWarning(DeprecationWarning): - pass + def __init__(self, tags=None): + super(Tag.UrlDeprecationWarning, self).__init__() + self.tags = tags categories_rev = { 'autor': 'author', @@ -157,24 +160,24 @@ class Tag(TagBase): has_description.boolean = True @staticmethod - def get_tag_list(tags): - if isinstance(tags, basestring): - if not tags: + def get_tag_list(tag_str): + if isinstance(tag_str, basestring): + if not tag_str: return [] - real_tags = [] + tags = [] ambiguous_slugs = [] category = None deprecated = False - tags_splitted = tags.split('/') + tags_splitted = tag_str.split('/') for name in tags_splitted: if category: - real_tags.append(Tag.objects.get(slug=name, category=category)) + tags.append(Tag.objects.get(slug=name, category=category)) category = None elif name in Tag.categories_rev: category = Tag.categories_rev[name] else: try: - real_tags.append(Tag.objects.get(slug=name)) + tags.append(Tag.objects.get(slug=name)) deprecated = True except Tag.MultipleObjectsReturned: ambiguous_slugs.append(name) @@ -185,16 +188,14 @@ class Tag(TagBase): if ambiguous_slugs: # some tags should be qualified e = Tag.MultipleObjectsReturned() - e.tags = real_tags + e.tags = tags e.ambiguous_slugs = ambiguous_slugs raise e if deprecated: - e = Tag.UrlDeprecationWarning() - e.tags = real_tags - raise e - return real_tags + raise Tag.UrlDeprecationWarning(tags=tags) + return tags else: - return TagBase.get_tag_list(tags) + return TagBase.get_tag_list(tag_str) @property def url_chunk(self): diff --git a/src/catalogue/templates/catalogue/audiobook_list.html b/src/catalogue/templates/catalogue/audiobook_list.html deleted file mode 100644 index 7eda46e4b..000000000 --- a/src/catalogue/templates/catalogue/audiobook_list.html +++ /dev/null @@ -1,21 +0,0 @@ -{% extends "base.html" %} -{% load i18n %} -{% load catalogue_tags switch_tag social_tags %} -{% load ssi_include from ssify %} - -{% block titleextra %}{% trans "Audiobooks" %}{% endblock %} - -{% block bodyid %}audiobooks{% endblock %} - -{% block body %} -

{% trans "Audiobooks" %}

- - {% work_list best %} -

{% trans "Listing of all audiobooks" %}

- {% plain_list books by_author=True %} - -

{% trans "Listing of all DAISY files" %}

- {% plain_list daisy by_author=True %} - - -{% endblock %} diff --git a/src/catalogue/templates/catalogue/book_wide.html b/src/catalogue/templates/catalogue/book_wide.html index 26ee55d71..aa78292a8 100644 --- a/src/catalogue/templates/catalogue/book_wide.html +++ b/src/catalogue/templates/catalogue/book_wide.html @@ -1,6 +1,6 @@ {% extends "catalogue/book_short.html" %} {% load i18n %} -{% load choose_fragment download_audio tag_list custom_pdf_link_li license_icon source_name from catalogue_tags %} +{% load choose_fragment download_audio custom_pdf_link_li license_icon source_name from catalogue_tags %} {% load choose_cite from social_tags %} {% load ssi_include from ssify %} diff --git a/src/catalogue/templates/catalogue/inline_tag_list.html b/src/catalogue/templates/catalogue/inline_tag_list.html index f6d59a788..802fcb19e 100755 --- a/src/catalogue/templates/catalogue/inline_tag_list.html +++ b/src/catalogue/templates/catalogue/inline_tag_list.html @@ -1,34 +1,27 @@ {% load i18n %} {% load catalogue_tags %} - {% if choices %} - {% if category_choices %} - - {% endif %} - {% if tags %} + +{% if choices %} + {% if category_choices %} - {% endif %} - {% else %} - {% if tags %} -