From: Radek Czajka Date: Wed, 31 Aug 2011 09:58:29 +0000 (+0200) Subject: Merge branch 'api' X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/3d1fb545dd1f49e0624d3adf20e5568b1c33d8ec?ds=sidebyside;hp=-c Merge branch 'api' --- 3d1fb545dd1f49e0624d3adf20e5568b1c33d8ec diff --combined apps/catalogue/models.py index cda5ba532,c1290fbcf..fd5dc987a --- a/apps/catalogue/models.py +++ b/apps/catalogue/models.py @@@ -2,6 -2,8 +2,8 @@@ # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # + from datetime import datetime + from django.db import models from django.db.models import permalink, Q from django.utils.translation import ugettext_lazy as _ @@@ -17,13 -19,12 +19,14 @@@ from django.conf import setting from newtagging.models import TagBase, tags_updated from newtagging import managers -from catalogue.fields import JSONField +from catalogue.fields import JSONField, OverwritingFileField +from catalogue.utils import ExistingFile from librarian import dcparser, html, epub, NoDublinCore +import mutagen from mutagen import id3 from slughifi import slughifi + from sortify import sortify TAG_CATEGORIES = ( @@@ -66,9 -67,9 +69,12 @@@ class Tag(TagBase) gazeta_link = models.CharField(blank=True, max_length=240) wiki_link = models.CharField(blank=True, max_length=240) + created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True) + changed_at = models.DateTimeField(_('creation date'), auto_now=True, db_index=True) + + class UrlDeprecationWarning(DeprecationWarning): + pass + categories_rev = { 'autor': 'author', 'epoka': 'epoch', @@@ -127,20 -128,19 +133,20 @@@ real_tags = [] ambiguous_slugs = [] category = None + deprecated = False tags_splitted = tags.split('/') - for index, name in enumerate(tags_splitted): - if name in Tag.categories_rev: + for name in tags_splitted: + if category: + real_tags.append(Tag.objects.get(slug=name, category=category)) + category = None + elif name in Tag.categories_rev: category = Tag.categories_rev[name] else: - if category: - real_tags.append(Tag.objects.get(slug=name, category=category)) - category = None - else: - try: - real_tags.append(Tag.objects.exclude(category='book').get(slug=name)) - except Tag.MultipleObjectsReturned, e: - ambiguous_slugs.append(name) + try: + real_tags.append(Tag.objects.exclude(category='book').get(slug=name)) + deprecated = True + except Tag.MultipleObjectsReturned, e: + ambiguous_slugs.append(name) if category: # something strange left off @@@ -151,11 -151,8 +157,11 @@@ e.tags = real_tags e.ambiguous_slugs = ambiguous_slugs raise e - else: - return real_tags + if deprecated: + e = Tag.UrlDeprecationWarning() + e.tags = real_tags + raise e + return real_tags else: return TagBase.get_tag_list(tags) @@@ -177,18 -174,24 +183,18 @@@ def book_upload_path(ext=None, maxlen=1 name = slughifi(filename.split(".")[0]) else: name = slughifi(media.name) - return 'lektura/%s.%s' % (name[:maxlen-len('lektura/.%s' % ext)-4], ext) + return 'book/%s/%s.%s' % (ext, name[:maxlen-len('book/%s/.%s' % (ext, ext))-4], ext) return get_dynamic_path class BookMedia(models.Model): - type = models.CharField(_('type'), choices=MEDIA_FORMATS, max_length="100") + type = models.CharField(_('type'), choices=MEDIA_FORMATS, max_length="100", editable=False) name = models.CharField(_('name'), max_length="100") - file = models.FileField(_('file'), upload_to=book_upload_path()) + file = OverwritingFileField(_('file'), upload_to=book_upload_path()) uploaded_at = models.DateTimeField(_('creation date'), auto_now_add=True, editable=False) - extra_info = JSONField(_('extra information'), default='{}') - - def book_count(self): - return self.book_set.count() - book_count.short_description = _('book count') - - def books(self): - return mark_safe('
'.join("%s" % (reverse('admin:catalogue_book_change', args=[b.id]), b.title) for b in self.book_set.all())) - books.short_description = _('books') + extra_info = JSONField(_('extra information'), default='{}', editable=False) + book = models.ForeignKey('Book', related_name='media') + source_sha1 = models.CharField(null=True, blank=True, max_length=40, editable=False) def __unicode__(self): return "%s (%s)" % (self.name, self.file.name.split("/")[-1]) @@@ -198,85 -201,37 +204,87 @@@ verbose_name = _('book media') verbose_name_plural = _('book media') - def save(self, force_insert=False, force_update=False, **kwargs): - media = super(BookMedia, self).save(force_insert, force_update, **kwargs) - if self.type == 'mp3': - file = self.file - extra_info = self.get_extra_info_value() - extra_info.update(self.get_mp3_info()) - self.set_extra_info_value(extra_info) - media = super(BookMedia, self).save(force_insert, force_update, **kwargs) - return media - - def get_mp3_info(self): - """Retrieves artist and director names from audio ID3 tags.""" + def save(self, *args, **kwargs): try: - audio = id3.ID3(self.file.path) - artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1')) - director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3')) - except: - artist_name = director_name = '' - return {'artist_name': artist_name, 'director_name': director_name} + old = BookMedia.objects.get(pk=self.pk) + except BookMedia.DoesNotExist, e: + pass + else: + # if name changed, change the file name, too + if slughifi(self.name) != slughifi(old.name): + self.file.save(None, ExistingFile(self.file.path), save=False, leave=True) + + super(BookMedia, self).save(*args, **kwargs) + extra_info = self.get_extra_info_value() + extra_info.update(self.read_meta()) + self.set_extra_info_value(extra_info) + self.source_sha1 = self.read_source_sha1(self.file.path, self.type) + return super(BookMedia, self).save(*args, **kwargs) + + def read_meta(self): + """ + Reads some metadata from the audiobook. + """ + + artist_name = director_name = project = funded_by = '' + if self.type == 'mp3': + try: + audio = id3.ID3(self.file.path) + artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1')) + director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3')) + project = ", ".join([t.data for t in audio.getall('PRIV') + if t.owner=='wolnelektury.pl?project']) + funded_by = ", ".join([t.data for t in audio.getall('PRIV') + if t.owner=='wolnelektury.pl?funded_by']) + except: + pass + elif self.type == 'ogg': + try: + audio = mutagen.File(self.file.path) + artist_name = ', '.join(audio.get('artist', [])) + director_name = ', '.join(audio.get('conductor', [])) + project = ", ".join(audio.get('project', [])) + funded_by = ", ".join(audio.get('funded_by', [])) + except: + pass + else: + return {} + return {'artist_name': artist_name, 'director_name': director_name, + 'project': project, 'funded_by': funded_by} + + @staticmethod + def read_source_sha1(filepath, filetype): + """ + Reads source file SHA1 from audiobok metadata. + """ + + if filetype == 'mp3': + try: + audio = id3.ID3(filepath) + return [t.data for t in audio.getall('PRIV') + if t.owner=='wolnelektury.pl?flac_sha1'][0] + except: + return None + elif filetype == 'ogg': + try: + audio = mutagen.File(filepath) + return audio.get('flac_sha1', [None])[0] + except: + return None + else: + return None class Book(models.Model): title = models.CharField(_('title'), max_length=120) + sort_key = models.CharField(_('sort_key'), max_length=120, db_index=True, editable=False) slug = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True) description = models.TextField(_('description'), blank=True) - created_at = models.DateTimeField(_('creation date'), auto_now_add=True) + created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True) + changed_at = models.DateTimeField(_('creation date'), auto_now=True, db_index=True) _short_html = models.TextField(_('short HTML'), editable=False) parent_number = models.IntegerField(_('parent number'), default=0) - extra_info = JSONField(_('extra information')) + extra_info = JSONField(_('extra information'), default='{}') gazeta_link = models.CharField(blank=True, max_length=240) wiki_link = models.CharField(blank=True, max_length=240) # files generated during publication @@@ -285,7 -240,9 +293,7 @@@ pdf_file = models.FileField(_('PDF file'), upload_to=book_upload_path('pdf'), blank=True) epub_file = models.FileField(_('EPUB file'), upload_to=book_upload_path('epub'), blank=True) txt_file = models.FileField(_('TXT file'), upload_to=book_upload_path('txt'), blank=True) - # other files - medias = models.ManyToManyField(BookMedia, blank=True) - + parent = models.ForeignKey('self', blank=True, null=True, related_name='children') objects = models.Manager() tagged = managers.ModelTaggedItemManager(Tag) @@@ -298,7 -255,7 +306,7 @@@ pass class Meta: - ordering = ('title',) + ordering = ('sort_key',) verbose_name = _('book') verbose_name_plural = _('books') @@@ -306,6 -263,8 +314,8 @@@ return self.title def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs): + self.sort_key = sortify(self.title) + if reset_short_html: # Reset _short_html during save update = {} @@@ -364,7 -323,7 +374,7 @@@ else: return False else: - if self.medias.filter(book=self, type=type).count() > 0: + if self.media.filter(type=type).exists(): return True else: return False @@@ -382,7 -341,7 +392,7 @@@ elif type == "pdf": return self.pdf_file else: - return self.medias.filter(book=self, type=type) + return self.media.filter(type=type) else: return None @@@ -416,7 -375,7 +426,7 @@@ if self.has_media("txt"): formats.append(u'TXT' % self.get_media('txt').url) # other files - for m in self.medias.order_by('type'): + for m in self.media.order_by('type'): formats.append(u'%s' % (m.file.url, m.type.upper())) formats = [mark_safe(format) for format in formats] @@@ -605,7 -564,7 +615,7 @@@ tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category) if created: tag.name = tag_name - tag.sort_key = tag_sort_key.lower() + tag.sort_key = sortify(tag_sort_key.lower()) tag.save() book_tags.append(tag) @@@ -665,8 -624,8 +675,8 @@@ new_fragment.save() new_fragment.tags = set(book_tags + themes + [book_tag] + ancestor_tags) - if not settings.NO_BUILD_TXT and build_txt: - book.build_txt() + if not settings.NO_BUILD_TXT and build_txt: + book.build_txt() if not settings.NO_BUILD_EPUB and build_epub: book.root_ancestor.build_epub() @@@ -751,6 -710,24 +761,24 @@@ return ', '.join(names) + @classmethod + def tagged_top_level(cls, tags): + """ Returns top-level books tagged with `tags'. + + It only returns those books which don't have ancestors which are + also tagged with those tags. + + """ + # get relevant books and their tags + objects = cls.tagged.with_all(tags) + # eliminate descendants + l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in objects]) + descendants_keys = [book.pk for book in cls.tagged.with_any(l_tags)] + if descendants_keys: + objects = objects.exclude(pk__in=descendants_keys) + + return objects + class Fragment(models.Model): text = models.TextField() @@@ -806,7 -783,8 +834,8 @@@ class FileRecord(models.Model) def _tags_updated_handler(sender, affected_tags, **kwargs): # reset tag global counter - Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None) + # we want Tag.changed_at updated for API to know the tag was touched + Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None, changed_at=datetime.now()) # if book tags changed, reset book tag counter if isinstance(sender, Book) and \ @@@ -821,14 -799,23 +850,14 @@@ tags_updated.connect(_tags_updated_handler) -def _m2m_changed_handler(sender, instance, action, reverse, pk_set, **kwargs): - """ refresh all the short_html stuff on BookMedia delete """ - if sender == Book.medias.through and reverse and action == 'pre_clear': - for book in instance.book_set.all(): - book.save() -m2m_changed.connect(_m2m_changed_handler) - def _pre_delete_handler(sender, instance, **kwargs): - """ explicitly clear m2m, so that Books can be refreshed """ + """ refresh Book on BookMedia delete """ if sender == BookMedia: - instance.book_set.clear() + instance.book.save() pre_delete.connect(_pre_delete_handler) def _post_save_handler(sender, instance, **kwargs): """ refresh all the short_html stuff on BookMedia update """ if sender == BookMedia: - for book in instance.book_set.all(): - book.save() + instance.book.save() post_save.connect(_post_save_handler) - diff --combined apps/catalogue/urls.py index e864cdba3,2ae24571f..7592ca408 --- a/apps/catalogue/urls.py +++ b/apps/catalogue/urls.py @@@ -25,13 -25,13 +25,14 @@@ urlpatterns = patterns('catalogue.views # tools url(r'^zegar/$', 'clock', name='clock'), url(r'^xmls.zip$', 'xmls', name='xmls'), + url(r'^liczniki/$', 'counters', name='catalogue_counters'), # Public interface. Do not change this URLs. url(r'^lektura/(?P[a-zA-Z0-9-]+)\.html$', 'book_text', name='book_text'), url(r'^lektura/(?P[a-zA-Z0-9-]+)/$', 'book_detail', name='book_detail'), url(r'^lektura/(?P[a-zA-Z0-9-]+)/motyw/(?P[a-zA-Z0-9-]+)/$', 'book_fragments', name='book_fragments'), + url(r'^(?P[a-zA-Z0-9-/]*)/$', 'tagged_object_list', name='tagged_object_list'), url(r'^audiobooki/(?Pmp3|ogg|daisy|all).xml$', AudiobookFeed(), name='audiobook_feed'), diff --combined apps/catalogue/views.py index f585aeaa1,00a3a6b60..d17c349c5 --- a/apps/catalogue/views.py +++ b/apps/catalogue/views.py @@@ -15,9 -15,9 +15,9 @@@ from datetime import datetim from django.conf import settings from django.template import RequestContext from django.shortcuts import render_to_response, get_object_or_404 -from django.http import HttpResponse, HttpResponseRedirect, Http404 +from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponsePermanentRedirect from django.core.urlresolvers import reverse -from django.db.models import Q +from django.db.models import Count, Sum, Q from django.contrib.auth.decorators import login_required, user_passes_test from django.utils.datastructures import SortedDict from django.views.decorators.http import require_POST @@@ -28,6 -28,7 +28,7 @@@ from django.utils.functional import Pro from django.utils.encoding import force_unicode from django.utils.http import urlquote_plus from django.views.decorators import cache + from django.utils import translation from django.utils.translation import ugettext as _ from django.views.generic.list_detail import object_list @@@ -37,7 -38,6 +38,7 @@@ from catalogue.utils import split_tag from newtagging import views as newtagging_views from pdcounter import models as pdcounter_models from pdcounter import views as pdcounter_views +from suggest.forms import PublishingSuggestForm from slughifi import slughifi @@@ -83,7 -83,7 +84,7 @@@ def book_list(request, filter=None, tem form = forms.SearchForm() books_by_parent = {} - books = models.Book.objects.all().order_by('parent_number', 'title').only('title', 'parent', 'slug') + books = models.Book.objects.all().order_by('parent_number', 'sort_key').only('title', 'parent', 'slug') if filter: books = books.filter(filter).distinct() book_ids = set((book.pk for book in books)) @@@ -119,36 -119,15 +120,36 @@@ def audiobook_list(request): - return book_list(request, Q(medias__type='mp3') | Q(medias__type='ogg'), + return book_list(request, Q(media__type='mp3') | Q(media__type='ogg'), template_name='catalogue/audiobook_list.html') def daisy_list(request): - return book_list(request, Q(medias__type='daisy'), + return book_list(request, Q(media__type='daisy'), template_name='catalogue/daisy_list.html') +def counters(request): + form = forms.SearchForm() + + books = models.Book.objects.count() + books_nonempty = models.Book.objects.exclude(html_file='').count() + books_empty = models.Book.objects.filter(html_file='').count() + books_root = models.Book.objects.filter(parent=None).count() + + media = models.BookMedia.objects.count() + media_types = models.BookMedia.objects.values('type').\ + annotate(count=Count('type')).\ + order_by('type') + for mt in media_types: + mt['size'] = sum(b.file.size for b in models.BookMedia.objects.filter(type=mt['type'])) + mt['deprecated'] = models.BookMedia.objects.filter( + type=mt['type'], source_sha1=None).count() if mt['type'] in ('mp3', 'ogg') else '-' + + return render_to_response('catalogue/counters.html', + locals(), context_instance=RequestContext(request)) + + def differentiate_tags(request, tags, ambiguous_slugs): beginning = '/'.join(tag.url_chunk for tag in tags) unparsed = '/'.join(ambiguous_slugs[1:]) @@@ -174,8 -153,6 +175,8 @@@ def tagged_object_list(request, tags='' raise Http404 except models.Tag.MultipleObjectsReturned, e: return differentiate_tags(request, e.tags, e.ambiguous_slugs) + except models.Tag.UrlDeprecationWarning, e: + return HttpResponsePermanentRedirect(reverse('tagged_object_list', args=['/'.join(tag.url_chunk for tag in e.tags)])) try: if len(tags) > settings.MAX_TAG_LIST: @@@ -217,14 -194,10 +218,10 @@@ objects = fragments else: - # get relevant books and their tags - objects = models.Book.tagged.with_all(tags) - if not shelf_is_set: - # eliminate descendants - l_tags = models.Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in objects]) - descendants_keys = [book.pk for book in models.Book.tagged.with_any(l_tags)] - if descendants_keys: - objects = objects.exclude(pk__in=descendants_keys) + if shelf_is_set: + objects = models.Book.tagged.with_all(tags) + else: + objects = models.Book.tagged_top_level(tags) # get related tags from `tag_counter` and `theme_counter` related_counts = {} @@@ -256,7 -229,6 +253,6 @@@ 'only_author': only_author, 'only_my_shelf': only_my_shelf, 'formats_form': forms.DownloadFormatsForm(), - 'tags': tags, } ) @@@ -282,7 -254,7 +278,7 @@@ def book_detail(request, slug) book_tag = book.book_tag() tags = list(book.tags.filter(~Q(category='set'))) categories = split_tags(tags) - book_children = book.children.all().order_by('parent_number', 'title') + book_children = book.children.all().order_by('parent_number', 'sort_key') _book = book parents = [] @@@ -298,18 -270,6 +294,18 @@@ extra_info = book.get_extra_info_value() + projects = set() + for m in book.media.filter(type='mp3'): + # ogg files are always from the same project + meta = m.get_extra_info_value() + project = meta.get('project') + if not project: + # temporary fallback + project = u'CzytamySłuchając' + + projects.add((project, meta.get('funded_by'))) + projects = sorted(projects) + form = forms.SearchForm() return render_to_response('catalogue/book_detail.html', locals(), context_instance=RequestContext(request)) @@@ -502,9 -462,7 +498,9 @@@ def search(request) {'tags':tag_list, 'prefix':prefix, 'results':((x, _get_result_link(x, tag_list), _get_result_type(x)) for x in result)}, context_instance=RequestContext(request)) else: - return render_to_response('catalogue/search_no_hits.html', {'tags':tag_list, 'prefix':prefix}, + form = PublishingSuggestForm(initial={"books": prefix + ", "}) + return render_to_response('catalogue/search_no_hits.html', + {'tags':tag_list, 'prefix':prefix, "pubsuggest_form": form}, context_instance=RequestContext(request)) @@@ -803,3 -761,18 +799,18 @@@ def xmls(request) temp.seek(0) response.write(temp.read()) return response + + + + # info views for API + + def book_info(request, id, lang='pl'): + book = get_object_or_404(models.Book, id=id) + # set language by hand + translation.activate(lang) + return render_to_response('catalogue/book_info.html', locals(), + context_instance=RequestContext(request)) + + def tag_info(request, id): + tag = get_object_or_404(models.Tag, id=id) + return HttpResponse(tag.description) diff --combined wolnelektury/settings.py index a47276378,dac4e6a3f..3c5faf8a2 --- a/wolnelektury/settings.py +++ b/wolnelektury/settings.py @@@ -81,14 -81,14 +81,14 @@@ TEMPLATE_LOADERS = # 'django.template.loaders.eggs.load_template_source', ] -TEMPLATE_CONTEXT_PROCESSORS = [ +TEMPLATE_CONTEXT_PROCESSORS = ( 'django.core.context_processors.auth', 'django.core.context_processors.debug', 'django.core.context_processors.i18n', 'django.core.context_processors.media', 'django.core.context_processors.request', 'wolnelektury.context_processors.extra_settings', -] +) MIDDLEWARE_CLASSES = [ 'django.middleware.cache.UpdateCacheMiddleware', @@@ -98,7 -98,6 +98,7 @@@ 'django.middleware.doc.XViewMiddleware', 'pagination.middleware.PaginationMiddleware', 'django.middleware.locale.LocaleMiddleware', + 'piwik.django.middleware.PiwikMiddleware', 'maintenancemode.middleware.MaintenanceModeMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.cache.FetchFromCacheMiddleware', @@@ -142,7 -141,6 +142,7 @@@ INSTALLED_APPS = 'lesmianator', 'opds', 'pdcounter', + 'piwik.django', ] #CACHE_BACKEND = 'locmem:///?max_entries=3000' @@@ -206,6 -204,10 +206,10 @@@ THUMBNAIL_PROCESSORS = TRANSLATION_REGISTRY = "wolnelektury.translation" + + # seconds until a changes appears in the changes api + API_WAIT = 100 + # limit number of filtering tags MAX_TAG_LIST = 6 diff --combined wolnelektury/templates/catalogue/book_text.html index 6c8edba6d,4dc6cb937..935cdf392 --- a/wolnelektury/templates/catalogue/book_text.html +++ b/wolnelektury/templates/catalogue/book_text.html @@@ -25,34 -25,7 +25,7 @@@
-

- {% if book.get_extra_info_value.license %} - {% trans "This work is licensed under:" %} - {{ book.get_extra_info_value.license_description }} - {% else %} - {% blocktrans %}This work isn't covered by copyright and is part of the - public domain, which means it can be freely used, published and - distributed. If there are any additional copyrighted materials - provided with this work (such as annotations, motifs etc.), those - materials are licensed under the - Creative Commons Attribution-ShareAlike 3.0 - license.{% endblocktrans %} - {% endif %} -

- - {% if book.get_extra_info_value.source_name %} -

{% trans "Text prepared based on:" %} {{ book.get_extra_info_value.source_name }}

- {% endif %} - - {% if book.get_extra_info_value.description %} -

{{ book.get_extra_info_value.description }}

- {% endif %} - - {% if book.get_extra_info_value.editor or book.get_extra_info_value.technical_editor %} -

{% trans "Edited and annotated by:" %} - {% all_editors book.get_extra_info_value %}.

- {% endif %} - + {% book_info book %}