367f382c66b18952e9602191f8b1b4bd0026a42e
[wolnelektury.git] / apps / catalogue / models.py
1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 from django.db import models
6 from django.db.models import permalink, Q
7 from django.utils.translation import ugettext_lazy as _
8 from django.contrib.auth.models import User
9 from django.core.files import File
10 from django.template.loader import render_to_string
11 from django.utils.safestring import mark_safe
12 from django.utils.translation import get_language
13 from django.core.urlresolvers import reverse
14 from datetime import datetime
15
16 from newtagging.models import TagBase
17 from newtagging import managers
18 from catalogue.fields import JSONField
19
20 from librarian import html, dcparser
21 from mutagen import id3
22
23
24 TAG_CATEGORIES = (
25     ('author', _('author')),
26     ('epoch', _('epoch')),
27     ('kind', _('kind')),
28     ('genre', _('genre')),
29     ('theme', _('theme')),
30     ('set', _('set')),
31     ('book', _('book')),
32 )
33
34
35 class TagSubcategoryManager(models.Manager):
36     def __init__(self, subcategory):
37         super(TagSubcategoryManager, self).__init__()
38         self.subcategory = subcategory
39
40     def get_query_set(self):
41         return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)
42
43
44 class Tag(TagBase):
45     name = models.CharField(_('name'), max_length=50, db_index=True)
46     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
47     sort_key = models.SlugField(_('sort key'), max_length=120, db_index=True)
48     category = models.CharField(_('category'), max_length=50, blank=False, null=False,
49         db_index=True, choices=TAG_CATEGORIES)
50     description = models.TextField(_('description'), blank=True)
51     main_page = models.BooleanField(_('main page'), default=False, db_index=True, help_text=_('Show tag on main page'))
52
53     user = models.ForeignKey(User, blank=True, null=True)
54     book_count = models.IntegerField(_('book count'), default=0, blank=False, null=False)
55     death = models.IntegerField(_(u'year of death'), blank=True, null=True)
56     gazeta_link = models.CharField(blank=True, max_length=240)
57     wiki_link = models.CharField(blank=True, max_length=240)
58
59     categories_rev = {
60         'autor': 'author',
61         'epoka': 'epoch',
62         'rodzaj': 'kind',
63         'gatunek': 'genre',
64         'motyw': 'theme',
65         'polka': 'set',
66     }
67     categories_dict = dict((item[::-1] for item in categories_rev.iteritems()))
68
69     class Meta:
70         ordering = ('sort_key',)
71         verbose_name = _('tag')
72         verbose_name_plural = _('tags')
73         unique_together = (("slug", "category"),)
74
75     def __unicode__(self):
76         return self.name
77
78     def __repr__(self):
79         return "Tag(slug=%r)" % self.slug
80
81     @permalink
82     def get_absolute_url(self):
83         return ('catalogue.views.tagged_object_list', [self.url_chunk])
84
85     def has_description(self):
86         return len(self.description) > 0
87     has_description.short_description = _('description')
88     has_description.boolean = True
89
90     def alive(self):
91         return self.death is None
92
93     def in_pd(self):
94         """ tests whether an author is in public domain """
95         return self.death is not None and self.goes_to_pd() <= datetime.now().year
96
97     def goes_to_pd(self):
98         """ calculates the year of public domain entry for an author """
99         return self.death + 71 if self.death is not None else None
100
101     @staticmethod
102     def get_tag_list(tags):
103         if isinstance(tags, basestring):
104             real_tags = []
105             ambiguous_slugs = []
106             category = None
107             tags_splitted = tags.split('/')
108             for index, name in enumerate(tags_splitted):
109                 if name in Tag.categories_rev:
110                     category = Tag.categories_rev[name]
111                 else:
112                     if category:
113                         real_tags.append(Tag.objects.get(slug=name, category=category))
114                         category = None
115                     else:
116                         try:
117                             real_tags.append(Tag.objects.exclude(category='book').get(slug=name))
118                         except Tag.MultipleObjectsReturned, e:
119                             ambiguous_slugs.append(name)
120
121             if category:
122                 # something strange left off
123                 raise Tag.DoesNotExist()
124             if ambiguous_slugs:
125                 # some tags should be qualified
126                 e = Tag.MultipleObjectsReturned()
127                 e.tags = real_tags
128                 e.ambiguous_slugs = ambiguous_slugs
129                 raise e
130             else:
131                 return real_tags
132         else:
133             return TagBase.get_tag_list(tags)
134
135     @property
136     def url_chunk(self):
137         return '/'.join((Tag.categories_dict[self.category], self.slug))
138
139
140 # TODO: why is this hard-coded ?
141 def book_upload_path(ext):
142     def get_dynamic_path(book, filename):
143         return 'lektura/%s.%s' % (book.slug, ext)
144     return get_dynamic_path
145
146
147 class Book(models.Model):
148     title = models.CharField(_('title'), max_length=120)
149     slug = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
150     description = models.TextField(_('description'), blank=True)
151     created_at = models.DateTimeField(_('creation date'), auto_now=True)
152     _short_html = models.TextField(_('short HTML'), editable=False)
153     parent_number = models.IntegerField(_('parent number'), default=0)
154     extra_info = JSONField(_('extra information'))
155     gazeta_link = models.CharField(blank=True, max_length=240)
156     wiki_link = models.CharField(blank=True, max_length=240)
157
158
159     # Formats
160     xml_file = models.FileField(_('XML file'), upload_to=book_upload_path('xml'), blank=True)
161     html_file = models.FileField(_('HTML file'), upload_to=book_upload_path('html'), blank=True)
162     pdf_file = models.FileField(_('PDF file'), upload_to=book_upload_path('pdf'), blank=True)
163     epub_file = models.FileField(_('EPUB file'), upload_to=book_upload_path('epub'), blank=True)
164     odt_file = models.FileField(_('ODT file'), upload_to=book_upload_path('odt'), blank=True)
165     txt_file = models.FileField(_('TXT file'), upload_to=book_upload_path('txt'), blank=True)
166     mp3_file = models.FileField(_('MP3 file'), upload_to=book_upload_path('mp3'), blank=True)
167     ogg_file = models.FileField(_('OGG file'), upload_to=book_upload_path('ogg'), blank=True)
168
169     parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
170
171     objects = models.Manager()
172     tagged = managers.ModelTaggedItemManager(Tag)
173     tags = managers.TagDescriptor(Tag)
174
175     _tag_counter = JSONField(null=True, editable=False)
176     _theme_counter = JSONField(null=True, editable=False)
177
178     class AlreadyExists(Exception):
179         pass
180
181     class Meta:
182         ordering = ('title',)
183         verbose_name = _('book')
184         verbose_name_plural = _('books')
185
186     def __unicode__(self):
187         return self.title
188
189     def save(self, force_insert=False, force_update=False, reset_short_html=True, refresh_mp3=True):
190         if reset_short_html:
191             # Reset _short_html during save
192             for key in filter(lambda x: x.startswith('_short_html'), self.__dict__):
193                 self.__setattr__(key, '')
194
195         book = super(Book, self).save(force_insert, force_update)
196
197         if refresh_mp3 and self.mp3_file:
198             print self.mp3_file, self.mp3_file.path
199             extra_info = self.get_extra_info_value()
200             extra_info.update(self.get_mp3_info())
201             self.set_extra_info_value(extra_info)
202             book = super(Book, self).save(force_insert, force_update)
203
204         return book
205
206     @permalink
207     def get_absolute_url(self):
208         return ('catalogue.views.book_detail', [self.slug])
209
210     @property
211     def name(self):
212         return self.title
213
214     def book_tag(self):
215         slug = ('l-' + self.slug)[:120]
216         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
217         if created:
218             book_tag.name = self.title[:50]
219             book_tag.sort_key = slug
220             book_tag.save()
221         return book_tag
222
223     def short_html(self):
224         key = '_short_html_%s' % get_language()
225         short_html = getattr(self, key)
226
227         if short_html and len(short_html):
228             return mark_safe(short_html)
229         else:
230             tags = self.tags.filter(~Q(category__in=('set', 'theme', 'book')))
231             tags = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name)) for tag in tags]
232
233             formats = []
234             if self.html_file:
235                 formats.append(u'<a href="%s">%s</a>' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online')))
236             if self.pdf_file:
237                 formats.append(u'<a href="%s">PDF</a>' % self.pdf_file.url)
238             if self.epub_file:
239                 formats.append(u'<a href="%s">EPUB</a>' % self.epub_file.url)
240             if self.odt_file:
241                 formats.append(u'<a href="%s">ODT</a>' % self.odt_file.url)
242             if self.txt_file:
243                 formats.append(u'<a href="%s">TXT</a>' % self.txt_file.url)
244             if self.mp3_file:
245                 formats.append(u'<a href="%s">MP3</a>' % self.mp3_file.url)
246             if self.ogg_file:
247                 formats.append(u'<a href="%s">OGG</a>' % self.ogg_file.url)
248
249             formats = [mark_safe(format) for format in formats]
250
251             setattr(self, key, unicode(render_to_string('catalogue/book_short.html',
252                 {'book': self, 'tags': tags, 'formats': formats})))
253             self.save(reset_short_html=False)
254             return mark_safe(getattr(self, key))
255
256
257     def get_mp3_info(self):
258         """Retrieves artist and director names from audio ID3 tags."""
259         audio = id3.ID3(self.mp3_file.path)
260         artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1'))
261         director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3'))
262         return {'artist_name': artist_name, 'director_name': director_name}
263
264     def has_description(self):
265         return len(self.description) > 0
266     has_description.short_description = _('description')
267     has_description.boolean = True
268
269     def has_pdf_file(self):
270         return bool(self.pdf_file)
271     has_pdf_file.short_description = 'PDF'
272     has_pdf_file.boolean = True
273
274     def has_epub_file(self):
275         return bool(self.epub_file)
276     has_epub_file.short_description = 'EPUB'
277     has_epub_file.boolean = True
278
279     def has_odt_file(self):
280         return bool(self.odt_file)
281     has_odt_file.short_description = 'ODT'
282     has_odt_file.boolean = True
283
284     def has_html_file(self):
285         return bool(self.html_file)
286     has_html_file.short_description = 'HTML'
287     has_html_file.boolean = True
288
289     @classmethod
290     def from_xml_file(cls, xml_file, overwrite=False):
291         # use librarian to parse meta-data
292         book_info = dcparser.parse(xml_file)
293
294         if not isinstance(xml_file, File):
295             xml_file = File(xml_file)
296
297         try:
298             return cls.from_text_and_meta(xml_file, book_info, overwrite)
299         finally:
300             xml_file.close()
301
302     @classmethod
303     def from_text_and_meta(cls, raw_file, book_info, overwrite=False):
304         from tempfile import NamedTemporaryFile
305         from slughifi import slughifi
306         from markupstring import MarkupString
307
308         # Read book metadata
309         book_base, book_slug = book_info.url.rsplit('/', 1)
310         book, created = Book.objects.get_or_create(slug=book_slug)
311
312         if created:
313             book_shelves = []
314         else:
315             if not overwrite:
316                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
317             # Save shelves for this book
318             book_shelves = list(book.tags.filter(category='set'))
319
320         book.title = book_info.title
321         book.set_extra_info_value(book_info.to_dict())
322         book._short_html = ''
323         book.save()
324
325         book_tags = []
326         for category in ('kind', 'genre', 'author', 'epoch'):
327             tag_name = getattr(book_info, category)
328             tag_sort_key = tag_name
329             if category == 'author':
330                 tag_sort_key = tag_name.last_name
331                 tag_name = ' '.join(tag_name.first_names) + ' ' + tag_name.last_name
332             tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
333             if created:
334                 tag.name = tag_name
335                 tag.sort_key = slughifi(tag_sort_key)
336                 tag.save()
337             book_tags.append(tag)
338
339         book.tags = book_tags
340
341         book_tag = book.book_tag()
342
343         if hasattr(book_info, 'parts'):
344             for n, part_url in enumerate(book_info.parts):
345                 base, slug = part_url.rsplit('/', 1)
346                 try:
347                     child_book = Book.objects.get(slug=slug)
348                     child_book.parent = book
349                     child_book.parent_number = n
350                     child_book.save()
351                 except Book.DoesNotExist, e:
352                     raise Book.DoesNotExist(_('Book with slug = "%s" does not exist.') % slug)
353
354         book_descendants = list(book.children.all())
355         while len(book_descendants) > 0:
356             child_book = book_descendants.pop(0)
357             child_book.tags = list(child_book.tags) + [book_tag]
358             child_book.save()
359             for fragment in child_book.fragments.all():
360                 fragment.tags = set(list(fragment.tags) + [book_tag])
361             book_descendants += list(child_book.children.all())
362
363         # Save XML and HTML files
364         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
365
366         html_file = NamedTemporaryFile()
367         if html.transform(book.xml_file.path, html_file, parse_dublincore=False):
368             book.html_file.save('%s.html' % book.slug, File(html_file), save=False)
369
370             # Extract fragments
371             closed_fragments, open_fragments = html.extract_fragments(book.html_file.path)
372             book_themes = []
373             for fragment in closed_fragments.values():
374                 text = fragment.to_string()
375                 short_text = ''
376                 if (len(MarkupString(text)) > 240):
377                     short_text = unicode(MarkupString(text)[:160])
378                 new_fragment, created = Fragment.objects.get_or_create(anchor=fragment.id, book=book,
379                     defaults={'text': text, 'short_text': short_text})
380
381                 try:
382                     theme_names = [s.strip() for s in fragment.themes.split(',')]
383                 except AttributeError:
384                     continue
385                 themes = []
386                 for theme_name in theme_names:
387                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
388                     if created:
389                         tag.name = theme_name
390                         tag.sort_key = slughifi(theme_name)
391                         tag.save()
392                     themes.append(tag)
393                 new_fragment.save()
394                 new_fragment.tags = set(list(book.tags) + themes + [book_tag])
395                 book_themes += themes
396
397             book_themes = set(book_themes)
398             book.tags = list(book.tags) + list(book_themes) + book_shelves
399
400         book.save()
401         return book
402
403
404     def refresh_tag_counter(self):
405         tags = {}
406         for child in self.children.all().order_by():
407             for tag_pk, value in child.tag_counter.iteritems():
408                 tags[tag_pk] = tags.get(tag_pk, 0) + value
409         for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by():
410             tags[tag.pk] = 1
411         self.set__tag_counter_value(tags)
412         self.save(reset_short_html=False, refresh_mp3=False)
413         return tags
414
415     @property
416     def tag_counter(self):
417         if self._tag_counter is None:
418             return self.refresh_tag_counter()
419         return dict((int(k), v) for k, v in self.get__tag_counter_value().iteritems())
420
421     def refresh_theme_counter(self):
422         tags = {}
423         for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by():
424             for tag in fragment.tags.filter(category='theme').order_by():
425                 tags[tag.pk] = tags.get(tag.pk, 0) + 1
426         self.set__theme_counter_value(tags)
427         self.save(reset_short_html=False, refresh_mp3=False)
428         return tags
429
430     @property
431     def theme_counter(self):
432         if self._theme_counter is None:
433             return self.refresh_theme_counter()
434         return dict((int(k), v) for k, v in self.get__theme_counter_value().iteritems())
435
436
437
438 class Fragment(models.Model):
439     text = models.TextField()
440     short_text = models.TextField(editable=False)
441     _short_html = models.TextField(editable=False)
442     anchor = models.CharField(max_length=120)
443     book = models.ForeignKey(Book, related_name='fragments')
444
445     objects = models.Manager()
446     tagged = managers.ModelTaggedItemManager(Tag)
447     tags = managers.TagDescriptor(Tag)
448
449     class Meta:
450         ordering = ('book', 'anchor',)
451         verbose_name = _('fragment')
452         verbose_name_plural = _('fragments')
453
454     def get_absolute_url(self):
455         return '%s#m%s' % (reverse('book_text', kwargs={'slug': self.book.slug}), self.anchor)
456
457     def short_html(self):
458         key = '_short_html_%s' % get_language()
459         short_html = getattr(self, key)
460         if short_html and len(short_html):
461             return mark_safe(short_html)
462         else:
463             book_authors = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name))
464                 for tag in self.book.tags if tag.category == 'author']
465
466             setattr(self, key, unicode(render_to_string('catalogue/fragment_short.html',
467                 {'fragment': self, 'book': self.book, 'book_authors': book_authors})))
468             self.save()
469             return mark_safe(getattr(self, key))
470
471
472 class BookStub(models.Model):
473     title = models.CharField(_('title'), max_length=120)
474     author = models.CharField(_('author'), max_length=120)
475     pd = models.IntegerField(_('goes to public domain'), null=True, blank=True)
476     slug = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
477     translator = models.TextField(_('translator'), blank=True)
478     translator_death = models.TextField(_('year of translator\'s death'), blank=True)
479
480     class Meta:
481         ordering = ('title',)
482         verbose_name = _('book stub')
483         verbose_name_plural = _('book stubs')
484
485     def __unicode__(self):
486         return self.title
487
488     @permalink
489     def get_absolute_url(self):
490         return ('catalogue.views.book_detail', [self.slug])
491
492     def in_pd(self):
493         return self.pd is not None and self.pd <= datetime.now().year
494
495     @property
496     def name(self):
497         return self.title
498
499