54a915227ce480f379a48c766a43195135b87ec8
[wolnelektury.git] / apps / catalogue / models.py
1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 from django.db import models
6 from django.db.models import permalink, Q
7 from django.utils.translation import ugettext_lazy as _
8 from django.contrib.auth.models import User
9 from django.core.files import File
10 from django.template.loader import render_to_string
11 from django.utils.safestring import mark_safe
12 from django.utils.translation import get_language
13 from django.core.urlresolvers import reverse
14
15 from newtagging.models import TagBase, tags_updated
16 from newtagging import managers
17 from catalogue.fields import JSONField
18
19 from librarian import dcparser, html, epub, NoDublinCore
20 from mutagen import id3
21
22
23 TAG_CATEGORIES = (
24     ('author', _('author')),
25     ('epoch', _('epoch')),
26     ('kind', _('kind')),
27     ('genre', _('genre')),
28     ('theme', _('theme')),
29     ('set', _('set')),
30     ('book', _('book')),
31 )
32
33
34 class TagSubcategoryManager(models.Manager):
35     def __init__(self, subcategory):
36         super(TagSubcategoryManager, self).__init__()
37         self.subcategory = subcategory
38
39     def get_query_set(self):
40         return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)
41
42
43 class Tag(TagBase):
44     name = models.CharField(_('name'), max_length=50, db_index=True)
45     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
46     sort_key = models.SlugField(_('sort key'), max_length=120, db_index=True)
47     category = models.CharField(_('category'), max_length=50, blank=False, null=False,
48         db_index=True, choices=TAG_CATEGORIES)
49     description = models.TextField(_('description'), blank=True)
50     main_page = models.BooleanField(_('main page'), default=False, db_index=True, help_text=_('Show tag on main page'))
51
52     user = models.ForeignKey(User, blank=True, null=True)
53     book_count = models.IntegerField(_('book count'), blank=False, null=True)
54     gazeta_link = models.CharField(blank=True, max_length=240)
55     wiki_link = models.CharField(blank=True, max_length=240)
56
57     categories_rev = {
58         'autor': 'author',
59         'epoka': 'epoch',
60         'rodzaj': 'kind',
61         'gatunek': 'genre',
62         'motyw': 'theme',
63         'polka': 'set',
64     }
65     categories_dict = dict((item[::-1] for item in categories_rev.iteritems()))
66
67     class Meta:
68         ordering = ('sort_key',)
69         verbose_name = _('tag')
70         verbose_name_plural = _('tags')
71         unique_together = (("slug", "category"),)
72
73     def __unicode__(self):
74         return self.name
75
76     def __repr__(self):
77         return "Tag(slug=%r)" % self.slug
78
79     @permalink
80     def get_absolute_url(self):
81         return ('catalogue.views.tagged_object_list', [self.url_chunk])
82
83     def has_description(self):
84         return len(self.description) > 0
85     has_description.short_description = _('description')
86     has_description.boolean = True
87
88     def get_count(self):
89         """ returns global book count for book tags, fragment count for themes """
90
91         if self.book_count is None:
92             if self.category == 'book':
93                 # never used
94                 objects = Book.objects.none()
95             elif self.category == 'theme':
96                 objects = Fragment.tagged.with_all((self,))
97             else:
98                 objects = Book.tagged.with_all((self,)).order_by()
99                 if self.category != 'set':
100                     # eliminate descendants
101                     l_tags = Tag.objects.filter(slug__in=[book.book_tag_slug() for book in objects])
102                     descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags)]
103                     if descendants_keys:
104                         objects = objects.exclude(pk__in=descendants_keys)
105             self.book_count = objects.count()
106             self.save()
107         return self.book_count
108
109     @staticmethod
110     def get_tag_list(tags):
111         if isinstance(tags, basestring):
112             real_tags = []
113             ambiguous_slugs = []
114             category = None
115             tags_splitted = tags.split('/')
116             for index, name in enumerate(tags_splitted):
117                 if name in Tag.categories_rev:
118                     category = Tag.categories_rev[name]
119                 else:
120                     if category:
121                         real_tags.append(Tag.objects.get(slug=name, category=category))
122                         category = None
123                     else:
124                         try:
125                             real_tags.append(Tag.objects.exclude(category='book').get(slug=name))
126                         except Tag.MultipleObjectsReturned, e:
127                             ambiguous_slugs.append(name)
128
129             if category:
130                 # something strange left off
131                 raise Tag.DoesNotExist()
132             if ambiguous_slugs:
133                 # some tags should be qualified
134                 e = Tag.MultipleObjectsReturned()
135                 e.tags = real_tags
136                 e.ambiguous_slugs = ambiguous_slugs
137                 raise e
138             else:
139                 return real_tags
140         else:
141             return TagBase.get_tag_list(tags)
142
143     @property
144     def url_chunk(self):
145         return '/'.join((Tag.categories_dict[self.category], self.slug))
146
147
148 # TODO: why is this hard-coded ?
149 def book_upload_path(ext):
150     def get_dynamic_path(book, filename):
151         return 'lektura/%s.%s' % (book.slug, ext)
152     return get_dynamic_path
153
154
155 class Book(models.Model):
156     title = models.CharField(_('title'), max_length=120)
157     slug = models.SlugField(_('slug'), max_length=120, unique=True, db_index=True)
158     description = models.TextField(_('description'), blank=True)
159     created_at = models.DateTimeField(_('creation date'), auto_now=True)
160     _short_html = models.TextField(_('short HTML'), editable=False)
161     parent_number = models.IntegerField(_('parent number'), default=0)
162     extra_info = JSONField(_('extra information'))
163     gazeta_link = models.CharField(blank=True, max_length=240)
164     wiki_link = models.CharField(blank=True, max_length=240)
165
166
167     # Formats
168     xml_file = models.FileField(_('XML file'), upload_to=book_upload_path('xml'), blank=True)
169     html_file = models.FileField(_('HTML file'), upload_to=book_upload_path('html'), blank=True)
170     pdf_file = models.FileField(_('PDF file'), upload_to=book_upload_path('pdf'), blank=True)
171     epub_file = models.FileField(_('EPUB file'), upload_to=book_upload_path('epub'), blank=True)
172     odt_file = models.FileField(_('ODT file'), upload_to=book_upload_path('odt'), blank=True)
173     txt_file = models.FileField(_('TXT file'), upload_to=book_upload_path('txt'), blank=True)
174     mp3_file = models.FileField(_('MP3 file'), upload_to=book_upload_path('mp3'), blank=True)
175     ogg_file = models.FileField(_('OGG file'), upload_to=book_upload_path('ogg'), blank=True)
176     daisy_file = models.FileField(_('DAISY file'), upload_to=book_upload_path('daisy.zip'), blank=True)
177
178     parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
179
180     objects = models.Manager()
181     tagged = managers.ModelTaggedItemManager(Tag)
182     tags = managers.TagDescriptor(Tag)
183
184     _tag_counter = JSONField(null=True, editable=False)
185     _theme_counter = JSONField(null=True, editable=False)
186
187     class AlreadyExists(Exception):
188         pass
189
190     class Meta:
191         ordering = ('title',)
192         verbose_name = _('book')
193         verbose_name_plural = _('books')
194
195     def __unicode__(self):
196         return self.title
197
198     def save(self, force_insert=False, force_update=False, reset_short_html=True, refresh_mp3=True, **kwargs):
199         if reset_short_html:
200             # Reset _short_html during save
201             update = {}
202             for key in filter(lambda x: x.startswith('_short_html'), self.__dict__):
203                 update[key] = ''
204                 self.__setattr__(key, '')
205             # Fragment.short_html relies on book's tags, so reset it here too
206             self.fragments.all().update(**update)
207
208         book = super(Book, self).save(force_insert, force_update)
209
210         if refresh_mp3 and self.mp3_file:
211             print self.mp3_file, self.mp3_file.path
212             extra_info = self.get_extra_info_value()
213             extra_info.update(self.get_mp3_info())
214             self.set_extra_info_value(extra_info)
215             book = super(Book, self).save(force_insert, force_update)
216
217         return book
218
219     @permalink
220     def get_absolute_url(self):
221         return ('catalogue.views.book_detail', [self.slug])
222
223     @property
224     def name(self):
225         return self.title
226
227     def book_tag_slug(self):
228         return ('l-' + self.slug)[:120]
229
230     def book_tag(self):
231         slug = self.book_tag_slug()
232         book_tag, created = Tag.objects.get_or_create(slug=slug, category='book')
233         if created:
234             book_tag.name = self.title[:50]
235             book_tag.sort_key = slug
236             book_tag.save()
237         return book_tag
238
239     def short_html(self):
240         key = '_short_html_%s' % get_language()
241         short_html = getattr(self, key)
242
243         if short_html and len(short_html):
244             return mark_safe(short_html)
245         else:
246             tags = self.tags.filter(~Q(category__in=('set', 'theme', 'book')))
247             tags = [mark_safe(u'<a href="%s">%s</a>' % (tag.get_absolute_url(), tag.name)) for tag in tags]
248
249             formats = []
250             if self.html_file:
251                 formats.append(u'<a href="%s">%s</a>' % (reverse('book_text', kwargs={'slug': self.slug}), _('Read online')))
252             if self.pdf_file:
253                 formats.append(u'<a href="%s">PDF</a>' % self.pdf_file.url)
254             if self.root_ancestor.epub_file:
255                 formats.append(u'<a href="%s">EPUB</a>' % self.root_ancestor.epub_file.url)
256             if self.odt_file:
257                 formats.append(u'<a href="%s">ODT</a>' % self.odt_file.url)
258             if self.txt_file:
259                 formats.append(u'<a href="%s">TXT</a>' % self.txt_file.url)
260             if self.mp3_file:
261                 formats.append(u'<a href="%s">MP3</a>' % self.mp3_file.url)
262             if self.ogg_file:
263                 formats.append(u'<a href="%s">OGG</a>' % self.ogg_file.url)
264             if self.daisy_file:
265                 formats.append(u'<a href="%s">DAISY</a>' % self.daisy_file.url)
266
267             formats = [mark_safe(format) for format in formats]
268
269             setattr(self, key, unicode(render_to_string('catalogue/book_short.html',
270                 {'book': self, 'tags': tags, 'formats': formats})))
271             self.save(reset_short_html=False)
272             return mark_safe(getattr(self, key))
273
274
275     @property
276     def root_ancestor(self):
277         """ returns the oldest ancestor """
278
279         if not hasattr(self, '_root_ancestor'):
280             book = self
281             while book.parent:
282                 book = book.parent
283             self._root_ancestor = book
284         return self._root_ancestor
285
286
287     def get_mp3_info(self):
288         """Retrieves artist and director names from audio ID3 tags."""
289         audio = id3.ID3(self.mp3_file.path)
290         artist_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE1'))
291         director_name = ', '.join(', '.join(tag.text) for tag in audio.getall('TPE3'))
292         return {'artist_name': artist_name, 'director_name': director_name}
293
294     def has_description(self):
295         return len(self.description) > 0
296     has_description.short_description = _('description')
297     has_description.boolean = True
298
299     def has_pdf_file(self):
300         return bool(self.pdf_file)
301     has_pdf_file.short_description = 'PDF'
302     has_pdf_file.boolean = True
303
304     def has_epub_file(self):
305         return bool(self.epub_file)
306     has_epub_file.short_description = 'EPUB'
307     has_epub_file.boolean = True
308
309     def has_odt_file(self):
310         return bool(self.odt_file)
311     has_odt_file.short_description = 'ODT'
312     has_odt_file.boolean = True
313
314     def has_html_file(self):
315         return bool(self.html_file)
316     has_html_file.short_description = 'HTML'
317     has_html_file.boolean = True
318
319     def build_epub(self, remove_descendants=True):
320         """ (Re)builds the epub file.
321             If book has a parent, does nothing.
322             Unless remove_descendants is False, descendants' epubs are removed.
323         """
324     
325         from StringIO import StringIO
326         from hashlib import sha1
327         from django.core.files.base import ContentFile
328         from librarian import DocProvider
329
330         class BookImportDocProvider(DocProvider):
331             """ used for joined EPUBs """
332
333             def __init__(self, book):
334                 self.book = book
335
336             def by_slug(self, slug):
337                 if slug == self.book.slug:
338                     return self.book.xml_file
339                 else:
340                     return Book.objects.get(slug=slug).xml_file
341
342         if self.parent:
343             # don't need an epub
344             return
345
346         epub_file = StringIO()
347         try:
348             epub.transform(BookImportDocProvider(self), self.slug, epub_file)
349             self.epub_file.save('%s.epub' % self.slug, ContentFile(epub_file.getvalue()), save=False)
350             self.save()
351             FileRecord(slug=self.slug, type='epub', sha1=sha1(epub_file.getvalue()).hexdigest()).save()
352         except NoDublinCore:
353             pass
354
355         book_descendants = list(self.children.all())
356         while len(book_descendants) > 0:
357             child_book = book_descendants.pop(0)
358             if remove_descendants and child_book.has_epub_file():
359                 child_book.epub_file.delete()
360             # save anyway, to refresh short_html
361             child_book.save()
362             book_descendants += list(child_book.children.all())
363
364
365     @classmethod
366     def from_xml_file(cls, xml_file, overwrite=False):
367         # use librarian to parse meta-data
368         book_info = dcparser.parse(xml_file)
369
370         if not isinstance(xml_file, File):
371             xml_file = File(open(xml_file))
372
373         try:
374             return cls.from_text_and_meta(xml_file, book_info, overwrite)
375         finally:
376             xml_file.close()
377
378     @classmethod
379     def from_text_and_meta(cls, raw_file, book_info, overwrite=False):
380         from tempfile import NamedTemporaryFile
381         from slughifi import slughifi
382         from markupstring import MarkupString
383         from django.core.files.storage import default_storage
384
385         # Read book metadata
386         book_base, book_slug = book_info.url.rsplit('/', 1)
387         book, created = Book.objects.get_or_create(slug=book_slug)
388
389         if created:
390             book_shelves = []
391         else:
392             if not overwrite:
393                 raise Book.AlreadyExists(_('Book %s already exists') % book_slug)
394             # Save shelves for this book
395             book_shelves = list(book.tags.filter(category='set'))
396
397         book.title = book_info.title
398         book.set_extra_info_value(book_info.to_dict())
399         book._short_html = ''
400         book.save()
401
402         book_tags = []
403         categories = (('kinds', 'kind'), ('genres', 'genre'), ('authors', 'author'), ('epochs', 'epoch'))
404         for field_name, category in categories:
405             try:
406                 tag_names = getattr(book_info, field_name)
407             except:
408                 tag_names = [getattr(book_info, category)]
409             for tag_name in tag_names:
410                 tag_sort_key = tag_name
411                 if category == 'author':
412                     tag_sort_key = tag_name.last_name
413                     tag_name = ' '.join(tag_name.first_names) + ' ' + tag_name.last_name
414                 tag, created = Tag.objects.get_or_create(slug=slughifi(tag_name), category=category)
415                 if created:
416                     tag.name = tag_name
417                     tag.sort_key = slughifi(tag_sort_key)
418                     tag.save()
419                 book_tags.append(tag)
420
421         book.tags = book_tags + book_shelves
422
423         book_tag = book.book_tag()
424
425         if hasattr(book_info, 'parts'):
426             for n, part_url in enumerate(book_info.parts):
427                 base, slug = part_url.rsplit('/', 1)
428                 try:
429                     child_book = Book.objects.get(slug=slug)
430                     child_book.parent = book
431                     child_book.parent_number = n
432                     child_book.save()
433                 except Book.DoesNotExist, e:
434                     raise Book.DoesNotExist(_('Book with slug = "%s" does not exist.') % slug)
435
436         # Save XML and HTML files
437         book.xml_file.save('%s.xml' % book.slug, raw_file, save=False)
438
439         # delete old fragments when overwriting
440         book.fragments.all().delete()
441
442         html_file = NamedTemporaryFile()
443         if html.transform(book.xml_file.path, html_file, parse_dublincore=False):
444             book.html_file.save('%s.html' % book.slug, File(html_file), save=False)
445
446             # get ancestor l-tags for adding to new fragments
447             ancestor_tags = []
448             p = book.parent
449             while p:
450                 ancestor_tags.append(p.book_tag())
451                 p = p.parent
452
453             # Extract fragments
454             closed_fragments, open_fragments = html.extract_fragments(book.html_file.path)
455             for fragment in closed_fragments.values():
456                 try:
457                     theme_names = [s.strip() for s in fragment.themes.split(',')]
458                 except AttributeError:
459                     continue
460                 themes = []
461                 for theme_name in theme_names:
462                     if not theme_name:
463                         continue
464                     tag, created = Tag.objects.get_or_create(slug=slughifi(theme_name), category='theme')
465                     if created:
466                         tag.name = theme_name
467                         tag.sort_key = slughifi(theme_name)
468                         tag.save()
469                     themes.append(tag)
470                 if not themes:
471                     continue
472
473                 text = fragment.to_string()
474                 short_text = ''
475                 if (len(MarkupString(text)) > 240):
476                     short_text = unicode(MarkupString(text)[:160])
477                 new_fragment, created = Fragment.objects.get_or_create(anchor=fragment.id, book=book,
478                     defaults={'text': text, 'short_text': short_text})
479
480                 new_fragment.save()
481                 new_fragment.tags = set(book_tags + themes + [book_tag] + ancestor_tags)
482
483         if not book.parent:
484             book.build_epub(remove_descendants=False)
485
486         book_descendants = list(book.children.all())
487         # add l-tag to descendants and their fragments
488         # delete unnecessary EPUB files
489         while len(book_descendants) > 0:
490             child_book = book_descendants.pop(0)
491             child_book.tags = list(child_book.tags) + [book_tag]
492             if child_book.has_epub_file():
493                 child_book.epub_file.delete()
494             child_book.save()
495             for fragment in child_book.fragments.all():
496                 fragment.tags = set(list(fragment.tags) + [book_tag])
497             book_descendants += list(child_book.children.all())
498
499         # refresh cache
500         book.reset_tag_counter()
501         book.reset_theme_counter()
502
503         book.save()
504         return book
505
506
507     def refresh_tag_counter(self):
508         tags = {}
509         for child in self.children.all().order_by():
510             for tag_pk, value in child.tag_counter.iteritems():
511                 tags[tag_pk] = tags.get(tag_pk, 0) + value
512         for tag in self.tags.exclude(category__in=('book', 'theme', 'set')).order_by():
513             tags[tag.pk] = 1
514         self.set__tag_counter_value(tags)
515         self.save(reset_short_html=False, refresh_mp3=False)
516         return tags
517
518     def reset_tag_counter(self):
519         self._tag_counter = None
520         self.save(reset_short_html=False, refresh_mp3=False)
521         if self.parent:
522             self.parent.reset_tag_counter()
523
524     @property
525     def tag_counter(self):
526         if self._tag_counter is None:
527             return self.refresh_tag_counter()
528         return dict((int(k), v) for k, v in self.get__tag_counter_value().iteritems())
529
530     def refresh_theme_counter(self):
531         tags = {}
532         for fragment in Fragment.tagged.with_any([self.book_tag()]).order_by():
533             for tag in fragment.tags.filter(category='theme').order_by():
534                 tags[tag.pk] = tags.get(tag.pk, 0) + 1
535         self.set__theme_counter_value(tags)
536         self.save(reset_short_html=False, refresh_mp3=False)
537         return tags
538
539     def reset_theme_counter(self):
540         self._theme_counter = None
541         self.save(reset_short_html=False, refresh_mp3=False)
542         if self.parent:
543             self.parent.reset_theme_counter()
544
545     @property
546     def theme_counter(self):
547         if self._theme_counter is None:
548             return self.refresh_theme_counter()
549         return dict((int(k), v) for k, v in self.get__theme_counter_value().iteritems())
550
551
552
553 class Fragment(models.Model):
554     text = models.TextField()
555     short_text = models.TextField(editable=False)
556     _short_html = models.TextField(editable=False)
557     anchor = models.CharField(max_length=120)
558     book = models.ForeignKey(Book, related_name='fragments')
559
560     objects = models.Manager()
561     tagged = managers.ModelTaggedItemManager(Tag)
562     tags = managers.TagDescriptor(Tag)
563
564     class Meta:
565         ordering = ('book', 'anchor',)
566         verbose_name = _('fragment')
567         verbose_name_plural = _('fragments')
568
569     def get_absolute_url(self):
570         return '%s#m%s' % (reverse('book_text', kwargs={'slug': self.book.slug}), self.anchor)
571
572     def short_html(self):
573         key = '_short_html_%s' % get_language()
574         short_html = getattr(self, key)
575         if short_html and len(short_html):
576             return mark_safe(short_html)
577         else:
578             setattr(self, key, unicode(render_to_string('catalogue/fragment_short.html',
579                 {'fragment': self})))
580             self.save()
581             return mark_safe(getattr(self, key))
582
583
584 class FileRecord(models.Model):
585     slug = models.SlugField(_('slug'), max_length=120, db_index=True)
586     type = models.CharField(_('type'), max_length=20, db_index=True)
587     sha1 = models.CharField(_('sha-1 hash'), max_length=40)
588     time = models.DateTimeField(_('time'), auto_now_add=True)
589
590     class Meta:
591         ordering = ('-time','-slug', '-type')
592         verbose_name = _('file record')
593         verbose_name_plural = _('file records')
594
595     def __unicode__(self):
596         return "%s %s.%s" % (self.sha1,  self.slug, self.type)
597
598
599 def _tags_updated_handler(sender, affected_tags, **kwargs):
600     # reset tag global counter
601     Tag.objects.filter(pk__in=[tag.pk for tag in affected_tags]).update(book_count=None)
602
603     # if book tags changed, reset book tag counter
604     if isinstance(sender, Book) and \
605                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
606                     exclude(category__in=('book', 'theme', 'set')).count():
607         sender.reset_tag_counter()
608     # if fragment theme changed, reset book theme counter
609     elif isinstance(sender, Fragment) and \
610                 Tag.objects.filter(pk__in=(tag.pk for tag in affected_tags)).\
611                     filter(category='theme').count():
612         sender.book.reset_theme_counter()
613 tags_updated.connect(_tags_updated_handler)
614