Celery 3.1, librarian update
[wolnelektury.git] / apps / picture / models.py
1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 from django.db import models, transaction
6 import catalogue.models
7 from django.db.models import permalink
8 from sorl.thumbnail import ImageField
9 from django.conf import settings
10 from django.core.files.storage import FileSystemStorage
11 from django.utils.datastructures import SortedDict
12 from django.template.loader import render_to_string
13 from django.utils.safestring import mark_safe
14 from django.core.cache import get_cache
15 from catalogue.utils import split_tags
16 from fnpdjango.utils.text.slughifi import slughifi
17 from picture import tasks
18 from StringIO import StringIO
19 import jsonfield
20 import itertools
21 import logging
22
23 from PIL import Image
24
25 from django.utils.translation import get_language, ugettext_lazy as _
26 from newtagging import managers
27 from os import path
28
29
30 permanent_cache = get_cache('permanent')
31
32 picture_storage = FileSystemStorage(location=path.join(
33         settings.MEDIA_ROOT, 'pictures'),
34         base_url=settings.MEDIA_URL + "pictures/")
35
36
37 class PictureArea(models.Model):
38     picture = models.ForeignKey('picture.Picture', related_name='areas')
39     area = jsonfield.JSONField(_('area'), default={}, editable=False)
40     kind = models.CharField(_('kind'), max_length=10, blank=False,
41                            null=False, db_index=True,
42                            choices=(('thing', _('thing')),
43                                     ('theme', _('theme'))))
44
45     objects     = models.Manager()
46     tagged      = managers.ModelTaggedItemManager(catalogue.models.Tag)
47     tags        = managers.TagDescriptor(catalogue.models.Tag)
48
49     @classmethod
50     def rectangle(cls, picture, kind, coords):
51         pa = PictureArea()
52         pa.picture = picture
53         pa.kind = kind
54         pa.area = coords
55         return pa
56
57     def reset_short_html(self):
58         if self.id is None:
59             return
60
61         cache_key = "PictureArea.short_html/%d/%s"
62         for lang, langname in settings.LANGUAGES:
63             permanent_cache.delete(cache_key % (self.id, lang))
64
65
66     def short_html(self):
67         if self.id:
68             cache_key = "PictureArea.short_html/%d/%s" % (self.id, get_language())
69             short_html = permanent_cache.get(cache_key)
70         else:
71             short_html = None
72
73         if short_html is not None:
74             return mark_safe(short_html)
75         else:
76             theme = self.tags.filter(category='theme')
77             theme = theme and theme[0] or None
78             thing = self.tags.filter(category='thing')
79             thing = thing and thing[0] or None
80             area = self
81             short_html = unicode(render_to_string(
82                     'picture/picturearea_short.html', locals()))
83             if self.id:
84                 permanent_cache.set(cache_key, short_html)
85             return mark_safe(short_html)
86
87
88 class Picture(models.Model):
89     """
90     Picture resource.
91
92     """
93     title       = models.CharField(_('title'), max_length=120)
94     slug        = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
95     sort_key    = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
96     sort_key_author = models.CharField(_('sort key by author'), max_length=120, db_index=True, editable=False, default=u'')
97     created_at  = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
98     changed_at  = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
99     xml_file    = models.FileField('xml_file', upload_to="xml", storage=picture_storage)
100     image_file  = ImageField(_('image_file'), upload_to="images", storage=picture_storage)
101     html_file   = models.FileField('html_file', upload_to="html", storage=picture_storage)
102     areas_json       = jsonfield.JSONField(_('picture areas JSON'), default={}, editable=False)
103     extra_info    = jsonfield.JSONField(_('extra information'), default={})
104     culturepl_link   = models.CharField(blank=True, max_length=240)
105     wiki_link     = models.CharField(blank=True, max_length=240)
106
107     _related_info = jsonfield.JSONField(blank=True, null=True, editable=False)
108
109     width       = models.IntegerField(null=True)
110     height      = models.IntegerField(null=True)
111
112     objects     = models.Manager()
113     tagged      = managers.ModelTaggedItemManager(catalogue.models.Tag)
114     tags        = managers.TagDescriptor(catalogue.models.Tag)
115
116     class AlreadyExists(Exception):
117         pass
118
119     class Meta:
120         ordering = ('sort_key',)
121
122         verbose_name = _('picture')
123         verbose_name_plural = _('pictures')
124
125     def save(self, force_insert=False, force_update=False, reset_short_html=True, **kwargs):
126         from sortify import sortify
127
128         self.sort_key = sortify(self.title)
129
130         ret = super(Picture, self).save(force_insert, force_update)
131
132         if reset_short_html:
133             self.reset_short_html()
134
135         return ret
136
137     def __unicode__(self):
138         return self.title
139
140     @permalink
141     def get_absolute_url(self):
142         return ('picture.views.picture_detail', [self.slug])
143
144     @classmethod
145     def from_xml_file(cls, xml_file, image_file=None, image_store=None, overwrite=False):
146         """
147         Import xml and it's accompanying image file.
148         If image file is missing, it will be fetched by librarian.picture.ImageStore
149         which looks for an image file in the same directory the xml is, with extension matching
150         its mime type.
151         """
152         from sortify import sortify
153         from django.core.files import File
154         from librarian.picture import WLPicture, ImageStore
155         close_xml_file = False
156         close_image_file = False
157
158
159         if image_file is not None and not isinstance(image_file, File):
160             image_file = File(open(image_file))
161             close_image_file = True
162
163         if not isinstance(xml_file, File):
164             xml_file = File(open(xml_file))
165             close_xml_file = True
166
167         try:
168             # use librarian to parse meta-data
169             if image_store is None:
170                 image_store = ImageStore(picture_storage.path('images'))
171             picture_xml = WLPicture.from_file(xml_file, image_store=image_store)
172
173             picture, created = Picture.objects.get_or_create(slug=picture_xml.slug)
174             if not created and not overwrite:
175                 raise Picture.AlreadyExists('Picture %s already exists' % picture_xml.slug)
176
177             picture.areas.all().delete()
178             picture.title = unicode(picture_xml.picture_info.title)
179             picture.extra_info = picture_xml.picture_info.to_dict()
180
181             picture_tags = set(catalogue.models.Tag.tags_from_info(picture_xml.picture_info))
182             motif_tags = set()
183             thing_tags = set()
184
185             area_data = {'themes':{}, 'things':{}}
186
187             for part in picture_xml.partiter():
188                 if picture_xml.frame:
189                     c = picture_xml.frame[0]
190                     part['coords'] = [[p[0] - c[0], p[1] - c[1]] for p in part['coords']]
191                 if part.get('object', None) is not None:
192                     objname = part['object']
193                     tag, created = catalogue.models.Tag.objects.get_or_create(slug=slughifi(objname), category='thing')
194                     if created:
195                         tag.name = objname
196                         tag.sort_key = sortify(tag.name)
197                         tag.save()
198                     #thing_tags.add(tag)
199                     area_data['things'][tag.slug] = {
200                         'object': part['object'],
201                         'coords': part['coords'],
202                         }
203                     area = PictureArea.rectangle(picture, 'thing', part['coords'])
204                     area.save()
205                     _tags = set()
206                     _tags.add(tag)
207                     area.tags = _tags
208                 else:
209                     _tags = set()
210                     for motif in part['themes']:
211                         tag, created = catalogue.models.Tag.objects.get_or_create(slug=slughifi(motif), category='theme')
212                         if created:
213                             tag.name = motif
214                             tag.sort_key = sortify(tag.name)
215                             tag.save()
216                         #motif_tags.add(tag)
217                         _tags.add(tag)
218                         area_data['themes'][tag.slug] = {
219                             'theme': motif,
220                             'coords': part['coords']
221                             }
222
223                     logging.debug("coords for theme: %s" % part['coords'])
224                     area = PictureArea.rectangle(picture, 'theme', part['coords'])
225                     area.save()
226                     area.tags = _tags.union(picture_tags)
227
228             picture.tags = picture_tags.union(motif_tags).union(thing_tags)
229             picture.areas_json = area_data
230
231             if image_file is not None:
232                 img = image_file
233             else:
234                 img = picture_xml.image_file()
235
236             modified = cls.crop_to_frame(picture_xml, img)
237             modified = cls.add_source_note(picture_xml, modified)
238
239             picture.width, picture.height = modified.size
240
241             modified_file = StringIO()
242             modified.save(modified_file, format='png', quality=95)
243             # FIXME: hardcoded extension - detect from DC format or orginal filename
244             picture.image_file.save(path.basename(picture_xml.image_path), File(modified_file))
245
246             picture.xml_file.save("%s.xml" % picture.slug, File(xml_file))
247             picture.save()
248             tasks.generate_picture_html(picture.id)
249
250         except Exception, ex:
251             logging.exception("Exception during import, rolling back")
252             transaction.rollback()
253             raise ex
254
255         finally:
256             if close_xml_file:
257                 xml_file.close()
258             if close_image_file:
259                 image_file.close()
260
261         transaction.commit()
262
263         return picture
264
265     @classmethod
266     def crop_to_frame(cls, wlpic, image_file):
267         img = Image.open(image_file)
268         if wlpic.frame is None:
269             return img
270         img = img.crop(itertools.chain(*wlpic.frame))
271         return img
272
273     @staticmethod
274     def add_source_note(wlpic, img):
275         from PIL import ImageDraw, ImageFont
276         from librarian import get_resource
277
278         annotated = Image.new(img.mode,
279                 (img.size[0], img.size[1] + 40),
280                 (255, 255, 255)
281             )
282         annotated.paste(img, (0, 0))
283         annotation = Image.new(img.mode, (3000, 120), (255, 255, 255))
284         ImageDraw.Draw(annotation).text(
285             (30, 15),
286             wlpic.picture_info.source_name,
287             (0, 0, 0),
288             font=ImageFont.truetype(get_resource("fonts/DejaVuSerif.ttf"), 75)
289         )
290         annotated.paste(annotation.resize((1000, 40), Image.ANTIALIAS), (0, img.size[1]))
291         return annotated
292
293     @classmethod
294     def picture_list(cls, filter=None):
295         """Generates a hierarchical listing of all pictures
296         Pictures are optionally filtered with a test function.
297         """
298
299         pics = cls.objects.all().order_by('sort_key')\
300             .only('title', 'slug', 'image_file')
301
302         if filter:
303             pics = pics.filter(filter).distinct()
304
305         pics_by_author = SortedDict()
306         orphans = []
307         for tag in catalogue.models.Tag.objects.filter(category='author'):
308             pics_by_author[tag] = []
309
310         for pic in pics.iterator():
311             authors = list(pic.tags.filter(category='author'))
312             if authors:
313                 for author in authors:
314                     pics_by_author[author].append(pic)
315             else:
316                 orphans.append(pic)
317
318         return pics_by_author, orphans
319
320     @property
321     def info(self):
322         if not hasattr(self, '_info'):
323             from librarian import dcparser
324             from librarian import picture
325             info = dcparser.parse(self.xml_file.path, picture.PictureInfo)
326             self._info = info
327         return self._info
328
329     def reset_short_html(self):
330         if self.id is None:
331             return
332
333         type(self).objects.filter(pk=self.pk).update(_related_info=None)
334         for area in self.areas.all().iterator():
335             area.reset_short_html()
336
337         try:
338             author = self.tags.filter(category='author')[0].sort_key
339         except IndexError:
340             author = u''
341         type(self).objects.filter(pk=self.pk).update(sort_key_author=author)
342
343         cache_key = "Picture.short_html/%d/%s"
344         for lang, langname in settings.LANGUAGES:
345             permanent_cache.delete(cache_key % (self.id, lang))
346
347     def short_html(self):
348         if self.id:
349             cache_key = "Picture.short_html/%d/%s" % (self.id, get_language())
350             short_html = get_cache('permanent').get(cache_key)
351         else:
352             short_html = None
353
354         if short_html is not None:
355             return mark_safe(short_html)
356         else:
357             tags = self.tags.filter(category__in=('author', 'kind', 'epoch', 'genre'))
358             tags = split_tags(tags)
359
360             short_html = unicode(render_to_string(
361                     'picture/picture_short.html',
362                     {'picture': self, 'tags': tags}))
363
364             if self.id:
365                 get_cache('permanent').set(cache_key, short_html)
366             return mark_safe(short_html)
367
368     def pretty_title(self, html_links=False):
369         picture = self
370         # TODO Add translations (related_tag_info)
371         names = [(tag.name,
372                   catalogue.models.Tag.create_url('author', tag.slug))
373                  for tag in self.tags.filter(category='author')]
374         names.append((self.title, self.get_absolute_url()))
375
376         if html_links:
377             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
378         else:
379             names = [tag[0] for tag in names]
380         return ', '.join(names)
381
382     def related_info(self):
383         """Keeps info about related objects (tags) in cache field."""
384         if self._related_info is not None:
385             return self._related_info
386         else:
387             rel = {'tags': {}}
388
389             tags = self.tags.filter(category__in=(
390                     'author', 'kind', 'genre', 'epoch'))
391             tags = split_tags(tags)
392             for category in tags:
393                 cat = []
394                 for tag in tags[category]:
395                     tag_info = {'slug': tag.slug, 'name': tag.name}
396                     for lc, ln in settings.LANGUAGES:
397                         tag_name = getattr(tag, "name_%s" % lc)
398                         if tag_name:
399                             tag_info["name_%s" % lc] = tag_name
400                     cat.append(tag_info)
401                 rel['tags'][category] = cat
402
403
404             if self.pk:
405                 type(self).objects.filter(pk=self.pk).update(_related_info=rel)
406             return rel
407
408     # copied from book.py, figure out
409     def related_themes(self):
410         # self.theme_counter hides a computation, so a line below actually makes sense
411         theme_counter = self.theme_counter
412         picture_themes = list(catalogue.models.Tag.objects.filter(pk__in=theme_counter.keys()))
413         for tag in picture_themes:
414             tag.count = theme_counter[tag.pk]
415         return picture_themes
416
417     def reset_tag_counter(self):
418         if self.id is None:
419             return
420
421         cache_key = "Picture.tag_counter/%d" % self.id
422         permanent_cache.delete(cache_key)
423         if self.parent:
424             self.parent.reset_tag_counter()
425
426     @property
427     def tag_counter(self):
428         if self.id:
429             cache_key = "Picture.tag_counter/%d" % self.id
430             tags = permanent_cache.get(cache_key)
431         else:
432             tags = None
433
434         if tags is None:
435             tags = {}
436             # do we need to do this? there are no children here.
437             for tag in self.tags.exclude(category__in=('book', 'theme', 'thing', 'set')).order_by().iterator():
438                 tags[tag.pk] = 1
439
440             if self.id:
441                 permanent_cache.set(cache_key, tags)
442         return tags
443
444     def reset_theme_counter(self):
445         if self.id is None:
446             return
447
448         cache_key = "Picture.theme_counter/%d" % self.id
449         permanent_cache.delete(cache_key)
450
451     @property
452     def theme_counter(self):
453         if self.id:
454             cache_key = "Picture.theme_counter/%d" % self.id
455             tags = permanent_cache.get(cache_key)
456         else:
457             tags = None
458
459         if tags is None:
460             tags = {}
461             for area in PictureArea.objects.filter(picture=self).order_by().iterator():
462                 for tag in area.tags.filter(category__in=('theme', 'thing')).order_by().iterator():
463                     tags[tag.pk] = tags.get(tag.pk, 0) + 1
464
465             if self.id:
466                 permanent_cache.set(cache_key, tags)
467         return tags