remove "we don't support full text search"
[wolnelektury.git] / src / picture / models.py
1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 from django.db import models, transaction
6 import catalogue.models
7 from django.db.models import permalink
8 from sorl.thumbnail import ImageField
9 from django.conf import settings
10 from django.contrib.contenttypes.fields import GenericRelation
11 from django.core.files.storage import FileSystemStorage
12 from django.utils.datastructures import SortedDict
13 from fnpdjango.utils.text.slughifi import slughifi
14 from ssify import flush_ssi_includes
15
16 from catalogue.models.tag import prefetched_relations
17 from catalogue.utils import split_tags
18 from picture import tasks
19 from StringIO import StringIO
20 import jsonfield
21 import itertools
22 import logging
23 import re
24
25 from PIL import Image
26
27 from django.utils.translation import ugettext_lazy as _
28 from newtagging import managers
29 from os import path
30
31
32 picture_storage = FileSystemStorage(location=path.join(
33         settings.MEDIA_ROOT, 'pictures'),
34         base_url=settings.MEDIA_URL + "pictures/")
35
36
37 class PictureArea(models.Model):
38     picture = models.ForeignKey('picture.Picture', related_name='areas')
39     area = jsonfield.JSONField(_('area'), default={}, editable=False)
40     kind = models.CharField(
41         _('kind'), max_length=10, blank=False, null=False, db_index=True,
42         choices=(('thing', _('thing')), ('theme', _('theme'))))
43
44     objects = models.Manager()
45     tagged = managers.ModelTaggedItemManager(catalogue.models.Tag)
46     tags = managers.TagDescriptor(catalogue.models.Tag)
47     tag_relations = GenericRelation(catalogue.models.Tag.intermediary_table_model)
48
49     short_html_url_name = 'picture_area_short'
50
51     @classmethod
52     def rectangle(cls, picture, kind, coords):
53         pa = PictureArea()
54         pa.picture = picture
55         pa.kind = kind
56         pa.area = coords
57         return pa
58
59     def flush_includes(self, languages=True):
60         if not languages:
61             return
62         if languages is True:
63             languages = [lc for (lc, _ln) in settings.LANGUAGES]
64         flush_ssi_includes([
65             template % (self.pk, lang)
66             for template in [
67                 '/katalog/pa/%d/short.%s.html',
68                 ]
69             for lang in languages
70             ])
71
72
73 class Picture(models.Model):
74     """
75     Picture resource.
76
77     """
78     title = models.CharField(_('title'), max_length=32767)
79     slug = models.SlugField(_('slug'), max_length=120, db_index=True, unique=True)
80     sort_key = models.CharField(_('sort key'), max_length=120, db_index=True, editable=False)
81     sort_key_author = models.CharField(
82         _('sort key by author'), max_length=120, db_index=True, editable=False, default=u'')
83     created_at = models.DateTimeField(_('creation date'), auto_now_add=True, db_index=True)
84     changed_at = models.DateTimeField(_('creation date'), auto_now=True, db_index=True)
85     xml_file = models.FileField(_('xml file'), upload_to="xml", storage=picture_storage)
86     image_file = ImageField(_('image file'), upload_to="images", storage=picture_storage)
87     html_file = models.FileField(_('html file'), upload_to="html", storage=picture_storage)
88     areas_json = jsonfield.JSONField(_('picture areas JSON'), default={}, editable=False)
89     extra_info = jsonfield.JSONField(_('extra information'), default={})
90     culturepl_link = models.CharField(blank=True, max_length=240)
91     wiki_link = models.CharField(blank=True, max_length=240)
92
93     width = models.IntegerField(null=True)
94     height = models.IntegerField(null=True)
95
96     objects = models.Manager()
97     tagged = managers.ModelTaggedItemManager(catalogue.models.Tag)
98     tags = managers.TagDescriptor(catalogue.models.Tag)
99     tag_relations = GenericRelation(catalogue.models.Tag.intermediary_table_model)
100
101     short_html_url_name = 'picture_short'
102
103     class AlreadyExists(Exception):
104         pass
105
106     class Meta:
107         ordering = ('sort_key_author', 'sort_key')
108
109         verbose_name = _('picture')
110         verbose_name_plural = _('pictures')
111
112     def save(self, force_insert=False, force_update=False, **kwargs):
113         from sortify import sortify
114
115         self.sort_key = sortify(self.title)[:120]
116
117         try:
118             author = self.authors().first().sort_key
119         except AttributeError:
120             author = u''
121         self.sort_key_author = author
122
123         ret = super(Picture, self).save(force_insert, force_update)
124
125         return ret
126
127     def __unicode__(self):
128         return self.title
129
130     def authors(self):
131         return self.tags.filter(category='author')
132
133     def tag_unicode(self, category):
134         relations = prefetched_relations(self, category)
135         if relations:
136             return ', '.join(rel.tag.name for rel in relations)
137         else:
138             return ', '.join(self.tags.filter(category=category).values_list('name', flat=True))
139
140     def author_unicode(self):
141         return self.tag_unicode('author')
142
143     def tags_by_category(self):
144         return split_tags(self.tags)
145
146     @permalink
147     def get_absolute_url(self):
148         return 'picture.views.picture_detail', [self.slug]
149
150     def get_initial(self):
151         try:
152             return re.search(r'\w', self.title, re.U).group(0)
153         except AttributeError:
154             return ''
155
156     def get_next(self):
157         try:
158             return type(self).objects.filter(sort_key__gt=self.sort_key)[0]
159         except IndexError:
160             return None
161
162     def get_previous(self):
163         try:
164             return type(self).objects.filter(sort_key__lt=self.sort_key).order_by('-sort_key')[0]
165         except IndexError:
166             return None
167
168     @classmethod
169     def from_xml_file(cls, xml_file, image_file=None, image_store=None, overwrite=False, search_index=True):
170         """
171         Import xml and it's accompanying image file.
172         If image file is missing, it will be fetched by librarian.picture.ImageStore
173         which looks for an image file in the same directory the xml is, with extension matching
174         its mime type.
175         """
176         from sortify import sortify
177         from django.core.files import File
178         from librarian.picture import WLPicture, ImageStore
179         close_xml_file = False
180         close_image_file = False
181
182         if image_file is not None and not isinstance(image_file, File):
183             image_file = File(open(image_file))
184             close_image_file = True
185
186         if not isinstance(xml_file, File):
187             xml_file = File(open(xml_file))
188             close_xml_file = True
189
190         with transaction.atomic():
191             # use librarian to parse meta-data
192             if image_store is None:
193                 image_store = ImageStore(picture_storage.path('images'))
194             picture_xml = WLPicture.from_file(xml_file, image_store=image_store)
195
196             picture, created = Picture.objects.get_or_create(slug=picture_xml.slug[:120])
197             if not created and not overwrite:
198                 raise Picture.AlreadyExists('Picture %s already exists' % picture_xml.slug)
199
200             picture.areas.all().delete()
201             picture.title = unicode(picture_xml.picture_info.title)
202             picture.extra_info = picture_xml.picture_info.to_dict()
203
204             picture_tags = set(catalogue.models.Tag.tags_from_info(picture_xml.picture_info))
205             for tag in picture_tags:
206                 if not tag.for_pictures:
207                     tag.for_pictures = True
208                     tag.save()
209
210             area_data = {'themes': {}, 'things': {}}
211
212             # Treat all names in picture XML as in default language.
213             lang = settings.LANGUAGE_CODE
214
215             for part in picture_xml.partiter():
216                 if picture_xml.frame:
217                     c = picture_xml.frame[0]
218                     part['coords'] = [[p[0] - c[0], p[1] - c[1]] for p in part['coords']]
219                 if part.get('object', None) is not None:
220                     _tags = set()
221                     for objname in part['object'].split(','):
222                         objname = objname.strip()
223                         assert objname, 'Empty object name'
224                         # str.capitalize() is wrong, because it also lowers letters
225                         objname = objname[0].upper() + objname[1:]
226                         tag, created = catalogue.models.Tag.objects.get_or_create(
227                             slug=slughifi(objname), category='thing')
228                         if created:
229                             tag.name = objname
230                             setattr(tag, 'name_%s' % lang, tag.name)
231                             tag.sort_key = sortify(tag.name)
232                             tag.for_pictures = True
233                             tag.save()
234                         area_data['things'][tag.slug] = {
235                             'object': objname,
236                             'coords': part['coords'],
237                             }
238
239                         _tags.add(tag)
240                         if not tag.for_pictures:
241                             tag.for_pictures = True
242                             tag.save()
243                     area = PictureArea.rectangle(picture, 'thing', part['coords'])
244                     area.save()
245                     # WTF thing area does not inherit tags from picture and theme area does, is it intentional?
246                     area.tags = _tags
247                 else:
248                     _tags = set()
249                     for motifs in part['themes']:
250                         for motif in motifs.split(','):
251                             tag, created = catalogue.models.Tag.objects.get_or_create(
252                                 slug=slughifi(motif), category='theme')
253                             if created:
254                                 tag.name = motif
255                                 tag.sort_key = sortify(tag.name)
256                                 tag.for_pictures = True
257                                 tag.save()
258                             # motif_tags.add(tag)
259                             _tags.add(tag)
260                             if not tag.for_pictures:
261                                 tag.for_pictures = True
262                                 tag.save()
263                             area_data['themes'][tag.slug] = {
264                                 'theme': motif,
265                                 'coords': part['coords']
266                                 }
267
268                     logging.debug("coords for theme: %s" % part['coords'])
269                     area = PictureArea.rectangle(picture, 'theme', part['coords'])
270                     area.save()
271                     area.tags = _tags.union(picture_tags)
272
273             picture.tags = picture_tags
274             picture.areas_json = area_data
275
276             if image_file is not None:
277                 img = image_file
278             else:
279                 img = picture_xml.image_file()
280
281             modified = cls.crop_to_frame(picture_xml, img)
282             modified = cls.add_source_note(picture_xml, modified)
283
284             picture.width, picture.height = modified.size
285
286             modified_file = StringIO()
287             modified.save(modified_file, format='JPEG', quality=95)
288             # FIXME: hardcoded extension - detect from DC format or orginal filename
289             picture.image_file.save(path.basename(picture_xml.image_path), File(modified_file))
290
291             picture.xml_file.save("%s.xml" % picture.slug, File(xml_file))
292             picture.save()
293             tasks.generate_picture_html(picture.id)
294             if not settings.NO_SEARCH_INDEX and search_index:
295                 tasks.index_picture.delay(picture.id, picture_info=picture_xml.picture_info)
296
297         if close_xml_file:
298             xml_file.close()
299         if close_image_file:
300             image_file.close()
301
302         return picture
303
304     @classmethod
305     def crop_to_frame(cls, wlpic, image_file):
306         img = Image.open(image_file)
307         if wlpic.frame is None or wlpic.frame == [[0, 0], [-1, -1]]:
308             return img
309         img = img.crop(itertools.chain(*wlpic.frame))
310         return img
311
312     @staticmethod
313     def add_source_note(wlpic, img):
314         from PIL import ImageDraw, ImageFont
315         from librarian import get_resource
316
317         annotated = Image.new(img.mode, (img.size[0], img.size[1] + 40), (255, 255, 255))
318         annotated.paste(img, (0, 0))
319         annotation = Image.new('RGB', (img.size[0] * 3, 120), (255, 255, 255))
320         ImageDraw.Draw(annotation).text(
321             (30, 15),
322             wlpic.picture_info.source_name,
323             (0, 0, 0),
324             font=ImageFont.truetype(get_resource("fonts/DejaVuSerif.ttf"), 75)
325         )
326         annotated.paste(annotation.resize((img.size[0], 40), Image.ANTIALIAS), (0, img.size[1]))
327         return annotated
328
329     # WTF/unused
330     @classmethod
331     def picture_list(cls, filter=None):
332         """Generates a hierarchical listing of all pictures
333         Pictures are optionally filtered with a test function.
334         """
335
336         pics = cls.objects.all().order_by('sort_key').only('title', 'slug', 'image_file')
337
338         if filter:
339             pics = pics.filter(filter).distinct()
340
341         pics_by_author = SortedDict()
342         orphans = []
343         for tag in catalogue.models.Tag.objects.filter(category='author'):
344             pics_by_author[tag] = []
345
346         for pic in pics.iterator():
347             authors = list(pic.authors().only('pk'))
348             if authors:
349                 for author in authors:
350                     pics_by_author[author].append(pic)
351             else:
352                 orphans.append(pic)
353
354         return pics_by_author, orphans
355
356     @property
357     def info(self):
358         if not hasattr(self, '_info'):
359             from librarian import dcparser
360             from librarian import picture
361             info = dcparser.parse(self.xml_file.path, picture.PictureInfo)
362             self._info = info
363         return self._info
364
365     def pretty_title(self, html_links=False):
366         names = [(tag.name, tag.get_absolute_url()) for tag in self.authors().only('name', 'category', 'slug')]
367         names.append((self.title, self.get_absolute_url()))
368
369         if html_links:
370             names = ['<a href="%s">%s</a>' % (tag[1], tag[0]) for tag in names]
371         else:
372             names = [tag[0] for tag in names]
373         return ', '.join(names)
374
375     def related_themes(self):
376         return catalogue.models.Tag.objects.usage_for_queryset(
377             self.areas.all(), counts=True).filter(category__in=('theme', 'thing'))
378
379     def flush_includes(self, languages=True):
380         if not languages:
381             return
382         if languages is True:
383             languages = [lc for (lc, _ln) in settings.LANGUAGES]
384         flush_ssi_includes([
385             template % (self.pk, lang)
386             for template in [
387                 '/katalog/p/%d/short.%s.html',
388                 '/katalog/p/%d/mini.%s.html',
389                 ]
390             for lang in languages
391             ])
392
393     def search_index(self, picture_info=None, index=None, index_tags=True, commit=True):
394         if index is None:
395             from search.index import Index
396             index = Index()
397         try:
398             index.index_picture(self, picture_info)
399             if index_tags:
400                 index.index_tags()
401             if commit:
402                 index.index.commit()
403         except Exception, e:
404             index.index.rollback()
405             raise e