Source fixes: avoid race and wait a minute after uploading to prevent unnecessary...
[redakcja.git] / src / documents / views.py
1 # This file is part of FNP-Redakcja, licensed under GNU Affero GPLv3 or later.
2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
3 #
4 from collections import defaultdict
5 from datetime import datetime, date, timedelta
6 import logging
7 import os
8 from urllib.parse import quote_plus, unquote, urlsplit, urlunsplit
9
10 from django.conf import settings
11 from django.contrib import auth
12 from django.contrib.auth.models import User
13 from django.contrib.auth.decorators import login_required, permission_required
14 from django.urls import reverse
15 from django.db.models import Count, Q
16 from django.db import transaction
17 from django import http
18 from django.http import Http404, HttpResponse, HttpResponseForbidden
19 from django.http.response import HttpResponseRedirect
20 from django.shortcuts import get_object_or_404, render
21 from django.utils.encoding import iri_to_uri
22 from django.utils.translation import gettext_lazy as _
23 from django.views.decorators.http import require_POST
24 from django_cas_ng.decorators import user_passes_test
25
26 from librarian import epubcheck
27 from apiclient import api_call, NotAuthorizedError
28 from . import forms
29 from . import helpers
30 from .helpers import active_tab
31 from .models import (Book, Chunk, Image, BookPublishRecord, 
32         ChunkPublishRecord, ImagePublishRecord, Project)
33 from fileupload.views import UploadView
34
35 #
36 # Quick hack around caching problems, TODO: use ETags
37 #
38 from django.views.decorators.cache import never_cache
39
40 logger = logging.getLogger("fnp.documents")
41
42
43 @active_tab('all')
44 @never_cache
45 def document_list(request):
46     return render(request, 'documents/document_list.html')
47
48
49 @active_tab('images')
50 @never_cache
51 def image_list(request, user=None):
52     return render(request, 'documents/image_list.html')
53
54
55 @never_cache
56 def user(request, username):
57     user = get_object_or_404(User, username=username)
58     return render(request, 'documents/user_page.html', {"viewed_user": user})
59
60
61 @login_required
62 @active_tab('my')
63 @never_cache
64 def my(request):
65     last_books = sorted(request.session.get("wiki_last_books", {}).items(),
66         key=lambda x: x[1]['time'], reverse=True)
67     for k, v in last_books:
68         v['time'] = datetime.fromtimestamp(v['time'])
69     try:
70         resp = api_call(request.user, 'username/')
71     except NotAuthorizedError:
72         wllogin = None
73     else:
74         wllogin = resp['username']
75
76     return render(request, 'documents/my_page.html', {
77         'last_books': last_books,
78         "logout_to": '/',
79         "wllogin": wllogin,
80         })
81
82
83 @active_tab('users')
84 def users(request):
85     return render(request, 'documents/user_list.html', {
86         'users': User.objects.all().annotate(count=Count('chunk')).order_by(
87             '-count', 'last_name', 'first_name'),
88     })
89
90
91 @active_tab('activity')
92 def activity(request, isodate=None):
93     today = date.today()
94     try:
95         day = helpers.parse_isodate(isodate)
96     except ValueError:
97         day = today
98
99     if day > today:
100         raise Http404
101     if day != today:
102         next_day = day + timedelta(1)
103     prev_day = day - timedelta(1)
104
105     return render(request, 'documents/activity.html', locals())
106
107
108 @never_cache
109 def logout_then_redirect(request):
110     auth.logout(request)
111     return http.HttpResponseRedirect(quote_plus(request.GET.get('next', '/'), safe='/?='))
112
113
114 @permission_required('documents.add_book')
115 @active_tab('create')
116 def create_missing(request, slug=None):
117     if slug is None:
118         slug = ''
119     slug = slug.replace(' ', '-')
120
121     if request.method == "POST":
122         form = forms.DocumentCreateForm(request.POST, request.FILES)
123         if form.is_valid():
124             
125             if request.user.is_authenticated:
126                 creator = request.user
127             else:
128                 creator = None
129             book = Book.create(
130                 text=form.cleaned_data['text'],
131                 creator=creator,
132                 slug=form.cleaned_data['slug'],
133                 title=form.cleaned_data['title'],
134                 gallery=form.cleaned_data['gallery'],
135             )
136
137             return http.HttpResponseRedirect(reverse("documents_book", args=[book.slug]))
138     else:
139         form = forms.DocumentCreateForm(initial={
140                 "slug": slug,
141                 "title": slug.replace('-', ' ').title(),
142                 "gallery": slug,
143         })
144
145     return render(request, "documents/document_create_missing.html", {
146         "slug": slug,
147         "form": form,
148
149         "logout_to": '/',
150     })
151
152
153 @permission_required('documents.add_book')
154 @active_tab('upload')
155 def upload(request):
156     if request.method == "POST":
157         form = forms.DocumentsUploadForm(request.POST, request.FILES)
158         if form.is_valid():
159             from slugify import slugify
160
161             if request.user.is_authenticated:
162                 creator = request.user
163             else:
164                 creator = None
165
166             zip = form.cleaned_data['zip']
167             skipped_list = []
168             ok_list = []
169             error_list = []
170             slugs = {}
171             existing = [book.slug for book in Book.objects.all()]
172             for filename in zip.namelist():
173                 if filename[-1] == '/':
174                     continue
175                 title = os.path.basename(filename)[:-4]
176                 slug = slugify(title)
177                 if not (slug and filename.endswith('.xml')):
178                     skipped_list.append(filename)
179                 elif slug in slugs:
180                     error_list.append((filename, slug, _('Slug already used for %s' % slugs[slug])))
181                 elif slug in existing:
182                     error_list.append((filename, slug, _('Slug already used in repository.')))
183                 else:
184                     try:
185                         zip.read(filename).decode('utf-8') # test read
186                         ok_list.append((filename, slug, title))
187                     except UnicodeDecodeError:
188                         error_list.append((filename, title, _('File should be UTF-8 encoded.')))
189                     slugs[slug] = filename
190
191             if not error_list:
192                 for filename, slug, title in ok_list:
193                     book = Book.create(
194                         text=zip.read(filename).decode('utf-8'),
195                         creator=creator,
196                         slug=slug,
197                         title=title,
198                     )
199
200             return render(request, "documents/document_upload.html", {
201                 "form": form,
202                 "ok_list": ok_list,
203                 "skipped_list": skipped_list,
204                 "error_list": error_list,
205
206                 "logout_to": '/',
207             })
208     else:
209         form = forms.DocumentsUploadForm()
210
211     return render(request, "documents/document_upload.html", {
212         "form": form,
213
214         "logout_to": '/',
215     })
216
217
218 def serve_xml(request, book, slug):
219     if not book.accessible(request):
220         return HttpResponseForbidden("Not authorized.")
221     xml = book.materialize(publishable=True)
222     response = http.HttpResponse(xml, content_type='application/xml')
223     response['Content-Disposition'] = 'attachment; filename=%s.xml' % slug
224     return response
225
226
227 @never_cache
228 def book_xml(request, slug):
229     book = get_object_or_404(Book, slug=slug)
230     return serve_xml(request, book, slug)
231
232
233 @never_cache
234 def book_xml_dc(request, slug):
235     book = get_object_or_404(Book, catalogue_book_id=slug)
236     return serve_xml(request, book, slug)
237
238
239 @never_cache
240 def book_txt(request, slug):
241     book = get_object_or_404(Book, slug=slug)
242     if not book.accessible(request):
243         return HttpResponseForbidden("Not authorized.")
244
245     doc = book.wldocument()
246     text = doc.as_text().get_bytes()
247     response = http.HttpResponse(text, content_type='text/plain')
248     response['Content-Disposition'] = 'attachment; filename=%s.txt' % slug
249     return response
250
251
252 @never_cache
253 def book_html(request, slug):
254     book = get_object_or_404(Book, slug=slug)
255     if not book.accessible(request):
256         return HttpResponseForbidden("Not authorized.")
257
258     doc = book.wldocument(parse_dublincore=False)
259     html = doc.as_html(options={'gallery': "'%s'" % book.gallery_url()})
260
261     html = html.get_bytes().decode('utf-8') if html is not None else ''
262     # response = http.HttpResponse(html, content_type='text/html')
263     # return response
264     # book_themes = {}
265     # for fragment in book.fragments.all().iterator():
266     #     for theme in fragment.tags.filter(category='theme').iterator():
267     #         book_themes.setdefault(theme, []).append(fragment)
268
269     # book_themes = book_themes.items()
270     # book_themes.sort(key=lambda s: s[0].sort_key)
271     return render(request, 'documents/book_text.html', locals())
272
273
274 @login_required
275 @never_cache
276 def book_pdf(request, slug, mobile=False):
277     book = get_object_or_404(Book, slug=slug)
278     if not book.accessible(request):
279         return HttpResponseForbidden("Not authorized.")
280
281     # TODO: move to celery
282     doc = book.wldocument()
283     # TODO: error handling
284     customizations = ['26pt', 'nothemes', 'nomargins', 'notoc'] if mobile else None
285     pdf_file = doc.as_pdf(cover=True, base_url=request.build_absolute_uri(book.gallery_path()), customizations=customizations)
286     from .ebook_utils import serve_file
287     return serve_file(pdf_file.get_filename(),
288                 book.slug + '.pdf', 'application/pdf')
289
290
291 @login_required
292 @never_cache
293 def book_epub(request, slug):
294     book = get_object_or_404(Book, slug=slug)
295     if not book.accessible(request):
296         return HttpResponseForbidden("Not authorized.")
297
298     # TODO: move to celery
299     doc = book.wldocument(librarian2=True)
300     # TODO: error handling
301
302     from librarian.builders import EpubBuilder
303     epub = EpubBuilder(
304         base_url='file://' + book.gallery_path() + '/',
305         debug=True
306     ).build(doc).get_bytes()
307     response = HttpResponse(content_type='application/epub+zip')
308     response['Content-Disposition'] = 'attachment; filename=%s' % book.slug + '.epub'
309     response.write(epub)
310     return response
311
312
313 @login_required
314 @never_cache
315 def book_epubcheck(request, slug):
316     book = get_object_or_404(Book, slug=slug)
317     if not book.accessible(request):
318         return HttpResponseForbidden("Not authorized.")
319
320     # TODO: move to celery
321     doc = book.wldocument(librarian2=True)
322     # TODO: error handling
323
324     from librarian.builders import EpubBuilder
325     epub = EpubBuilder(
326         base_url='file://' + book.gallery_path() + '/',
327         debug=True
328     ).build(doc)
329     fname = epub.get_filename()
330
331     messages = epubcheck.epubcheck(fname)
332     for message in messages:
333         for location in message.get('locations', []):
334             if 'wl_chunk' in location:
335                 location['wl_chunk'] = book[location['wl_chunk']]
336     return render(request, 'documents/book_epubcheck.html', {
337         'messages': messages,
338         'book': book,
339     })
340
341
342 @login_required
343 @never_cache
344 def book_mobi(request, slug):
345     book = get_object_or_404(Book, slug=slug)
346     if not book.accessible(request):
347         return HttpResponseForbidden("Not authorized.")
348
349     # TODO: move to celery
350     doc = book.wldocument(librarian2=True)
351     # TODO: error handling
352     from librarian.builders import MobiBuilder
353     mobi = MobiBuilder(
354         base_url='file://' + book.gallery_path() + '/'
355     ).build(doc).get_bytes()
356     response = HttpResponse(content_type='application/x-mobipocket-ebook')
357     response['Content-Disposition'] = 'attachment; filename=%s' % book.slug + '.mobi'
358     response.write(mobi)
359     return response
360
361
362 @never_cache
363 def revision(request, slug, chunk=None):
364     try:
365         doc = Chunk.get(slug, chunk)
366     except (Chunk.MultipleObjectsReturned, Chunk.DoesNotExist):
367         raise Http404
368     if not doc.book.accessible(request):
369         return HttpResponseForbidden("Not authorized.")
370     return http.HttpResponse(str(doc.revision()))
371
372
373 def book(request, slug):
374     book = get_object_or_404(Book, slug=slug)
375     if not book.accessible(request):
376         return HttpResponseForbidden("Not authorized.")
377
378     if request.user.has_perm('documents.change_book'):
379         if request.method == "POST":
380             form = forms.BookForm(request.POST, instance=book)
381             if form.is_valid():
382                 form.save()
383                 return http.HttpResponseRedirect(book.get_absolute_url())
384         else:
385             form = forms.BookForm(instance=book)
386         publish_options_form = forms.PublishOptionsForm()
387         editable = True
388     else:
389         form = forms.ReadonlyBookForm(instance=book)
390         publish_options_form = forms.PublishOptionsForm()
391         editable = False
392
393     publish_error = book.publishable_error()
394     publishable = publish_error is None
395
396     stats = None
397     try:
398         doc = book.wldocument(librarian2=True)
399     except:
400         doc = None
401     else:
402         try:
403             stats = doc.get_statistics()
404         except:
405             pass
406
407     return render(request, "documents/book_detail.html", {
408         "book": book,
409         "doc": doc,
410         "stats": stats,
411         "publishable": publishable,
412         "publishable_error": publish_error,
413         "form": form,
414         "publish_options_form": publish_options_form,
415         "editable": editable,
416     })
417
418
419 def image(request, slug):
420     image = get_object_or_404(Image, slug=slug)
421     if not image.accessible(request):
422         return HttpResponseForbidden("Not authorized.")
423
424     if request.user.has_perm('documents.change_image'):
425         if request.method == "POST":
426             form = forms.ImageForm(request.POST, instance=image)
427             if form.is_valid():
428                 form.save()
429                 return http.HttpResponseRedirect(image.get_absolute_url())
430         else:
431             form = forms.ImageForm(instance=image)
432         editable = True
433     else:
434         form = forms.ReadonlyImageForm(instance=image)
435         editable = False
436
437     publish_error = image.publishable_error()
438     publishable = publish_error is None
439
440     return render(request, "documents/image_detail.html", {
441         "object": image,
442         "publishable": publishable,
443         "publishable_error": publish_error,
444         "form": form,
445         "editable": editable,
446     })
447
448
449 @permission_required('documents.add_chunk')
450 def chunk_add(request, slug, chunk):
451     try:
452         doc = Chunk.get(slug, chunk)
453     except (Chunk.MultipleObjectsReturned, Chunk.DoesNotExist):
454         raise Http404
455     if not doc.book.accessible(request):
456         return HttpResponseForbidden("Not authorized.")
457
458     if request.method == "POST":
459         form = forms.ChunkAddForm(request.POST, instance=doc)
460         if form.is_valid():
461             if request.user.is_authenticated:
462                 creator = request.user
463             else:
464                 creator = None
465             doc.split(creator=creator,
466                 slug=form.cleaned_data['slug'],
467                 title=form.cleaned_data['title'],
468                 gallery_start=form.cleaned_data['gallery_start'],
469                 user=form.cleaned_data['user'],
470                 stage=form.cleaned_data['stage']
471             )
472
473             return http.HttpResponseRedirect(doc.book.get_absolute_url())
474     else:
475         form = forms.ChunkAddForm(initial={
476                 "slug": str(doc.number + 1),
477                 "title": "cz. %d" % (doc.number + 1, ),
478         })
479
480     return render(request, "documents/chunk_add.html", {
481         "chunk": doc,
482         "form": form,
483     })
484
485
486 @login_required
487 def chunk_edit(request, slug, chunk):
488     try:
489         doc = Chunk.get(slug, chunk)
490     except (Chunk.MultipleObjectsReturned, Chunk.DoesNotExist):
491         raise Http404
492     if not doc.book.accessible(request):
493         return HttpResponseForbidden("Not authorized.")
494
495     if request.method == "POST":
496         form = forms.ChunkForm(request.POST, instance=doc)
497         if form.is_valid():
498             form.save()
499             go_next = request.GET.get('next', None)
500             if go_next:
501                 go_next = quote_plus(unquote(iri_to_uri(go_next)), safe='/?=&')
502             else:
503                 go_next = doc.book.get_absolute_url()
504             return http.HttpResponseRedirect(go_next)
505     else:
506         form = forms.ChunkForm(instance=doc)
507
508     referer = request.META.get('HTTP_REFERER')
509     if referer:
510         parts = urlsplit(referer)
511         parts = ['', ''] + list(parts[2:])
512         go_next = quote_plus(urlunsplit(parts))
513     else:
514         go_next = ''
515
516     return render(request, "documents/chunk_edit.html", {
517         "chunk": doc,
518         "form": form,
519         "go_next": go_next,
520     })
521
522
523 @transaction.atomic
524 @login_required
525 @require_POST
526 def chunk_mass_edit(request):
527     ids = [int(i) for i in request.POST.get('ids').split(',') if i.strip()]
528     chunks = list(Chunk.objects.filter(id__in=ids))
529     
530     stage = request.POST.get('stage')
531     if stage:
532         try:
533             stage = Chunk.tag_model.objects.get(slug=stage)
534         except Chunk.DoesNotExist as e:
535             stage = None
536        
537         for c in chunks: c.stage = stage
538
539     username = request.POST.get('user')
540     logger.info("username: %s" % username)
541     logger.info(request.POST)
542     if username:
543         try:
544             user = User.objects.get(username=username)
545         except User.DoesNotExist as e:
546             user = None
547             
548         for c in chunks: c.user = user
549
550     project_id = request.POST.get('project')
551     if project_id:
552         try:
553             project = Project.objects.get(pk=int(project_id))
554         except (Project.DoesNotExist, ValueError) as e:
555             project = None
556         for c in chunks:
557             book = c.book
558             book.project = project
559             book.save()
560
561     for c in chunks: c.save()
562
563     return HttpResponse("", content_type="text/plain")
564
565
566 @transaction.atomic
567 @login_required
568 @require_POST
569 def image_mass_edit(request):
570     ids = map(int, filter(lambda i: i.strip()!='', request.POST.get('ids').split(',')))
571     images = map(lambda i: Image.objects.get(id=i), ids)
572     
573     stage = request.POST.get('stage')
574     if stage:
575         try:
576             stage = Image.tag_model.objects.get(slug=stage)
577         except Image.DoesNotExist as e:
578             stage = None
579        
580         for c in images: c.stage = stage
581
582     username = request.POST.get('user')
583     logger.info("username: %s" % username)
584     logger.info(request.POST)
585     if username:
586         try:
587             user = User.objects.get(username=username)
588         except User.DoesNotExist as e:
589             user = None
590             
591         for c in images: c.user = user
592
593     project_id = request.POST.get('project')
594     if project_id:
595         try:
596             project = Project.objects.get(pk=int(project_id))
597         except (Project.DoesNotExist, ValueError) as e:
598             project = None
599         for c in images:
600             c.project = project
601
602     for c in images: c.save()
603
604     return HttpResponse("", content_type="text/plain")
605
606
607 @permission_required('documents.change_book')
608 def book_append(request, slug):
609     book = get_object_or_404(Book, slug=slug)
610     if not book.accessible(request):
611         return HttpResponseForbidden("Not authorized.")
612
613     if request.method == "POST":
614         form = forms.BookAppendForm(book, request.POST)
615         if form.is_valid():
616             append_to = form.cleaned_data['append_to']
617             append_to.append(book)
618             return http.HttpResponseRedirect(append_to.get_absolute_url())
619     else:
620         form = forms.BookAppendForm(book)
621     return render(request, "documents/book_append_to.html", {
622         "book": book,
623         "form": form,
624
625         "logout_to": '/',
626     })
627
628
629 @require_POST
630 @login_required
631 def publish(request, slug):
632     form = forms.PublishOptionsForm(request.POST)
633     if form.is_valid():
634         days = form.cleaned_data['days']
635         beta = form.cleaned_data['beta']
636         hidden = form.cleaned_data['hidden']
637     else:
638         days = 0
639         beta = False
640         hidden = False
641     book = get_object_or_404(Book, slug=slug)
642     if not book.accessible(request):
643         return HttpResponseForbidden("Not authorized.")
644
645     try:
646         protocol = 'https://' if request.is_secure() else 'http://'
647         book.publish(request.user, host=protocol + request.get_host(), days=days, beta=beta, hidden=hidden)
648     except NotAuthorizedError:
649         return http.HttpResponseRedirect(reverse('apiclient_oauth' if not beta else 'apiclient_beta_oauth'))
650     except BaseException as e:
651         return http.HttpResponse(repr(e))
652     else:
653         return http.HttpResponseRedirect(book.get_absolute_url())
654
655
656 @require_POST
657 @login_required
658 def publish_image(request, slug):
659     image = get_object_or_404(Image, slug=slug)
660     if not image.accessible(request):
661         return HttpResponseForbidden("Not authorized.")
662
663     try:
664         image.publish(request.user)
665     except NotAuthorizedError:
666         return http.HttpResponseRedirect(reverse('apiclient_oauth'))
667     except BaseException as e:
668         return http.HttpResponse(e)
669     else:
670         return http.HttpResponseRedirect(image.get_absolute_url())
671
672
673 class GalleryView(UploadView):
674     def get_object(self, request, slug):
675         book = get_object_or_404(Book, slug=slug)
676         if not book.gallery:
677             raise Http404
678         return book
679
680     def breadcrumbs(self):
681         return [
682             (_('books'), reverse('documents_document_list')),
683             (self.object.title, self.object.get_absolute_url()),
684             (_('scan gallery'),),
685         ]
686
687     def get_directory(self):
688         return "%s%s/" % (settings.IMAGE_DIR, self.object.gallery)
689
690
691 def active_users_list(request, csv=False):
692     year = int(request.GET.get('y', date.today().year))
693     by_user = defaultdict(lambda: 0)
694     by_email = defaultdict(lambda: 0)
695     names_by_email = defaultdict(set)
696     for change_model in (Chunk.change_model, Image.change_model):
697         for c in change_model.objects.filter(
698                 created_at__year=year).order_by(
699                 'author', 'author_email', 'author_name').values(
700                 'author', 'author_name', 'author_email').annotate(
701                 c=Count('author'), ce=Count('author_email')).distinct():
702             if c['author']:
703                 by_user[c['author']] += c['c']
704             else:
705                 by_email[c['author_email']] += c['ce']
706                 if (c['author_name'] or '').strip():
707                     names_by_email[c['author_email']].add(c['author_name'])
708     for user in User.objects.filter(pk__in=by_user):
709         by_email[user.email] += by_user[user.pk]
710         names_by_email[user.email].add("%s %s" % (user.first_name, user.last_name))
711
712     active_users = []
713     for email, count in by_email.items():
714         active_users.append((email, names_by_email[email], count))
715     active_users.sort(key=lambda x: -x[2])
716     if csv:
717         return http.HttpResponse(
718             '\n'.join((
719                 ','.join(
720                     (str(x[2]), x[0], ','.join(x[1]))
721                 )
722                 for x in active_users
723             )),
724             content_type='text/csv',
725             headers={
726                 'Content-Disposition': f'attachment; filename=redakcja-{year}.csv',
727             }
728         )
729     else:
730         return render(request, 'documents/active_users_list.html', {
731             'users': active_users,
732             'year': year,
733         })
734
735
736 @user_passes_test(lambda u: u.is_superuser)
737 def mark_final(request):
738     if request.method == 'POST':
739         form = forms.MarkFinalForm(data=request.POST)
740         if form.is_valid():
741             form.save()
742             return HttpResponseRedirect(reverse('mark_final_completed'))
743     else:
744         form = forms.MarkFinalForm()
745     return render(request, 'documents/mark_final.html', {'form': form})
746
747
748 def mark_final_completed(request):
749     return render(request, 'documents/mark_final_completed.html')