From: Radek Czajka Date: Thu, 22 Mar 2012 14:56:18 +0000 (+0100) Subject: celery waiter for custompdf X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/aae5369e6dab510705fdfa1265cf9e095d24a97d celery waiter for custompdf --- diff --git a/apps/catalogue/forms.py b/apps/catalogue/forms.py index 75d9ab997..d191d46db 100644 --- a/apps/catalogue/forms.py +++ b/apps/catalogue/forms.py @@ -35,54 +35,46 @@ class DownloadFormatsForm(forms.Form): widget=forms.CheckboxSelectMultiple) def __init__(self, *args, **kwargs): - super(DownloadFormatsForm, self).__init__(*args, **kwargs) + super(DownloadFormatsForm, self).__init__(*args, **kwargs) -PDF_PAGE_SIZES = ( - ('a4paper', _('A4')), - ('a5paper', _('A5')), -) - - -PDF_LEADINGS = ( - ('', _('Normal leading')), - ('onehalfleading', _('One and a half leading')), - ('doubleleading', _('Double leading')), +CUSTOMIZATION_FLAGS = ( + ('nofootnotes', _("Don't show footnotes")), + ('nothemes', _("Don't disply themes")), + ('nowlfont', _("Don't use our custom font")), ) - -PDF_FONT_SIZES = ( - ('11pt', _('Default')), - ('13pt', _('Big')) +CUSTOMIZATION_OPTIONS = ( + ('leading', _("Leading"), ( + ('defaultleading', _('Normal leading')), + ('onehalfleading', _('One and a half leading')), + ('doubleleading', _('Double leading')), + )), + ('fontsize', _("Font size"), ( + ('11pt', _('Default')), + ('13pt', _('Big')) + )), +# ('pagesize', _("Paper size"), ( +# ('a4paper', _('A4')), +# ('a5paper', _('A5')), +# )), ) class CustomPDFForm(forms.Form): - nofootnotes = forms.BooleanField(required=False, label=_("Don't show footnotes")) - nothemes = forms.BooleanField(required=False, label=_("Don't disply themes")) - nowlfont = forms.BooleanField(required=False, label=_("Don't use our custom font")) - ## pagesize = forms.ChoiceField(PDF_PAGE_SIZES, required=True, label=_("Paper size")) - leading = forms.ChoiceField(PDF_LEADINGS, required=False, label=_("Leading")) - fontsize = forms.ChoiceField(PDF_FONT_SIZES, required=True, label=_("Font size")) + def __init__(self, *args, **kwargs): + super(CustomPDFForm, self).__init__(*args, **kwargs) + for name, label in CUSTOMIZATION_FLAGS: + self.fields[name] = forms.BooleanField(required=False, label=label) + for name, label, choices in CUSTOMIZATION_OPTIONS: + self.fields[name] = forms.ChoiceField(choices, label=label) @property def customizations(self): c = [] - if self.cleaned_data['nofootnotes']: - c.append('nofootnotes') - - if self.cleaned_data['nothemes']: - c.append('nothemes') - - if self.cleaned_data['nowlfont']: - c.append('nowlfont') - - ## c.append(self.cleaned_data['pagesize']) - c.append(self.cleaned_data['fontsize']) - - if self.cleaned_data['leading']: - c.append(self.cleaned_data['leading']) - + for name, label in CUSTOMIZATION_FLAGS: + if self.cleaned_data.get(name): + c.append(name) + for name, label, choices in CUSTOMIZATION_OPTIONS: + c.append(self.cleaned_data[name]) c.sort() - return c - diff --git a/apps/catalogue/utils.py b/apps/catalogue/utils.py index 185f5fa34..949ac96b2 100644 --- a/apps/catalogue/utils.py +++ b/apps/catalogue/utils.py @@ -140,7 +140,7 @@ class AttachmentHttpResponse(HttpResponse): for chunk in read_chunks(f): self.write(chunk) -@task +@task(rate_limit=settings.CATALOGUE_CUSTOMPDF_RATE_LIMIT) def async_build_pdf(book_id, customizations, file_name): """ A celery task to generate pdf files. diff --git a/apps/catalogue/views.py b/apps/catalogue/views.py index c8549c26f..0c05d17a8 100644 --- a/apps/catalogue/views.py +++ b/apps/catalogue/views.py @@ -22,7 +22,7 @@ from ajaxable.utils import JSONResponse, AjaxableFormView from catalogue import models from catalogue import forms -from catalogue.utils import (split_tags, AttachmentHttpResponse, +from catalogue.utils import (split_tags, async_build_pdf, MultiQuerySet) from pdcounter import models as pdcounter_models from pdcounter import views as pdcounter_views @@ -543,7 +543,7 @@ def download_custom_pdf(request, slug, method='GET'): url = WaitedFile.order(pdf_file, lambda p: async_build_pdf.delay(book.id, cust, p), - "%s: %s" % (book.pretty_title(), ", ".join(cust)) + book.pretty_title() ) return redirect(url) else: diff --git a/apps/waiter/__init__.py b/apps/waiter/__init__.py index 5aabbd62d..d3696b741 100644 --- a/apps/waiter/__init__.py +++ b/apps/waiter/__init__.py @@ -1,5 +1,8 @@ """ -Waiting for and serving files generated in Celery to the user. +Celery waiter. + +Takes orders for files generated by async Celery tasks. +Serves the file when ready. Kindly asks the user to wait if not. Author: Radek Czajka """ \ No newline at end of file diff --git a/apps/waiter/locale/pl/LC_MESSAGES/django.mo b/apps/waiter/locale/pl/LC_MESSAGES/django.mo new file mode 100644 index 000000000..8f8afb7de Binary files /dev/null and b/apps/waiter/locale/pl/LC_MESSAGES/django.mo differ diff --git a/apps/waiter/locale/pl/LC_MESSAGES/django.po b/apps/waiter/locale/pl/LC_MESSAGES/django.po new file mode 100644 index 000000000..bbc9f8dfa --- /dev/null +++ b/apps/waiter/locale/pl/LC_MESSAGES/django.po @@ -0,0 +1,67 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2012-03-22 15:54+0100\n" +"PO-Revision-Date: 2012-03-22 15:54+0100\n" +"Last-Translator: Radek Czajka \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2)\n" + +#: templates/waiter/wait.html:7 +#: templates/waiter/wait.html.py:33 +msgid "The file is ready for download!" +msgstr "Plik jest gotowy do pobrania!" + +#: templates/waiter/wait.html:10 +#: templates/waiter/wait.html.py:42 +msgid "Your file is being prepared, please wait." +msgstr "Plik jest generowany, proszę czekać." + +#: templates/waiter/wait.html:12 +#: templates/waiter/wait.html.py:51 +msgid "Something went wrong." +msgstr "Coś poszło nie tak." + +#: templates/waiter/wait.html:36 +#, python-format +msgid "" +"Your file is ready!\n" +" If the download doesn't start in a few seconds,\n" +" feel free to use this direct link." +msgstr "" +"Twój plik jest gotowy!\n" +"Jeśli pobieranie nie zacznie się w ciągu kilku sekund,\n" +"skorzystaj z tego bezpośredniego linku." + +#: templates/waiter/wait.html:45 +#, python-format +msgid "The file you requested was: %(d)s." +msgstr "Zamówiony plik to: %(d)s." + +#: templates/waiter/wait.html:47 +msgid "" +"Be aware: Generating the file can take a while.\n" +" Please be patient, or bookmark this page and come back later.

" +msgstr "" +"Uwaga: Generowanie pliku może trwać dłuższą chwilę.\n" +"Poczekaj cierpliwie, albo dodaj tę stronę do zakładek i wróć później.

" + +#: templates/waiter/wait.html:55 +#, python-format +msgid "" +"Something seems to have gone wrong while generating your file.\n" +" Please order it again or complain to us about it." +msgstr "" +"Wygląda na to, że coś poszło źle podczas generowania Twojego pliku.\n" +"Spróbuj zamówić go jeszcze raz albo napisz do nas." + diff --git a/apps/waiter/models.py b/apps/waiter/models.py index fd9730131..26a9a6d4d 100644 --- a/apps/waiter/models.py +++ b/apps/waiter/models.py @@ -1,7 +1,9 @@ from os.path import join, abspath, exists +from django.core.urlresolvers import reverse from django.db import models from waiter.settings import WAITER_ROOT, WAITER_URL -from django.core.urlresolvers import reverse +from djcelery.models import TaskMeta + class WaitedFile(models.Model): path = models.CharField(max_length=255, unique=True, db_index=True) @@ -23,31 +25,44 @@ class WaitedFile(models.Model): Won't open a path leading outside of WAITER_ROOT. """ abs_path = cls.abspath(path) - # Pre-fetch objects to avoid minor race condition + # Pre-fetch objects for deletion to avoid minor race condition relevant = [o.id for o in cls.objects.filter(path=path)] - print abs_path if exists(abs_path): cls.objects.filter(id__in=relevant).delete() return True else: return False + def is_stale(self): + if self.task is None: + # Race; just let the other task roll. + return False + try: + meta = TaskMeta.objects.get(task_id=self.task) + assert meta.status in (u'PENDING', u'STARTED', u'SUCCESS', u'RETRY') + except TaskMeta.DoesNotExist: + # Might happen it's not yet there. + pass + except AssertionError: + return True + return False + @classmethod def order(cls, path, task_creator, description=None): """ Returns an URL for the user to follow. If the file is ready, returns download URL. If not, starts preparing it and returns waiting URL. + + task_creator: function taking a path and generating the file; + description: a string or string proxy with a description for user; """ already = cls.exists(path) if not already: waited, created = cls.objects.get_or_create(path=path) - if created: - # TODO: makedirs + if created or waited.is_stale(): waited.task = task_creator(cls.abspath(path)) - print waited.task waited.description = description waited.save() - # TODO: it the task exists, if stale delete, send some mail and restart return reverse("waiter", args=[path]) return join(WAITER_URL, path) diff --git a/apps/waiter/templates/waiter/wait.html b/apps/waiter/templates/waiter/wait.html index 7c5a88428..f4dedc77a 100644 --- a/apps/waiter/templates/waiter/wait.html +++ b/apps/waiter/templates/waiter/wait.html @@ -1,21 +1,92 @@ {% extends "base.html" %} +{% load i18n %} +{% load url from future %} -{% block titleextra %}Prosię ciekać{% endblock %} +{% block titleextra %} +{% if file_url %} + {% trans "The file is ready for download!" %} +{% else %} + {% if waiting %} + {% trans "Your file is being prepared, please wait." %} + {% else %} + {% trans "Something went wrong." %} + {% endif %} +{% endif %} +{% endblock %} {% block extrahead %} - {# TODO: Not like that! What are you, caveman?! #} - +{% if file_url %} + +{% else %} + {% if waiting %} + + {% endif %} +{% endif %} {% endblock %} {% block body %} - {% if file_url %} -

Plik jest gotowy! Jeśli pobieranie nie zacznie się automatycznie, - oto bezpośredni link.

- {% else %} -

Idź na kawę. To trochę potrwa.

- -

{{ waiting_for.description }}

- {% endif %} +{% if file_url %} +

{% trans "The file is ready for download!" %}

+ +
+

{% blocktrans %}Your file is ready! + If the download doesn't start in a few seconds, + feel free to use this direct link.{% endblocktrans %}

+
+{% else %} + {% if waiting %} +

{% trans "Your file is being prepared, please wait." %}

+ +
+

{% blocktrans with d=waiting.description %}The file you requested was: {{d}}.{% endblocktrans %}

+ +

{% blocktrans %}Be aware: Generating the file can take a while. + Please be patient, or bookmark this page and come back later.

{% endblocktrans %} +
+ {% else %} +

{% trans "Something went wrong." %}

+ +
+ {% url 'suggest' as s %} +

{% blocktrans %}Something seems to have gone wrong while generating your file. + Please order it again or complain to us about it.{% endblocktrans %}

+
+ {% endif %} +{% endif %} + +{% endblock %} + +{% block extrabody %} +{% if waiting %} + +{% endif %} {% endblock %} diff --git a/apps/waiter/views.py b/apps/waiter/views.py index 81cd3b49d..e38bd8f1d 100644 --- a/apps/waiter/views.py +++ b/apps/waiter/views.py @@ -1,12 +1,19 @@ from os.path import join from waiter.models import WaitedFile from waiter.settings import WAITER_URL -from django.shortcuts import get_object_or_404, render, redirect +from django.shortcuts import render, get_object_or_404 +from django.http import HttpResponse def wait(request, path): if WaitedFile.exists(path): file_url = join(WAITER_URL, path) else: - waiting_for = get_object_or_404(WaitedFile, path=path) - # TODO: check if not stale, inform the user and send some mail if so. - return render(request, "waiter/wait.html", locals()) + file_url = "" + waiting = get_object_or_404(WaitedFile, path=path) + if waiting.is_stale(): + waiting = None + + if request.is_ajax(): + return HttpResponse(file_url) + else: + return render(request, "waiter/wait.html", locals()) diff --git a/lib/librarian b/lib/librarian index 9e13b0c99..b8e34e0e6 160000 --- a/lib/librarian +++ b/lib/librarian @@ -1 +1 @@ -Subproject commit 9e13b0c994e9d481008bef7006a74609adfd16f8 +Subproject commit b8e34e0e6730ef76a353a15ff653faa9e8c88a77 diff --git a/wolnelektury/settings/celery.py b/wolnelektury/settings/celery.py index cb5dde97d..ad37707ee 100644 --- a/wolnelektury/settings/celery.py +++ b/wolnelektury/settings/celery.py @@ -9,3 +9,4 @@ BROKER_PASSWORD = "guest" BROKER_VHOST = "/" CELERY_EAGER_PROPAGATES_EXCEPTIONS = True +CELERY_SEND_TASK_ERROR_EMAILS = True diff --git a/wolnelektury/settings/custom.py b/wolnelektury/settings/custom.py index 3031f4ed8..97d958802 100644 --- a/wolnelektury/settings/custom.py +++ b/wolnelektury/settings/custom.py @@ -21,3 +21,5 @@ ALL_MOBI_ZIP = 'wolnelektury_pl_mobi' CATALOGUE_DEFAULT_LANGUAGE = 'pol' PUBLISH_PLAN_FEED = 'http://redakcja.wolnelektury.pl/documents/track/editor-proofreading/?published=false' + +CATALOGUE_CUSTOMPDF_RATE_LIMIT = '6/m'