From: Radek Czajka Date: Thu, 17 Nov 2011 15:40:32 +0000 (+0100) Subject: report bad audiobooks, X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/af77776a9ff93ac89cfe941c8a97e3e4cac1bd2e?ds=inline;hp=--cc report bad audiobooks, generate catalogue in pdf --- af77776a9ff93ac89cfe941c8a97e3e4cac1bd2e diff --git a/apps/catalogue/models.py b/apps/catalogue/models.py index d90299611..bb207747d 100644 --- a/apps/catalogue/models.py +++ b/apps/catalogue/models.py @@ -12,6 +12,7 @@ from django.utils.translation import ugettext_lazy as _ from django.contrib.auth.models import User from django.core.files import File from django.template.loader import render_to_string +from django.utils.datastructures import SortedDict from django.utils.safestring import mark_safe from django.utils.translation import get_language from django.core.urlresolvers import reverse @@ -306,6 +307,7 @@ class Book(models.Model): tags = managers.TagDescriptor(Tag) html_built = django.dispatch.Signal() + published = django.dispatch.Signal() class AlreadyExists(Exception): pass @@ -734,6 +736,7 @@ class Book(models.Model): book.reset_tag_counter() book.reset_theme_counter() + cls.published.send(sender=book) return book def reset_tag_counter(self): @@ -827,6 +830,43 @@ class Book(models.Model): return objects + @classmethod + def book_list(cls, filter=None): + """Generates a hierarchical listing of all books. + + Books are optionally filtered with a test function. + + """ + + books_by_parent = {} + books = cls.objects.all().order_by('parent_number', 'sort_key').only('title', 'parent', 'slug') + if filter: + books = books.filter(filter).distinct() + book_ids = set((book.pk for book in books)) + for book in books: + parent = book.parent_id + if parent not in book_ids: + parent = None + books_by_parent.setdefault(parent, []).append(book) + else: + for book in books: + books_by_parent.setdefault(book.parent_id, []).append(book) + + orphans = [] + books_by_author = SortedDict() + for tag in Tag.objects.filter(category='author'): + books_by_author[tag] = [] + + for book in books_by_parent.get(None,()): + authors = list(book.tags.filter(category='author')) + if authors: + for author in authors: + books_by_author[author].append(book) + else: + orphans.append(book) + + return books_by_author, orphans, books_by_parent + def _has_factory(ftype): has = lambda self: bool(getattr(self, "%s_file" % ftype)) diff --git a/apps/catalogue/templatetags/catalogue_tags.py b/apps/catalogue/templatetags/catalogue_tags.py index e433b8e32..ba70f7b57 100644 --- a/apps/catalogue/templatetags/catalogue_tags.py +++ b/apps/catalogue/templatetags/catalogue_tags.py @@ -140,6 +140,20 @@ def book_tree(book_list, books_by_parent): else: return '' +@register.simple_tag +def book_tree_texml(book_list, books_by_parent, depth=0): + return "".join(""" + %(depth)dem%(title)s + %(audiobook)s + + %(children)s + """ % { + "depth": depth, + "title": book.title, + "audiobook": "audiobook" if book.has_media('mp3') else "", + "children": book_tree_texml(books_by_parent.get(book.id, ()), books_by_parent, depth + 1) + } for book in book_list) + @register.simple_tag def all_editors(extra_info): diff --git a/apps/catalogue/views.py b/apps/catalogue/views.py index 64aada533..5f0b01652 100644 --- a/apps/catalogue/views.py +++ b/apps/catalogue/views.py @@ -83,34 +83,8 @@ def book_list(request, filter=None, template_name='catalogue/book_list.html'): form = forms.SearchForm() - books_by_parent = {} - books = models.Book.objects.all().order_by('parent_number', 'sort_key').only('title', 'parent', 'slug') - if filter: - books = books.filter(filter).distinct() - book_ids = set((book.pk for book in books)) - for book in books: - parent = book.parent_id - if parent not in book_ids: - parent = None - books_by_parent.setdefault(parent, []).append(book) - else: - for book in books: - books_by_parent.setdefault(book.parent_id, []).append(book) - - orphans = [] - books_by_author = SortedDict() + books_by_author, orphans, books_by_parent = models.Book.book_list(filter) books_nav = SortedDict() - for tag in models.Tag.objects.filter(category='author'): - books_by_author[tag] = [] - - for book in books_by_parent.get(None,()): - authors = list(book.tags.filter(category='author')) - if authors: - for author in authors: - books_by_author[author].append(book) - else: - orphans.append(book) - for tag in books_by_author: if books_by_author[tag]: books_nav.setdefault(tag.sort_key[0], []).append(tag) diff --git a/apps/reporting/__init__.py b/apps/reporting/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/reporting/models.py b/apps/reporting/models.py new file mode 100644 index 000000000..740b9276e --- /dev/null +++ b/apps/reporting/models.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. +# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# + + +# import views here, so that signals are attached correctly +from reporting.views import catalogue_pdf diff --git a/apps/reporting/templates/reporting/catalogue.texml b/apps/reporting/templates/reporting/catalogue.texml new file mode 100755 index 000000000..78e655135 --- /dev/null +++ b/apps/reporting/templates/reporting/catalogue.texml @@ -0,0 +1,116 @@ +{% load catalogue_tags %} + + + \documentclass[a4paper, oneside, 11pt]{book} + +\usepackage[MeX]{polski} + +\usepackage[xetex]{graphicx} +\usepackage{xunicode} +\usepackage{xltxtra} + +\usepackage{scalefnt} +\usepackage[colorlinks=true,linkcolor=black,setpagesize=false,urlcolor=black,xetex]{hyperref} + +\setmainfont [ +%ExternalLocation, +UprightFont = JunicodeWL-Regular, +ItalicFont = JunicodeWL-Italic, +BoldFont = JunicodeWL-Regular, +BoldItalicFont = JunicodeWL-Italic, +SmallCapsFont = JunicodeWL-Regular, +SmallCapsFeatures = {Letters={SmallCaps,UppercaseSmallCaps}}, +Numbers=OldStyle, +Scale=1.04, +LetterSpace=-1.0 +] {JunicodeWL} + +\pagestyle{plain} +\usepackage{fancyhdr} + +\makeatletter + +\usepackage{color} +\definecolor{theme}{gray}{.3} + +\setlength{\hoffset}{-1cm} +\setlength{\oddsidemargin}{0pt} +\setlength{\marginparsep}{0pt} +\setlength{\marginparwidth}{0pt} + +\setlength{\voffset}{0pt} +\setlength{\topmargin}{0pt} +\setlength{\headheight}{0pt} +\setlength{\headsep}{0pt} +\setlength{\leftmargin}{0em} +\setlength{\rightmargin}{0em} +\setlength{\textheight}{24cm} +\setlength{\textwidth}{18cm} + + +\pagestyle{fancy} +\fancyhf{} +\renewcommand{\headrulewidth}{0pt} +\renewcommand{\footrulewidth}{0pt} +\lfoot{\footnotesize Katalog biblioteki internetowej WolneLektury.pl, \today} +\cfoot{} +\rfoot{\footnotesize \thepage} + +\clubpenalty=100000 +\widowpenalty=100000 + + +% see http://osdir.com/ml/tex.xetex/2005-10/msg00003.html +\newsavebox{\ximagebox}\newlength{\ximageheight} +\newsavebox{\xglyphbox}\newlength{\xglyphheight} +\newcommand{\xbox}[1] +{\savebox{\ximagebox}{#1}\settoheight{\ximageheight}{\usebox {\ximagebox}}% +\savebox{\xglyphbox}{\char32}\settoheight{\xglyphheight}{\usebox {\xglyphbox}}% +\raisebox{\ximageheight}[0pt][0pt]{%\raisebox{-\xglyphheight}[0pt] [0pt]{% +\makebox[0pt][l]{\usebox{\xglyphbox}}}%}% +\usebox{\ximagebox}% +\raisebox{0pt}[0pt][0pt]{\makebox[0pt][r]{\usebox{\xglyphbox}}}} + + +\newcommand{\name}[1]{% +\vspace{.5em}\Large{#1}% +} + + + \begin{document} + + \noindent \begin{minipage}[t]{.35\textwidth}\vspace{0pt} + \href{http://www.wolnelektury.pl}{\xbox{\includegraphics[width=\textwidth]{wl-logo.png}}} + \vspace{1em} + \end{minipage} + + \begin{minipage}[t]{.65\textwidth}\vspace{0pt} + + \begin{flushright} + + \section*{Katalog biblioteki internetowej \href{http://www.wolnelektury.pl/}{WolneLektury.pl}.} + stan na \today + + \end{flushright} + + \end{minipage} + + + \begin{tabular}{p{12cm} p{2cm}} + + + {% book_tree_texml orphans books_by_parent %} + {% for author, group in books_by_author.items %} + {% if group %} + {{ author }} + + + {% book_tree_texml group books_by_parent %} + {% endif %} + {% endfor %} + + + \end{tabular} + \end{document} + + \ No newline at end of file diff --git a/apps/reporting/templates/reporting/main.html b/apps/reporting/templates/reporting/main.html new file mode 100755 index 000000000..bbfca9964 --- /dev/null +++ b/apps/reporting/templates/reporting/main.html @@ -0,0 +1,37 @@ +{% extends "base.html" %} +{% load i18n %} +{% load reporting_stats catalogue_tags %} + +{% block title %}Statystyka w WolneLektury.pl{% endblock %} + +{% block bodyid %}reports-stats{% endblock %} + + +{% block body %} +

Statystyka

+ {% search_form %} + +

Katalog biblioteki w formacie PDF.

+ + + + + + + + + + {% for mt in media_types %} + + + + + + {% endfor %} +
Utwory
Wszystkie utwory:{% count_books_all %}
Utwory z własną treścią:{% count_books_nonempty %}
Utwory bez własnej treści:{% count_books_empty %}
Niezależne książki:{% count_books_root %}
MediaLiczbaRozmiarDo wymiany
{{ mt.type }}:{{ mt.count }}{{ mt.size|filesizeformat }}{{ mt.deprecated }} + {% for m in mt.deprecated_files %} +
{{ m }} + {% endfor %} +
+ +{% endblock %} diff --git a/apps/reporting/templatetags/__init__.py b/apps/reporting/templatetags/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/apps/reporting/templatetags/reporting_stats.py b/apps/reporting/templatetags/reporting_stats.py new file mode 100755 index 000000000..dceee0001 --- /dev/null +++ b/apps/reporting/templatetags/reporting_stats.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. +# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# +import feedparser +from functools import wraps +import datetime + +from django import template + +from catalogue.models import Book, BookMedia + + +register = template.Library() + +class StatsNode(template.Node): + def __init__(self, value, varname=None): + self.value = value + self.varname = varname + + def render(self, context): + if self.varname: + context[self.varname] = self.value + return '' + else: + return self.value + + +def register_counter(f): + """Turns a simple counting function into a registered counter tag. + + You can run a counter tag as a simple {% tag_name %} tag, or + as {% tag_name var_name %} to store the result in a variable. + + """ + @wraps(f) + def wrapped(parser, token): + try: + tag_name, args = token.contents.split(None, 1) + except ValueError: + args = None + return StatsNode(f(), args) + + return register.tag(wrapped) + + +@register_counter +def count_books_all(): + return Book.objects.all().count() + +@register_counter +def count_books_nonempty(): + return Book.objects.exclude(html_file='').count() + +@register_counter +def count_books_empty(): + return Book.objects.filter(html_file='').count() + +@register_counter +def count_books_root(): + return Book.objects.filter(parent=None).count() diff --git a/apps/reporting/urls.py b/apps/reporting/urls.py new file mode 100755 index 000000000..f50921513 --- /dev/null +++ b/apps/reporting/urls.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. +# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# +from django.conf.urls.defaults import * + + +urlpatterns = patterns('reporting.views', + url(r'^$', 'stats_page', name='reporting_stats'), + url(r'^katalog.pdf$', 'catalogue_pdf', name='reporting_catalogue_pdf'), +) + diff --git a/apps/reporting/utils.py b/apps/reporting/utils.py new file mode 100755 index 000000000..919e2edc1 --- /dev/null +++ b/apps/reporting/utils.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. +# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# +import os +import os.path +import shutil +import subprocess +from tempfile import mkdtemp +from StringIO import StringIO +from django.conf import settings +import logging +from django.http import HttpResponse +from django.template.loader import render_to_string + +logger = logging.getLogger(__name__) + + +def render_to_pdf(output_path, template, context=None, add_files=None): + """Renders a TeXML document into a PDF file. + + :param str output_path: is where the PDF file should go + :param str template: is a TeXML template path + :param context: is context for rendering the template + :param dict add_files: a dictionary of additional files XeTeX will need + """ + + import Texml.processor + rendered = render_to_string(template, context) + texml = StringIO(rendered.encode('utf-8')) + tempdir = mkdtemp(prefix = "render_to_pdf-") + tex_path = os.path.join(tempdir, "doc.tex") + with open(tex_path, 'w') as tex_file: + Texml.processor.process(texml, tex_file, encoding="utf-8") + + if add_files: + for add_name, src_file in add_files.items(): + add_path = os.path.join(tempdir, add_name) + if hasattr(src_file, "read"): + with open(add_path, 'w') as add_file: + add_file.write(add_file.read()) + else: + shutil.copy(src_file, add_path) + + cwd = os.getcwd() + os.chdir(tempdir) + try: + subprocess.check_call(['xelatex', '-interaction=batchmode', tex_path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + os.makedirs(os.path.dirname(output_path)) + except: + pass + shutil.move(os.path.join(tempdir, "doc.pdf"), output_path) + finally: + os.chdir(cwd) + shutil.rmtree(tempdir) + + +def read_chunks(f, size=8192): + chunk = f.read(size) + while chunk: + yield chunk + chunk = f.read(size) + + +def generated_file_view(file_name, mime_type, send_name=None, signals=None): + file_path = os.path.join(settings.MEDIA_ROOT, file_name) + file_url = os.path.join(settings.MEDIA_URL, file_name) + if send_name is None: + send_name = os.path.basename(file_name) + + def signal_handler(*args, **kwargs): + os.unlink(file_path) + + if signals: + for signal in signals: + signal.connect(signal_handler, weak=False) + + def decorator(func): + def view(request, *args, **kwargs): + if not os.path.exists(file_path) or True: + func(file_path, *args, **kwargs) + + if hasattr(send_name, "__call__"): + name = send_name() + else: + name = send_name + + response = HttpResponse(mimetype=mime_type) + response['Content-Disposition'] = 'attachment; filename=%s' % name + with open(file_path) as f: + for chunk in read_chunks(f): + response.write(chunk) + return response + return view + return decorator diff --git a/apps/reporting/views.py b/apps/reporting/views.py new file mode 100644 index 000000000..02038637e --- /dev/null +++ b/apps/reporting/views.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. +# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# +import os.path +from datetime import date +from django.conf import settings +from django.db.models import Count +from django.shortcuts import render_to_response +from django.template import RequestContext + +from catalogue.models import Book, BookMedia +from reporting.utils import render_to_pdf, generated_file_view + + +def stats_page(request): + media = BookMedia.objects.count() + media_types = BookMedia.objects.values('type').\ + annotate(count=Count('type')).\ + order_by('type') + for mt in media_types: + mt['size'] = sum(b.file.size for b in BookMedia.objects.filter(type=mt['type'])) + if mt['type'] in ('mp3', 'ogg'): + deprecated = BookMedia.objects.filter( + type=mt['type'], source_sha1=None) + mt['deprecated'] = deprecated.count() + mt['deprecated_files'] = deprecated.order_by('book', 'name') + else: + mt['deprecated'] = '-' + + return render_to_response('reporting/main.html', + locals(), context_instance=RequestContext(request)) + + +@generated_file_view('reports/katalog.pdf', 'application/pdf', + send_name=lambda: 'wolnelektury_%s.pdf' % date.today(), + signals=[Book.published]) +def catalogue_pdf(path): + books_by_author, orphans, books_by_parent = Book.book_list() + print books_by_parent + render_to_pdf(path, 'reporting/catalogue.texml', locals(), { + "wl-logo.png": os.path.join(settings.STATIC_ROOT, "img/logo-big.png"), + }) diff --git a/apps/stats/templates/stats/main.html b/apps/stats/templates/stats/main.html deleted file mode 100755 index 2be411d0e..000000000 --- a/apps/stats/templates/stats/main.html +++ /dev/null @@ -1,30 +0,0 @@ -{% extends "base.html" %} -{% load i18n %} -{% load stats catalogue_tags %} - -{% block title %}Statystyka w WolneLektury.pl{% endblock %} - -{% block bodyid %}tagged-object-list{% endblock %} - -{% block body %} -

Statystyka

- {% search_form %} - - - - - - - - - - {% for mt in media_types %} - - - - - - {% endfor %} -
Utwory
Wszystkie utwory:{% count_books_all %}
Utwory z własną treścią:{% count_books_nonempty %}
Utwory bez własnej treści:{% count_books_empty %}
Niezależne książki:{% count_books_root %}
MediaLiczbaRozmiarDo wymiany
{{ mt.type }}:{{ mt.count }}{{ mt.size|filesizeformat }}{{ mt.deprecated }}
- -{% endblock %} diff --git a/apps/stats/templatetags/__init__.py b/apps/stats/templatetags/__init__.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/apps/stats/templatetags/stats.py b/apps/stats/templatetags/stats.py deleted file mode 100755 index dceee0001..000000000 --- a/apps/stats/templatetags/stats.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. -# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. -# -import feedparser -from functools import wraps -import datetime - -from django import template - -from catalogue.models import Book, BookMedia - - -register = template.Library() - -class StatsNode(template.Node): - def __init__(self, value, varname=None): - self.value = value - self.varname = varname - - def render(self, context): - if self.varname: - context[self.varname] = self.value - return '' - else: - return self.value - - -def register_counter(f): - """Turns a simple counting function into a registered counter tag. - - You can run a counter tag as a simple {% tag_name %} tag, or - as {% tag_name var_name %} to store the result in a variable. - - """ - @wraps(f) - def wrapped(parser, token): - try: - tag_name, args = token.contents.split(None, 1) - except ValueError: - args = None - return StatsNode(f(), args) - - return register.tag(wrapped) - - -@register_counter -def count_books_all(): - return Book.objects.all().count() - -@register_counter -def count_books_nonempty(): - return Book.objects.exclude(html_file='').count() - -@register_counter -def count_books_empty(): - return Book.objects.filter(html_file='').count() - -@register_counter -def count_books_root(): - return Book.objects.filter(parent=None).count() diff --git a/apps/stats/urls.py b/apps/stats/urls.py deleted file mode 100755 index 3b624099b..000000000 --- a/apps/stats/urls.py +++ /dev/null @@ -1,11 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. -# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. -# -from django.conf.urls.defaults import * - - -urlpatterns = patterns('stats.views', - url(r'^$', 'stats_page', name='stats'), -) - diff --git a/apps/stats/utils.py b/apps/stats/utils.py index 7ee4cfdbe..1218e00fe 100644 --- a/apps/stats/utils.py +++ b/apps/stats/utils.py @@ -1,4 +1,7 @@ - +# -*- coding: utf-8 -*- +# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. +# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# from django.contrib.sites.models import Site from piwik.django.models import PiwikSite from django.conf import settings diff --git a/apps/stats/views.py b/apps/stats/views.py deleted file mode 100644 index b4fd44bed..000000000 --- a/apps/stats/views.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. -# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. -# - -from django.db.models import Count -from django.shortcuts import render_to_response -from django.template import RequestContext - -from catalogue.models import Book, BookMedia - - -def stats_page(request): - media = BookMedia.objects.count() - media_types = BookMedia.objects.values('type').\ - annotate(count=Count('type')).\ - order_by('type') - for mt in media_types: - mt['size'] = sum(b.file.size for b in BookMedia.objects.filter(type=mt['type'])) - mt['deprecated'] = BookMedia.objects.filter( - type=mt['type'], source_sha1=None).count() if mt['type'] in ('mp3', 'ogg') else '-' - - return render_to_response('stats/main.html', - locals(), context_instance=RequestContext(request)) diff --git a/wolnelektury/settings.py b/wolnelektury/settings.py index 527b70277..45dfe3709 100644 --- a/wolnelektury/settings.py +++ b/wolnelektury/settings.py @@ -147,6 +147,7 @@ INSTALLED_APPS = [ 'newtagging', 'opds', 'pdcounter', + 'reporting', 'sponsors', 'stats', 'suggest', diff --git a/wolnelektury/static/css/master.css b/wolnelektury/static/css/master.css index f4a93e73b..e788e5562 100644 --- a/wolnelektury/static/css/master.css +++ b/wolnelektury/static/css/master.css @@ -1185,3 +1185,9 @@ div.shown-tags p, div.all-tags p { #footnotes .pagination { margin-top: 1em; } + + +/* report */ +.stats td { + vertical-align: top; +} \ No newline at end of file diff --git a/wolnelektury/urls.py b/wolnelektury/urls.py index f379c9ca1..dd3c92cb6 100644 --- a/wolnelektury/urls.py +++ b/wolnelektury/urls.py @@ -19,7 +19,7 @@ urlpatterns = patterns('', url(r'^sugestia/', include('suggest.urls')), url(r'^lesmianator/', include('lesmianator.urls')), url(r'^przypisy/', include('dictionary.urls')), - url(r'^statystyka/', include('stats.urls')), + url(r'^raporty/', include('reporting.urls')), # Static pages url(r'^mozesz-nam-pomoc/$', 'infopages.views.infopage', {'slug': 'help_us'}, name='help_us'),