From c30cd74f1f4dcf0bfbabb0e5a739bcdf236b4946 Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Mon, 20 Dec 2021 12:52:55 +0100 Subject: [PATCH 1/1] Add depot for building packages. --- requirements/requirements.txt | 2 +- src/depot/__init__.py | 0 src/depot/admin.py | 10 ++ src/depot/apps.py | 5 + src/depot/migrations/0001_initial.py | 29 +++++ src/depot/migrations/__init__.py | 0 src/depot/models.py | 111 ++++++++++++++++++ src/depot/tests.py | 3 + src/depot/views.py | 3 + src/documents/locale/pl/LC_MESSAGES/django.mo | Bin 7892 -> 8140 bytes src/documents/locale/pl/LC_MESSAGES/django.po | 47 ++++++-- src/documents/models/book.py | 17 ++- .../templates/documents/book_detail.html | 30 +++++ src/documents/views.py | 12 +- src/redakcja/settings/__init__.py | 1 + 15 files changed, 250 insertions(+), 20 deletions(-) create mode 100644 src/depot/__init__.py create mode 100644 src/depot/admin.py create mode 100644 src/depot/apps.py create mode 100644 src/depot/migrations/0001_initial.py create mode 100644 src/depot/migrations/__init__.py create mode 100644 src/depot/models.py create mode 100644 src/depot/tests.py create mode 100644 src/depot/views.py diff --git a/requirements/requirements.txt b/requirements/requirements.txt index db084ea2..56201a62 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -10,7 +10,7 @@ python-slugify python-docx==0.8.10 Wikidata==0.6.1 -librarian==2.2 +librarian==2.3.1 ## Django Django==3.1.13 diff --git a/src/depot/__init__.py b/src/depot/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/depot/admin.py b/src/depot/admin.py new file mode 100644 index 00000000..35f721bd --- /dev/null +++ b/src/depot/admin.py @@ -0,0 +1,10 @@ +from django.contrib import admin +from . import models + + +@admin.register(models.Package) +class PackageAdmin(admin.ModelAdmin): + filter_horizontal = ['books'] + pass + +# Register your models here. diff --git a/src/depot/apps.py b/src/depot/apps.py new file mode 100644 index 00000000..1e16d99e --- /dev/null +++ b/src/depot/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class DepotConfig(AppConfig): + name = 'depot' diff --git a/src/depot/migrations/0001_initial.py b/src/depot/migrations/0001_initial.py new file mode 100644 index 00000000..9fbf24e3 --- /dev/null +++ b/src/depot/migrations/0001_initial.py @@ -0,0 +1,29 @@ +# Generated by Django 3.1.13 on 2021-12-17 15:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('documents', '0006_auto_20210706_0130'), + ] + + operations = [ + migrations.CreateModel( + name='Package', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('placed_at', models.DateTimeField(blank=True, null=True)), + ('finished_at', models.DateTimeField(blank=True, null=True)), + ('definition_json', models.TextField(blank=True)), + ('status_json', models.TextField(blank=True)), + ('logo', models.FileField(blank=True, upload_to='depot/logo')), + ('file', models.FileField(blank=True, upload_to='depot/package/')), + ('books', models.ManyToManyField(to='documents.Book')), + ], + ), + ] diff --git a/src/depot/migrations/__init__.py b/src/depot/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/depot/models.py b/src/depot/models.py new file mode 100644 index 00000000..12bd9c2a --- /dev/null +++ b/src/depot/models.py @@ -0,0 +1,111 @@ +import json +import os +import tempfile +import zipfile +from datetime import datetime +from django.db import models +from librarian.cover import make_cover +from librarian.builders import EpubBuilder, MobiBuilder + + +class Package(models.Model): + created_at = models.DateTimeField(auto_now_add=True) + placed_at = models.DateTimeField(null=True, blank=True) + finished_at = models.DateTimeField(null=True, blank=True) + definition_json = models.TextField(blank=True) + books = models.ManyToManyField('documents.Book') + status_json = models.TextField(blank=True) + logo = models.FileField(blank=True, upload_to='depot/logo') + file = models.FileField(blank=True, upload_to='depot/package/') + + def save(self, *args, **kwargs): + try: + self.set_status(self.get_status()) + except: + pass + + try: + self.set_definition(self.get_definition()) + except: + pass + + super().save(*args, **kwargs) + + def get_status(self): + return json.loads(self.status_json) + + def set_status(self, status): + self.status_json = json.dumps(status, indent=4) + + def get_definition(self): + return json.loads(self.definition_json) + + def set_definition(self, definition): + self.definition_json = json.dumps(definition, indent=4) + + def build(self): + f = tempfile.NamedTemporaryFile(prefix='depot-', suffix='.zip', mode='wb', delete=False) + with zipfile.ZipFile(f, 'w') as z: + for book in self.books.all(): + self.build_for(book, z) + f.close() + with open(f.name, 'rb') as ff: + self.file.save('package-{}.zip'.format(datetime.now().isoformat(timespec='seconds')), ff) + os.unlink(f.name) + + def build_for(self, book, z): + wldoc2 = book.wldocument(librarian2=True) + slug = wldoc2.meta.url.slug + for item in self.get_definition(): + wldoc = book.wldocument() + wldoc2 = book.wldocument(librarian2=True) + base_url = 'file://' + book.gallery_path() + '/' + + ext = item['type'] + + if item['type'] == 'cover': + kwargs = {} + if self.logo: + kwargs['cover_logo'] = self.logo.path + for k in 'format', 'width', 'height', 'cover_class': + if k in item: + kwargs[k] = item[k] + cover = make_cover(wldoc.book_info, **kwargs) + output = cover.output_file() + ext = cover.ext() + + elif item['type'] == 'pdf': + cover_kwargs = {} + if 'cover_class' in item: + cover_kwargs['cover_class'] = item['cover_class'] + if self.logo: + cover_kwargs['cover_logo'] = self.logo.path + cover = lambda *args, **kwargs: make_cover(*args, **kwargs, **cover_kwargs) + output = wldoc.as_pdf(cover=cover, base_url=base_url) + + elif item['type'] == 'epub': + cover_kwargs = {} + if 'cover_class' in item: + cover_kwargs['cover_class'] = item['cover_class'] + if self.logo: + cover_kwargs['cover_logo'] = self.logo.path + cover = lambda *args, **kwargs: make_cover(*args, **kwargs, **cover_kwargs) + + output = EpubBuilder( + cover=cover, + base_url=base_url, +# fundraising=[] + ).build(wldoc2) + + elif item['type'] == 'mobi': + output = MobiBuilder( + cover=cover, + base_url=base_url, + ).build(wldoc2) + + fname = f'{slug}/{slug}.{ext}' + + z.writestr( + fname, + output.get_bytes() + ) diff --git a/src/depot/tests.py b/src/depot/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/src/depot/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/src/depot/views.py b/src/depot/views.py new file mode 100644 index 00000000..91ea44a2 --- /dev/null +++ b/src/depot/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/src/documents/locale/pl/LC_MESSAGES/django.mo b/src/documents/locale/pl/LC_MESSAGES/django.mo index dde8ac158d040540d6924e8ea53596d130a77c76..453940f330a06bdf1382e8b125b5624fb7f77c91 100644 GIT binary patch delta 3101 zcmZYAd2EzL7{~EhXsPAs1#EQLBiF7>F9L03P4pb~Yh8>9e1AX6K#bdFS1J zJpW9o^HoaBK||>w(uwjIW5T%Q20kcrM;J2}i*P(H$C2nmb=i(X(ZP7^vh_!hKl7+< ze-6h`KWf|GLZ)&|pY70(qv-G%s^bN`5wBnZCNb+|%*0_>hU&lA)>mON^;+Z>vlZ39 z1Nk#Y_+UcwGHRjWNV{Xq+guE%;eFJ=pI`=_!?}0`3$TFMwBt=klW9TS*MVB#L#X?o z#5g>J%GirI8c(1W_7ST8*O)4%qr()XI;d7WgJAg?*@n^`laG9+jC( z$X?9vsGY~KIyYvZ?k_=&UxC_abrR2C4eM#pvuj2-?#4xU#MUpOb~u7gnqUqp)#b=$ zOa;c`8q~z=Fc;e}6<ey59+>ssQZpMT)74;|zkv$rxoC{4{g_>wBY5=coZ?Nr6r~!kv-LdW6 zs5A61e&b?CIFou8+na`A)Q-PE-TxD6o@>aXbIcI#)(Vr6drcau<4jb_=b>60iBf;RDB|jXMU5z1=%xYn2S}Y1$3e^u^*MH zgQ$s*q88F?zkdfcQ6EmlOIV6yIEgytm8kJ*Q4459jn|ID^#1STLLCnvf94P$I<>v1 zg?)%*%Ur-ryowqyZNgycr=jX4$X8}DDsy$npKr{5o za1-vwS$NEL96){lqpe@F#xa^saSAH+nYa+ku>kktt#|^}?;1wV00$*V_v!J3EYQ(!7M~f6BHGpmuoC)_+Ew?yIQGx!9G)8H>tHCTg4_+g|C| ziyG988&Rp-V(Yt*^JKbFH$ID+u-ATn9QFF1LOsgUs7Ld;{eA$||9jN9m+bfdpcdxD zaiX-tB-Cq`ib_!yYQ?irr+yhKGqtD*w^`d!6F9cM$JY0v9^GT8!}+{ze+4zp8_3Uv z6Pb*QPtB8$6B@r!pL+&QJ2|WVsGFtS5w3N{4j}~QD z^P5SwWggZLLv1}D^~`T3I*3IC=PFVb@>yiBD^QO>Wf{>#DD^!=Ju!#aLa1~SdVG4+ zO9;(x))CW)oL5LJX;t#V>IF`vjKR3;B%*5FEFwXKVLue;V| z<6zw}_i+&<>WCf0VuHhIZX?`832`TJ7r|>B`F)syDr*V-ka$#~OeE$K`tgdE1}>%$ z#l&tRKHN_k7)`jW=K~vWdy>w@_`a%uvjedWq*&nJ8W{00jUg1h_^#?WtP2}@` zJr4YkQkCS%^UTWg6uLd0;)255d{21U=pC-11vB!)XU6Ot7MZy%7;0pmxH#h(S4Qx?PJgS{4Dz2I=3f~`Ds2D& delta 2853 zcmZA3e@xVM9LMp`5kE9g;SjU{y%bS6f#c)=1-vpv$VgI9@&^zFnL?DIpk${;tRH66 z<`%8SDA$U9%yyZp^+&F?9G1muCaYnKT+#Y9SF5cVJzsa9+g#t>`ySuV=ll77e!Rb* z@4+YjFUtdC&gf%?V?U9uh&JX8IfXBd&#yHm6-RJ3{)h?ah&M(}rsFj@3#Ve5EiXVm zCfnBga3*EHt*=Ff3YaF_parMVpaa!#AEw|@oQ|*Hd_0A*_%*8iceeaHCQ<$i`IxE9 zQteYv6JE*}1DXodMAwAs1IE;G!lFzws^fOdz)mc|qqqdmqEpGWmSZvAZr&tK2%AKP#m%c-QChc4_wt?XG;#~-3nJc3N#T)-Io z0X5(+=*Cz&y&k=&iB+Q}z5%tM&8SRtB$0o0&`X6negc(=BdGd*+i(E2;|mLv>V!8eoOB%3iNQO{@X6qE>so1C`R< zs7G-SNrE|qO8E=O9Lx}E;P+AEoJNfk_}E@JXD@t#>TuN7U&3*RF+X8G_2*dDLi`uC z!Yr1j_NAzfSEC+HHEKdzP$_RkwcCqn*oQ1UU|!@z4NoB3z`Td*co_8>UbJ38b(F@A z&BT0ECTdXwZnfn$)Lz(y%-I}9Hx8mE@F{BLqnO0_=37oQ@MY9QqIg%-AOST{GG<{3 z`mqu9nhu~k970Xt9aO46Lf!urs@)iBLKCP>8AU-8%fJ-9{~k`}Vg;(hR%9+_r!7B@ ze9SSvl)5+ZIvhrI_&xG5lYDt_HWxLaDpV%xQ4?uL_1kUhpT@u=R2;Vzi|9?8tQ>X2 zDpW>lkS1mmYK5Js751V2W;}&zKZIJ@Nz^7hgL7~cz4$k(otstYdQlqrSH)^7v|HDq zQr?Ia*o#Z>ESBLUs$ChQ>e*Le=ss&B(!{i3F+PTD8}k9G{dv@aE+TU=6Y1n%H$=13 zbisvMS&l6ipl&QhWnd-h#s*uz$9fR8`=3N*=!h-9ggkchD(b$|sQ$mP*T(~#=)Igk zJ+og?kKl@J5JUcSLp-WuC+d18YGR8~6ZfHhNBpRaRAcCop*D9XDigh^@t(5=1~}0G z$8E)HwtNEhj8368!0aW|-bW~Ng~WZtJ;XyqE1~}f9wx}LSx@MAAdIQU{lq*X zSMUGT#~upZwqhUZ@Au8bQbId?0dXg>khp`miC9DESVY_qI%7X!fjy6maIYf<61cZ)dR-xBTM;mcBQ9-zfb;K=%QooDP50{SWFrja4{8Y`Z z$X4prX(x6PO@z|lMbr{HHWOP2Jx3imgg\n" "Language-Team: Fundacja Nowoczesna Polska \n" @@ -17,7 +17,7 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 " "|| n%100>=20) ? 1 : 2);\n" -"X-Generator: Poedit 2.2.4\n" +"X-Generator: Poedit 3.0\n" #: documents/forms.py:41 msgid "Text file must be UTF-8 encoded." @@ -73,44 +73,45 @@ msgstr "publiczna" msgid "scan gallery name" msgstr "nazwa galerii skanów" -#: documents/models/book.py:34 +#: documents/models/book.py:33 msgid "parent" msgstr "rodzic" -#: documents/models/book.py:35 +#: documents/models/book.py:34 msgid "parent number" msgstr "numeracja rodzica" -#: documents/models/book.py:53 documents/models/chunk.py:19 +#: documents/models/book.py:60 documents/models/chunk.py:19 #: documents/models/publish_log.py:15 +#: documents/templates/documents/book_detail.html:145 msgid "book" msgstr "książka" -#: documents/models/book.py:54 documents/views.py:619 +#: documents/models/book.py:61 documents/views.py:627 msgid "books" msgstr "książki" -#: documents/models/book.py:263 +#: documents/models/book.py:271 msgid "No chunks in the book." msgstr "Książka nie ma części." -#: documents/models/book.py:267 +#: documents/models/book.py:275 msgid "Not all chunks have publishable revisions." msgstr "Niektóre części nie są gotowe do publikacji." -#: documents/models/book.py:274 documents/models/image.py:83 +#: documents/models/book.py:282 documents/models/image.py:83 msgid "Invalid XML" msgstr "Nieprawidłowy XML" -#: documents/models/book.py:276 documents/models/image.py:85 +#: documents/models/book.py:284 documents/models/image.py:85 msgid "No Dublin Core found." msgstr "Brak sekcji Dublin Core." -#: documents/models/book.py:278 documents/models/image.py:87 +#: documents/models/book.py:286 documents/models/image.py:87 msgid "Invalid Dublin Core" msgstr "Nieprawidłowy Dublin Core" -#: documents/models/book.py:281 documents/models/image.py:91 +#: documents/models/book.py:289 documents/models/image.py:91 msgid "rdf:about is not" msgstr "rdf:about jest różny od" @@ -294,6 +295,26 @@ msgstr "Zaloguj się, aby opublikować." msgid "This book can't be published yet, because:" msgstr "Ta książka nie może jeszcze zostać opublikowana. Powód:" +#: documents/templates/documents/book_detail.html:138 +msgid "Statistics" +msgstr "Statystyki" + +#: documents/templates/documents/book_detail.html:147 +msgid "characters" +msgstr "znaki" + +#: documents/templates/documents/book_detail.html:148 +msgid "words" +msgstr "słowa" + +#: documents/templates/documents/book_detail.html:149 +msgid "characters (with footnotes)" +msgstr "znaki (z przypisami)" + +#: documents/templates/documents/book_detail.html:150 +msgid "words (with footnotes)" +msgstr "słowa (z przypisami)" + #: documents/templates/documents/book_edit.html:5 msgid "Edit book" msgstr "Edytuj książkę" @@ -603,7 +624,7 @@ msgstr "Dokument o tym slugu już istnieje w repozytorium." msgid "File should be UTF-8 encoded." msgstr "Plik powinien mieć kodowanie UTF-8." -#: documents/views.py:621 +#: documents/views.py:629 msgid "scan gallery" msgstr "galeria skanów" diff --git a/src/documents/models/book.py b/src/documents/models/book.py index 4255530a..0e696e40 100644 --- a/src/documents/models/book.py +++ b/src/documents/models/book.py @@ -17,6 +17,7 @@ from documents.models import BookPublishRecord, ChunkPublishRecord, Project from documents.signals import post_publish from documents.xml_tools import compile_text, split_xml from cover.models import Image +from io import BytesIO import os import shutil import re @@ -407,13 +408,21 @@ class Book(models.Model): return compile_text(change.materialize() for change in changes) def wldocument(self, publishable=True, changes=None, - parse_dublincore=True, strict=False): + parse_dublincore=True, strict=False, librarian2=False): from documents.ebook_utils import RedakcjaDocProvider from librarian.parser import WLDocument - + from librarian.document import WLDocument as WLDocument2 + + provider = RedakcjaDocProvider(publishable=publishable), + xml = self.materialize(publishable=publishable, changes=changes).encode('utf-8') + + if librarian2: + return WLDocument2( + BytesIO(xml), + provider=provider) return WLDocument.from_bytes( - self.materialize(publishable=publishable, changes=changes).encode('utf-8'), - provider=RedakcjaDocProvider(publishable=publishable), + xml, + provider=provider, parse_dublincore=parse_dublincore, strict=strict) diff --git a/src/documents/templates/documents/book_detail.html b/src/documents/templates/documents/book_detail.html index be17ec14..85da536e 100644 --- a/src/documents/templates/documents/book_detail.html +++ b/src/documents/templates/documents/book_detail.html @@ -129,4 +129,34 @@ Okładka w rozmiarze + + + +{% if doc %} +
+
+

{% trans "Statistics" %}

+
+
+ + + + + + + + + + + + {% with stats=book.wldocument.get_statistics %} + {% include 'documents/book_stats.html' with book=book stats=stats depth=0 %} + {% endwith %} + +
+ {% trans "book" %} + {% trans "characters" %}{% trans "words" %}{% trans "characters (with footnotes)" %}{% trans "words (with footnotes)" %}
+
+
+{% endif %} {% endblock content %} diff --git a/src/documents/views.py b/src/documents/views.py index bfcd0139..38e69c1b 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -288,7 +288,9 @@ def book_epub(request, slug): # TODO: move to celery doc = book.wldocument() # TODO: error handling - epub = doc.as_epub(base_url=request.build_absolute_uri(book.gallery_path())).get_bytes() + + #### Problemas: images in children. + epub = doc.as_epub(base_url='file://' + book.gallery_path() + '/').get_bytes() response = HttpResponse(content_type='application/epub+zip') response['Content-Disposition'] = 'attachment; filename=%s' % book.slug + '.epub' response.write(epub) @@ -304,7 +306,7 @@ def book_mobi(request, slug): # TODO: move to celery doc = book.wldocument() # TODO: error handling - mobi = doc.as_mobi(base_url=request.build_absolute_uri(book.gallery_path())).get_bytes() + mobi = doc.as_mobi(base_url='file://' + book.gallery_path() + '/').get_bytes() response = HttpResponse(content_type='application/x-mobipocket-ebook') response['Content-Disposition'] = 'attachment; filename=%s' % book.slug + '.mobi' response.write(mobi) @@ -345,8 +347,14 @@ def book(request, slug): publish_error = book.publishable_error() publishable = publish_error is None + try: + doc = book.wldocument() + except: + doc = None + return render(request, "documents/book_detail.html", { "book": book, + "doc": doc, "publishable": publishable, "publishable_error": publish_error, "form": form, diff --git a/src/redakcja/settings/__init__.py b/src/redakcja/settings/__init__.py index 4259b319..46b80da1 100644 --- a/src/redakcja/settings/__init__.py +++ b/src/redakcja/settings/__init__.py @@ -93,6 +93,7 @@ INSTALLED_APPS = ( 'redakcja.api', 'catalogue', + 'depot', 'documents', 'cover', 'dvcs', -- 2.20.1