From 51d914bc0007135058a61623a2d057d2a7626a28 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Rekucki?= Date: Tue, 29 Sep 2009 16:13:26 +0200 Subject: [PATCH 1/1] The actual commit. --- apps/api/forms.py | 4 +- apps/api/handlers/library_handlers.py | 113 ++++++++++++++++++----- apps/api/models.py | 3 + apps/api/resources.py | 2 + apps/api/tests/__init__.py | 2 +- apps/api/urls.py | 10 +- apps/api/utils.py | 17 +++- apps/explorer/admin.py | 3 +- apps/explorer/models.py | 20 ++-- apps/explorer/views.py | 2 +- lib/wlrepo/__init__.py | 15 ++- lib/wlrepo/mercurial_backend/__init__.py | 11 ++- lib/wlrepo/mercurial_backend/document.py | 24 +++-- lib/wlrepo/mercurial_backend/library.py | 22 ++--- project/static/js/models.js | 45 ++++++++- project/templates/explorer/editor.html | 21 +++++ 16 files changed, 249 insertions(+), 65 deletions(-) diff --git a/apps/api/forms.py b/apps/api/forms.py index af1b6b5a..d55e6291 100644 --- a/apps/api/forms.py +++ b/apps/api/forms.py @@ -16,14 +16,14 @@ class MergeRequestForm(forms.Form): target_revision = forms.RegexField('[0-9a-f]{40}') # any additional comments that user wants to add to the change - comment = forms.CharField(required=False) + message = forms.CharField(required=False) class DocumentUploadForm(forms.Form): ocr_file = forms.FileField(label='Source OCR file', required=False) ocr_data = forms.CharField(widget=forms.HiddenInput(), required=False) bookname = forms.RegexField(regex=r'[0-9\.\w_-]+', \ - label='Publication name', help_text='Example: slowacki-beniowski') + label='Publication name', help_text='Example: słowacki__beniowski__pieśń_1') generate_dc = forms.BooleanField(required=False, \ initial=True, label=u"Generate DublinCore template") diff --git a/apps/api/handlers/library_handlers.py b/apps/api/handlers/library_handlers.py index d2c6ec32..2170b44b 100644 --- a/apps/api/handlers/library_handlers.py +++ b/apps/api/handlers/library_handlers.py @@ -1,3 +1,4 @@ +import os.path # -*- encoding: utf-8 -*- __author__= "Łukasz Rekucki" @@ -16,15 +17,18 @@ import librarian import librarian.html from librarian import dcparser -from wlrepo import RevisionNotFound, LibraryException, DocumentAlreadyExists -from explorer.models import PullRequest +from wlrepo import * +from explorer.models import PullRequest, GalleryForDocument # internal imports import api.forms as forms import api.response as response -from api.utils import validate_form, hglibrary +from api.utils import validate_form, hglibrary, natural_order from api.models import PartCache +# +import settings + # # Document List Handlers # @@ -76,8 +80,15 @@ class LibraryHandler(BaseHandler): # not top-level anymore document_tree.pop(part) parent['parts'].append(child) + + # sort the right way + + + for doc in documents.itervalues(): + doc['parts'].sort(key=natural_order(lambda d: d['name'])) - return {'documents': sorted(document_tree.values()) } + return {'documents': sorted(document_tree.itervalues(), + key=natural_order(lambda d: d['name']) ) } @validate_form(forms.DocumentUploadForm, 'POST') @hglibrary @@ -163,8 +174,9 @@ class DocumentHandler(BaseHandler): try: doc = lib.document(docid) udoc = doc.take(request.user.username) - except RevisionNotFound: - return request.EnityNotFound().django_response() + except RevisionNotFound, e: + return response.EntityNotFound().django_response({ + 'exception': type(e), 'message': e.message}) # is_shared = udoc.ancestorof(doc) # is_uptodate = is_shared or shared.ancestorof(document) @@ -174,9 +186,12 @@ class DocumentHandler(BaseHandler): 'html_url': reverse('dochtml_view', args=[udoc.id,udoc.revision]), 'text_url': reverse('doctext_view', args=[udoc.id,udoc.revision]), 'dc_url': reverse('docdc_view', args=[udoc.id,udoc.revision]), - #'gallery_url': reverse('docdc_view', args=[udoc.id,udoc.revision]), + 'gallery_url': reverse('docgallery_view', args=[udoc.id]), + 'merge_url': reverse('docmerge_view', args=[udoc.id]), 'user_revision': udoc.revision, - 'public_revision': doc.revision, + 'user_timestamp': udoc.revision.timestamp, + 'public_revision': doc.revision, + 'public_timestamp': doc.revision.timestamp, } return result @@ -189,7 +204,7 @@ class DocumentHandler(BaseHandler): # # class DocumentHTMLHandler(BaseHandler): - allowed_methods = ('GET', 'PUT') + allowed_methods = ('GET') @hglibrary def read(self, request, docid, revision, lib): @@ -200,9 +215,51 @@ class DocumentHTMLHandler(BaseHandler): else: document = lib.document_for_rev(revision) + if document.id != docid: + return response.BadRequest().django_response({'reason': 'name-mismatch', + 'message': 'Provided revision refers, to document "%s", but provided "%s"' % (document.id, docid) }) + return librarian.html.transform(document.data('xml'), is_file=False) - except RevisionNotFound: - return response.EntityNotFound().django_response() + except (EntryNotFound, RevisionNotFound), e: + return response.EntityNotFound().django_response({ + 'exception': type(e), 'message': e.message}) + + +# +# Image Gallery +# +from django.core.files.storage import FileSystemStorage + +class DocumentGalleryHandler(BaseHandler): + allowed_methods = ('GET') + + def read(self, request, docid): + """Read meta-data about scans for gallery of this document.""" + galleries = [] + + for assoc in GalleryForDocument.objects.filter(document=docid): + dirpath = os.path.join(settings.MEDIA_ROOT, assoc.subpath) + + if not os.path.isdir(dirpath): + print u"[WARNING]: missing gallery %s" % dirpath + continue + + gallery = {'name': assoc.name, 'pages': []} + + for file in sorted(os.listdir(dirpath), key=natural_order()): + print file + name, ext = os.path.splitext(os.path.basename(file)) + + if ext.lower() not in ['.png', '.jpeg', '.jpg']: + print "Ignoring:", name, ext + continue + + url = settings.MEDIA_URL + assoc.subpath + u'/' + file.decode('utf-8'); + gallery['pages'].append(url) + + galleries.append(gallery) + + return galleries # # Document Text View @@ -223,11 +280,16 @@ class DocumentTextHandler(BaseHandler): document = lib.document(docid) else: document = lib.document_for_rev(revision) + + if document.id != docid: + return response.BadRequest().django_response({'reason': 'name-mismatch', + 'message': 'Provided revision is not valid for this document'}) # TODO: some finer-grained access control return document.data('xml') - except RevisionNotFound: - return response.EntityNotFound().django_response() + except (EntryNotFound, RevisionNotFound), e: + return response.EntityNotFound().django_response({ + 'exception': type(e), 'message': e.message}) @hglibrary def update(self, request, docid, revision, lib): @@ -289,7 +351,8 @@ class DocumentTextHandler(BaseHandler): "document": ndoc.id, "subview": "xml", "previous_revision": current.revision, - "updated_revision": ndoc.revision, + "revision": ndoc.revision, + 'timestamp': ndoc.revision.timestamp, "url": reverse("doctext_view", args=[ndoc.id, ndoc.revision]) }) except Exception, e: @@ -316,11 +379,17 @@ class DocumentDublinCoreHandler(BaseHandler): doc = lib.document(docid) else: doc = lib.document_for_rev(revision) + + + if document.id != docid: + return response.BadRequest().django_response({'reason': 'name-mismatch', + 'message': 'Provided revision is not valid for this document'}) bookinfo = dcparser.BookInfo.from_string(doc.data('xml')) return bookinfo.serialize() - except RevisionNotFound: - return response.EntityNotFound().django_response() + except (EntryNotFound, RevisionNotFound), e: + return response.EntityNotFound().django_response({ + 'exception': type(e), 'message': e.message}) @hglibrary def update(self, request, docid, revision, lib): @@ -354,7 +423,8 @@ class DocumentDublinCoreHandler(BaseHandler): "document": ndoc.id, "subview": "dc", "previous_revision": current.revision, - "updated_revision": ndoc.revision, + "revision": ndoc.revision, + 'timestamp': ndoc.revision.timestamp, "url": reverse("docdc_view", args=[ndoc.id, ndoc.revision]) } except Exception, e: @@ -412,7 +482,7 @@ class MergeHandler(BaseHandler): document=docid, source_revision = str(udoc.revision), status="N", - comment = form.cleaned_data['comment'] or '$AUTO$ Document shared.' + comment = form.cleaned_data['message'] or '$AUTO$ Document shared.' ) prq.save() @@ -426,10 +496,10 @@ class MergeHandler(BaseHandler): success, changed = udoc.update(request.user.username) if form.cleaned_data['type'] == 'share': - success, changed = udoc.share(form.cleaned_data['comment']) + success, changed = udoc.share(form.cleaned_data['message']) if not success: - return response.EntityConflict().django_response() + return response.EntityConflict().django_response({}) if not changed: return response.SuccessNoContent().django_response() @@ -440,5 +510,6 @@ class MergeHandler(BaseHandler): "name": udoc.id, "parent_user_resivion": udoc.revision, "parent_revision": doc.revision, - "revision": udoc.revision, + "revision": ndoc.revision, + 'timestamp': ndoc.revision.timestamp, }) diff --git a/apps/api/models.py b/apps/api/models.py index c08f38db..ac694880 100644 --- a/apps/api/models.py +++ b/apps/api/models.py @@ -20,6 +20,9 @@ class PartCache(models.Model): for part in created: me.objects.create(user_id=userid, document_id=docid, part_id=part) + + + diff --git a/apps/api/resources.py b/apps/api/resources.py index 51127e48..103933d5 100644 --- a/apps/api/resources.py +++ b/apps/api/resources.py @@ -19,6 +19,7 @@ document_resource = Resource(dh.DocumentHandler, **authdata) document_text_resource = Resource(dh.DocumentTextHandler, **authdata) document_html_resource = Resource(dh.DocumentHTMLHandler, **authdata) document_dc_resource = Resource(dh.DocumentDublinCoreHandler, **authdata) +document_gallery = Resource(dh.DocumentGalleryHandler, **authdata) document_merge = Resource(dh.MergeHandler, **authdata) import api.handlers.manage_handlers as mh @@ -41,6 +42,7 @@ __all__ = [ 'document_text_resource', 'document_html_resource', 'document_dc_resource', + 'document_gallery', 'document_merge', 'toolbar_buttons', 'scriptlets', diff --git a/apps/api/tests/__init__.py b/apps/api/tests/__init__.py index 751a085c..2044fce7 100644 --- a/apps/api/tests/__init__.py +++ b/apps/api/tests/__init__.py @@ -145,7 +145,7 @@ class SimpleTest(TestCase): @temprepo('simple') - def test_document_text_update(self): + def test_document_text_save(self): self.assertTrue(self.client.login(username='admin', password='admin')) TEXT = u"Ala ma kota i psa" diff --git a/apps/api/urls.py b/apps/api/urls.py index 6a95cf0c..18fc02c4 100644 --- a/apps/api/urls.py +++ b/apps/api/urls.py @@ -41,15 +41,22 @@ urlpatterns = patterns('', url(urlpath(r'documents', DOC, format=False), document_resource, {'emitter_format': 'json'}, name="document_view"), - + + url(urlpath(r'documents', DOC, 'gallery', format=False), + document_gallery, {'emitter_format': 'json'}, + name="docgallery_view"), + + # XML url(urlpath(r'documents', DOC, 'text', REVISION, format=False), document_text_resource, {'emitter_format': 'rawxml'}, name="doctext_view"), + # HTML url(urlpath(r'documents', DOC, 'html', REVISION, format=False), document_html_resource, {'emitter_format': 'rawhtml'}, name="dochtml_view"), + # DC url(urlpath(r'documents', DOC, 'dc', REVISION), document_dc_resource, name="docdc_view_withformat"), @@ -58,6 +65,7 @@ urlpatterns = patterns('', document_dc_resource, {'emitter_format': 'json'}, name="docdc_view"), + # MERGE url(urlpath(r'documents', DOC, 'revision', format=False), document_merge, {'emitter_format': 'json'}, name="docmerge_view") diff --git a/apps/api/utils.py b/apps/api/utils.py index 93e54be0..9b45a320 100644 --- a/apps/api/utils.py +++ b/apps/api/utils.py @@ -11,7 +11,7 @@ from piston.utils import rc import api.response -from wlrepo import MercurialLibrary +import wlrepo import settings class TextEmitter(Emitter): @@ -50,8 +50,19 @@ def validate_form(formclass, source='GET'): def hglibrary(func): @wraps(func) def decorated(self, *args, **kwargs): - l = MercurialLibrary(settings.REPOSITORY_PATH) + l = wlrepo.open_library(settings.REPOSITORY_PATH, 'hg') kwargs['lib'] = l return func(self, *args, **kwargs) - return decorated + return decorated + + + +import re +NAT_EXPR = re.compile(r'(\d+)', re.LOCALE | re.UNICODE) +def natural_order(get_key=lambda x: x): + def getter(key): + key = [int(x) if n%2 else x for (n,x) in enumerate(NAT_EXPR.split(get_key(key))) ] + return key + + return getter diff --git a/apps/explorer/admin.py b/apps/explorer/admin.py index b496893f..034eb3bb 100644 --- a/apps/explorer/admin.py +++ b/apps/explorer/admin.py @@ -5,4 +5,5 @@ import explorer.models admin.site.register(explorer.models.EditorSettings) admin.site.register(explorer.models.EditorPanel) -admin.site.register(explorer.models.PullRequest) \ No newline at end of file +admin.site.register(explorer.models.PullRequest) +admin.site.register(explorer.models.GalleryForDocument) \ No newline at end of file diff --git a/apps/explorer/models.py b/apps/explorer/models.py index 48d0247d..5181e167 100644 --- a/apps/explorer/models.py +++ b/apps/explorer/models.py @@ -20,7 +20,8 @@ class EditorSettings(models.Model): Przykład: { 'panels': [ - {'name': 'htmleditor', 'ratio': 0.5}, + {'name': 'htmleditor', + 'ratio': 0.5}, {'name': 'gallery', 'ratio': 0.5} ], 'recentFiles': [ @@ -87,16 +88,15 @@ class PullRequest(models.Model): # revision number in which the changes were merged (if any) merged_rev = models.CharField(max_length=40, blank=True, null=True) - def __unicode__(self): return unicode(self.comitter) + u':' + self.document - -def get_image_folders(): - return sorted(fn for fn in os.listdir(os.path.join(settings.MEDIA_ROOT, settings.IMAGE_DIR)) if not fn.startswith('.')) - -def get_images_from_folder(folder): - return sorted(settings.MEDIA_URL + settings.IMAGE_DIR + u'/' + folder + u'/' + fn.decode('utf-8') for fn - in os.listdir(os.path.join(settings.MEDIA_ROOT, settings.IMAGE_DIR, folder)) - if not fn.decode('utf-8').startswith('.')) +# Yes, this is intentionally unnormalized ! +class GalleryForDocument(models.Model): + name = models.CharField(max_length=100) + + # directory containing scans under MEDIA_ROOT/ + subpath = models.CharField(max_length=255) + # document associated with the gallery + document = models.CharField(max_length=255) diff --git a/apps/explorer/views.py b/apps/explorer/views.py index 28082e64..23b70592 100644 --- a/apps/explorer/views.py +++ b/apps/explorer/views.py @@ -38,7 +38,7 @@ def file_path(fileid): def with_repo(view): """Open a repository for this view""" def view_with_repo(request, *args, **kwargs): - kwargs['repo'] = wlrepo.MercurialLibrary(settings.REPOSITORY_PATH) + kwargs['repo'] = wlrepo.open_library(settings.REPOSITORY_PATH, 'hg') return view(request, *args, **kwargs) return view_with_repo diff --git a/lib/wlrepo/__init__.py b/lib/wlrepo/__init__.py index 7e878984..ab6f319f 100644 --- a/lib/wlrepo/__init__.py +++ b/lib/wlrepo/__init__.py @@ -108,10 +108,21 @@ class LibraryException(Exception): class RevisionNotFound(LibraryException): def __init__(self, rev): LibraryException.__init__(self, "Revision %r not found." % rev) - pass + +class EntryNotFound(LibraryException): + def __init__(self, rev, entry, guesses=[]): + LibraryException.__init__(self, \ + u"Entry '%s' at revision %r not found. %s" % (entry, rev, \ + (u"Posible values:\n" + u',\n'.join(guesses)) if len(guesses) else u'') ) class DocumentAlreadyExists(LibraryException): pass # import backends to local namespace -from mercurial_backend.library import MercurialLibrary \ No newline at end of file + +def open_library(path, proto, *args, **kwargs): + if proto == 'hg': + import wlrepo.mercurial_backend + return wlrepo.mercurial_backend.MercurialLibrary(path, *args, **kwargs) + + raise NotImplemented() \ No newline at end of file diff --git a/lib/wlrepo/mercurial_backend/__init__.py b/lib/wlrepo/mercurial_backend/__init__.py index 2d0ce828..9a22395e 100644 --- a/lib/wlrepo/mercurial_backend/__init__.py +++ b/lib/wlrepo/mercurial_backend/__init__.py @@ -28,11 +28,11 @@ class MercurialRevision(wlrepo.Revision): @property def document_name(self): - return self._docname + return self._docname.decode('utf-8') @property def user_name(self): - return self._username + return self._username.decode('utf-8') def hgrev(self): return self._changectx.node() @@ -43,9 +43,16 @@ class MercurialRevision(wlrepo.Revision): def hgbranch(self): return self._changectx.branch() + @property + def timestamp(self): + return self._changectx.date()[0] + def __unicode__(self): return u"%s" % self._changectx.hex() + def __str__(self): + return self.__unicode__().encode('utf-8') + def __repr__(self): return "%s" % self._changectx.hex() diff --git a/lib/wlrepo/mercurial_backend/document.py b/lib/wlrepo/mercurial_backend/document.py index 51a2014c..3f94097a 100644 --- a/lib/wlrepo/mercurial_backend/document.py +++ b/lib/wlrepo/mercurial_backend/document.py @@ -5,13 +5,18 @@ __date__ = "$2009-09-25 09:35:06$" __doc__ = "Module documentation." import wlrepo +import mercurial.error class MercurialDocument(wlrepo.Document): def data(self, entry): - path = self._revision._docname + '.' + entry - return self._library._filectx(path, \ - self._revision.hgrev()).data() + path = self._library._sanitize_string(self.id + u'.' + entry) + try: + return self._library._filectx(path, \ + self._revision.hgrev()).data().decode('utf-8') + except mercurial.error.LookupError, e: + fl = [x.decode('utf-8') for x in self._revision._changectx] + raise wlrepo.EntryNotFound(self._revision, path.decode('utf-8'), fl) def quickwrite(self, entry, data, msg, user=None): user = user or self.owner @@ -32,16 +37,16 @@ class MercurialDocument(wlrepo.Document): f.close() l._fileadd(r(entry)) - return self.invoke_and_commit(write, lambda d: (msg, self.owner)) + return self.invoke_and_commit(write, lambda d: (msg, \ + self._library._sanitize_string(self.owner)) ) - def invoke_and_commit(self, ops, - commit_info): + def invoke_and_commit(self, ops, commit_info): lock = self._library.lock() try: self._library._checkout(self._revision.hgrev()) def entry_path(entry): - return self.id + '.' + entry + return self._library._sanitize_string(self.id + u'.' + entry) ops(self._library, entry_path) message, user = commit_info(self) @@ -184,9 +189,12 @@ class MercurialDocument(wlrepo.Document): finally: lock.release() - def __str__(self): + def __unicode__(self): return u"Document(%s:%s)" % (self.name, self.owner) + def __str__(self): + return self.__unicode__().encode('utf-8') + def __eq__(self, other): return (self._revision == other._revision) and (self.name == other.name) diff --git a/lib/wlrepo/mercurial_backend/library.py b/lib/wlrepo/mercurial_backend/library.py index 648ebe90..042fda29 100644 --- a/lib/wlrepo/mercurial_backend/library.py +++ b/lib/wlrepo/mercurial_backend/library.py @@ -10,8 +10,8 @@ from mercurial import ui as hgui from mercurial import error import wlrepo -from wlrepo.mercurial_backend.document import MercurialDocument from wlrepo.mercurial_backend import MercurialRevision +from wlrepo.mercurial_backend.document import MercurialDocument class MergeStatus(object): def __init__(self, mstatus): @@ -75,13 +75,14 @@ class MercurialLibrary(wlrepo.Library): @property def ospath(self): - return self._ospath + return self._ospath.decode('utf-8') def document_for_rev(self, revision): if revision is None: raise ValueError("Revision can't be None.") if not isinstance(revision, MercurialRevision): + revision = self._sanitize_string(unicode(revision)) rev = self.get_revision(revision) else: rev = revision @@ -111,14 +112,11 @@ class MercurialLibrary(wlrepo.Library): return MercurialRevision(self, ctx) - def fulldocid(self, docid, user=None): - docid = self._sanitize_string(docid) - user = self._sanitize_string(user) - - fulldocid = '' + def fulldocid(self, docid, user=None): + fulldocid = u'' if user is not None: - fulldocid += '$user:' + user - fulldocid += '$doc:' + docid + fulldocid += u'$user:' + user + fulldocid += u'$doc:' + docid return fulldocid @@ -130,16 +128,16 @@ class MercurialLibrary(wlrepo.Library): return False def document_create(self, docid): - docid = self._sanitize_string(docid) + # check if it already exists fullid = self.fulldocid(docid) if self.has_revision(fullid): - raise wlrepo.DocumentAlreadyExists("Document %s already exists!" % docid); + raise wlrepo.DocumentAlreadyExists(u"Document %s already exists!" % docid); # doesn't exist - self._create_branch(fullid) + self._create_branch(self._sanitize_string(fullid)) return self.document_for_rev(fullid) # diff --git a/project/static/js/models.js b/project/static/js/models.js index 9542eac5..d0de300e 100644 --- a/project/static/js/models.js +++ b/project/static/js/models.js @@ -172,6 +172,48 @@ Editor.HTMLModel = Editor.Model.extend({ }); +Editor.ImageGalleryModel = Editor.Model.extend({ + _className: 'Editor.ImageGalleryModel', + serverURL: null, + state: 'empty', + + init: function(serverURL) { + this._super(); + this.set('state', 'empty'); + this.serverURL = serverURL; + // olewać data + this.pages = [] + }, + + load: function() { + if (this.get('state') == 'empty') { + this.set('state', 'loading'); + $.ajax({ + url: this.serverURL, + dataType: 'json', + success: this.loadingSucceeded.bind(this) + }); + } + }, + + loadingSucceeded: function(data) { + if (this.get('state') != 'loading') { + alert('erroneous state:', this.get('state')); + } + + this.set('pages', data.pages) + this.set('state', 'synced'); + }, + + set: function(property, value) { + if (property == 'state') { + console.log(this.description(), ':', property, '=', value); + } + return this._super(property, value); + } +}); + + Editor.DocumentModel = Editor.Model.extend({ _className: 'Editor.DocumentModel', data: null, // name, text_url, latest_rev, latest_shared_rev, parts_url, dc_url, size @@ -201,7 +243,8 @@ Editor.DocumentModel = Editor.Model.extend({ this.set('state', 'synced'); this.contentModels = { 'xml': new Editor.XMLModel(data.text_url), - 'html': new Editor.HTMLModel(data.html_url) + 'html': new Editor.HTMLModel(data.html_url), + 'gallery': new Editor.ImageGalleryModel(data.gallery_url) }; for (var key in this.contentModels) { this.contentModels[key].addObserver(this, 'state', this.contentModelStateChanged.bind(this)); diff --git a/project/templates/explorer/editor.html b/project/templates/explorer/editor.html index d0756788..7cb23a7b 100644 --- a/project/templates/explorer/editor.html +++ b/project/templates/explorer/editor.html @@ -18,6 +18,7 @@ + @@ -43,6 +44,26 @@
+ +