# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
from datetime import datetime
-from django.shortcuts import get_object_or_404
-from django.contrib.auth.decorators import login_required, user_passes_test
-from django.utils import simplejson as json
from piston.handler import BaseHandler
-from piston.utils import rc, validate
-from api.models import Deleted
from api.helpers import timestamp
+from api.models import Deleted
from catalogue.models import Book, Tag
-from catalogue.forms import BookImportForm
-from catalogue.views import tagged_object_list
-"""
-class TagHandler(BaseHandler):
- allowed_methods = ('GET',)
- model = Tag
-
- def read(self, request, tags=''):
- if tags == '':
- return Tag.objects.all()
- else:
- return tagged_object_list(request, tags, api=True)
-
-class BookHandler(BaseHandler):
- model = Book
- #fields = ('slug', 'title')
-
- def read(self, request, slug=None):
- if slug:
- return get_object_or_404(Book, slug=slug)
- else:
- return Book.objects.all()
-"""
-
-class WLHandler(BaseHandler):
+class CatalogueHandler(BaseHandler):
@staticmethod
def fields(request, name):
return fields_str.split(',') if fields_str is not None else None
@staticmethod
- def book_dict(book, fields=None, extra_fields=None):
+ def book_dict(book, fields=None):
+ all_fields = ('url', 'title', 'description',
+ 'gazeta_link', 'wiki_link',
+ 'xml', 'epub', 'txt', 'pdf', 'html',
+ 'mp3', 'ogg', 'daisy',
+ 'parent', 'parent_number',
+ 'tags',
+ 'license', 'license_description', 'source_name',
+ 'technical_editors', 'editors',
+ )
+ if fields:
+ fields = (f for f in fields if f in all_fields)
+ else:
+ fields = all_fields
+
+ extra_info = book.get_extra_info_value()
+
obj = {}
- for field in ('slug', 'title', 'description',
- 'extra_info', 'gazeta_link', 'wiki_link'):
- if getattr(book, field):
- obj[field] = getattr(book, field)
- for field in ('created_at', 'changed_at'):
- obj[field] = timestamp(getattr(book, field))
- for field in ('xml', 'epub', 'txt', 'pdf', 'html'):
- f = getattr(book, field+'_file')
- if f:
- obj[field] = f.url
- for media in book.medias.all():
- obj.setdefault(media.type, []).append(media.file.url)
- if book.parent:
- obj['parent'] = book.parent.id
- obj['parent_number'] = book.parent_number
- if fields is not None:
- for key in obj.keys():
- if key not in fields:
- del obj[key]
-
- # if there's still extra_info, we can parse it
- if 'extra_info' in obj:
- extra = json.loads(obj['extra_info'])
- if extra_fields is not None:
- for key in extra.keys():
- if key not in extra_fields:
- del extra[key]
- obj['extra_info'] = extra
+ for field in fields:
+
+ if field in ('xml', 'epub', 'txt', 'pdf', 'html'):
+ f = getattr(book, field+'_file')
+ if f:
+ obj[field] = {
+ 'url': f.url,
+ 'size': f.size,
+ }
+
+ elif field in ('mp3', 'ogg', 'daisy'):
+ media = []
+ for m in book.medias.filter(type=''):
+ files.append({
+ 'url': m.file.get_absolute_url(),
+ 'size': m.file.size,
+ })
+ if media:
+ obj[field] = media
+
+ elif field == 'url':
+ obj[field] = book.get_absolute_url()
+
+ elif field == 'tags':
+ obj[field] = [t.id for t in book.tags.exclude(category__in=('book', 'set'))]
+
+ elif field in ('license', 'license_description', 'source_name',
+ 'technical_editors', 'editors'):
+ f = extra_info.get(field)
+ if f:
+ obj[field] = f
+
+ else:
+ f = getattr(book, field)
+ if f:
+ obj[field] = f
obj['id'] = book.id
return obj
@classmethod
- def book_changes(cls, since=0, request=None):
- since = datetime.fromtimestamp(float(since))
- book_fields = cls.fields(request, 'book_fields')
- extra_fields = cls.fields(request, 'extra_fields')
+ def book_changes(cls, since=0, request=None, fields=None):
+ since = datetime.fromtimestamp(int(since))
+ if not fields:
+ fields = cls.fields(request, 'book_fields')
added = []
- changed = []
+ updated = []
deleted = []
last_change = since
- for book in Book.objects.filter(changed_at__gt=since):
- if book.changed_at > last_change:
- last_change = book.changed_at
- book_d = cls.book_dict(book, book_fields, extra_fields)
- if book.created_at > since:
- added.append(book_d)
- else:
- changed.append(book_d)
+ for book in Book.objects.filter(changed_at__gte=since):
+ book_d = cls.book_dict(book, fields)
+ updated.append(book_d)
- for book in Deleted.objects.filter(type='Book', deleted_at__gt=since, created_at__lte=since):
- if book.deleted_at > last_change:
- last_change = book.deleted_at
+ for book in Deleted.objects.filter(content_type=Book, deleted_at__gte=since, created_at__lt=since):
deleted.append(book.id)
- return {'added': added, 'changed': changed, 'deleted': deleted, 'last_change': timestamp(last_change)}
+ return {'updated': updated, 'deleted': deleted}
@staticmethod
def tag_dict(tag, fields=None):
+ all_fields = ('name', 'category', 'sort_key', 'description',
+ 'gazeta_link', 'wiki_link',
+ 'url',
+ )
+
+ if fields:
+ fields = (f for f in fields if f in all_fields)
+ else:
+ fields = all_fields
+
obj = {}
- for field in ('name', 'slug', 'sort_key', 'category', 'description', 'main_page', #'created_at', 'changed_at',
- 'gazeta_link', 'wiki_link'):
- if getattr(tag, field):
- obj[field] = getattr(tag, field)
- if fields is not None:
- for key in obj.keys():
- if key not in fields:
- del obj[key]
+ for field in fields:
+
+ if field == 'url':
+ obj[field] = tag.get_absolute_url()
+
+ else:
+ f = getattr(tag, field)
+ if f:
+ obj[field] = f
+
obj['id'] = tag.id
return obj
@classmethod
- def tag_changes(cls, since=0, request=None):
- since = datetime.fromtimestamp(float(since))
- tag_fields = cls.fields(request, 'tag_fields')
+ def tag_changes(cls, since=0, request=None, fields=None, categories=None):
+ since = datetime.fromtimestamp(int(since))
+ if not fields:
+ fields = cls.fields(request, 'tag_fields')
+ if not categories:
+ categories = cls.fields(request, 'tag_categories')
+
+ all_categories = ('author', 'theme', 'epoch', 'kind', 'genre')
+ if categories:
+ categories = (c for c in categories if c in all_categories)
+ else:
+ categories = all_categories
- added = []
- changed = []
+ updated = []
deleted = []
- last_change = since
- for tag in Tag.objects.filter(changed_at__gt=since):
- if tag.changed_at > last_change:
- last_change = tag.changed_at
- tag_d = cls.tag_dict(tag, tag_fields)
- if tag.created_at > since:
- added.append(tag_d)
- else:
- changed.append(tag_d)
+ for tag in Tag.objects.filter(category__in=categories, changed_at__gte=since):
+ tag_d = cls.tag_dict(tag, fields)
+ updated.append(tag_d)
- for tag in Deleted.objects.filter(type='Tag', deleted_at__gt=since, created_at__lte=since):
- if tag.deleted_at > last_change:
- last_change = tag.deleted_at
+ for tag in Deleted.objects.filter(category__in=categories,
+ content_type=Tag, deleted_at__gte=since, created_at__lt=since):
deleted.append(tag.id)
- return {'added': added, 'changed': changed, 'deleted': deleted, 'last_change': timestamp(last_change)}
+ return {'updated': updated, 'deleted': deleted}
+
+ @classmethod
+ def changes(cls, since=0, request=None, book_fields=None,
+ tag_fields=None, tag_categories=None):
+ changes = {
+ 'time_checked': timestamp(datetime.now())
+ }
+
+ changes_by_type = {
+ 'books': cls.book_changes(since, request, book_fields),
+ 'tags': cls.tag_changes(since, request, tag_fields, tag_categories),
+ }
+ for model in changes_by_type:
+ for field in changes_by_type[model]:
+ changes.setdefault(field, {})[model] = changes_by_type[model][field]
+ return changes
-class BookChangesHandler(WLHandler):
+
+class BookChangesHandler(CatalogueHandler):
allowed_methods = ('GET',)
def read(self, request, since):
return self.book_changes(since, request)
-class TagChangesHandler(WLHandler):
+class TagChangesHandler(CatalogueHandler):
allowed_methods = ('GET',)
def read(self, request, since):
return self.tag_changes(since, request)
-class ChangesHandler(WLHandler):
+class ChangesHandler(CatalogueHandler):
allowed_methods = ('GET',)
def read(self, request, since):
- changes = {
- 'books': self.book_changes(since, request),
- 'tags': self.tag_changes(since, request),
- }
-
- last_change = 0
- changes_rev = {}
- for model in changes:
- for field in changes[model]:
- if field == 'last_change':
- if changes[model][field] > last_change:
- last_change = changes[model][field]
- else:
- changes_rev.setdefault(field, {})[model] = changes[model][field]
- changes_rev['last_change'] = last_change
- return changes_rev
-
-
-
-# old
-"""
-staff_required = user_passes_test(lambda user: user.is_staff)
-
-class BookHandler(BaseHandler):
- model = Book
- fields = ('slug', 'title')
-
- @staff_required
- def read(self, request, slug=None):
- if slug:
- return get_object_or_404(Book, slug=slug)
- else:
- return Book.objects.all()
-
- @staff_required
- def create(self, request):
- form = BookImportForm(request.POST, request.FILES)
- if form.is_valid():
- form.save()
- return rc.CREATED
- else:
- return rc.BAD_REQUEST
-"""
+ return self.changes(since, request)
from time import mktime
def timestamp(dtime):
- "converts a datetime.datetime object to a timestamp with fractional part"
- return mktime(dtime.timetuple()) + dtime.microsecond / 1000000.0
+ "converts a datetime.datetime object to a timestamp int"
+ return int(mktime(dtime.timetuple()))
--- /dev/null
+# -*- coding: utf-8 -*-
+# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
+# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
+#
+from datetime import datetime
+import os
+import os.path
+import re
+import sqlite3
+from django.core.management.base import BaseCommand
+from slughifi import char_map
+
+from api.helpers import timestamp
+from api.settings import MOBILE_INIT_DB
+from catalogue.models import Book, Tag
+from catalogue.views import tagged_object_list # this should be somewhere else
+
+
+class Command(BaseCommand):
+ help = 'Creates an initial SQLite file for the mobile app.'
+
+ def handle(self, **options):
+ # those should be versioned
+ last_checked = timestamp(datetime.now())
+ db = init_db(last_checked)
+ for b in Book.objects.all():
+ add_book(db, b)
+ for t in Tag.objects.exclude(category__in=('book', 'set', 'theme')):
+ add_tag(db, t)
+ db.commit()
+ db.close()
+ current(last_checked)
+
+
+def pretty_size(size):
+ """ Turns size in bytes into a prettier string.
+
+ >>> pretty_size(100000)
+ '97 KiB'
+ """
+ if not size:
+ return None
+ units = ['B', 'KiB', 'MiB', 'GiB']
+ size = float(size)
+ unit = units.pop(0)
+ while size > 1000 and units:
+ size /= 1024
+ unit = units.pop(0)
+ if size < 10:
+ return "%.1f %s" % (size, unit)
+ return "%d %s" % (size, unit)
+
+
+special_marks = {'ż': '|', 'Ż': '|',}
+def replace_char(m):
+ char = m.group()
+ if char_map.has_key(char):
+ special = special_marks.get(char, '{')
+ return char_map[char] + special
+ else:
+ return char
+
+def sortify(value):
+ """
+ Turns Unicode into ASCII-sortable str
+
+ Examples :
+
+ >>> slughifi('aa') < slughifi('a a') < slughifi('ą') < slughifi('b')
+ True
+
+ """
+
+ if not isinstance(value, unicode):
+ value = unicode(value, 'utf-8')
+
+ # try to replace chars
+ value = re.sub('[^a-zA-Z0-9\\s\\-]{1}', replace_char, value)
+ value = value.lower()
+ value = re.sub(r'[^a-z0-9{|}]+', '~', value)
+
+ return value.encode('ascii', 'ignore')
+
+
+
+def init_db(last_checked):
+ if not os.path.isdir(MOBILE_INIT_DB):
+ os.makedirs(MOBILE_INIT_DB)
+ db = sqlite3.connect(os.path.join(MOBILE_INIT_DB, 'initial.db-%d' % last_checked))
+
+ schema = """
+CREATE TABLE book (
+ id INTEGER PRIMARY KEY,
+ title VARCHAR,
+ html_file VARCHAR,
+ html_file_size INTEGER,
+ parent INTEGER,
+ parent_number INTEGER,
+
+ sort_key VARCHAR,
+ pretty_size VARCHAR,
+ authors VARCHAR,
+ _local BOOLEAN
+);
+CREATE INDEX IF NOT EXISTS book_title_index ON book (sort_key);
+CREATE INDEX IF NOT EXISTS book_title_index ON book (title);
+CREATE INDEX IF NOT EXISTS book_parent_index ON book (parent);
+
+CREATE TABLE tag (
+ id INTEGER PRIMARY KEY,
+ name VARCHAR,
+ category VARCHAR,
+ sort_key VARCHAR,
+ books VARCHAR);
+CREATE INDEX IF NOT EXISTS tag_name_index ON tag (name);
+CREATE INDEX IF NOT EXISTS tag_category_index ON tag (category);
+CREATE INDEX IF NOT EXISTS tag_sort_key_index ON tag (sort_key);
+
+CREATE TABLE book_tag (book INTEGER, tag INTEGER);
+CREATE INDEX IF NOT EXISTS book_tag_book ON book_tag (book);
+CREATE INDEX IF NOT EXISTS book_tag_tag_index ON book_tag (tag);
+
+CREATE TABLE state (last_checked INTEGER);
+"""
+
+ db.executescript(schema)
+ db.execute("INSERT INTO state VALUES (:last_checked)", locals())
+ return db
+
+
+def current(last_checked):
+ target = os.path.join(MOBILE_INIT_DB, 'initial.db')
+ os.unlink(target)
+ os.symlink(
+ 'initial.db-%d' % last_checked,
+ target,
+ )
+
+
+
+book_sql = """
+ INSERT INTO book
+ (id, title, html_file, html_file_size, parent, parent_number, sort_key, pretty_size, authors)
+ VALUES
+ (:id, :title, :html_file, :html_file_size, :parent, :parent_number, :sort_key, :size_str, :authors);
+"""
+book_tag_sql = "INSERT INTO book_tag (book, tag) VALUES (:book, :tag);"
+tag_sql = """
+ INSERT INTO tag
+ (id, category, name, sort_key, books)
+ VALUES
+ (:id, :category, :name, :sort_key, :book_ids);
+"""
+categories = {'author': 'autor',
+ 'epoch': 'epoka',
+ 'genre': 'gatunek',
+ 'kind': 'rodzaj',
+ 'theme': 'motyw'
+ }
+
+
+def add_book(db, book):
+ id = book.id
+ title = book.title
+ if book.html_file:
+ html_file = book.html_file.url
+ html_file_size = book.html_file.size
+ else:
+ html_file = html_file_size = None
+ parent = book.parent
+ parent_number = book.parent_number
+ sort_key = sortify(title)
+ size_str = pretty_size(html_file_size)
+ authors = ", ".join(t.name for t in book.tags.filter(category='author'))
+ db.execute(book_sql, locals())
+
+
+def add_tag(db, tag):
+ id = tag.id
+ category = categories[tag.category]
+ name = tag.name
+ sort_key = sortify(tag.sort_key)
+
+ books = list(tagged_object_list(None, [tag], api=True))
+ book_ids = ','.join(str(b.id) for b in books)
+ db.execute(tag_sql, locals())
+
+ for b in books:
+ db.execute(book_tag_sql, {'book': b.id, 'tag': tag.id})
db.create_table('api_deleted', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('object_id', self.gf('django.db.models.fields.IntegerField')()),
- ('type', self.gf('django.db.models.fields.CharField')(max_length='50')),
- ('created_at', self.gf('django.db.models.fields.DateTimeField')()),
- ('deleted_at', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+ ('content_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['contenttypes.ContentType'])),
+ ('created_at', self.gf('django.db.models.fields.DateTimeField')(db_index=True)),
+ ('deleted_at', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, db_index=True, blank=True)),
))
db.send_create_signal('api', ['Deleted'])
- # Adding unique constraint on 'Deleted', fields ['type', 'object_id']
- db.create_unique('api_deleted', ['type', 'object_id'])
+ # Adding unique constraint on 'Deleted', fields ['content_type', 'object_id']
+ db.create_unique('api_deleted', ['content_type_id', 'object_id'])
def backwards(self, orm):
- # Removing unique constraint on 'Deleted', fields ['type', 'object_id']
- db.delete_unique('api_deleted', ['type', 'object_id'])
+ # Removing unique constraint on 'Deleted', fields ['content_type', 'object_id']
+ db.delete_unique('api_deleted', ['content_type_id', 'object_id'])
# Deleting model 'Deleted'
db.delete_table('api_deleted')
models = {
'api.deleted': {
- 'Meta': {'unique_together': "(('type', 'object_id'),)", 'object_name': 'Deleted'},
- 'created_at': ('django.db.models.fields.DateTimeField', [], {}),
- 'deleted_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'Meta': {'unique_together': "(('content_type', 'object_id'),)", 'object_name': 'Deleted'},
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+ 'created_at': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'deleted_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
- 'object_id': ('django.db.models.fields.IntegerField', [], {}),
- 'type': ('django.db.models.fields.CharField', [], {'max_length': "'50'"})
+ 'object_id': ('django.db.models.fields.IntegerField', [], {})
+ },
+ 'contenttypes.contenttype': {
+ 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+ 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
}
}
--- /dev/null
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+
+ # Adding field 'Deleted.category'
+ db.add_column('api_deleted', 'category', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=64, null=True, blank=True), keep_default=False)
+
+
+ def backwards(self, orm):
+
+ # Deleting field 'Deleted.category'
+ db.delete_column('api_deleted', 'category')
+
+
+ models = {
+ 'api.deleted': {
+ 'Meta': {'unique_together': "(('content_type', 'object_id'),)", 'object_name': 'Deleted'},
+ 'category': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '64', 'null': 'True', 'blank': 'True'}),
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+ 'created_at': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'deleted_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'object_id': ('django.db.models.fields.IntegerField', [], {})
+ },
+ 'contenttypes.contenttype': {
+ 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+ 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ }
+ }
+
+ complete_apps = ['api']
# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
#
+from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models.signals import pre_delete
class Deleted(models.Model):
object_id = models.IntegerField()
- type = models.CharField(max_length="50", db_index=True)
+ content_type = models.ForeignKey(ContentType)
+ category = models.CharField(max_length=64, null=True, blank=True, db_index=True)
created_at = models.DateTimeField(editable=False, db_index=True)
deleted_at = models.DateTimeField(auto_now_add=True, db_index=True)
class Meta:
- unique_together = (('type', 'object_id'),)
-
+ unique_together = (('content_type', 'object_id'),)
def _pre_delete_handler(sender, instance, **kwargs):
""" save deleted objects for change history purposes """
if sender in (Book, Tag):
- Deleted.objects.create(type=sender.__name__, object_id=instance.id, created_at=instance.created_at)
+ if sender == Tag:
+ if instance.category in ('book', 'set'):
+ return
+ else:
+ category = instance.category
+ else:
+ category = None
+ Deleted.objects.create(type=sender, object_id=instance.id,
+ created_at=instance.created_at, category=category)
pre_delete.connect(_pre_delete_handler)
--- /dev/null
+import os.path
+from django.conf import settings
+
+
+try:
+ MOBILE_INIT_DB = settings.API_MOBILE_INIT_DB
+except AttributeError:
+ MOBILE_INIT_DB = os.path.abspath(os.path.join(settings.MEDIA_ROOT, 'api/mobile/initial/'))
#auth = OAuthAuthentication(realm='API')
-book_changes_resource = Resource(handler=handlers.BookChangesHandler)
-tag_changes_resource = Resource(handler=handlers.TagChangesHandler)#, authentication=auth)
+#book_changes_resource = Resource(handler=handlers.BookChangesHandler)
+#tag_changes_resource = Resource(handler=handlers.TagChangesHandler)#, authentication=auth)
changes_resource = Resource(handler=handlers.ChangesHandler)
urlpatterns = patterns('',
- url(r'^book_changes/(?P<since>\d*(\.\d*)?)\.(?P<emitter_format>xml|json|yaml)$', book_changes_resource),
- url(r'^tag_changes/(?P<since>\d*(\.\d*)?)\.(?P<emitter_format>xml|json|yaml)$', tag_changes_resource),
+ #url(r'^book_changes/(?P<since>\d*(\.\d*)?)\.(?P<emitter_format>xml|json|yaml)$', book_changes_resource),
+ #url(r'^tag_changes/(?P<since>\d*(\.\d*)?)\.(?P<emitter_format>xml|json|yaml)$', tag_changes_resource),
url(r'^changes/(?P<since>\d*(\.\d*)?)\.(?P<emitter_format>xml|json|yaml)$', changes_resource),
#url(r'^books/(?P<id>[\d,]+)\.(?P<emitter_format>xml|json|yaml)$', book_resource),
url(r'^oauth/authorize/$','oauth_user_auth'),
url(r'^oauth/access_token/$','oauth_access_token'),
)
-"""
\ No newline at end of file
+"""
objects = models.Book.objects.none()
if api:
- print objects
return objects
else:
return object_list(
fnp / wolnelektury.git / commitdiff