From 9ff5f2ac22360b1ff3ef2473c15d4acfd1e7769b Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Fri, 10 Sep 2021 15:46:34 +0200 Subject: [PATCH] Simple API for connecting bots. --- requirements/requirements.txt | 2 + scripts/bot-run.py | 103 ++++++++++++++++++++ src/documents/api/serializers.py | 92 +++++++++++++++++ src/documents/api/urls.py | 18 ++++ src/documents/api/views.py | 74 ++++++++++++++ src/documents/models/book.py | 6 ++ src/documents/models/chunk.py | 14 +++ src/dvcs/models.py | 2 +- src/redakcja/api/admin.py | 7 ++ src/redakcja/api/auth.py | 13 +++ src/redakcja/api/migrations/0001_initial.py | 32 ++++++ src/redakcja/api/migrations/__init__.py | 0 src/redakcja/api/models.py | 10 ++ src/redakcja/api/serializers.py | 8 ++ src/redakcja/api/urls.py | 9 ++ src/redakcja/api/views.py | 11 +++ src/redakcja/settings/__init__.py | 15 +++ src/redakcja/urls.py | 2 + 18 files changed, 417 insertions(+), 1 deletion(-) create mode 100644 scripts/bot-run.py create mode 100644 src/documents/api/serializers.py create mode 100644 src/documents/api/urls.py create mode 100644 src/documents/api/views.py create mode 100644 src/redakcja/api/admin.py create mode 100644 src/redakcja/api/auth.py create mode 100644 src/redakcja/api/migrations/0001_initial.py create mode 100644 src/redakcja/api/migrations/__init__.py create mode 100644 src/redakcja/api/models.py create mode 100644 src/redakcja/api/serializers.py create mode 100644 src/redakcja/api/urls.py create mode 100644 src/redakcja/api/views.py diff --git a/requirements/requirements.txt b/requirements/requirements.txt index ed33ac02..1ff1b461 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -26,5 +26,7 @@ django-bootstrap4==3.0.1 libsasscompiler==0.1.8 django-debug-toolbar==3.2.1 django-admin-numeric-filter==0.1.6 +djangorestframework==3.12.4 +django-filter==2.4.0 sentry-sdk==0.12.2 diff --git a/scripts/bot-run.py b/scripts/bot-run.py new file mode 100644 index 00000000..58e14181 --- /dev/null +++ b/scripts/bot-run.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +""" +Script for running a simple bot. +""" +import json +import subprocess +from urllib.parse import urljoin +from urllib.request import Request, urlopen + + +class API: + def __init__(self, base_url, token): + self.base_url = base_url + self.token = token + + def request(self, url, method='GET', data=None): + url = urljoin(self.base_url, url) + if data: + data = json.dumps(data).encode('utf-8') + else: + data = None + + headers = { + "Content-type": "application/json", + } + if self.token: + headers['Authorization'] = 'Token ' + self.token + + req = Request(url, data=data, method=method, headers=headers) + try: + resp = urlopen(req) + except Exception as e: + print(e.reason) + print(e.read().decode('utf-8')) + raise + else: + return json.load(resp) + + def my_chunks(self): + me = self.request('me/')['id'] + return self.request('documents/chunks/?user={}'.format(me)) + + +def process_chunk(chunk, api, executable): + print(chunk['id']) + head = chunk['head'] + text = api.request(head)['text'] + text = text.encode('utf-8') + + try: + p = subprocess.run( + [executable], + input=text, + capture_output=True, + check=True + ) + except subprocess.CalledProcessError as e: + print('Ditching the update. Bot exited with error code {} and output:'.format(e.returncode)) + print(e.stderr.decode('utf-8')) + return + result_text = p.stdout.decode('utf-8') + stderr_text = p.stderr.decode('utf-8') + api.request(chunk['revisions'], 'POST', { + "parent": head, + "description": stderr_text or 'Automatic update.', + "text": result_text + }) + # Remove the user assignment. + api.request(chunk['id'], 'PUT', { + "user": None + }) + + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser( + description='Runs a bot for Redakcja. ' + 'You need to provide an executable which will take current revision ' + 'of text as stdin, and output the new version on stdout. ' + 'Any output given on stderr will be used as revision description. ' + 'If bot exits with non-zero return code, the update will be ditched.' + ) + parser.add_argument( + 'token', metavar='TOKEN', help='A Redakcja API token.' + ) + parser.add_argument( + 'executable', metavar='EXECUTABLE', help='An executable to run as bot.' + ) + parser.add_argument( + '--api', metavar='API', help='A base URL for the API.', + default='https://redakcja.wolnelektury.pl/api/', + ) + args = parser.parse_args() + + + api = API(args.api, args.token) + + chunks = api.my_chunks() + if chunks: + for chunk in api.my_chunks(): + process_chunk(chunk, api, args.executable) + else: + print('No assigned chunks found.') diff --git a/src/documents/api/serializers.py b/src/documents/api/serializers.py new file mode 100644 index 00000000..9ac59101 --- /dev/null +++ b/src/documents/api/serializers.py @@ -0,0 +1,92 @@ +from rest_framework import serializers +from .. import models + + +class TextField(serializers.Field): + def get_attribute(self, instance): + return instance + + def to_representation(self, value): + return value.materialize() + + def to_internal_value(self, data): + return data + + +class BookSerializer(serializers.ModelSerializer): + id = serializers.HyperlinkedIdentityField(view_name='documents_api_book') + + class Meta: + model = models.Book + fields = [ + 'id', + 'title' + ] + + +class ChunkSerializer(serializers.ModelSerializer): + id = serializers.HyperlinkedIdentityField(view_name='documents_api_chunk') + book = serializers.HyperlinkedRelatedField(view_name='documents_api_book', read_only=True) + revisions = serializers.HyperlinkedIdentityField(view_name='documents_api_chunk_revision_list') + head = serializers.HyperlinkedRelatedField(view_name='documents_api_revision', read_only=True) + ## RelatedField + + class Meta: + model = models.Chunk + fields = ['id', 'book', 'revisions', 'head', 'user', 'stage'] + + +class RHRF(serializers.HyperlinkedRelatedField): + def get_queryset(self): + return self.context['chunk'].change_set.all(); + + +class RevisionSerializer(serializers.ModelSerializer): + id = serializers.HyperlinkedIdentityField(view_name='documents_api_revision') + parent = RHRF( + view_name='documents_api_revision', + queryset=models.Chunk.change_model.objects.all() + ) + merge_parent = RHRF( + view_name='documents_api_revision', + read_only=True + ) + chunk = serializers.HyperlinkedRelatedField(view_name='documents_api_chunk', read_only=True, source='tree') + author = serializers.SerializerMethodField() + + class Meta: + model = models.Chunk.change_model + fields = ['id', 'chunk', 'created_at', 'author', 'author_email', 'author_name', 'parent', 'merge_parent'] + read_only_fields = ['author_email', 'author_name'] + + def get_author(self, obj): + return obj.author.username if obj.author is not None else None + + +class BookDetailSerializer(BookSerializer): + chunk = ChunkSerializer(many=True, source='chunk_set') + + class Meta: + model = models.Book + fields = BookSerializer.Meta.fields + ['chunk'] + + + +class ChunkDetailSerializer(ChunkSerializer): + pass + + +class RevisionDetailSerializer(RevisionSerializer): + text = TextField() + + class Meta(RevisionSerializer.Meta): + fields = RevisionSerializer.Meta.fields + ['description', 'text'] + + def create(self, validated_data): + chunk = self.context['chunk'] + return chunk.commit( + validated_data['text'], + author=self.context['request'].user, # what if anonymous? + description=validated_data['description'], + parent=validated_data.get('parent'), + ) diff --git a/src/documents/api/urls.py b/src/documents/api/urls.py new file mode 100644 index 00000000..1654034f --- /dev/null +++ b/src/documents/api/urls.py @@ -0,0 +1,18 @@ +from django.urls import path +from . import views + + +urlpatterns = [ + path('books/', views.BookList.as_view(), + name='documents_api_book_list'), + path('books//', views.BookDetail.as_view(), + name='documents_api_book'), + path('chunks/', views.ChunkList.as_view(), + name='documents_api_chunk_list'), + path('chunks//', views.ChunkDetail.as_view(), + name='documents_api_chunk'), + path('chunks//revisions/', views.ChunkRevisionList.as_view(), + name='documents_api_chunk_revision_list'), + path('revisions//', views.RevisionDetail.as_view(), + name='documents_api_revision'), +] diff --git a/src/documents/api/views.py b/src/documents/api/views.py new file mode 100644 index 00000000..a2f3529d --- /dev/null +++ b/src/documents/api/views.py @@ -0,0 +1,74 @@ +from rest_framework.generics import RetrieveAPIView, RetrieveUpdateAPIView, ListAPIView, ListCreateAPIView +from rest_framework.permissions import IsAuthenticatedOrReadOnly +from django.http import Http404 +from .. import models +from . import serializers + + +class BookList(ListAPIView): + serializer_class = serializers.BookSerializer + search_fields = ['title'] + + def get_queryset(self): + return models.Book.get_visible_for(self.request.user) + + +class BookDetail(RetrieveAPIView): + serializer_class = serializers.BookDetailSerializer + + def get_queryset(self): + return models.Book.get_visible_for(self.request.user) + + +class ChunkList(ListAPIView): + queryset = models.Chunk.objects.all() + serializer_class = serializers.ChunkSerializer + filter_fields = ['user', 'stage'] + search_fields = ['book__title'] + + def get_queryset(self): + return models.Chunk.get_visible_for(self.request.user) + + +class ChunkDetail(RetrieveUpdateAPIView): + permission_classes = [IsAuthenticatedOrReadOnly] + serializer_class = serializers.ChunkDetailSerializer + + def get_queryset(self): + return models.Chunk.get_visible_for(self.request.user) + + +class ChunkRevisionList(ListCreateAPIView): + permission_classes = [IsAuthenticatedOrReadOnly] + serializer_class = serializers.RevisionSerializer + + def get_serializer_class(self): + if self.request.method == 'POST': + return serializers.RevisionDetailSerializer + else: + return serializers.RevisionSerializer + + def get_serializer_context(self): + ctx = super().get_serializer_context() + try: + ctx["chunk"] = models.Chunk.objects.get(pk=self.kwargs['pk']) + except models.Chunk.DoesNotExist: + raise Http404 + return ctx + + def get_queryset(self): + try: + return models.Chunk.get_visible_for(self.request.user).get( + pk=self.kwargs['pk'] + ).change_set.all() + except models.Chunk.DoesNotExist: + raise Http404() + + +class RevisionDetail(RetrieveAPIView): + queryset = models.Chunk.change_model.objects.all() + serializer_class = serializers.RevisionDetailSerializer + + def get_queryset(self): + return models.Chunk.get_revisions_visible_for(self.request.user) + diff --git a/src/documents/models/book.py b/src/documents/models/book.py index 42ea33a1..794c570a 100644 --- a/src/documents/models/book.py +++ b/src/documents/models/book.py @@ -60,6 +60,12 @@ class Book(models.Model): verbose_name = _('book') verbose_name_plural = _('books') + @classmethod + def get_visible_for(cls, user): + qs = cls.objects.all() + if not user.is_authenticated: + qs = qs.filter(public=True) + return qs # Representing # ============ diff --git a/src/documents/models/chunk.py b/src/documents/models/chunk.py index 61aa5177..2698b58f 100644 --- a/src/documents/models/chunk.py +++ b/src/documents/models/chunk.py @@ -39,6 +39,20 @@ class Chunk(dvcs_models.Document): verbose_name_plural = _('chunks') permissions = [('can_pubmark', 'Can mark for publishing')] + @classmethod + def get_visible_for(cls, user): + qs = cls.objects.all() + if not user.is_authenticated: + qs = qs.filter(book__public=True) + return qs + + @classmethod + def get_revisions_visible_for(cls, user): + qs = cls.change_model.objects.all() + if not user.is_authenticated: + qs = qs.filter(tree__book__public=True) + return qs + # Representing # ============ diff --git a/src/dvcs/models.py b/src/dvcs/models.py index ed771c43..978f0720 100644 --- a/src/dvcs/models.py +++ b/src/dvcs/models.py @@ -94,7 +94,7 @@ class Change(models.Model): unique_together = ['tree', 'revision'] def __str__(self): - return u"Id: %r, Tree %r, Parent %r, Data: %s" % (self.id, self.tree_id, self.parent_id, self.data) + return "rev. {} @ {}".format(self.revision, self.created_at) def author_str(self): if self.author: diff --git a/src/redakcja/api/admin.py b/src/redakcja/api/admin.py new file mode 100644 index 00000000..bb0f4a3c --- /dev/null +++ b/src/redakcja/api/admin.py @@ -0,0 +1,7 @@ +from django.contrib import admin +from . import models + + +@admin.register(models.Token) +class TokenAdmin(admin.ModelAdmin): + readonly_fields = ['key', 'created', 'last_seen_at'] diff --git a/src/redakcja/api/auth.py b/src/redakcja/api/auth.py new file mode 100644 index 00000000..ccc58639 --- /dev/null +++ b/src/redakcja/api/auth.py @@ -0,0 +1,13 @@ +from django.utils.timezone import now +from rest_framework.authentication import TokenAuthentication as BaseTokenAuthentication +from . import models + + +class TokenAuthentication(BaseTokenAuthentication): + model = models.Token + + def authenticate_credentials(self, key): + user, token = super().authenticate_credentials(key) + token.last_seen_at = now() + token.save(update_fields=['last_seen_at']) + return (user, token) diff --git a/src/redakcja/api/migrations/0001_initial.py b/src/redakcja/api/migrations/0001_initial.py new file mode 100644 index 00000000..cc194d1d --- /dev/null +++ b/src/redakcja/api/migrations/0001_initial.py @@ -0,0 +1,32 @@ +# Generated by Django 3.1.13 on 2021-09-10 15:04 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Token', + fields=[ + ('key', models.CharField(max_length=40, primary_key=True, serialize=False, verbose_name='Key')), + ('created', models.DateTimeField(auto_now_add=True, verbose_name='Created')), + ('last_seen_at', models.DateTimeField(blank=True, null=True)), + ('api_version', models.IntegerField(choices=[(1, '1')])), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='auth_token', to=settings.AUTH_USER_MODEL, verbose_name='User')), + ], + options={ + 'verbose_name': 'Token', + 'verbose_name_plural': 'Tokens', + 'abstract': False, + }, + ), + ] diff --git a/src/redakcja/api/migrations/__init__.py b/src/redakcja/api/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/redakcja/api/models.py b/src/redakcja/api/models.py new file mode 100644 index 00000000..797aa6d4 --- /dev/null +++ b/src/redakcja/api/models.py @@ -0,0 +1,10 @@ +from django.conf import settings +from django.db import models +from rest_framework.authtoken.models import Token as TokenBase + + +class Token(TokenBase): + last_seen_at = models.DateTimeField(blank=True, null=True) + api_version = models.IntegerField(choices=[ + (1, '1'), + ]) diff --git a/src/redakcja/api/serializers.py b/src/redakcja/api/serializers.py new file mode 100644 index 00000000..63d5cbbf --- /dev/null +++ b/src/redakcja/api/serializers.py @@ -0,0 +1,8 @@ +from django.contrib.auth.models import User +from rest_framework import serializers + + +class UserSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = ['id', 'username'] diff --git a/src/redakcja/api/urls.py b/src/redakcja/api/urls.py new file mode 100644 index 00000000..7f6e1bb6 --- /dev/null +++ b/src/redakcja/api/urls.py @@ -0,0 +1,9 @@ +from django.urls import include, path +from . import views + + +urlpatterns = [ + path('documents/', include('documents.api.urls')), + + path('me/', views.MeView.as_view()), +] diff --git a/src/redakcja/api/views.py b/src/redakcja/api/views.py new file mode 100644 index 00000000..65ce839f --- /dev/null +++ b/src/redakcja/api/views.py @@ -0,0 +1,11 @@ +from rest_framework.generics import RetrieveAPIView +from rest_framework.permissions import IsAuthenticated +from . import serializers + + +class MeView(RetrieveAPIView): + permission_classes = [IsAuthenticated] + serializer_class = serializers.UserSerializer + + def get_object(self): + return self.request.user diff --git a/src/redakcja/settings/__init__.py b/src/redakcja/settings/__init__.py index 9bca2631..4259b319 100644 --- a/src/redakcja/settings/__init__.py +++ b/src/redakcja/settings/__init__.py @@ -88,7 +88,10 @@ INSTALLED_APPS = ( 'fnpdjango', 'django_cas_ng', 'bootstrap4', + 'rest_framework', + 'django_filters', + 'redakcja.api', 'catalogue', 'documents', 'cover', @@ -266,6 +269,18 @@ PIPELINE = { SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer' +REST_FRAMEWORK = { + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.SessionAuthentication', + 'redakcja.api.auth.TokenAuthentication', + ], + 'DEFAULT_FILTER_BACKENDS': [ + 'django_filters.rest_framework.DjangoFilterBackend', + 'rest_framework.filters.SearchFilter', + ] +} + + try: SENTRY_DSN except NameError: diff --git a/src/redakcja/urls.py b/src/redakcja/urls.py index 8050ae1e..6e607305 100644 --- a/src/redakcja/urls.py +++ b/src/redakcja/urls.py @@ -27,6 +27,8 @@ urlpatterns = [ url(r'^images/', include('wiki_img.urls')), url(r'^cover/', include('cover.urls')), url(r'^wlxml/', include('wlxml.urls')), + + path('api/', include('redakcja.api.urls')), ] -- 2.20.1