From: Radek Czajka Date: Tue, 29 Jul 2025 12:54:10 +0000 (+0200) Subject: new sync X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/044ee915f89f30b2935dbbcb4a5d5e4a91cbfca9?ds=inline;hp=106aa5a8226eb09b4c59c55662b7903a01108f71 new sync --- diff --git a/src/bookmarks/api/views.py b/src/bookmarks/api/views.py index 1bd49633f..87314c651 100644 --- a/src/bookmarks/api/views.py +++ b/src/bookmarks/api/views.py @@ -17,10 +17,11 @@ from api.fields import AbsoluteURLField class BookmarkSerializer(serializers.ModelSerializer): book = serializers.SlugRelatedField(queryset=catalogue.models.Book.objects.all(), slug_field='slug') href = AbsoluteURLField(view_name='api_bookmark', view_args=['uuid']) + timestamp = serializers.IntegerField() class Meta: model = models.Bookmark - fields = ['book', 'anchor', 'note', 'href', 'uuid', 'location'] + fields = ['book', 'anchor', 'note', 'href', 'uuid', 'location', 'timestamp'] read_only_fields = ['uuid'] diff --git a/src/bookmarks/models.py b/src/bookmarks/models.py index 88b3e8553..234d0519d 100644 --- a/src/bookmarks/models.py +++ b/src/bookmarks/models.py @@ -13,6 +13,10 @@ class Bookmark(models.Model): def __str__(self): return str(self.uuid) + @property + def timestamp(self): + return self.created_at.timestamp() + def location(self): return f'{self.book.slug}/{self.anchor}' diff --git a/src/social/api/urls2.py b/src/social/api/urls2.py index 039a6fea7..1a6d97e2c 100644 --- a/src/social/api/urls2.py +++ b/src/social/api/urls2.py @@ -22,7 +22,10 @@ urlpatterns = [ path('progress//text/', views.TextProgressView.as_view()), path('progress//audio/', views.AudioProgressView.as_view()), - path('sync/', views.SyncView.as_view()), + path('sync/progress/', views.ProgressSyncView.as_view()), + path('sync/userlist/', views.UserListSyncView.as_view()), + path('sync/userlistitem/', views.UserListItemSyncView.as_view()), + path('sync/bookmark/', views.BookmarkSyncView.as_view()), ] diff --git a/src/social/api/views.py b/src/social/api/views.py index 36c40e135..b10ea562b 100644 --- a/src/social/api/views.py +++ b/src/social/api/views.py @@ -18,6 +18,8 @@ from catalogue.models import Book import catalogue.models from social.views import get_sets_for_book_ids from social import models +import bookmarks.models +from bookmarks.api.views import BookmarkSerializer @never_cache @@ -94,12 +96,27 @@ class UserListItemsField(serializers.Field): class UserListSerializer(serializers.ModelSerializer): - books = UserListItemsField(source='*') + client_id = serializers.CharField(write_only=True, required=False) + books = UserListItemsField(source='*', required=False) + timestamp = serializers.IntegerField(required=False) class Meta: model = models.UserList - fields = ['name', 'slug', 'books'] - read_only_fields = ['slug'] + fields = [ + 'timestamp', + 'client_id', + 'name', + 'slug', + 'favorites', + 'deleted', + 'books', + ] + read_only_fields = ['favorites'] + extra_kwargs = { + 'slug': { + 'required': False + } + } def create(self, validated_data): instance = models.UserList.get_by_name( @@ -124,6 +141,47 @@ class UserListBooksSerializer(UserListSerializer): fields = ['books'] +class UserListItemSerializer(serializers.ModelSerializer): + client_id = serializers.CharField(write_only=True, required=False) + favorites = serializers.BooleanField(required=False) + list_slug = serializers.SlugRelatedField( + queryset=models.UserList.objects.all(), + source='list', + slug_field='slug', + required=False, + ) + timestamp = serializers.IntegerField(required=False) + book_slug = serializers.SlugRelatedField( + queryset=Book.objects.all(), + source='book', + slug_field='slug', + required=False + ) + + class Meta: + model = models.UserListItem + fields = [ + 'client_id', + 'uuid', + 'order', + 'list_slug', + 'timestamp', + 'favorites', + 'deleted', + + 'book_slug', + 'fragment', + 'quote', + 'bookmark', + 'note', + ] + extra_kwargs = { + 'order': { + 'required': False + } + } + + @never_cache class ListsView(ListCreateAPIView): permission_classes = [IsAuthenticated] @@ -218,19 +276,31 @@ class ProgressSerializer(serializers.ModelSerializer): view_name='catalogue_api_book', lookup_field='slug' ) - book_slug = serializers.SlugRelatedField(source='book', read_only=True, slug_field='slug') + book_slug = serializers.SlugRelatedField( + queryset=Book.objects.all(), + source='book', + slug_field='slug') + timestamp = serializers.IntegerField(required=False) class Meta: model = models.Progress - fields = ['book', 'book_slug', 'last_mode', 'text_percent', - 'text_anchor', - 'audio_percent', - 'audio_timestamp', - 'implicit_text_percent', - 'implicit_text_anchor', - 'implicit_audio_percent', - 'implicit_audio_timestamp', - ] + fields = [ + 'timestamp', + 'book', 'book_slug', 'last_mode', 'text_percent', + 'text_anchor', + 'audio_percent', + 'audio_timestamp', + 'implicit_text_percent', + 'implicit_text_anchor', + 'implicit_audio_percent', + 'implicit_audio_timestamp', + ] + extra_kwargs = { + 'last_mode': { + 'required': False, + 'default': 'text', + } + } class TextProgressSerializer(serializers.ModelSerializer): @@ -295,26 +365,12 @@ class AudioProgressView(ProgressMixin, RetrieveUpdateAPIView): -class SyncSerializer(serializers.Serializer): - timestamp = serializers.IntegerField() - type = serializers.CharField() - id = serializers.CharField() - - def to_representation(self, instance): - rep = super().to_representation(instance) - rep['object'] = instance['object'].data - return rep - - def to_internal_value(self, data): - ret = super().to_internal_value(data) - ret['object'] = data['object'] - return ret - - @never_cache class SyncView(ListAPIView): permission_classes = [IsAuthenticated] - serializer_class = SyncSerializer + sync_id_field = 'slug' + sync_id_serializer_field = 'slug' + sync_user_field = 'user' def get_queryset(self): try: @@ -325,47 +381,90 @@ class SyncView(ListAPIView): timestamp = datetime.fromtimestamp(timestamp, tz=utc) data = [] - for p in models.Progress.objects.filter( - user=self.request.user, - updated_at__gt=timestamp).order_by('updated_at'): - data.append({ - 'timestamp': p.updated_at.timestamp(), - 'type': 'progress', - 'id': p.book.slug, - 'object': ProgressSerializer( - p, context={'request': self.request} - ) if not p.deleted else None - }) - - for p in models.UserList.objects.filter( - user=self.request.user, - updated_at__gt=timestamp).order_by('updated_at'): - data.append({ - 'timestamp': p.updated_at.timestamp(), - 'type': 'user-list', - 'id': p.slug, - 'object': UserListSerializer( - p, context={'request': self.request} - ) if not p.deleted else None - }) - - data.sort(key=lambda x: x['timestamp']) - return data + return self.get_queryset_for_ts(timestamp) + + def get_queryset_for_ts(self, timestamp): + return self.model.objects.filter( + updated_at__gt=timestamp, + **{ + self.sync_user_field: self.request.user + } + ).order_by('updated_at') + + def get_instance(self, user, data): + sync_id = data.get(self.sync_id_serializer_field) + if not sync_id: + return None + return self.model.objects.filter(**{ + self.sync_user_field: user, + self.sync_id_field: sync_id + }).first() def post(self, request): + new_ids = [] data = request.data for item in data: - ser = SyncSerializer(data=item) + instance = self.get_instance(request.user, item) + ser = self.get_serializer( + instance=instance, + data=item + ) ser.is_valid(raise_exception=True) - d = ser.validated_data - if d['type'] == 'progress': - if d['object'] is not None: - objser = ProgressSerializer(data=d['object']) - objser.is_valid(raise_exception=True) - models.Progress.sync( - user=request.user, - slug=d['id'], - ts=datetime.fromtimestamp(d['timestamp'], tz=utc), - data=d['object'] - ) - return Response() + synced_instance = self.model.sync( + request.user, + instance, + ser.validated_data + ) + if instance is None and 'client_id' in ser.validated_data and synced_instance is not None: + new_ids.append({ + 'client_id': ser.validated_data['client_id'], + self.sync_id_serializer_field: getattr(synced_instance, self.sync_id_field), + }) + return Response(new_ids) + + +class ProgressSyncView(SyncView): + model = models.Progress + serializer_class = ProgressSerializer + + sync_id_field = 'book__slug' + sync_id_serializer_field = 'book_slug' + + +class UserListSyncView(SyncView): + model = models.UserList + serializer_class = UserListSerializer + + +class UserListItemSyncView(SyncView): + model = models.UserListItem + serializer_class = UserListItemSerializer + + sync_id_field = 'uuid' + sync_id_serializer_field = 'uuid' + sync_user_field = 'list__user' + + def get_queryset_for_ts(self, timestamp): + qs = self.model.objects.filter( + updated_at__gt=timestamp, + **{ + self.sync_user_field: self.request.user + } + ) + if self.request.query_params.get('favorites'): + qs = qs.filter(list__favorites=True) + return qs.order_by('updated_at') + + +class BookmarkSyncView(SyncView): + model = bookmarks.models.Bookmark + serializer_class = BookmarkSerializer + + sync_id_field = 'uuid' + sync_id_serializer_field = 'uuid' + + def get_queryset_for_ts(self, timestamp): + return self.model.objects.filter( + user=self.request.user, + created_at__gt=timestamp + ).order_by('created_at') diff --git a/src/social/migrations/0022_userlist_reported_timestamp_and_more.py b/src/social/migrations/0022_userlist_reported_timestamp_and_more.py new file mode 100644 index 000000000..2e21005f9 --- /dev/null +++ b/src/social/migrations/0022_userlist_reported_timestamp_and_more.py @@ -0,0 +1,42 @@ +# Generated by Django 4.0.8 on 2025-07-22 13:09 + +from django.db import migrations, models +import django.utils.timezone +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('social', '0021_move_sets'), + ] + + operations = [ + migrations.AddField( + model_name='userlist', + name='reported_timestamp', + field=models.DateTimeField(default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='userlistitem', + name='reported_timestamp', + field=models.DateTimeField(default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='userlistitem', + name='uuid', + field=models.UUIDField(editable=False, null=True), + ), + migrations.AlterField( + model_name='userlist', + name='updated_at', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='userlistitem', + name='updated_at', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/src/social/migrations/0023_auto_20250722_1513.py b/src/social/migrations/0023_auto_20250722_1513.py new file mode 100644 index 000000000..a7120e85b --- /dev/null +++ b/src/social/migrations/0023_auto_20250722_1513.py @@ -0,0 +1,26 @@ +# Generated by Django 4.0.8 on 2025-07-22 13:13 + +from django.db import migrations, transaction +import uuid + + +def gen_uuid(apps, schema_editor): + UserListItem = apps.get_model("social", "UserListItem") + while UserListItem.objects.filter(uuid__isnull=True).exists(): + print(UserListItem.objects.filter(uuid__isnull=True).count(), 'rows left') + with transaction.atomic(): + for row in UserListItem.objects.filter(uuid__isnull=True)[:1000]: + row.uuid = uuid.uuid4() + row.save(update_fields=["uuid"]) + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ('social', '0022_userlist_reported_timestamp_and_more'), + ] + + operations = [ + migrations.RunPython(gen_uuid, reverse_code=migrations.RunPython.noop), + ] diff --git a/src/social/migrations/0024_auto_20250722_1513.py b/src/social/migrations/0024_auto_20250722_1513.py new file mode 100644 index 000000000..c0a8b0c1d --- /dev/null +++ b/src/social/migrations/0024_auto_20250722_1513.py @@ -0,0 +1,19 @@ +# Generated by Django 4.0.8 on 2025-07-22 13:13 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('social', '0023_auto_20250722_1513'), + ] + + operations = [ + migrations.AlterField( + model_name='userlistitem', + name='uuid', + field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True), + ), + ] diff --git a/src/social/migrations/0025_progress_reported_timestamp_alter_userlistitem_uuid.py b/src/social/migrations/0025_progress_reported_timestamp_alter_userlistitem_uuid.py new file mode 100644 index 000000000..56923c0ac --- /dev/null +++ b/src/social/migrations/0025_progress_reported_timestamp_alter_userlistitem_uuid.py @@ -0,0 +1,26 @@ +# Generated by Django 4.0.8 on 2025-07-29 12:44 + +from django.db import migrations, models +import django.utils.timezone +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('social', '0024_auto_20250722_1513'), + ] + + operations = [ + migrations.AddField( + model_name='progress', + name='reported_timestamp', + field=models.DateTimeField(default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AlterField( + model_name='userlistitem', + name='uuid', + field=models.UUIDField(blank=True, default=uuid.uuid4, editable=False, unique=True), + ), + ] diff --git a/src/social/models.py b/src/social/models.py index dc9a09a1b..0b366fc29 100644 --- a/src/social/models.py +++ b/src/social/models.py @@ -1,7 +1,10 @@ # This file is part of Wolne Lektury, licensed under GNU Affero GPLv3 or later. # Copyright © Fundacja Wolne Lektury. See NOTICE for more information. # +from datetime import datetime +import uuid from oauthlib.common import urlencode, generate_token +from pytz import utc from random import randint from django.db import models from django.conf import settings @@ -11,6 +14,7 @@ from django.core.mail import send_mail from django.urls import reverse from django.utils.timezone import now from catalogue.models import Book +from catalogue.utils import get_random_hash from wolnelektury.utils import cached_render, clear_cached_renders @@ -204,11 +208,49 @@ class UserConfirmation(models.Model): -class Progress(models.Model): +class Syncable: + @classmethod + def sync(cls, user, instance, data): + ts = data.get('timestamp') + if ts is None: + ts = now() + else: + ts = datetime.fromtimestamp(ts, tz=utc) + + if instance is not None: + if ts and ts < instance.reported_timestamp: + return + + if instance is None: + if data.get('deleted'): + return + instance = cls.create_from_data(user, data) + if instance is None: + return + + instance.reported_timestamp = ts + for f in cls.syncable_fields: + if f in data: + setattr(instance, f, data[f]) + + instance.save() + return instance + + @property + def timestamp(self): + return self.updated_at.timestamp() + + @classmethod + def create_from_data(cls, user, data): + raise NotImplementedError + + +class Progress(Syncable, models.Model): user = models.ForeignKey(User, models.CASCADE) book = models.ForeignKey('catalogue.Book', models.CASCADE) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) + reported_timestamp = models.DateTimeField() deleted = models.BooleanField(default=False) last_mode = models.CharField(max_length=64, choices=[ ('text', 'text'), @@ -223,23 +265,30 @@ class Progress(models.Model): implicit_audio_percent = models.FloatField(null=True, blank=True) implicit_audio_timestamp = models.FloatField(null=True, blank=True) + syncable_fields = [ + 'deleted', + 'last_mode', 'text_anchor', 'audio_timestamp' + ] + class Meta: unique_together = [('user', 'book')] + @property + def timestamp(self): + return self.updated_at.timestamp() + @classmethod - def sync(cls, user, slug, ts, data): - obj, _created = cls.objects.get_or_create(user=user, book__slug=slug) - if _created or obj.updated_at < ts: - if data is not None: - obj.deleted = False - for k, v in data.items(): - setattr(obj, k, v) - else: - obj.deleted = True - obj.save() + def create_from_data(cls, user, data): + return cls.objects.create( + user=user, + book=data['book'] + ) def save(self, *args, **kwargs): - audio_l = self.book.get_audio_length() + try: + audio_l = self.book.get_audio_length() + except: + audio_l = 60 if self.text_anchor: self.text_percent = 33 if audio_l: @@ -255,7 +304,7 @@ class Progress(models.Model): return super().save(*args, **kwargs) -class UserList(models.Model): +class UserList(Syncable, models.Model): slug = models.SlugField(unique=True) user = models.ForeignKey(User, models.CASCADE) name = models.CharField(max_length=1024) @@ -263,8 +312,11 @@ class UserList(models.Model): public = models.BooleanField(default=False) deleted = models.BooleanField(default=False) created_at = models.DateTimeField(auto_now_add=True) - updated_at = models.DateTimeField() + updated_at = models.DateTimeField(auto_now=True) + reported_timestamp = models.DateTimeField() + syncable_fields = ['name', 'public', 'deleted'] + def get_absolute_url(self): return reverse( 'tagged_object_list', @@ -273,18 +325,24 @@ class UserList(models.Model): def __str__(self): return self.name - + @property def url_chunk(self): return f'polka/{self.slug}' + @classmethod + def create_from_data(cls, user, data): + return cls.create(user, data['name']) + @classmethod def create(cls, user, name): + n = now() return cls.objects.create( user=user, name=name, slug=get_random_hash(name), - updated_at=now() + updated_at=n, + reported_timestamp=n, ) @classmethod @@ -333,12 +391,15 @@ class UserList(models.Model): def append(self, book): # TODO: check for duplicates? - self.userlistitem_set.create( + n = now() + item = self.userlistitem_set.create( book=book, - order=self.userlistitem_set.aggregate(m=models.Max('order'))['m'] + 1, - updated_at=now(), + order=(self.userlistitem_set.aggregate(m=models.Max('order'))['m'] or 0) + 1, + updated_at=n, + reported_timestamp=n, ) book.update_popularity() + return item def remove(self, book): self.userlistitem_set.filter(book=book).update( @@ -362,12 +423,14 @@ class UserList(models.Model): return [item.book for item in self.userlistitem_set.exclude(deleted=True).exclude(book=None)] -class UserListItem(models.Model): +class UserListItem(Syncable, models.Model): list = models.ForeignKey(UserList, models.CASCADE) + uuid = models.UUIDField(unique=True, default=uuid.uuid4, editable=False, blank=True) order = models.IntegerField() deleted = models.BooleanField(default=False) created_at = models.DateTimeField(auto_now_add=True) - updated_at = models.DateTimeField() + updated_at = models.DateTimeField(auto_now=True) + reported_timestamp = models.DateTimeField() book = models.ForeignKey('catalogue.Book', models.SET_NULL, null=True, blank=True) fragment = models.ForeignKey('catalogue.Fragment', models.SET_NULL, null=True, blank=True) @@ -375,3 +438,21 @@ class UserListItem(models.Model): bookmark = models.ForeignKey('bookmarks.Bookmark', models.SET_NULL, null=True, blank=True) note = models.TextField(blank=True) + + syncable_fields = ['order', 'deleted', 'book', 'fragment', 'quote', 'bookmark', 'note'] + + @classmethod + def create_from_data(cls, user, data): + if data.get('favorites'): + l = UserList.get_favorites_list(user, create=True) + else: + l = data['list'] + try: + assert l.user == user + except AssertionError: + return + return l.append(book=data['book']) + + @property + def favorites(self): + return self.list.favorites