From 2dd9e6498b5e0bd5c0bd882db02be9768c34c019 Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Wed, 7 Feb 2024 12:48:48 +0100 Subject: [PATCH 1/1] Source fixes: avoid race and wait a minute after uploading to prevent unnecessary duplicates. --- src/sources/management/commands/sources_process.py | 6 ++++-- src/sources/models.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sources/management/commands/sources_process.py b/src/sources/management/commands/sources_process.py index 8b3efa91..7053e5fc 100644 --- a/src/sources/management/commands/sources_process.py +++ b/src/sources/management/commands/sources_process.py @@ -1,12 +1,14 @@ +from datetime import timedelta from django.core.management.base import BaseCommand from django.db.models import F +from django.utils.timezone import now from sources.models import Source class Command(BaseCommand): def handle(self, **options): - for s in Source.objects.exclude( - modified_at=None + for s in Source.objects.filter( + modified_at__lt=now() - timedelta(seconds=60) ).exclude(processed_at__gt=F('modified_at')).order_by('modified_at'): print(s) try: diff --git a/src/sources/models.py b/src/sources/models.py index 165cb67b..efa086a5 100644 --- a/src/sources/models.py +++ b/src/sources/models.py @@ -67,6 +67,7 @@ class Source(models.Model): return os.path.isdir(d) and os.listdir(d) def process(self): + processed_at = now() updir = os.path.join( settings.MEDIA_ROOT, self.get_upload_directory() @@ -83,7 +84,7 @@ class Source(models.Model): self.build_view_directory(updir, d) with utils.replace_dir(ocr_dir) as d: self.build_ocr_directory(updir, d) - self.processed_at = now() + self.processed_at = processed_at self.save(update_fields=['processed_at']) def build_view_directory(self, srcpath, targetpath): -- 2.20.1