Source fixes: avoid race and wait a minute after uploading to prevent unnecessary...

diff --git a/src/sources/management/commands/sources_process.py b/src/sources/management/commands/sources_process.py
index 8b3efa9..7053e5f 100644 (file)
--- a/src/sources/management/commands/sources_process.py
+++ b/src/sources/management/commands/sources_process.py
@@ -1,12 +1,14 @@
+from datetime import timedelta

 from django.core.management.base import BaseCommand
 from django.db.models import F
 from django.core.management.base import BaseCommand
 from django.db.models import F

+from django.utils.timezone import now

 from sources.models import Source
 
 
 class Command(BaseCommand):
     def handle(self, **options):
 from sources.models import Source
 
 
 class Command(BaseCommand):
     def handle(self, **options):

-        for s in Source.objects.exclude(
-                modified_at=None
+        for s in Source.objects.filter(
+            modified_at__lt=now() - timedelta(seconds=60)

         ).exclude(processed_at__gt=F('modified_at')).order_by('modified_at'):
             print(s)
             try:
         ).exclude(processed_at__gt=F('modified_at')).order_by('modified_at'):
             print(s)
             try:

diff --git a/src/sources/models.py b/src/sources/models.py
index 165cb67..efa086a 100644 (file)
--- a/src/sources/models.py
+++ b/src/sources/models.py
@@ -67,6 +67,7 @@ class Source(models.Model):
         return os.path.isdir(d) and os.listdir(d)
 
     def process(self):
         return os.path.isdir(d) and os.listdir(d)
 
     def process(self):

+        processed_at = now()

         updir = os.path.join(
             settings.MEDIA_ROOT,
             self.get_upload_directory()
         updir = os.path.join(
             settings.MEDIA_ROOT,
             self.get_upload_directory()


@@ -83,7 +84,7 @@ class Source(models.Model):
             self.build_view_directory(updir, d)
         with utils.replace_dir(ocr_dir) as d:
             self.build_ocr_directory(updir, d)
             self.build_view_directory(updir, d)
         with utils.replace_dir(ocr_dir) as d:
             self.build_ocr_directory(updir, d)

-        self.processed_at = now()
+        self.processed_at = processed_at

         self.save(update_fields=['processed_at'])
     
     def build_view_directory(self, srcpath, targetpath):
         self.save(update_fields=['processed_at'])
     
     def build_view_directory(self, srcpath, targetpath):