Docker and testing changes

diff --git a/.dockerignore b/.dockerignore
new file mode 100644 (file)
index 0000000..6d8372f
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,7 @@
+**/__pycache__
+**/*~
+**/#*#
+**/.#*
+src/wolnelektury/localsettings.py
+src/wolnelektury/dev.db
+

diff --git a/.gitignore b/.gitignore
index 3d746f9..6c61236 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 localsettings.py
 localsettings.py

+.env

 dev.sqlite
 *.db
 *.db-journal
 dev.sqlite
 *.db
 *.db-journal

diff --git a/Dockerfile b/Dockerfile
index 2fc76de..7d4c875 100644 (file)
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,10 +1,33 @@
-# Base system and requirements.
+# Wolne Lektury images.
+#
+# Stages marked with ##### are build targets:
+#   deps      dev image (docker-compose.yml, code mounted)
+#   test      docker-compose.test.yml
+#   prod      production app image (default, last stage)
+#   nginx     production nginx with static files
+#   postgres  Postgres with Polish dictionary
+# Stages marked with ===== are intermediate.

 
 

-FROM python:3.9-trixie AS base

 
 

+# Global build arguments. Stages using UID/GID must redeclare them
+# (bare `ARG UID`) to bring them into scope; the defaults are set here.
+
+ARG PYTHON_IMAGE=python:3.9-trixie

 ARG UID=8100
 ARG GID=8100
 
 ARG UID=8100
 ARG GID=8100
 

+
+##########
+#  deps  #############################################################
+#
+#  Base system and requirements.
+#  Used directly as the dev image, with code mounted.
+#
+FROM ${PYTHON_IMAGE} AS deps
+
+ARG UID
+ARG GID
+

 RUN     apt-get update && apt-get install -y \
        git \
        calibre \
 RUN     apt-get update && apt-get install -y \
        git \
        calibre \


@@ -40,19 +63,13 @@ WORKDIR /app/src
 RUN mkdir /app/.ipython
 
 
 RUN mkdir /app/.ipython
 
 

-
-# dev
-# Assumes code will be mounted.
-
-FROM base AS dev
-
-#RUN pip install --no-cache-dir coverage
-
-
-# prod-base
-# Installs code and production requirements.
-
-FROM base AS prod-base
+#=======#
+#  app  #=============================================================
+#
+#  Code and production requirements. Not production yet: needs
+#  staticfiles.json manifest from static-builder.
+#
+FROM deps AS app

 
 USER root
 
 
 USER root
 


@@ -63,22 +80,27 @@ USER app
 COPY src /app/src
 
 
 COPY src /app/src
 
 

-# static-builder
-# Static files are collected here. Statics are then copied into an Nginx image,
-# and staticfiles.json manifest file is copied back into production image.
-
-FROM prod-base AS static-builder
+#==================#
+#  static-builder  #==================================================
+#
+#  Static files are collected here. Statics are then copied into
+#  the nginx image, and staticfiles.json manifest is copied back
+#  into the test and prod images.
+#
+FROM app AS static-builder

 
 RUN python manage.py collectstatic --noinput
 
 
 
 RUN python manage.py collectstatic --noinput
 
 

-# nginx
-# Contains static files and nginx config.
-
+###########
+#  nginx  ############################################################
+#
+#  Static files for production nginx. Config is mounted.
+#

 FROM nginx:alpine AS nginx
 
 FROM nginx:alpine AS nginx
 

-ARG UID=8100
-ARG GID=8100
+ARG UID
+ARG GID

 
 # COPY nginx.conf conf.d/...
 
 
 # COPY nginx.conf conf.d/...
 


@@ -86,13 +108,13 @@ RUN addgroup -g $GID app && addgroup nginx app
 COPY --from=static-builder /app/var/static /app/var/static
 
 
 COPY --from=static-builder /app/var/static /app/var/static
 
 

-
-
-# Postgres with Polish hunspell dictionary withoud diacritics
-# to allow full text search without diacritics.
-
-# --- Stage 1: prepare dictionary files ---
-FROM python:3.13-slim-trixie AS polish-dict
+#===============#
+#  polish-dict  #=====================================================
+#
+#  Polish hunspell dictionary without diacritics, to allow
+#  full text search without diacritics.
+#
+FROM ${PYTHON_IMAGE} AS polish-dict

 
 RUN apt-get update \
  && apt-get install -y --no-install-recommends hunspell-pl \
 
 RUN apt-get update \
  && apt-get install -y --no-install-recommends hunspell-pl \


@@ -109,7 +131,12 @@ RUN set -eu; \
         /out/polish.dict /out/polish.affix; \
     dpkg-query -W -f='hunspell-pl ${Version}\n' hunspell-pl > /out/polish.version
 
         /out/polish.dict /out/polish.affix; \
     dpkg-query -W -f='hunspell-pl ${Version}\n' hunspell-pl > /out/polish.version
 

-# --- Stage 2: actual Postgres image with the dictionary ---
+
+##############
+#  postgres  #########################################################
+#
+#  Postgres with the dictionary from polish-dict.
+#

 FROM postgres:18 AS postgres
 
 COPY --from=polish-dict /out/ /tmp/polish-dict/
 FROM postgres:18 AS postgres
 
 COPY --from=polish-dict /out/ /tmp/polish-dict/


@@ -119,10 +146,28 @@ RUN mv /tmp/polish-dict/* "/usr/share/postgresql/$PG_MAJOR/tsearch_data/" \
 COPY scripts/postgres-polish.sql /docker-entrypoint-initdb.d/10-postgres-polish.sql
 
 
 COPY scripts/postgres-polish.sql /docker-entrypoint-initdb.d/10-postgres-polish.sql
 
 

+##########
+#  test  #############################################################
+#
+#  Production image plus test requirements. Defined before `prod`,
+#  so that `prod` stays the default (last) stage.
+#
+FROM app AS test
+
+USER root
+COPY requirements/requirements-test.txt /tmp/
+RUN pip install --no-cache-dir -r /tmp/requirements-test.txt
+USER app
+RUN mkdir -p /app/var/counters
+COPY --from=static-builder /app/var/static/staticfiles.json /app/var/static/staticfiles.json
+CMD ["./manage.py", "test"]

 
 

-# prod
-# That's the production image with staticfiles.json manifest file added.

 
 

-FROM prod-base AS prod
+##########
+#  prod  #############################################################
+#
+#  Production image: app + staticfiles.json manifest.
+#
+FROM app AS prod

 
 COPY --from=static-builder /app/var/static/staticfiles.json /app/var/static/staticfiles.json
 
 COPY --from=static-builder /app/var/static/staticfiles.json /app/var/static/staticfiles.json

diff --git a/Makefile b/Makefile
index 59e222a..3f1b3c4 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -12,15 +12,14 @@ deploy: src/wolnelektury/localsettings.py
        src/manage.py collectstatic --noinput
 
 
        src/manage.py collectstatic --noinput
 
 

-.ONESHELL:
+TEST_COMPOSE := docker compose -f docker-compose.test.yml
+

 test:
 test:

-       cd src
-       python -Wall -m coverage run --branch --source='.' ./manage.py test; true
-       coverage html -d ../htmlcov.new
-       rm -rf ../htmlcov
-       mv ../htmlcov.new ../htmlcov
-       coverage report
-       rm .coverage
+       mkdir -p htmlcov
+       UID=$(UID) GID=$(GID) $(TEST_COMPOSE) run --rm --build test sh -c '\
+         python -Wall -m coverage run --branch --source=. ./manage.py test $(ARGS); s=$$?; \
+         coverage report --skip-covered; coverage html -d /app/htmlcov; exit $$s'; \
+       status=$$?; $(TEST_COMPOSE) down; exit $$status

 
 
 up:
 
 
 up:

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
new file mode 100644 (file)
index 0000000..11c6697
--- /dev/null
+++ b/docker-compose.test.yml
@@ -0,0 +1,31 @@
+#   make test
+#   make test ARGS=catalogue
+
+name: wolnelektury-test
+
+x-env: &env
+  POSTGRES_USER: wl
+  POSTGRES_PASSWORD: wl
+
+services:
+  test:
+    build:
+      context: .
+      target: test
+      args:
+        UID: ${UID:-1000}
+        GID: ${GID:-1000}
+    environment:
+      <<: *env
+      SECRET_KEY: test
+      ALLOWED_HOSTS: "*"
+      COVERAGE_FILE: /tmp/.coverage
+    volumes:
+      - ./htmlcov:/app/htmlcov
+    depends_on: [db]
+
+  db:
+    build: {context: ., target: postgres}
+    environment: *env
+    tmpfs: [/var/lib/postgresql]
+

diff --git a/docker-compose.yml b/docker-compose.yml
index b01e970..55a09d8 100644 (file)
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,7 +1,7 @@
 x-app-base: &app-base
   build:
     context: .
 x-app-base: &app-base
   build:
     context: .

-    target: dev
+    target: deps

     args:
       UID: ${UID:-1000}
       GID: ${GID:-1000}
     args:
       UID: ${UID:-1000}
       GID: ${GID:-1000}


@@ -27,6 +27,9 @@ services:
   web:
     <<: *app-base
     command: ./manage.py runserver 0.0.0.0:8000
   web:
     <<: *app-base
     command: ./manage.py runserver 0.0.0.0:8000

+    ports:
+      # Dev: use http://localhost:8000/ directly
+      - "127.0.0.1:8000:8000"

 
   celery:
     <<: *app-base
 
   celery:
     <<: *app-base


@@ -48,6 +51,9 @@ services:
     image: memcached:1.6-alpine
 
   nginx:
     image: memcached:1.6-alpine
 
   nginx:

+    # Just for testing production-like environment.
+    # docker compose --profile nginx up
+    profiles: ["nginx"]

     build:
       context: .
       target: nginx
     build:
       context: .
       target: nginx

diff --git a/src/catalogue/test_utils.py b/src/catalogue/test_utils.py
index a145522..6ad67c7 100644 (file)
--- a/src/catalogue/test_utils.py
+++ b/src/catalogue/test_utils.py
@@ -38,7 +38,7 @@ class PersonStub:
 
 
 class BookInfoStub:
 
 
 class BookInfoStub:

-    _empty_fields = ['cover_url', 'variant_of']
+    _empty_fields = ['cover_url', 'variant_of', 'license']

     # allow single definition for multiple-value fields
     _salias = {
         'authors': 'author',
     # allow single definition for multiple-value fields
     _salias = {
         'authors': 'author',

diff --git a/src/search/views.py b/src/search/views.py
index 6066cd9..6a9e1ea 100644 (file)
--- a/src/search/views.py
+++ b/src/search/views.py
@@ -29,7 +29,7 @@ def get_hints(prefix, user=None, limit=10):
     data = []
     if len(data) < limit:
         authors = catalogue.models.Tag.objects.filter(
     data = []
     if len(data) < limit:
         authors = catalogue.models.Tag.objects.filter(

-            category='author', name_pl__iregex='\m' + prefix).only('name', 'id', 'slug', 'category')
+            category='author', name_pl__iregex=r'\m' + prefix).only('name', 'id', 'slug', 'category')

         data.extend([
             {
                 'type': 'author',
         data.extend([
             {
                 'type': 'author',


@@ -44,7 +44,7 @@ def get_hints(prefix, user=None, limit=10):
     
     if user is not None and user.is_authenticated and len(data) < limit:
         tags = social.models.UserList.objects.filter(
     
     if user is not None and user.is_authenticated and len(data) < limit:
         tags = social.models.UserList.objects.filter(

-            user=user, name__iregex='\m' + prefix).only('name', 'id', 'slug')
+            user=user, name__iregex=r'\m' + prefix).only('name', 'id', 'slug')

         data.extend([
             {
                 'type': 'userlist',
         data.extend([
             {
                 'type': 'userlist',


@@ -56,7 +56,7 @@ def get_hints(prefix, user=None, limit=10):
         ])
     if len(data) < limit:
         tags = catalogue.models.Tag.objects.filter(
         ])
     if len(data) < limit:
         tags = catalogue.models.Tag.objects.filter(

-            category__in=('theme', 'genre', 'epoch', 'kind'), name_pl__iregex='\m' + prefix).only('name', 'id', 'slug', 'category')
+            category__in=('theme', 'genre', 'epoch', 'kind'), name_pl__iregex=r'\m' + prefix).only('name', 'id', 'slug', 'category')

         data.extend([
             {
                 'type': tag.category,
         data.extend([
             {
                 'type': tag.category,


@@ -69,7 +69,7 @@ def get_hints(prefix, user=None, limit=10):
         ])
     if len(data) < limit:
         collections = catalogue.models.Collection.objects.filter(
         ])
     if len(data) < limit:
         collections = catalogue.models.Collection.objects.filter(

-            title_pl__iregex='\m' + prefix).only('title', 'slug')
+            title_pl__iregex=r'\m' + prefix).only('title', 'slug')

         data.extend([
             {
                 'type': 'collection',
         data.extend([
             {
                 'type': 'collection',


@@ -80,7 +80,7 @@ def get_hints(prefix, user=None, limit=10):
             for collection in collections[:limit - len(data)]
         ])
     if len(data) < limit:
             for collection in collections[:limit - len(data)]
         ])
     if len(data) < limit:

-        for b in catalogue.models.Book.objects.filter(findable=True, title__iregex='\m' + prefix)[:limit-len(data)]:
+        for b in catalogue.models.Book.objects.filter(findable=True, title__iregex=r'\m' + prefix)[:limit-len(data)]:

             author_str = b.author_unicode()
             translator = b.translator()
             if translator:
             author_str = b.author_unicode()
             translator = b.translator()
             if translator:


@@ -99,7 +99,7 @@ def get_hints(prefix, user=None, limit=10):
         infos = infopages.models.InfoPage.objects.filter(
             published=True,
             findable=True,
         infos = infopages.models.InfoPage.objects.filter(
             published=True,
             findable=True,

-            title_pl__iregex='\m' + prefix).only('title', 'id', 'slug')
+            title_pl__iregex=r'\m' + prefix).only('title', 'id', 'slug')

         data.extend([
             {
                 'type': 'info',
         data.extend([
             {
                 'type': 'info',

diff --git a/src/stats/models.py b/src/stats/models.py
index ca0b55d..5dcd551 100644 (file)
--- a/src/stats/models.py
+++ b/src/stats/models.py
@@ -23,7 +23,7 @@ class VisitsBase(models.Model):
         data = urlopen(url).read().decode('utf-16')
         lines = data.split('\n')[1:]
         for line in lines:
         data = urlopen(url).read().decode('utf-16')
         lines = data.split('\n')[1:]
         for line in lines:

-            m = re.match('^/katalog/lektura/([^,]+)\.html,', line)
+            m = re.match(r'^/katalog/lektura/([^,]+)\.html,', line)

             if m is not None:
                 slug = m.group(1)
                 _url, uviews, views, _rest = line.split(',', 3)
             if m is not None:
                 slug = m.group(1)
                 _url, uviews, views, _rest = line.split(',', 3)

diff --git a/src/wolnelektury/settings/basic.py b/src/wolnelektury/settings/basic.py
index 84503ed..50e0edd 100644 (file)
--- a/src/wolnelektury/settings/basic.py
+++ b/src/wolnelektury/settings/basic.py
@@ -4,6 +4,7 @@
 import os
 
 DEBUG = os.environ.get('DEBUG', '')
 import os
 
 DEBUG = os.environ.get('DEBUG', '')

+SECRET_KEY = os.environ.get('SECRET_KEY', None)

 
 ADMINS = [
     # ('Your Name', 'your_email@domain.com'),
 
 ADMINS = [
     # ('Your Name', 'your_email@domain.com'),

diff --git a/src/wolnelektury/settings/test.py b/src/wolnelektury/settings/test.py
index f5d70b2..42a453e 100644 (file)
--- a/src/wolnelektury/settings/test.py
+++ b/src/wolnelektury/settings/test.py
@@ -10,3 +10,11 @@ MEDIA_URL = '/media/'
 SEARCH_CONFIG = 'english'
 SEARCH_CONFIG_SIMPLE = 'simple'
 SEARCH_USE_UNACCENT = False
 SEARCH_CONFIG = 'english'
 SEARCH_CONFIG_SIMPLE = 'simple'
 SEARCH_USE_UNACCENT = False

+
+# Infrastructure: tests don't need Redis or Memcached.
+CELERY_BROKER_URL = 'memory://'
+CELERY_TASK_ALWAYS_EAGER = True
+CACHES = {
+    'default': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'},
+    'template_fragments': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'},
+}