From 2f63ddbb8cb9d393f99a03ee057291d23c6655de Mon Sep 17 00:00:00 2001 From: Radek Czajka Date: Sat, 7 Nov 2026 00:47:06 +0100 Subject: [PATCH] Docker and testing changes --- .dockerignore | 7 ++ .gitignore | 1 + Dockerfile | 117 ++++++++++++++++++++--------- Makefile | 15 ++-- docker-compose.test.yml | 31 ++++++++ docker-compose.yml | 8 +- src/catalogue/test_utils.py | 2 +- src/search/views.py | 12 +-- src/stats/models.py | 2 +- src/wolnelektury/settings/basic.py | 1 + src/wolnelektury/settings/test.py | 8 ++ 11 files changed, 151 insertions(+), 53 deletions(-) create mode 100644 .dockerignore create mode 100644 docker-compose.test.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..6d8372f6d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +**/__pycache__ +**/*~ +**/#*# +**/.#* +src/wolnelektury/localsettings.py +src/wolnelektury/dev.db + diff --git a/.gitignore b/.gitignore index 3d746f904..6c612361e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ localsettings.py +.env dev.sqlite *.db *.db-journal diff --git a/Dockerfile b/Dockerfile index 2fc76debf..7d4c87540 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,33 @@ -# Base system and requirements. +# Wolne Lektury images. +# +# Stages marked with ##### are build targets: +# deps dev image (docker-compose.yml, code mounted) +# test docker-compose.test.yml +# prod production app image (default, last stage) +# nginx production nginx with static files +# postgres Postgres with Polish dictionary +# Stages marked with ===== are intermediate. -FROM python:3.9-trixie AS base +# Global build arguments. Stages using UID/GID must redeclare them +# (bare `ARG UID`) to bring them into scope; the defaults are set here. + +ARG PYTHON_IMAGE=python:3.9-trixie ARG UID=8100 ARG GID=8100 + +########## +# deps ############################################################# +# +# Base system and requirements. +# Used directly as the dev image, with code mounted. +# +FROM ${PYTHON_IMAGE} AS deps + +ARG UID +ARG GID + RUN apt-get update && apt-get install -y \ git \ calibre \ @@ -40,19 +63,13 @@ WORKDIR /app/src RUN mkdir /app/.ipython - -# dev -# Assumes code will be mounted. - -FROM base AS dev - -#RUN pip install --no-cache-dir coverage - - -# prod-base -# Installs code and production requirements. - -FROM base AS prod-base +#=======# +# app #============================================================= +# +# Code and production requirements. Not production yet: needs +# staticfiles.json manifest from static-builder. +# +FROM deps AS app USER root @@ -63,22 +80,27 @@ USER app COPY src /app/src -# static-builder -# Static files are collected here. Statics are then copied into an Nginx image, -# and staticfiles.json manifest file is copied back into production image. - -FROM prod-base AS static-builder +#==================# +# static-builder #================================================== +# +# Static files are collected here. Statics are then copied into +# the nginx image, and staticfiles.json manifest is copied back +# into the test and prod images. +# +FROM app AS static-builder RUN python manage.py collectstatic --noinput -# nginx -# Contains static files and nginx config. - +########### +# nginx ############################################################ +# +# Static files for production nginx. Config is mounted. +# FROM nginx:alpine AS nginx -ARG UID=8100 -ARG GID=8100 +ARG UID +ARG GID # COPY nginx.conf conf.d/... @@ -86,13 +108,13 @@ RUN addgroup -g $GID app && addgroup nginx app COPY --from=static-builder /app/var/static /app/var/static - - -# Postgres with Polish hunspell dictionary withoud diacritics -# to allow full text search without diacritics. - -# --- Stage 1: prepare dictionary files --- -FROM python:3.13-slim-trixie AS polish-dict +#===============# +# polish-dict #===================================================== +# +# Polish hunspell dictionary without diacritics, to allow +# full text search without diacritics. +# +FROM ${PYTHON_IMAGE} AS polish-dict RUN apt-get update \ && apt-get install -y --no-install-recommends hunspell-pl \ @@ -109,7 +131,12 @@ RUN set -eu; \ /out/polish.dict /out/polish.affix; \ dpkg-query -W -f='hunspell-pl ${Version}\n' hunspell-pl > /out/polish.version -# --- Stage 2: actual Postgres image with the dictionary --- + +############## +# postgres ######################################################### +# +# Postgres with the dictionary from polish-dict. +# FROM postgres:18 AS postgres COPY --from=polish-dict /out/ /tmp/polish-dict/ @@ -119,10 +146,28 @@ RUN mv /tmp/polish-dict/* "/usr/share/postgresql/$PG_MAJOR/tsearch_data/" \ COPY scripts/postgres-polish.sql /docker-entrypoint-initdb.d/10-postgres-polish.sql +########## +# test ############################################################# +# +# Production image plus test requirements. Defined before `prod`, +# so that `prod` stays the default (last) stage. +# +FROM app AS test + +USER root +COPY requirements/requirements-test.txt /tmp/ +RUN pip install --no-cache-dir -r /tmp/requirements-test.txt +USER app +RUN mkdir -p /app/var/counters +COPY --from=static-builder /app/var/static/staticfiles.json /app/var/static/staticfiles.json +CMD ["./manage.py", "test"] -# prod -# That's the production image with staticfiles.json manifest file added. -FROM prod-base AS prod +########## +# prod ############################################################# +# +# Production image: app + staticfiles.json manifest. +# +FROM app AS prod COPY --from=static-builder /app/var/static/staticfiles.json /app/var/static/staticfiles.json diff --git a/Makefile b/Makefile index 59e222af6..3f1b3c4c9 100644 --- a/Makefile +++ b/Makefile @@ -12,15 +12,14 @@ deploy: src/wolnelektury/localsettings.py src/manage.py collectstatic --noinput -.ONESHELL: +TEST_COMPOSE := docker compose -f docker-compose.test.yml + test: - cd src - python -Wall -m coverage run --branch --source='.' ./manage.py test; true - coverage html -d ../htmlcov.new - rm -rf ../htmlcov - mv ../htmlcov.new ../htmlcov - coverage report - rm .coverage + mkdir -p htmlcov + UID=$(UID) GID=$(GID) $(TEST_COMPOSE) run --rm --build test sh -c '\ + python -Wall -m coverage run --branch --source=. ./manage.py test $(ARGS); s=$$?; \ + coverage report --skip-covered; coverage html -d /app/htmlcov; exit $$s'; \ + status=$$?; $(TEST_COMPOSE) down; exit $$status up: diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 000000000..11c669785 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,31 @@ +# make test +# make test ARGS=catalogue + +name: wolnelektury-test + +x-env: &env + POSTGRES_USER: wl + POSTGRES_PASSWORD: wl + +services: + test: + build: + context: . + target: test + args: + UID: ${UID:-1000} + GID: ${GID:-1000} + environment: + <<: *env + SECRET_KEY: test + ALLOWED_HOSTS: "*" + COVERAGE_FILE: /tmp/.coverage + volumes: + - ./htmlcov:/app/htmlcov + depends_on: [db] + + db: + build: {context: ., target: postgres} + environment: *env + tmpfs: [/var/lib/postgresql] + diff --git a/docker-compose.yml b/docker-compose.yml index b01e97071..55a09d874 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ x-app-base: &app-base build: context: . - target: dev + target: deps args: UID: ${UID:-1000} GID: ${GID:-1000} @@ -27,6 +27,9 @@ services: web: <<: *app-base command: ./manage.py runserver 0.0.0.0:8000 + ports: + # Dev: use http://localhost:8000/ directly + - "127.0.0.1:8000:8000" celery: <<: *app-base @@ -48,6 +51,9 @@ services: image: memcached:1.6-alpine nginx: + # Just for testing production-like environment. + # docker compose --profile nginx up + profiles: ["nginx"] build: context: . target: nginx diff --git a/src/catalogue/test_utils.py b/src/catalogue/test_utils.py index a14552287..6ad67c749 100644 --- a/src/catalogue/test_utils.py +++ b/src/catalogue/test_utils.py @@ -38,7 +38,7 @@ class PersonStub: class BookInfoStub: - _empty_fields = ['cover_url', 'variant_of'] + _empty_fields = ['cover_url', 'variant_of', 'license'] # allow single definition for multiple-value fields _salias = { 'authors': 'author', diff --git a/src/search/views.py b/src/search/views.py index 6066cd97a..6a9e1ea84 100644 --- a/src/search/views.py +++ b/src/search/views.py @@ -29,7 +29,7 @@ def get_hints(prefix, user=None, limit=10): data = [] if len(data) < limit: authors = catalogue.models.Tag.objects.filter( - category='author', name_pl__iregex='\m' + prefix).only('name', 'id', 'slug', 'category') + category='author', name_pl__iregex=r'\m' + prefix).only('name', 'id', 'slug', 'category') data.extend([ { 'type': 'author', @@ -44,7 +44,7 @@ def get_hints(prefix, user=None, limit=10): if user is not None and user.is_authenticated and len(data) < limit: tags = social.models.UserList.objects.filter( - user=user, name__iregex='\m' + prefix).only('name', 'id', 'slug') + user=user, name__iregex=r'\m' + prefix).only('name', 'id', 'slug') data.extend([ { 'type': 'userlist', @@ -56,7 +56,7 @@ def get_hints(prefix, user=None, limit=10): ]) if len(data) < limit: tags = catalogue.models.Tag.objects.filter( - category__in=('theme', 'genre', 'epoch', 'kind'), name_pl__iregex='\m' + prefix).only('name', 'id', 'slug', 'category') + category__in=('theme', 'genre', 'epoch', 'kind'), name_pl__iregex=r'\m' + prefix).only('name', 'id', 'slug', 'category') data.extend([ { 'type': tag.category, @@ -69,7 +69,7 @@ def get_hints(prefix, user=None, limit=10): ]) if len(data) < limit: collections = catalogue.models.Collection.objects.filter( - title_pl__iregex='\m' + prefix).only('title', 'slug') + title_pl__iregex=r'\m' + prefix).only('title', 'slug') data.extend([ { 'type': 'collection', @@ -80,7 +80,7 @@ def get_hints(prefix, user=None, limit=10): for collection in collections[:limit - len(data)] ]) if len(data) < limit: - for b in catalogue.models.Book.objects.filter(findable=True, title__iregex='\m' + prefix)[:limit-len(data)]: + for b in catalogue.models.Book.objects.filter(findable=True, title__iregex=r'\m' + prefix)[:limit-len(data)]: author_str = b.author_unicode() translator = b.translator() if translator: @@ -99,7 +99,7 @@ def get_hints(prefix, user=None, limit=10): infos = infopages.models.InfoPage.objects.filter( published=True, findable=True, - title_pl__iregex='\m' + prefix).only('title', 'id', 'slug') + title_pl__iregex=r'\m' + prefix).only('title', 'id', 'slug') data.extend([ { 'type': 'info', diff --git a/src/stats/models.py b/src/stats/models.py index ca0b55dab..5dcd55139 100644 --- a/src/stats/models.py +++ b/src/stats/models.py @@ -23,7 +23,7 @@ class VisitsBase(models.Model): data = urlopen(url).read().decode('utf-16') lines = data.split('\n')[1:] for line in lines: - m = re.match('^/katalog/lektura/([^,]+)\.html,', line) + m = re.match(r'^/katalog/lektura/([^,]+)\.html,', line) if m is not None: slug = m.group(1) _url, uviews, views, _rest = line.split(',', 3) diff --git a/src/wolnelektury/settings/basic.py b/src/wolnelektury/settings/basic.py index 84503ed5f..50e0edd75 100644 --- a/src/wolnelektury/settings/basic.py +++ b/src/wolnelektury/settings/basic.py @@ -4,6 +4,7 @@ import os DEBUG = os.environ.get('DEBUG', '') +SECRET_KEY = os.environ.get('SECRET_KEY', None) ADMINS = [ # ('Your Name', 'your_email@domain.com'), diff --git a/src/wolnelektury/settings/test.py b/src/wolnelektury/settings/test.py index f5d70b26e..42a453e19 100644 --- a/src/wolnelektury/settings/test.py +++ b/src/wolnelektury/settings/test.py @@ -10,3 +10,11 @@ MEDIA_URL = '/media/' SEARCH_CONFIG = 'english' SEARCH_CONFIG_SIMPLE = 'simple' SEARCH_USE_UNACCENT = False + +# Infrastructure: tests don't need Redis or Memcached. +CELERY_BROKER_URL = 'memory://' +CELERY_TASK_ALWAYS_EAGER = True +CACHES = { + 'default': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}, + 'template_fragments': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}, +} -- 2.20.1