From 322bf6f49e49d78331694a6670004b8dae2ba022 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Tue, 23 Jun 2026 23:14:28 +0200 Subject: [PATCH 1/8] feat(cache): shared two-tier (L1+L2) read-through cache for singleton config - new dojo/caching.py: dojo_settings_cache(key, timeout) read-through over an in-process L1 tier (off|thread|process, TTL'd) on top of django.core.cache (L2). Resolves L1 -> L2 -> getter; None is never cached. invalidate_dojo_settings_cache; model_to_cache_dict/cache_dict_to_model (store dicts, not pickled instances). Configurable: DD_SETTINGS_CACHE_ENABLED / DD_CACHE_L1_MODE / DD_CACHE_L1_TTL / DD_CACHE_L2_TTL (-1 disables a tier). - System_Settings read path served through the cache (dict, rebuilt per call); write paths (no_cache=True) unchanged; busted on save via post_save receiver. - disabled in unit tests (compose overrides) so assertNumQueries counts stay deterministic; the cache itself is covered by unittests/test_caching.py. No new dependencies; no behavior change on non-cache paths. --- docker-compose.override.unit_tests.yml | 4 + docker-compose.override.unit_tests_cicd.yml | 5 + dojo/caching.py | 163 ++++++++++++++++++++ dojo/celery.py | 9 ++ dojo/middleware.py | 57 ++++++- dojo/settings/settings.dist.py | 9 ++ unittests/test_caching.py | 129 ++++++++++++++++ unittests/test_system_settings.py | 9 +- 8 files changed, 379 insertions(+), 6 deletions(-) create mode 100644 dojo/caching.py create mode 100644 unittests/test_caching.py diff --git a/docker-compose.override.unit_tests.yml b/docker-compose.override.unit_tests.yml index 565ff78d955..70a0668960b 100644 --- a/docker-compose.override.unit_tests.yml +++ b/docker-compose.override.unit_tests.yml @@ -22,6 +22,10 @@ services: DD_DATABASE_USER: ${DD_DATABASE_USER:-defectdojo} DD_DATABASE_PASSWORD: ${DD_DATABASE_PASSWORD:-defectdojo} DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite' + # Disable the singleton settings cache (dojo/caching.py) during tests so + # assertNumQueries counts stay deterministic. Covered by unittests/test_caching.py. + DD_SETTINGS_CACHE_L1_TTL: '-1' + DD_SETTINGS_CACHE_L2_TTL: '-1' DD_JIRA_EXTRA_ISSUE_TYPES: 'Vulnerability' # Shouldn't trigger a migration error celerybeat: !reset celeryworker: !reset diff --git a/docker-compose.override.unit_tests_cicd.yml b/docker-compose.override.unit_tests_cicd.yml index 01be14baa27..f233193f1cf 100644 --- a/docker-compose.override.unit_tests_cicd.yml +++ b/docker-compose.override.unit_tests_cicd.yml @@ -23,6 +23,11 @@ services: DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite' DD_JIRA_EXTRA_ISSUE_TYPES: 'Vulnerability' # Shouldn't trigger a migration error DD_V3_FEATURE_LOCATIONS: ${DD_V3_FEATURE_LOCATIONS:-False} + # Disable the singleton settings cache (dojo/caching.py) during tests so + # assertNumQueries counts stay deterministic (it dedupes System_Settings reads + # across requests/tests). The cache is covered by unittests/test_caching.py. + DD_SETTINGS_CACHE_L1_TTL: '-1' + DD_SETTINGS_CACHE_L2_TTL: '-1' celerybeat: !reset celeryworker: !reset initializer: !reset diff --git a/dojo/caching.py b/dojo/caching.py new file mode 100644 index 00000000000..16d234a3eb4 --- /dev/null +++ b/dojo/caching.py @@ -0,0 +1,163 @@ +""" +Two-tier read-through cache for global, low-cardinality singleton config. + +One in-process **L1** tier sits on top of the shared **L2** tier +(``django.core.cache``, Redis in deployments). A getter is resolved L1 → L2 → DB: +the first tier with a value wins; a ``None`` anywhere means "not cached, compute +it" and is never stored. This is deliberately simple — it is only for global, +user-INDEPENDENT, signal-invalidated singletons (feature flags, system settings, +and the like), never per-user or per-object data. + +Values are stored as plain dicts/scalars (see ``model_to_cache_dict`` / +``cache_dict_to_model``), never pickled model instances. + +Two independent tiers, each turned off by setting its TTL to ``-1`` (both off makes +the decorator a pass-through). Configuration (Django settings, wired from env in +``settings.dist.py``): + +* ``SETTINGS_CACHE_L1_TTL`` — per-thread in-process freshness budget in seconds + (``-1`` disables L1). Keep it short — it bounds cross-process staleness. L1 is + reset at each request/task boundary, so it is effectively request/task scoped. +* ``SETTINGS_CACHE_L2_TTL`` — L2 timeout in seconds (``-1`` disables L2). +""" + +import threading +import time +from functools import wraps + +from django.conf import settings +from django.core.cache import cache + + +class _L1Store: + + """ + Per-thread in-process store, TTL-stamped. ``get`` returns the value or ``None`` + (absent, expired, or L1 disabled via ``SETTINGS_CACHE_L1_TTL`` < 0). + + Per-thread (not shared across threads) so it needs no locking, and is reset at + each request/task boundary (see ``reset``) — making it effectively request/task + scoped on a reused worker/uwsgi thread. + """ + + def __init__(self): + self._local = threading.local() + + def _bucket(self): + bucket = getattr(self._local, "b", None) + if bucket is None: + bucket = self._local.b = {} + return bucket + + def get(self, key): + if getattr(settings, "SETTINGS_CACHE_L1_TTL", 30) < 0: + return None + entry = self._bucket().get(key) + if entry is None: + return None + value, expiry = entry + if time.monotonic() >= expiry: + self._bucket().pop(key, None) + return None + return value + + def set(self, key, value): + ttl = getattr(settings, "SETTINGS_CACHE_L1_TTL", 30) + if ttl < 0: + return + self._bucket()[key] = (value, time.monotonic() + ttl) + + def invalidate(self, key): + # Only this thread; other threads/processes self-heal within the L1 TTL + # (and reset at their next request/task boundary). + self._bucket().pop(key, None) + + def reset(self): + # Clear THIS thread's L1 bucket. Called at request/task boundaries so a + # reused worker/uwsgi thread never serves a value cached during a prior + # request or task (e.g. a since-changed System_Settings). + self._bucket().clear() + + def clear(self): + # Test helper: drop this thread's entries. + self._local = threading.local() + + +_L1_STORE = _L1Store() + + +def dojo_settings_cache(*, key: str): + """ + Read-through L1+L2 cache for a fixed-key singleton getter. + + Resolves L1 → L2 → wrapped function. A ``None`` result is treated as "no + value" and is not cached (so the next call retries). Becomes a pass-through + when both tiers are disabled (``SETTINGS_CACHE_L1_TTL=-1`` and + ``SETTINGS_CACHE_L2_TTL=-1``). + """ + + def decorator(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + l2_ttl = getattr(settings, "SETTINGS_CACHE_L2_TTL", 300) + l2_on = l2_ttl >= 0 # SETTINGS_CACHE_L2_TTL == -1 disables L2 + + value = _L1_STORE.get(key) # ---- L1 ---- + if value is not None: + return value + + if l2_on: # ---- L2 ---- + value = cache.get(key) + if value is not None: + _L1_STORE.set(key, value) + return value + + value = fn(*args, **kwargs) # ---- miss: compute ---- + if value is not None: + if l2_on: + cache.set(key, value, timeout=l2_ttl) + _L1_STORE.set(key, value) + return value + + return wrapper + + return decorator + + +def invalidate_dojo_settings_cache(key: str) -> None: + """Drop a cached singleton from L2 (all processes) and L1 (this process).""" + cache.delete(key) + _L1_STORE.invalidate(key) + + +def reset_l1_cache() -> None: + """ + Reset the current thread's L1 tier. + + Call at request/task boundaries (reused worker/uwsgi threads) so the + in-process L1 is effectively request/task-scoped and never serves a value + cached during a prior request or task. + """ + _L1_STORE.reset() + + +def model_to_cache_dict(instance) -> dict: + """ + Flatten a model instance to a plain dict of concrete field values. + + Keyed by ``attname`` (so a relation ``foo`` becomes ``foo_id``) and includes + the primary key. M2M and reverse relations are skipped. Storing this instead + of the model instance keeps the cache free of pickled model graphs; rebuild a + live instance with ``cache_dict_to_model``. + """ + return {f.attname: f.value_from_object(instance) for f in instance._meta.concrete_fields} + + +def cache_dict_to_model(model_cls, data: dict): + """ + Rebuild an in-memory model instance from a ``model_to_cache_dict`` dict. + + For read-only use; callers that persist changes must fetch a fresh DB instance + rather than saving a cache-derived one. + """ + return model_cls(**data) diff --git a/dojo/celery.py b/dojo/celery.py index 81dd44095d3..da9aa63619b 100644 --- a/dojo/celery.py +++ b/dojo/celery.py @@ -28,6 +28,12 @@ def __call__(self, *args, **kwargs): """ Restore user context in the celery worker via crum.impersonate. + Also resets the request/task-scoped L1 settings cache at the start of every + task (including eager): a prefork worker reuses its thread across tasks, so an + L1 value cached during a prior task (e.g. System_Settings) would otherwise be + served stale even after another process changed and saved it. The shared L2 + tier is left intact, so a reset task re-reads each singleton once from L2. + The apply_async method injects ``async_user_id`` into kwargs when a task is dispatched. Here we pop it, resolve to a user instance, and set it as the current user in thread-local storage so that all downstream @@ -39,6 +45,9 @@ def __call__(self, *args, **kwargs): intact so that callers who already set a user (e.g. via crum.impersonate in tests or request middleware) are not disrupted. """ + from dojo.caching import reset_l1_cache # noqa: PLC0415 + reset_l1_cache() + if "async_user_id" not in kwargs: return super().__call__(*args, **kwargs) diff --git a/dojo/middleware.py b/dojo/middleware.py index a576244312a..ee608231ca3 100644 --- a/dojo/middleware.py +++ b/dojo/middleware.py @@ -8,16 +8,59 @@ import pghistory.middleware from django.conf import settings from django.db import models +from django.dispatch import receiver from django.http import HttpResponseRedirect from django.urls import reverse from watson.middleware import SearchContextMiddleware from watson.search import search_context_manager +from dojo.caching import ( + cache_dict_to_model, + dojo_settings_cache, + invalidate_dojo_settings_cache, + model_to_cache_dict, + reset_l1_cache, +) from dojo.models import Dojo_User from dojo.product_announcements import LongRunningRequestProductAnnouncement logger = logging.getLogger(__name__) +# Two-tier (in-process L1 + django.core.cache L2) read-through cache for the +# System_Settings singleton, via the shared dojo_settings_cache decorator. Stored +# as a plain dict (no pickled model graph) and rebuilt into a read-only instance +# per call. Write paths use ``objects.get(no_cache=True)`` and are unaffected. +SYSTEM_SETTINGS_CACHE_KEY = "dojo.system_settings.singleton" + + +@dojo_settings_cache(key=SYSTEM_SETTINGS_CACHE_KEY) +def _cached_system_settings_dict(): + from dojo.models import System_Settings # noqa: PLC0415 circular import + settings_obj = System_Settings.objects.get_from_db() + # ``get_from_db`` returns an unsaved defaults instance (pk None) when the row + # can't be read; returning None keeps that out of the cache so the next call + # retries the DB instead of serving stale defaults. + if settings_obj.pk is None: + return None + return model_to_cache_dict(settings_obj) + + +def get_cached_system_settings(): + from dojo.models import System_Settings # noqa: PLC0415 circular import + data = _cached_system_settings_dict() + if not isinstance(data, dict): + return System_Settings() + return cache_dict_to_model(System_Settings, data) + + +@receiver(models.signals.post_save, sender="dojo.System_Settings") +def _invalidate_system_settings_cache(*args, **kwargs): + # Connected at import time (string sender avoids the circular import) so the + # bust fires in requests, Celery, commands and tests -- not only when a + # middleware instance is constructed. + invalidate_dojo_settings_cache(SYSTEM_SETTINGS_CACHE_KEY) + + EXEMPT_URLS = [re.compile(settings.LOGIN_URL.lstrip("/"))] if hasattr(settings, "LOGIN_EXEMPT_URLS"): EXEMPT_URLS += [re.compile(expr) for expr in settings.LOGIN_EXEMPT_URLS] @@ -84,6 +127,10 @@ def __init__(self, get_response): models.signals.post_save.connect(DojoSytemSettingsMiddleware.cleanup, sender=System_Settings) def __call__(self, request): + # uwsgi/gunicorn reuse threads across requests, so the in-process L1 cache + # (threading.local) would otherwise persist between requests. Reset it at + # the start of each request so cached singletons are request-scoped. + reset_l1_cache() self.load() try: # Store error in request for context processor to display @@ -117,8 +164,9 @@ def cleanup(cls, *args, **kwargs): # noqa: ARG003 def load(cls): # cleanup any existing settings first to ensure fresh state cls.cleanup() - from dojo.models import System_Settings # noqa: PLC0415 circular import - system_settings = System_Settings.objects.get(no_cache=True) + # Read through the shared L1/L2 cache so requests don't each hit the DB + # for the singleton. Freshness is bounded by the L1 TTL and busted on save. + system_settings = get_cached_system_settings() cls._thread_local.system_settings = system_settings return system_settings @@ -151,8 +199,9 @@ def get(self, no_cache=False, *args, **kwargs): # noqa: FBT002 - this is bit ha from_cache = DojoSytemSettingsMiddleware.get_system_settings() if not from_cache: - # logger.debug('no cached value found, loading system settings from db') - return self.get_from_db(*args, **kwargs) + # No per-request thread-local (e.g. outside a request / in Celery): + # fall back to the shared L1/L2 cache. + return get_cached_system_settings() return from_cache diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index 2d92588ad70..6ad153b279e 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -266,6 +266,11 @@ DD_V3_FEATURE_LOCATIONS=(bool, True), # Dictates if v3 org/asset relabeling (+url routing) will be enabled (on by default as of 3.0.0; set to False to restore Product/Product Type labels and URLs) DD_ENABLE_V3_ORGANIZATION_ASSET_RELABEL=(bool, True), + # Two-tier read-through cache for global singleton getters (see dojo/caching.py). + # Per-thread in-process L1 freshness budget in seconds; -1 disables L1. + DD_SETTINGS_CACHE_L1_TTL=(int, 30), + # L2 (django.core.cache) timeout in seconds; -1 disables L2. + DD_SETTINGS_CACHE_L2_TTL=(int, 300), # Notification env-vars (SLA notify, alert refresh/counter/cap, system-level trump). Defined in dojo.notifications.settings. **NOTIFICATIONS_ENV_DEFAULTS, ) @@ -314,6 +319,10 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param # Raises django's ImproperlyConfigured exception if SECRET_KEY not in os.environ SECRET_KEY = env("DD_SECRET_KEY") +# Two-tier singleton cache (dojo/caching.py) +SETTINGS_CACHE_L1_TTL = env("DD_SETTINGS_CACHE_L1_TTL") +SETTINGS_CACHE_L2_TTL = env("DD_SETTINGS_CACHE_L2_TTL") + # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name # although not all choices may be available on all operating systems. diff --git a/unittests/test_caching.py b/unittests/test_caching.py new file mode 100644 index 00000000000..84dcd380b47 --- /dev/null +++ b/unittests/test_caching.py @@ -0,0 +1,129 @@ +from unittest import mock + +from django.test import TestCase, override_settings + +from dojo import caching +from dojo.caching import ( + _L1_STORE, # noqa: PLC2701 test needs the in-process store + cache_dict_to_model, + dojo_settings_cache, + invalidate_dojo_settings_cache, + model_to_cache_dict, +) +from dojo.models import System_Settings + +from .dojo_test_case import DojoTestCase + + +class _FakeCache: + + """Dict-backed stand-in for ``django.core.cache.cache`` that counts access.""" + + def __init__(self): + self.store = {} + self.gets = 0 + self.sets = 0 + + def get(self, key, default=None): + self.gets += 1 + return self.store.get(key, default) + + def set(self, key, value, timeout=None): + self.sets += 1 + self.store[key] = value + + def delete(self, key): + self.store.pop(key, None) + + +@override_settings(SETTINGS_CACHE_L2_TTL=300, SETTINGS_CACHE_L1_TTL=30) +class DojoSettingsCacheTest(TestCase): + + """Unit tests for the simplified L1+L2 read-through decorator (dojo/caching.py).""" + + def setUp(self): + self.fake = _FakeCache() + self._patch = mock.patch.object(caching, "cache", self.fake) + self._patch.start() + _L1_STORE.clear() + + def tearDown(self): + self._patch.stop() + _L1_STORE.clear() + + def _build_getter(self, *, key="k", returns=1): + calls = {"n": 0} + + @dojo_settings_cache(key=key) + def getter(): + calls["n"] += 1 + return returns + + return getter, calls + + @override_settings(SETTINGS_CACHE_L1_TTL=-1) + def test_l1_disabled_consults_l2_every_call(self): + getter, calls = self._build_getter() + self.assertEqual(getter(), 1) + self.assertEqual(getter(), 1) # served from L2, fn not re-run + self.assertEqual(calls["n"], 1) # computed once + self.assertEqual(self.fake.gets, 2) # L2 consulted every call + + def test_l1_serves_without_l2(self): + getter, calls = self._build_getter() + getter() + getter() + getter() + self.assertEqual(calls["n"], 1) + self.assertEqual(self.fake.gets, 1) # only the first call reaches L2 + + @override_settings(SETTINGS_CACHE_L2_TTL=-1) + def test_l2_ttl_negative_disables_l2(self): + getter, calls = self._build_getter() + getter() + getter() + self.assertEqual(self.fake.gets, 0) # L2 never consulted + self.assertEqual(self.fake.sets, 0) + self.assertEqual(calls["n"], 1) # but L1 still memoizes in-process + + @override_settings(SETTINGS_CACHE_L1_TTL=-1, SETTINGS_CACHE_L2_TTL=-1) + def test_both_tiers_disabled_is_passthrough(self): + getter, calls = self._build_getter() + getter() + getter() + self.assertEqual(calls["n"], 2) # recomputed every call + self.assertEqual(self.fake.gets, 0) # L2 never consulted + + def test_reset_clears_l1(self): + getter, calls = self._build_getter() + getter() + _L1_STORE.reset() + getter() + self.assertEqual(calls["n"], 1) # reset drops L1, but L2 still serves + self.assertEqual(self.fake.gets, 2) # 2nd call re-reads L2 after reset + + def test_none_result_is_not_cached(self): + getter, calls = self._build_getter(returns=None) + self.assertIsNone(getter()) + self.assertIsNone(getter()) + self.assertEqual(calls["n"], 2) # None recomputed each call (never stored) + self.assertEqual(self.fake.sets, 0) + + def test_invalidate_clears_l1_and_l2(self): + getter, calls = self._build_getter(returns=7) + self.assertEqual(getter(), 7) + invalidate_dojo_settings_cache("k") + getter() + self.assertEqual(calls["n"], 2) # recomputed after invalidation + + +class ModelDictRoundTripTest(DojoTestCase): + + def test_round_trip_preserves_field_values(self): + settings_obj = System_Settings.objects.get(no_cache=True) + data = model_to_cache_dict(settings_obj) + self.assertIsInstance(data, dict) + self.assertEqual(data["id"], settings_obj.pk) + rebuilt = cache_dict_to_model(System_Settings, data) + self.assertEqual(rebuilt.pk, settings_obj.pk) + self.assertEqual(rebuilt.enable_deduplication, settings_obj.enable_deduplication) diff --git a/unittests/test_system_settings.py b/unittests/test_system_settings.py index e127bec8f59..1d125c2db24 100644 --- a/unittests/test_system_settings.py +++ b/unittests/test_system_settings.py @@ -6,7 +6,8 @@ from django.urls import reverse from django.utils.timezone import now -from dojo.middleware import DojoSytemSettingsMiddleware +from dojo.caching import invalidate_dojo_settings_cache +from dojo.middleware import SYSTEM_SETTINGS_CACHE_KEY, DojoSytemSettingsMiddleware from dojo.models import ( Engagement, Finding, @@ -225,8 +226,9 @@ def get_response_with_cache_check(request): def test_multiple_get_calls_use_cache(self): """Test that multiple calls to System_Settings.objects.get() use cache instead of multiple DB queries.""" - # Ensure cache is empty + # Ensure both the per-request thread-local and the shared L1/L2 cache are empty DojoSytemSettingsMiddleware.cleanup() + invalidate_dojo_settings_cache(SYSTEM_SETTINGS_CACHE_KEY) # First call should hit DB (cache is empty) with self.assertNumQueries(1): @@ -265,6 +267,9 @@ def get_response_with_multiple_gets(request): # Process request - should only hit DB once (when loading cache) # Then all subsequent get() calls should use cache request = self.factory.get("/test/") + # Start cold so the request's load is a real DB read (the shared L1/L2 cache + # otherwise persists across tests and would serve it with 0 queries). + invalidate_dojo_settings_cache(SYSTEM_SETTINGS_CACHE_KEY) with self.assertNumQueries(1): # Only one query to load settings into cache middleware(request) From 15a4c4e984c3ef7f4856722751c20df347d6e5d5 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Wed, 24 Jun 2026 16:00:12 +0200 Subject: [PATCH 2/8] feat(cache): wire shared Redis cache backend for L2 (cross-process invalidation) The L1+L2 singleton cache (dojo/caching.py) needs its L2 tier to be a cross-process store so cache invalidation (e.g. on System_Settings save in uwsgi) propagates to celery workers. Without it Django defaults to per-process LocMemCache, so a worker keeps serving a stale System_Settings (e.g. enable_deduplication=False) and async dedup silently no-ops -> dedupe/ close_old_findings_dedupe/questionnaire integration tests fail. - Add DD_CACHE_URL env; when set, configure django.core.cache RedisCache. - Default DD_CACHE_URL=redis://valkey:6379/1 in compose (uwsgi/celery/beat). - Mount source into the integration celeryworker so it runs the same code as uwsgi (enables local reproduction of worker-side cache behavior). --- docker-compose.override.integration_tests.yml | 3 +++ docker-compose.yml | 3 +++ dojo/settings/settings.dist.py | 16 ++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/docker-compose.override.integration_tests.yml b/docker-compose.override.integration_tests.yml index 71085eedd68..3f2ae51df41 100644 --- a/docker-compose.override.integration_tests.yml +++ b/docker-compose.override.integration_tests.yml @@ -43,6 +43,9 @@ services: DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/test_defectdojo} celeryworker: entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-celery-worker-dev.sh'] + volumes: + - '.:/app:z' + - defectdojo_media_integration_tests:${DD_MEDIA_ROOT:-/app/media} environment: DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/test_defectdojo} DD_V3_FEATURE_LOCATIONS: ${DD_V3_FEATURE_LOCATIONS:-False} diff --git a/docker-compose.yml b/docker-compose.yml index bfd3ad50743..e749c671700 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,7 @@ services: DD_ALLOWED_HOSTS: "${DD_ALLOWED_HOSTS:-*}" DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo} DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0} + DD_CACHE_URL: ${DD_CACHE_URL:-redis://valkey:6379/1} DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" @@ -71,6 +72,7 @@ services: environment: DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo} DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0} + DD_CACHE_URL: ${DD_CACHE_URL:-redis://valkey:6379/1} DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" @@ -91,6 +93,7 @@ services: environment: DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo} DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0} + DD_CACHE_URL: ${DD_CACHE_URL:-redis://valkey:6379/1} DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index 6ad153b279e..a50edf22830 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -266,6 +266,11 @@ DD_V3_FEATURE_LOCATIONS=(bool, True), # Dictates if v3 org/asset relabeling (+url routing) will be enabled (on by default as of 3.0.0; set to False to restore Product/Product Type labels and URLs) DD_ENABLE_V3_ORGANIZATION_ASSET_RELABEL=(bool, True), + # Shared cache backend (django.core.cache). When set, used as the L2 tier for + # dojo/caching.py. MUST be a cross-process store (e.g. redis://valkey:6379/1) + # so cache invalidation propagates across uwsgi/celery processes; when empty + # Django falls back to per-process LocMemCache (single-process only). + DD_CACHE_URL=(str, ""), # Two-tier read-through cache for global singleton getters (see dojo/caching.py). # Per-thread in-process L1 freshness budget in seconds; -1 disables L1. DD_SETTINGS_CACHE_L1_TTL=(int, 30), @@ -319,6 +324,17 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param # Raises django's ImproperlyConfigured exception if SECRET_KEY not in os.environ SECRET_KEY = env("DD_SECRET_KEY") +# Shared cache backend. A cross-process store (Redis) is required for cache +# invalidation to propagate across uwsgi/celery; otherwise Django defaults to +# per-process LocMemCache (fine only single-process). +if env("DD_CACHE_URL"): + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": env("DD_CACHE_URL"), + }, + } + # Two-tier singleton cache (dojo/caching.py) SETTINGS_CACHE_L1_TTL = env("DD_SETTINGS_CACHE_L1_TTL") SETTINGS_CACHE_L2_TTL = env("DD_SETTINGS_CACHE_L2_TTL") From 5feb99b139f8268d3d70f2c9bc0afefd92068278 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Wed, 24 Jun 2026 16:14:42 +0200 Subject: [PATCH 3/8] fix(ci): drop integration celeryworker source mount (media volume race) Sharing defectdojo_media_integration_tests between uwsgi and celeryworker races on mkdir media/threat at startup (file exists), flaking the worker container. The mount was only a local-repro aid and is unnecessary in CI, where the worker runs the image built from PR source. Revert to the original worker definition; the Redis L2 fix is unaffected. --- docker-compose.override.integration_tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.override.integration_tests.yml b/docker-compose.override.integration_tests.yml index 3f2ae51df41..71085eedd68 100644 --- a/docker-compose.override.integration_tests.yml +++ b/docker-compose.override.integration_tests.yml @@ -43,9 +43,6 @@ services: DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/test_defectdojo} celeryworker: entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-celery-worker-dev.sh'] - volumes: - - '.:/app:z' - - defectdojo_media_integration_tests:${DD_MEDIA_ROOT:-/app/media} environment: DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/test_defectdojo} DD_V3_FEATURE_LOCATIONS: ${DD_V3_FEATURE_LOCATIONS:-False} From b6ed8f15a8cbbc15bc3e57a63936fbcb52179408 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Wed, 24 Jun 2026 16:33:56 +0200 Subject: [PATCH 4/8] fix(cache): don't hit L2 backend on invalidate when L2 disabled Unit tests run with no Redis (DD_SETTINGS_CACHE_L2_TTL=-1) but inherited DD_CACHE_URL from base compose, so System_Settings post_save invalidation called cache.delete against a nonexistent valkey -> rest-framework setUpClass ConnectionError. invalidate_dojo_settings_cache now skips cache.delete when L2 is disabled (consistent with the read path, which already skips L2). Also blank DD_CACHE_URL in the unit-test overrides so the backend is LocMemCache. Adds test_invalidate_skips_l2_when_disabled. --- docker-compose.override.unit_tests.yml | 2 ++ docker-compose.override.unit_tests_cicd.yml | 2 ++ dojo/caching.py | 6 +++++- unittests/test_caching.py | 9 +++++++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docker-compose.override.unit_tests.yml b/docker-compose.override.unit_tests.yml index 70a0668960b..127a92294cf 100644 --- a/docker-compose.override.unit_tests.yml +++ b/docker-compose.override.unit_tests.yml @@ -22,6 +22,8 @@ services: DD_DATABASE_USER: ${DD_DATABASE_USER:-defectdojo} DD_DATABASE_PASSWORD: ${DD_DATABASE_PASSWORD:-defectdojo} DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite' + # No shared cache backend in unit tests (no Redis/valkey here) -> LocMemCache. + DD_CACHE_URL: '' # Disable the singleton settings cache (dojo/caching.py) during tests so # assertNumQueries counts stay deterministic. Covered by unittests/test_caching.py. DD_SETTINGS_CACHE_L1_TTL: '-1' diff --git a/docker-compose.override.unit_tests_cicd.yml b/docker-compose.override.unit_tests_cicd.yml index f233193f1cf..ff96c808426 100644 --- a/docker-compose.override.unit_tests_cicd.yml +++ b/docker-compose.override.unit_tests_cicd.yml @@ -23,6 +23,8 @@ services: DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite' DD_JIRA_EXTRA_ISSUE_TYPES: 'Vulnerability' # Shouldn't trigger a migration error DD_V3_FEATURE_LOCATIONS: ${DD_V3_FEATURE_LOCATIONS:-False} + # No shared cache backend in unit tests (no Redis/valkey here) -> LocMemCache. + DD_CACHE_URL: '' # Disable the singleton settings cache (dojo/caching.py) during tests so # assertNumQueries counts stay deterministic (it dedupes System_Settings reads # across requests/tests). The cache is covered by unittests/test_caching.py. diff --git a/dojo/caching.py b/dojo/caching.py index 16d234a3eb4..259b1340fcb 100644 --- a/dojo/caching.py +++ b/dojo/caching.py @@ -126,7 +126,11 @@ def wrapper(*args, **kwargs): def invalidate_dojo_settings_cache(key: str) -> None: """Drop a cached singleton from L2 (all processes) and L1 (this process).""" - cache.delete(key) + # Only touch L2 when it is enabled; when SETTINGS_CACHE_L2_TTL == -1 the L2 + # tier is off and the backend (``cache``) may not even be reachable (e.g. + # unit tests run with no Redis configured), so skip the delete entirely. + if getattr(settings, "SETTINGS_CACHE_L2_TTL", 300) >= 0: + cache.delete(key) _L1_STORE.invalidate(key) diff --git a/unittests/test_caching.py b/unittests/test_caching.py index 84dcd380b47..f6080428e8c 100644 --- a/unittests/test_caching.py +++ b/unittests/test_caching.py @@ -23,6 +23,7 @@ def __init__(self): self.store = {} self.gets = 0 self.sets = 0 + self.deletes = 0 def get(self, key, default=None): self.gets += 1 @@ -33,6 +34,7 @@ def set(self, key, value, timeout=None): self.store[key] = value def delete(self, key): + self.deletes += 1 self.store.pop(key, None) @@ -116,6 +118,13 @@ def test_invalidate_clears_l1_and_l2(self): getter() self.assertEqual(calls["n"], 2) # recomputed after invalidation + @override_settings(SETTINGS_CACHE_L2_TTL=-1) + def test_invalidate_skips_l2_when_disabled(self): + # With L2 off the backend may be unreachable (no Redis in unit tests); + # invalidation must not call cache.delete. + invalidate_dojo_settings_cache("k") + self.assertEqual(self.fake.deletes, 0) + class ModelDictRoundTripTest(DojoTestCase): From cc63ddba7d64c6899dd3f5f9cf4bb3c6b3c36e7e Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Wed, 24 Jun 2026 18:14:55 +0200 Subject: [PATCH 5/8] refactor(cache): slim System_Settings middleware to L1-reset + error banner The decorator (dojo.caching) now owns all System_Settings caching, so the middleware no longer keeps its own per-request thread-local instance. It only: (a) resets the request-scoped L1 tier at request start, (b) surfaces a DB-read error for the banner. Renamed DojoSytemSettingsMiddleware -> DojoSettingsManagerMiddleware (it governs the shared settings cache lifecycle, not just System_Settings). Manager.get() reads straight through the cache. To keep assertNumQueries deterministic without the thread-local, unit tests now run with L1 ON (request/test-scoped) and L2 OFF instead of both disabled: L1 is reset per request (middleware) and per test (dojo_test_case setUp via the new refresh_system_settings_cache helper, replacing middleware.load()). This mirrors the old thread-local's per-request memoization. Rewrote the middleware tests for the new contract (cache via decorator, instance rebuilt per call, L1 reset, error banner). --- docker-compose.override.unit_tests.yml | 7 +- docker-compose.override.unit_tests_cicd.yml | 9 +- dojo/middleware.py | 76 ++----- dojo/settings/settings.dist.py | 2 +- unittests/dojo_test_case.py | 26 ++- unittests/test_system_settings.py | 228 +++++--------------- 6 files changed, 106 insertions(+), 242 deletions(-) diff --git a/docker-compose.override.unit_tests.yml b/docker-compose.override.unit_tests.yml index 127a92294cf..66a36a68b83 100644 --- a/docker-compose.override.unit_tests.yml +++ b/docker-compose.override.unit_tests.yml @@ -24,9 +24,10 @@ services: DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite' # No shared cache backend in unit tests (no Redis/valkey here) -> LocMemCache. DD_CACHE_URL: '' - # Disable the singleton settings cache (dojo/caching.py) during tests so - # assertNumQueries counts stay deterministic. Covered by unittests/test_caching.py. - DD_SETTINGS_CACHE_L1_TTL: '-1' + # L1 (in-process, request/test-scoped) stays ON for deterministic + # assertNumQueries counts; reset per request (middleware) and per test + # (dojo_test_case setUp). L2 is OFF (no shared backend; would persist). + DD_SETTINGS_CACHE_L1_TTL: '30' DD_SETTINGS_CACHE_L2_TTL: '-1' DD_JIRA_EXTRA_ISSUE_TYPES: 'Vulnerability' # Shouldn't trigger a migration error celerybeat: !reset diff --git a/docker-compose.override.unit_tests_cicd.yml b/docker-compose.override.unit_tests_cicd.yml index ff96c808426..d9711143a3f 100644 --- a/docker-compose.override.unit_tests_cicd.yml +++ b/docker-compose.override.unit_tests_cicd.yml @@ -25,10 +25,11 @@ services: DD_V3_FEATURE_LOCATIONS: ${DD_V3_FEATURE_LOCATIONS:-False} # No shared cache backend in unit tests (no Redis/valkey here) -> LocMemCache. DD_CACHE_URL: '' - # Disable the singleton settings cache (dojo/caching.py) during tests so - # assertNumQueries counts stay deterministic (it dedupes System_Settings reads - # across requests/tests). The cache is covered by unittests/test_caching.py. - DD_SETTINGS_CACHE_L1_TTL: '-1' + # L1 (in-process, request/test-scoped) stays ON so a singleton is read once + # per request/test and assertNumQueries counts are deterministic; it is reset + # per request (middleware) and per test (dojo_test_case setUp). L2 is OFF + # (no shared backend here, and it would persist across tests/processes). + DD_SETTINGS_CACHE_L1_TTL: '30' DD_SETTINGS_CACHE_L2_TTL: '-1' celerybeat: !reset celeryworker: !reset diff --git a/dojo/middleware.py b/dojo/middleware.py index ee608231ca3..5516b672b18 100644 --- a/dojo/middleware.py +++ b/dojo/middleware.py @@ -117,58 +117,31 @@ def __call__(self, request): return response -class DojoSytemSettingsMiddleware: +class DojoSettingsManagerMiddleware: + # Caching of the System_Settings singleton lives in dojo.caching (L1+L2). This + # middleware only (a) resets the request-scoped L1 tier and (b) surfaces a + # System_Settings DB-read error as a banner. The thread-local carries just that + # error message (set by System_Settings_Manager.get_from_db). _thread_local = local() def __init__(self, get_response): self.get_response = get_response - from dojo.models import System_Settings # noqa: PLC0415 circular import - # Use classmethod directly to avoid keeping reference to middleware instance - models.signals.post_save.connect(DojoSytemSettingsMiddleware.cleanup, sender=System_Settings) def __call__(self, request): # uwsgi/gunicorn reuse threads across requests, so the in-process L1 cache # (threading.local) would otherwise persist between requests. Reset it at # the start of each request so cached singletons are request-scoped. reset_l1_cache() - self.load() - try: - # Store error in request for context processor to display - # (We can't use messages here because MessageMiddleware runs after this middleware) - if hasattr(self._thread_local, "system_settings_error"): - request.system_settings_error = self._thread_local.system_settings_error - # Clear from thread-local after copying to request - delattr(self._thread_local, "system_settings_error") - return self.get_response(request) - finally: - # ensure cleanup happens even if an exception occurs - self.cleanup() - - def process_exception(self, request, exception): - self.cleanup() - - @classmethod - def get_system_settings(cls): - if hasattr(cls._thread_local, "system_settings"): - return cls._thread_local.system_settings - return None - - @classmethod - def cleanup(cls, *args, **kwargs): # noqa: ARG003 - if hasattr(cls._thread_local, "system_settings"): - del cls._thread_local.system_settings - if hasattr(cls._thread_local, "system_settings_error"): - delattr(cls._thread_local, "system_settings_error") - - @classmethod - def load(cls): - # cleanup any existing settings first to ensure fresh state - cls.cleanup() - # Read through the shared L1/L2 cache so requests don't each hit the DB - # for the singleton. Freshness is bounded by the L1 TTL and busted on save. - system_settings = get_cached_system_settings() - cls._thread_local.system_settings = system_settings - return system_settings + # Drop any error left on this reused thread by a previous request. + if hasattr(self._thread_local, "system_settings_error"): + delattr(self._thread_local, "system_settings_error") + # Warm the cache once; this also captures any DB-read error (via + # get_from_db) so the context processor can display it as a banner. + # (We can't use messages here because MessageMiddleware runs after this.) + get_cached_system_settings() + if hasattr(self._thread_local, "system_settings_error"): + request.system_settings_error = self._thread_local.system_settings_error + return self.get_response(request) class System_Settings_Manager(models.Manager): @@ -181,11 +154,11 @@ def get_from_db(self, *args, **kwargs): except Exception as e: # Store error message in thread-local for middleware to display error_msg = str(e) - if hasattr(DojoSytemSettingsMiddleware._thread_local, "system_settings_error"): + if hasattr(DojoSettingsManagerMiddleware._thread_local, "system_settings_error"): # Only store the first error to avoid duplicates pass else: - DojoSytemSettingsMiddleware._thread_local.system_settings_error = error_msg + DojoSettingsManagerMiddleware._thread_local.system_settings_error = error_msg # Return defaults so app can still start - error will be displayed as warning message # logger.debug('unable to get system_settings from database, returning defaults. Exception was:', exc_info=True) return System_Settings() @@ -193,17 +166,12 @@ def get_from_db(self, *args, **kwargs): def get(self, no_cache=False, *args, **kwargs): # noqa: FBT002 - this is bit hard to fix nice have this universally fixed if no_cache: - # logger.debug('no_cache specified or cached value found, loading system settings from db') + # logger.debug('no_cache specified, loading system settings from db') return self.get_from_db(*args, **kwargs) - - from_cache = DojoSytemSettingsMiddleware.get_system_settings() - - if not from_cache: - # No per-request thread-local (e.g. outside a request / in Celery): - # fall back to the shared L1/L2 cache. - return get_cached_system_settings() - - return from_cache + # Read through the shared L1/L2 cache (dojo.caching). L1 is request/task + # scoped (reset by the middleware and the Celery task base), so repeated + # reads within a request are served in-process. + return get_cached_system_settings() class APITrailingSlashMiddleware: diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index a50edf22830..8e5b088d5a7 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -816,7 +816,7 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param DJANGO_MIDDLEWARE_CLASSES = [ "django.middleware.common.CommonMiddleware", "dojo.middleware.APITrailingSlashMiddleware", - "dojo.middleware.DojoSytemSettingsMiddleware", + "dojo.middleware.DojoSettingsManagerMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.middleware.security.SecurityMiddleware", diff --git a/unittests/dojo_test_case.py b/unittests/dojo_test_case.py index 9f8cf55f89f..9e4db142715 100644 --- a/unittests/dojo_test_case.py +++ b/unittests/dojo_test_case.py @@ -16,12 +16,13 @@ from rest_framework.test import APIClient, APITestCase from vcr_unittest import VCRTestCase +from dojo.caching import invalidate_dojo_settings_cache from dojo.importers.location_manager import LocationManager from dojo.jira import helper as jira_helper from dojo.jira.views import get_custom_field from dojo.location.models import Location, LocationFindingReference from dojo.location.status import FindingLocationStatus -from dojo.middleware import DojoSytemSettingsMiddleware +from dojo.middleware import SYSTEM_SETTINGS_CACHE_KEY, get_cached_system_settings from dojo.models import ( SEVERITIES, DojoMeta, @@ -44,6 +45,17 @@ logger = logging.getLogger(__name__) +def refresh_system_settings_cache(): + """ + Make the next ``System_Settings.objects.get()`` reflect DB changes made in a + test. Tests mutate settings via ``.update()`` (which fires no post_save) so + the L1/L2 cache isn't auto-busted; drop it and re-warm so a subsequent cached + read is served in-process. Replaces the old middleware ``load()``. + """ + invalidate_dojo_settings_cache(SYSTEM_SETTINGS_CACHE_KEY) + get_cached_system_settings() + + def get_unit_tests_path(): return Path(__file__).parent @@ -61,14 +73,14 @@ def wrapper(*args, **kwargs): # Set the flag to the specified value System_Settings.objects.update(**{flag_name: value}) # Reinitialize middleware with updated settings as this doesn't happen automatically during django tests - DojoSytemSettingsMiddleware.load() + refresh_system_settings_cache() try: return test_func(*args, **kwargs) finally: # Reset the flag to its original state after the test System_Settings.objects.update(**{flag_name: not value}) # Reinitialize middleware with updated settings as this doesn't happen automatically during django tests - DojoSytemSettingsMiddleware.load() + refresh_system_settings_cache() return wrapper return decorator @@ -84,14 +96,14 @@ def wrapper(*args, **kwargs): # Set the flag to the specified value System_Settings.objects.update(**{field: value}) # Reinitialize middleware with updated settings as this doesn't happen automatically during django tests - DojoSytemSettingsMiddleware.load() + refresh_system_settings_cache() try: return test_func(*args, **kwargs) finally: # Reset the flag to its original state after the test System_Settings.objects.update(**{field: old_value}) # Reinitialize middleware with updated settings as this doesn't happen automatically during django tests - DojoSytemSettingsMiddleware.load() + refresh_system_settings_cache() return wrapper @@ -131,7 +143,7 @@ def system_settings(self, **kwargs): setattr(ss, key, value) ss.save() # Refresh the cache with the new settings - DojoSytemSettingsMiddleware.load() + refresh_system_settings_cache() def create_product_type(self, name, *args, description="dummy description", **kwargs): product_type = Product_Type(name=name, description=description) @@ -566,7 +578,7 @@ def __init__(self, *args, **kwargs): def setUp(self): super().setUp() # Initialize middleware with fresh settings from db - DojoSytemSettingsMiddleware.load() + refresh_system_settings_cache() def common_check_finding(self, finding): self.assertIn(finding.severity, SEVERITIES) diff --git a/unittests/test_system_settings.py b/unittests/test_system_settings.py index 1d125c2db24..6d8ba7b2793 100644 --- a/unittests/test_system_settings.py +++ b/unittests/test_system_settings.py @@ -1,13 +1,12 @@ -from unittest.mock import Mock +from unittest import mock -from django.db import models from django.http import HttpResponse from django.test import RequestFactory, TestCase, override_settings from django.urls import reverse from django.utils.timezone import now -from dojo.caching import invalidate_dojo_settings_cache -from dojo.middleware import SYSTEM_SETTINGS_CACHE_KEY, DojoSytemSettingsMiddleware +from dojo.caching import invalidate_dojo_settings_cache, reset_l1_cache +from dojo.middleware import SYSTEM_SETTINGS_CACHE_KEY, DojoSettingsManagerMiddleware, get_cached_system_settings from dojo.models import ( Engagement, Finding, @@ -94,189 +93,72 @@ def test_post_request_initializes_form_with_finding_instance(self): self.assertIn(response.status_code, [200, 302]) +@override_settings(SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) class TestSystemSettingsMiddlewareIntegration(DojoTestCase): - """Integration tests for DojoSytemSettingsMiddleware using RequestFactory.""" + """ + Integration tests for DojoSettingsManagerMiddleware + System_Settings_Manager. + + Caching lives in dojo.caching (decorator); the middleware only resets the + request-scoped L1 tier and surfaces a load error. These tests pin L1 on / L2 + off via override_settings so they don't depend on the compose env. + """ def setUp(self): - """Set up test environment.""" super().setUp() self.factory = RequestFactory() - # Ensure signal is connected - models.signals.post_save.disconnect(DojoSytemSettingsMiddleware.cleanup, sender=System_Settings) - models.signals.post_save.connect(DojoSytemSettingsMiddleware.cleanup, sender=System_Settings) - - def test_middleware_loads_cache_on_request(self): - """Test that middleware loads settings into cache when processing a request.""" - # Ensure cache is empty - DojoSytemSettingsMiddleware.cleanup() - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - # Create middleware with mock get_response - mock_response = HttpResponse("OK") - mock_get_response = Mock(return_value=mock_response) - middleware = DojoSytemSettingsMiddleware(mock_get_response) - - # Create a request - request = self.factory.get("/test/") - - # Process request through middleware - response = middleware(request) - - # Verify response is returned - self.assertEqual(response, mock_response) - mock_get_response.assert_called_once_with(request) - - # Verify cache was populated during request processing - # Note: cache should be cleaned up after request, but we can check during processing - # Since cleanup happens in finally block, cache should be empty after __call__ returns - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - def test_middleware_cleans_up_cache_after_request(self): - """Test that middleware cleans up cache after request processing.""" - # Manually load cache first - DojoSytemSettingsMiddleware.load() - self.assertIsNotNone(DojoSytemSettingsMiddleware.get_system_settings()) - - # Create middleware - middleware = DojoSytemSettingsMiddleware(lambda _r: HttpResponse("OK")) - - # Process request - request = self.factory.get("/test/") - middleware(request) - - # Verify cache is cleaned up after request - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - def test_middleware_cleans_up_cache_on_exception(self): - """Test that middleware cleans up cache even when exception occurs.""" - # Load cache first - DojoSytemSettingsMiddleware.load() - self.assertIsNotNone(DojoSytemSettingsMiddleware.get_system_settings()) - - # Create middleware that raises an exception - def failing_get_response(request): - msg = "Test exception" - raise ValueError(msg) - - middleware = DojoSytemSettingsMiddleware(failing_get_response) - - # Process request - should raise exception - request = self.factory.get("/test/") - with self.assertRaises(ValueError): - middleware(request) - - # Verify cache is cleaned up even after exception - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - def test_middleware_process_exception_cleans_up_cache(self): - """Test that process_exception method cleans up cache.""" - # Load cache first - DojoSytemSettingsMiddleware.load() - self.assertIsNotNone(DojoSytemSettingsMiddleware.get_system_settings()) - - # Create middleware - middleware = DojoSytemSettingsMiddleware(lambda _r: HttpResponse("OK")) - - # Call process_exception directly - request = self.factory.get("/test/") - exception = ValueError("Test exception") - middleware.process_exception(request, exception) - # Verify cache is cleaned up - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) + def test_no_cache_always_hits_db(self): + # no_cache bypasses both tiers: every read is a fresh query. + with self.assertNumQueries(2): + System_Settings.objects.get(no_cache=True) + System_Settings.objects.get(no_cache=True) - def test_middleware_cache_isolation_between_requests(self): - """Test that cache is isolated between requests (thread-local).""" - # Create middleware - middleware = DojoSytemSettingsMiddleware(lambda _r: HttpResponse("OK")) - - # First request - request1 = self.factory.get("/test1/") - middleware(request1) - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - # Second request - cache should be empty at start - request2 = self.factory.get("/test2/") - middleware(request2) - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - def test_middleware_cache_during_request_processing(self): - """Test that cache is available during request processing.""" - # Track if cache was available during request - cache_available_during_request = [] - - def get_response_with_cache_check(request): - # Check if cache is available during request processing - cached = DojoSytemSettingsMiddleware.get_system_settings() - cache_available_during_request.append(cached is not None) - return HttpResponse("OK") - - middleware = DojoSytemSettingsMiddleware(get_response_with_cache_check) - - # Process request - request = self.factory.get("/test/") - middleware(request) - - # Verify cache was available during request processing - self.assertTrue(cache_available_during_request[0], "Cache should be available during request processing") - - # But cleaned up after request - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - def test_multiple_get_calls_use_cache(self): - """Test that multiple calls to System_Settings.objects.get() use cache instead of multiple DB queries.""" - # Ensure both the per-request thread-local and the shared L1/L2 cache are empty - DojoSytemSettingsMiddleware.cleanup() + def test_repeated_cached_get_served_from_l1(self): + # Cold start, warm once (1 query), then repeated cached reads do no queries. + reset_l1_cache() invalidate_dojo_settings_cache(SYSTEM_SETTINGS_CACHE_KEY) - - # First call should hit DB (cache is empty) with self.assertNumQueries(1): - settings1 = System_Settings.objects.get() - - # Load into cache via middleware - DojoSytemSettingsMiddleware.load() - - # Now multiple calls should use cache (no additional DB queries) + get_cached_system_settings() with self.assertNumQueries(0): - settings2 = System_Settings.objects.get() - settings3 = System_Settings.objects.get() - settings4 = System_Settings.objects.get() - - # All calls should return the same cached object instance - self.assertEqual(settings1.id, settings2.id) - self.assertEqual(settings2.id, settings3.id) - self.assertEqual(settings3.id, settings4.id) - # Verify they're the same object instance (same memory address) - self.assertIs(settings2, settings3) - self.assertIs(settings3, settings4) - - def test_multiple_get_calls_within_request_use_cache(self): - """Test that multiple get() calls within a single request use cache.""" - retrieved_settings = [] - - def get_response_with_multiple_gets(request): - # Make multiple calls to get() during request processing - retrieved_settings.append(System_Settings.objects.get()) - retrieved_settings.append(System_Settings.objects.get()) - retrieved_settings.append(System_Settings.objects.get()) + s2 = System_Settings.objects.get() + s3 = System_Settings.objects.get() + # Rebuilt per call from the cached dict: equal data, not the same instance. + self.assertEqual(s2.pk, s3.pk) + + def test_middleware_resets_l1_each_request(self): + # Warm L1, change the row underneath via update() (no post_save signal, so + # L1 is not auto-busted), then a request must still see the new value + # because the middleware resets L1 at request start. + get_cached_system_settings() + System_Settings.objects.update(enable_deduplication=True) + seen = [] + + def view(_request): + seen.append(System_Settings.objects.get().enable_deduplication) return HttpResponse("OK") - middleware = DojoSytemSettingsMiddleware(get_response_with_multiple_gets) + middleware = DojoSettingsManagerMiddleware(view) + middleware(self.factory.get("/test/")) + self.assertEqual(seen, [True]) - # Process request - should only hit DB once (when loading cache) - # Then all subsequent get() calls should use cache - request = self.factory.get("/test/") - # Start cold so the request's load is a real DB read (the shared L1/L2 cache - # otherwise persists across tests and would serve it with 0 queries). + def test_middleware_surfaces_load_error(self): + # When the DB read fails, get_from_db stashes an error on the thread-local + # and returns defaults; the middleware copies the error onto the request + # for the banner context processor. invalidate_dojo_settings_cache(SYSTEM_SETTINGS_CACHE_KEY) - with self.assertNumQueries(1): # Only one query to load settings into cache - middleware(request) + reset_l1_cache() + captured = {} + + def view(request): + captured["err"] = getattr(request, "system_settings_error", None) + return HttpResponse("OK") - # Verify we got 3 settings objects - self.assertEqual(len(retrieved_settings), 3) + def failing_get_from_db(*args, **kwargs): + DojoSettingsManagerMiddleware._thread_local.system_settings_error = "boom" + return System_Settings() # defaults (pk None) -> not cached - # All should be the same cached instance - self.assertIs(retrieved_settings[0], retrieved_settings[1]) - self.assertIs(retrieved_settings[1], retrieved_settings[2]) - self.assertEqual(retrieved_settings[0].id, retrieved_settings[1].id) + middleware = DojoSettingsManagerMiddleware(view) + with mock.patch.object(System_Settings.objects, "get_from_db", side_effect=failing_get_from_db): + middleware(self.factory.get("/test/")) + self.assertEqual(captured["err"], "boom") From 73705c69b155fe411e8d4268f921a64f733f5f15 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Wed, 24 Jun 2026 19:44:05 +0200 Subject: [PATCH 6/8] test: rebaseline query counts for per-task L1 cache reset The L1 settings cache is reset per celery task (DojoAsyncTask), so eager-mode tests now faithfully mirror real async workers: each task re-reads System_Settings (from L2) instead of sharing one request-wide read as the old middleware thread-local did. assertNumQueries baselines shift accordingly: - Single cached reads now save a query (metrics 27->25/41->40, fp_history 7->6). - Bulk create/import where many per-finding tasks run inline read once per task (tag_inheritance create_100 3124->3224; zap import/reimport +3-4; importer performance +2-8). In production these run in workers, off the request path. Counts verified locally against the CI unit-test cache config (L1 on, L2 off). --- .../test_false_positive_history_logic.py | 10 ++--- unittests/test_importers_performance.py | 44 +++++++++---------- unittests/test_metrics_queries.py | 4 +- unittests/test_tag_inheritance_perf.py | 20 ++++----- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/unittests/test_false_positive_history_logic.py b/unittests/test_false_positive_history_logic.py index 8748239bedd..e02722bab3d 100644 --- a/unittests/test_false_positive_history_logic.py +++ b/unittests/test_false_positive_history_logic.py @@ -1684,7 +1684,7 @@ def test_fp_history_batch_issues_single_candidate_query(self): # 4 lazy-load chain: findings[0].test / .engagement / .product / .test_type # 1 candidates SELECT (with .only()) # 1 bulk UPDATE - with self.assertNumQueries(7): + with self.assertNumQueries(6): do_false_positive_history_batch(batch) # One candidate-fetch call for the whole batch — not one per finding. self.assertEqual(mock_fetch.call_count, 1, "Expected exactly one call to _fetch_fp_candidates_for_batch") @@ -1713,7 +1713,7 @@ def test_fp_history_batch_retroactive_marks_existing_active_fp(self): # 4 lazy-load chain: findings[0].test / .engagement / .product / .test_type # 1 candidates SELECT (with .only()) # 1 bulk UPDATE - with self.assertNumQueries(7): + with self.assertNumQueries(6): do_false_positive_history_batch(batch) # The pre-existing active finding must now be retroactively marked FP. @@ -1721,9 +1721,9 @@ def test_fp_history_batch_retroactive_marks_existing_active_fp(self): def test_fp_history_batch_query_count_does_not_grow_with_affected_findings(self): """ - Query count must stay flat (7) no matter how many findings are retroactively marked. + Query count must stay flat (6) no matter how many findings are retroactively marked. - With the old per-finding approach this would have been 7 + N queries where N is the + With the old per-finding approach this would have been 6 + N queries where N is the number of pre-existing findings that get marked as FP. With the batch approach it is always 7: System_Settings, 4 lazy-load chain, candidates SELECT, one bulk UPDATE. """ @@ -1748,7 +1748,7 @@ def test_fp_history_batch_query_count_does_not_grow_with_affected_findings(self) # 4 lazy-load chain: findings[0].test / .engagement / .product / .test_type # 1 candidates SELECT (with .only()) # 1 bulk UPDATE covering all retroactively marked findings - with self.assertNumQueries(7): + with self.assertNumQueries(6): do_false_positive_history_batch(batch) # All pre-existing findings must now be marked as FP. diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 30ecfeb00f3..54366ebc98f 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -349,7 +349,7 @@ def test_import_reimport_reimport_performance_pghistory_async(self): expected_num_async_tasks2=1, expected_num_queries3=28, expected_num_async_tasks3=1, - expected_num_queries4=99, + expected_num_queries4=105, expected_num_async_tasks4=0, ) @@ -367,13 +367,13 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=170, + expected_num_queries1=172, expected_num_async_tasks1=2, - expected_num_queries2=129, + expected_num_queries2=130, expected_num_async_tasks2=1, - expected_num_queries3=36, + expected_num_queries3=37, expected_num_async_tasks3=1, - expected_num_queries4=99, + expected_num_queries4=105, expected_num_async_tasks4=0, ) @@ -392,13 +392,13 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=180, + expected_num_queries1=184, expected_num_async_tasks1=4, - expected_num_queries2=139, + expected_num_queries2=142, expected_num_async_tasks2=3, - expected_num_queries3=43, + expected_num_queries3=46, expected_num_async_tasks3=3, - expected_num_queries4=108, + expected_num_queries4=116, expected_num_async_tasks4=2, ) @@ -545,9 +545,9 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=106, + expected_num_queries1=108, expected_num_async_tasks1=2, - expected_num_queries2=87, + expected_num_queries2=89, expected_num_async_tasks2=2, ) @@ -639,7 +639,7 @@ def test_import_reimport_reimport_performance_pghistory_async(self): expected_num_async_tasks2=1, expected_num_queries3=36, expected_num_async_tasks3=1, - expected_num_queries4=100, + expected_num_queries4=106, expected_num_async_tasks4=0, ) @@ -657,13 +657,13 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=179, + expected_num_queries1=181, expected_num_async_tasks1=2, - expected_num_queries2=140, + expected_num_queries2=141, expected_num_async_tasks2=1, - expected_num_queries3=46, + expected_num_queries3=47, expected_num_async_tasks3=1, - expected_num_queries4=100, + expected_num_queries4=106, expected_num_async_tasks4=0, ) @@ -682,13 +682,13 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=192, + expected_num_queries1=196, expected_num_async_tasks1=4, - expected_num_queries2=153, + expected_num_queries2=156, expected_num_async_tasks2=3, - expected_num_queries3=53, + expected_num_queries3=56, expected_num_async_tasks3=3, - expected_num_queries4=112, + expected_num_queries4=120, expected_num_async_tasks4=2, ) @@ -809,8 +809,8 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=115, + expected_num_queries1=117, expected_num_async_tasks1=2, - expected_num_queries2=198, + expected_num_queries2=200, expected_num_async_tasks2=2, ) diff --git a/unittests/test_metrics_queries.py b/unittests/test_metrics_queries.py index 4a0cf3c87eb..2cc023b2bf1 100644 --- a/unittests/test_metrics_queries.py +++ b/unittests/test_metrics_queries.py @@ -90,7 +90,7 @@ def test_finding_queries(self, mock_timezone): # Queries over Finding (legacy auth: fewer auth-layer queries # than RBAC since per-action role-permission lookups are gone). - with self.assertNumQueries(27): + with self.assertNumQueries(25): product_types = [] finding_queries = utils.finding_queries( product_types, @@ -302,7 +302,7 @@ def test_endpoint_queries(self, mock_now): # Queries over Finding and Endpoint_Status (legacy auth: fewer # auth-layer queries than RBAC). - with self.assertNumQueries(41): + with self.assertNumQueries(40): product_types = Product_Type.objects.all() endpoint_queries = utils.endpoint_queries( product_types, diff --git a/unittests/test_tag_inheritance_perf.py b/unittests/test_tag_inheritance_perf.py index 698b1126a85..07151044f1c 100644 --- a/unittests/test_tag_inheritance_perf.py +++ b/unittests/test_tag_inheritance_perf.py @@ -405,10 +405,10 @@ def test_baseline_product_tag_remove_propagates_to_100_locations_v3(self): EXPECTED_PRODUCT_TAG_REMOVE_100_V2 = 53 EXPECTED_PRODUCT_TAG_REMOVE_100_V3 = 53 - EXPECTED_CREATE_ONE_FINDING_V2 = 55 - EXPECTED_CREATE_ONE_FINDING_V3 = 55 - EXPECTED_CREATE_100_FINDINGS_V2 = 3124 - EXPECTED_CREATE_100_FINDINGS_V3 = 3124 + EXPECTED_CREATE_ONE_FINDING_V2 = 56 + EXPECTED_CREATE_ONE_FINDING_V3 = 56 + EXPECTED_CREATE_100_FINDINGS_V2 = 3224 + EXPECTED_CREATE_100_FINDINGS_V3 = 3224 EXPECTED_FINDING_ADD_USER_TAG_V2 = 17 EXPECTED_FINDING_ADD_USER_TAG_V3 = 17 @@ -590,9 +590,9 @@ def test_baseline_zap_scan_reimport_with_new_findings_v3(self): # the async watson indexer, executed inline under CELERY_TASK_ALWAYS_EAGER); # +5 reimport (no-change + with-new) queries from removal of # WATSON_ASYNC_INDEX_UPDATE_THRESHOLD making async dispatch unconditional. - EXPECTED_ZAP_IMPORT_V2 = 287 - EXPECTED_ZAP_IMPORT_V3 = 311 - EXPECTED_ZAP_REIMPORT_NO_CHANGE_V2 = 74 - EXPECTED_ZAP_REIMPORT_NO_CHANGE_V3 = 86 - EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 148 - EXPECTED_ZAP_REIMPORT_WITH_NEW_V3 = 177 + EXPECTED_ZAP_IMPORT_V2 = 291 + EXPECTED_ZAP_IMPORT_V3 = 315 + EXPECTED_ZAP_REIMPORT_NO_CHANGE_V2 = 77 + EXPECTED_ZAP_REIMPORT_NO_CHANGE_V3 = 89 + EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 151 + EXPECTED_ZAP_REIMPORT_WITH_NEW_V3 = 180 From fb1becd45be212d40e26969913be1b35b35ff67b Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Wed, 24 Jun 2026 21:21:08 +0200 Subject: [PATCH 7/8] test: pin L2 off via override_settings on query-count tests The query-count assertions must measure in-process (L1) behavior deterministically, independent of whether a shared L2 (Redis) cache is configured/warm in the run environment. Relying on the unit-test compose env (DD_SETTINGS_CACHE_L2_TTL=-1) left the counts env-dependent (e.g. running via run-unittest in dev mode, or the manage.py-test perf path, would see L2 on). Add an explicit @override_settings(SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) on the count-asserting classes so they are self-contained. No count change (compose already disabled L2 in CI). --- unittests/test_false_positive_history_logic.py | 2 ++ unittests/test_importers_performance.py | 4 ++-- unittests/test_metrics_queries.py | 4 +++- unittests/test_tag_inheritance_perf.py | 4 ++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/unittests/test_false_positive_history_logic.py b/unittests/test_false_positive_history_logic.py index e02722bab3d..9ca6b8a91c1 100644 --- a/unittests/test_false_positive_history_logic.py +++ b/unittests/test_false_positive_history_logic.py @@ -4,6 +4,7 @@ from crum import impersonate from django.conf import settings +from django.test import override_settings from dojo.finding.deduplication import do_false_positive_history_batch from dojo.finding.views import EditFinding @@ -126,6 +127,7 @@ @versioned_fixtures +@override_settings(SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) class TestFalsePositiveHistoryLogic(DojoTestCase): fixtures = ["dojo_testdata.json"] diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 54366ebc98f..55686172bbd 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -275,7 +275,7 @@ def _import_reimport_performance( @tag("performance") -@override_settings(V3_FEATURE_LOCATIONS=False) +@override_settings(V3_FEATURE_LOCATIONS=False, SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) class TestDojoImporterPerformanceSmall(TestDojoImporterPerformanceBase): """Performance tests using small sample files (StackHawk, ~6 findings).""" @@ -553,7 +553,7 @@ def test_deduplication_performance_pghistory_no_async(self): @tag("performance") -@override_settings(V3_FEATURE_LOCATIONS=True) +@override_settings(V3_FEATURE_LOCATIONS=True, SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) class TestDojoImporterPerformanceSmallLocations(TestDojoImporterPerformanceBase): r""" diff --git a/unittests/test_metrics_queries.py b/unittests/test_metrics_queries.py index 2cc023b2bf1..94ec80f2035 100644 --- a/unittests/test_metrics_queries.py +++ b/unittests/test_metrics_queries.py @@ -4,7 +4,7 @@ from datetime import date, datetime from unittest.mock import patch -from django.test import RequestFactory +from django.test import RequestFactory, override_settings from django.urls import reverse from dojo.metrics import utils @@ -47,6 +47,7 @@ def add(*args, **kwargs): @versioned_fixtures +@override_settings(SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) class FindingQueriesTest(DojoTestCase): fixtures = ["dojo_testdata.json", "unit_metrics_additional_data.json"] @@ -267,6 +268,7 @@ def test_closed_findings_filtered_by_mitigated_date(self, mock_timezone): # TODO: Delete this after the move to Locations @skip_unless_v2 +@override_settings(SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) class EndpointQueriesTest(DojoTestCase): fixtures = ["dojo_testdata.json"] diff --git a/unittests/test_tag_inheritance_perf.py b/unittests/test_tag_inheritance_perf.py index 07151044f1c..1dfaf83965b 100644 --- a/unittests/test_tag_inheritance_perf.py +++ b/unittests/test_tag_inheritance_perf.py @@ -99,6 +99,8 @@ def _make_locations(product: Product, n: int) -> None: @override_settings( CELERY_TASK_ALWAYS_EAGER=True, CELERY_TASK_EAGER_PROPAGATES=True, + SETTINGS_CACHE_L1_TTL=30, + SETTINGS_CACHE_L2_TTL=-1, ) class TagInheritancePerfBaselines(DojoTestCase): @@ -433,6 +435,8 @@ def test_baseline_product_tag_remove_propagates_to_100_locations_v3(self): CELERY_TASK_ALWAYS_EAGER=True, CELERY_TASK_EAGER_PROPAGATES=True, SECURE_SSL_REDIRECT=False, + SETTINGS_CACHE_L1_TTL=30, + SETTINGS_CACHE_L2_TTL=-1, ) @versioned_fixtures class TagInheritanceImportPerfBaselines(DojoAPITestCase): From a4556344578e54940317b78bb8d77802a5195df0 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Wed, 24 Jun 2026 23:08:28 +0200 Subject: [PATCH 8/8] feat(cache): drop L2 (shared) tier for singleton settings cache; L1-only The singleton settings cache (dojo/caching.py) becomes a single in-process L1 tier; the shared django.core.cache (L2) tier is removed. Cross-process/worker freshness is provided entirely by resetting L1 at each request and task boundary (middleware + DojoAsyncTask), so each request/task reads a singleton from the DB at most once and never serves a value cached during a prior request/task. Why no L2 for settings singletons: - The dedupe correctness fix only needs each task to see current config, which the per-task L1 reset already guarantees by re-reading the DB -- a shared L2 + cross-process invalidation is not required for correctness. - L2 added real cost and complexity for little gain on these tiny, hot singletons: a Redis dependency, pickled-vs-dict storage and version-keying, cross-process invalidation, and Redis-down fragility -- while the headline win (avoiding repeated reads within a bulk loop) comes from L1, not L2. - Trade-off: one small indexed DB read per singleton per request/task instead of amortising across processes via Redis. Acceptable for low-cardinality config. Removes SETTINGS_CACHE_L2_TTL and the L2 read/set/delete paths. DD_CACHE_URL and the Redis default django cache backend stay (used by other framework caching). Unit-test overrides drop the now-defunct L2 TTL var. test_caching rewritten to the L1-only contract. --- docker-compose.override.unit_tests.yml | 8 +- docker-compose.override.unit_tests_cicd.yml | 10 +-- dojo/caching.py | 67 ++++++++--------- dojo/settings/settings.dist.py | 24 +++--- unittests/test_caching.py | 83 ++++----------------- unittests/test_system_settings.py | 8 +- 6 files changed, 67 insertions(+), 133 deletions(-) diff --git a/docker-compose.override.unit_tests.yml b/docker-compose.override.unit_tests.yml index 66a36a68b83..ba243c56d60 100644 --- a/docker-compose.override.unit_tests.yml +++ b/docker-compose.override.unit_tests.yml @@ -22,13 +22,11 @@ services: DD_DATABASE_USER: ${DD_DATABASE_USER:-defectdojo} DD_DATABASE_PASSWORD: ${DD_DATABASE_PASSWORD:-defectdojo} DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite' - # No shared cache backend in unit tests (no Redis/valkey here) -> LocMemCache. + # No Redis/valkey in unit tests -> default django cache is LocMemCache. DD_CACHE_URL: '' - # L1 (in-process, request/test-scoped) stays ON for deterministic - # assertNumQueries counts; reset per request (middleware) and per test - # (dojo_test_case setUp). L2 is OFF (no shared backend; would persist). + # In-process singleton cache (dojo/caching.py) stays ON for deterministic + # assertNumQueries counts; reset per request (middleware) and per test. DD_SETTINGS_CACHE_L1_TTL: '30' - DD_SETTINGS_CACHE_L2_TTL: '-1' DD_JIRA_EXTRA_ISSUE_TYPES: 'Vulnerability' # Shouldn't trigger a migration error celerybeat: !reset celeryworker: !reset diff --git a/docker-compose.override.unit_tests_cicd.yml b/docker-compose.override.unit_tests_cicd.yml index d9711143a3f..511f76ebcdb 100644 --- a/docker-compose.override.unit_tests_cicd.yml +++ b/docker-compose.override.unit_tests_cicd.yml @@ -23,14 +23,12 @@ services: DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite' DD_JIRA_EXTRA_ISSUE_TYPES: 'Vulnerability' # Shouldn't trigger a migration error DD_V3_FEATURE_LOCATIONS: ${DD_V3_FEATURE_LOCATIONS:-False} - # No shared cache backend in unit tests (no Redis/valkey here) -> LocMemCache. + # No Redis/valkey in unit tests -> default django cache is LocMemCache. DD_CACHE_URL: '' - # L1 (in-process, request/test-scoped) stays ON so a singleton is read once - # per request/test and assertNumQueries counts are deterministic; it is reset - # per request (middleware) and per test (dojo_test_case setUp). L2 is OFF - # (no shared backend here, and it would persist across tests/processes). + # In-process singleton cache (dojo/caching.py) stays ON: a singleton is read + # once per request/test (deterministic assertNumQueries), reset per request + # (middleware) and per test (dojo_test_case setUp). DD_SETTINGS_CACHE_L1_TTL: '30' - DD_SETTINGS_CACHE_L2_TTL: '-1' celerybeat: !reset celeryworker: !reset initializer: !reset diff --git a/dojo/caching.py b/dojo/caching.py index 259b1340fcb..7bdc2a9fee5 100644 --- a/dojo/caching.py +++ b/dojo/caching.py @@ -1,24 +1,29 @@ """ -Two-tier read-through cache for global, low-cardinality singleton config. - -One in-process **L1** tier sits on top of the shared **L2** tier -(``django.core.cache``, Redis in deployments). A getter is resolved L1 → L2 → DB: -the first tier with a value wins; a ``None`` anywhere means "not cached, compute -it" and is never stored. This is deliberately simple — it is only for global, -user-INDEPENDENT, signal-invalidated singletons (feature flags, system settings, -and the like), never per-user or per-object data. +In-process read-through cache for global, low-cardinality singleton config. + +A single per-thread **L1** tier resolves a getter L1 → DB: a hit wins; a ``None`` +result means "not cached, compute it" and is never stored. This is deliberately +simple — it is only for global, user-INDEPENDENT, signal-invalidated singletons +(feature flags, system settings, and the like), never per-user or per-object data. + +There is intentionally **no shared/cross-process (L2) tier**: freshness is provided +by resetting L1 at every request and task boundary (middleware + the Celery task +base), so each request/task reads the singleton from the DB at most once and never +serves a value cached during a prior request/task (e.g. a since-changed +``System_Settings``). This keeps the design free of a Redis dependency, pickled +model graphs, and cross-process invalidation — at the cost of one DB read per +singleton per request/task. (The default ``django.core.cache`` backend may still be +Redis for other uses; this module no longer reads or writes it.) Values are stored as plain dicts/scalars (see ``model_to_cache_dict`` / ``cache_dict_to_model``), never pickled model instances. -Two independent tiers, each turned off by setting its TTL to ``-1`` (both off makes -the decorator a pass-through). Configuration (Django settings, wired from env in -``settings.dist.py``): +Configuration (Django setting, wired from env in ``settings.dist.py``): * ``SETTINGS_CACHE_L1_TTL`` — per-thread in-process freshness budget in seconds - (``-1`` disables L1). Keep it short — it bounds cross-process staleness. L1 is - reset at each request/task boundary, so it is effectively request/task scoped. -* ``SETTINGS_CACHE_L2_TTL`` — L2 timeout in seconds (``-1`` disables L2). + (``-1`` disables the cache, making the decorator a pass-through). Keep it short. + L1 is reset at each request/task boundary, so it is effectively request/task + scoped. """ import threading @@ -26,7 +31,6 @@ from functools import wraps from django.conf import settings -from django.core.cache import cache class _L1Store: @@ -88,34 +92,23 @@ def clear(self): def dojo_settings_cache(*, key: str): """ - Read-through L1+L2 cache for a fixed-key singleton getter. + Read-through in-process (L1) cache for a fixed-key singleton getter. - Resolves L1 → L2 → wrapped function. A ``None`` result is treated as "no - value" and is not cached (so the next call retries). Becomes a pass-through - when both tiers are disabled (``SETTINGS_CACHE_L1_TTL=-1`` and - ``SETTINGS_CACHE_L2_TTL=-1``). + Resolves L1 → wrapped function. A ``None`` result is treated as "no value" and + is not cached (so the next call retries). Becomes a pass-through when L1 is + disabled (``SETTINGS_CACHE_L1_TTL=-1``). Freshness across processes comes from + resetting L1 each request/task (see ``reset_l1_cache``), not a shared tier. """ def decorator(fn): @wraps(fn) def wrapper(*args, **kwargs): - l2_ttl = getattr(settings, "SETTINGS_CACHE_L2_TTL", 300) - l2_on = l2_ttl >= 0 # SETTINGS_CACHE_L2_TTL == -1 disables L2 - value = _L1_STORE.get(key) # ---- L1 ---- if value is not None: return value - if l2_on: # ---- L2 ---- - value = cache.get(key) - if value is not None: - _L1_STORE.set(key, value) - return value - value = fn(*args, **kwargs) # ---- miss: compute ---- if value is not None: - if l2_on: - cache.set(key, value, timeout=l2_ttl) _L1_STORE.set(key, value) return value @@ -125,12 +118,12 @@ def wrapper(*args, **kwargs): def invalidate_dojo_settings_cache(key: str) -> None: - """Drop a cached singleton from L2 (all processes) and L1 (this process).""" - # Only touch L2 when it is enabled; when SETTINGS_CACHE_L2_TTL == -1 the L2 - # tier is off and the backend (``cache``) may not even be reachable (e.g. - # unit tests run with no Redis configured), so skip the delete entirely. - if getattr(settings, "SETTINGS_CACHE_L2_TTL", 300) >= 0: - cache.delete(key) + """ + Drop a cached singleton from L1 (this thread). + + With no shared tier, other threads/processes self-heal at their next + request/task boundary (L1 reset), so there is nothing cross-process to drop. + """ _L1_STORE.invalidate(key) diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index 8e5b088d5a7..ced429d3b46 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -266,16 +266,15 @@ DD_V3_FEATURE_LOCATIONS=(bool, True), # Dictates if v3 org/asset relabeling (+url routing) will be enabled (on by default as of 3.0.0; set to False to restore Product/Product Type labels and URLs) DD_ENABLE_V3_ORGANIZATION_ASSET_RELABEL=(bool, True), - # Shared cache backend (django.core.cache). When set, used as the L2 tier for - # dojo/caching.py. MUST be a cross-process store (e.g. redis://valkey:6379/1) - # so cache invalidation propagates across uwsgi/celery processes; when empty - # Django falls back to per-process LocMemCache (single-process only). + # Shared cache backend (django.core.cache). When set, Django uses RedisCache + # (e.g. redis://valkey:6379/1); when empty it falls back to LocMemCache. Used + # by general framework caching; the singleton settings cache (dojo/caching.py) + # is in-process only and does not read or write this backend. DD_CACHE_URL=(str, ""), - # Two-tier read-through cache for global singleton getters (see dojo/caching.py). - # Per-thread in-process L1 freshness budget in seconds; -1 disables L1. + # In-process (L1) read-through cache for global singleton getters (see + # dojo/caching.py). Per-thread freshness budget in seconds; -1 disables it. + # Reset every request/task, so each request/task reads the singleton once. DD_SETTINGS_CACHE_L1_TTL=(int, 30), - # L2 (django.core.cache) timeout in seconds; -1 disables L2. - DD_SETTINGS_CACHE_L2_TTL=(int, 300), # Notification env-vars (SLA notify, alert refresh/counter/cap, system-level trump). Defined in dojo.notifications.settings. **NOTIFICATIONS_ENV_DEFAULTS, ) @@ -324,9 +323,9 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param # Raises django's ImproperlyConfigured exception if SECRET_KEY not in os.environ SECRET_KEY = env("DD_SECRET_KEY") -# Shared cache backend. A cross-process store (Redis) is required for cache -# invalidation to propagate across uwsgi/celery; otherwise Django defaults to -# per-process LocMemCache (fine only single-process). +# Default cache backend (django.core.cache). Redis when DD_CACHE_URL is set, +# else per-process LocMemCache. General framework caching only; the singleton +# settings cache (dojo/caching.py) is in-process and does not use this backend. if env("DD_CACHE_URL"): CACHES = { "default": { @@ -335,9 +334,8 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param }, } -# Two-tier singleton cache (dojo/caching.py) +# In-process singleton cache (dojo/caching.py) SETTINGS_CACHE_L1_TTL = env("DD_SETTINGS_CACHE_L1_TTL") -SETTINGS_CACHE_L2_TTL = env("DD_SETTINGS_CACHE_L2_TTL") # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name diff --git a/unittests/test_caching.py b/unittests/test_caching.py index f6080428e8c..c39dec90fb4 100644 --- a/unittests/test_caching.py +++ b/unittests/test_caching.py @@ -1,8 +1,5 @@ -from unittest import mock - from django.test import TestCase, override_settings -from dojo import caching from dojo.caching import ( _L1_STORE, # noqa: PLC2701 test needs the in-process store cache_dict_to_model, @@ -15,42 +12,20 @@ from .dojo_test_case import DojoTestCase -class _FakeCache: - - """Dict-backed stand-in for ``django.core.cache.cache`` that counts access.""" - - def __init__(self): - self.store = {} - self.gets = 0 - self.sets = 0 - self.deletes = 0 - - def get(self, key, default=None): - self.gets += 1 - return self.store.get(key, default) - - def set(self, key, value, timeout=None): - self.sets += 1 - self.store[key] = value - - def delete(self, key): - self.deletes += 1 - self.store.pop(key, None) - - -@override_settings(SETTINGS_CACHE_L2_TTL=300, SETTINGS_CACHE_L1_TTL=30) +@override_settings(SETTINGS_CACHE_L1_TTL=30) class DojoSettingsCacheTest(TestCase): - """Unit tests for the simplified L1+L2 read-through decorator (dojo/caching.py).""" + """ + Unit tests for the in-process (L1) read-through decorator (dojo/caching.py). + + There is no shared/L2 tier: the decorator only memoizes in-process and relies + on L1 reset at request/task boundaries for cross-process freshness. + """ def setUp(self): - self.fake = _FakeCache() - self._patch = mock.patch.object(caching, "cache", self.fake) - self._patch.start() _L1_STORE.clear() def tearDown(self): - self._patch.stop() _L1_STORE.clear() def _build_getter(self, *, key="k", returns=1): @@ -63,68 +38,40 @@ def getter(): return getter, calls - @override_settings(SETTINGS_CACHE_L1_TTL=-1) - def test_l1_disabled_consults_l2_every_call(self): - getter, calls = self._build_getter() - self.assertEqual(getter(), 1) - self.assertEqual(getter(), 1) # served from L2, fn not re-run - self.assertEqual(calls["n"], 1) # computed once - self.assertEqual(self.fake.gets, 2) # L2 consulted every call - - def test_l1_serves_without_l2(self): + def test_l1_memoizes_in_process(self): getter, calls = self._build_getter() getter() getter() getter() - self.assertEqual(calls["n"], 1) - self.assertEqual(self.fake.gets, 1) # only the first call reaches L2 + self.assertEqual(calls["n"], 1) # computed once, then served from L1 - @override_settings(SETTINGS_CACHE_L2_TTL=-1) - def test_l2_ttl_negative_disables_l2(self): - getter, calls = self._build_getter() - getter() - getter() - self.assertEqual(self.fake.gets, 0) # L2 never consulted - self.assertEqual(self.fake.sets, 0) - self.assertEqual(calls["n"], 1) # but L1 still memoizes in-process - - @override_settings(SETTINGS_CACHE_L1_TTL=-1, SETTINGS_CACHE_L2_TTL=-1) - def test_both_tiers_disabled_is_passthrough(self): + @override_settings(SETTINGS_CACHE_L1_TTL=-1) + def test_l1_disabled_is_passthrough(self): getter, calls = self._build_getter() getter() getter() - self.assertEqual(calls["n"], 2) # recomputed every call - self.assertEqual(self.fake.gets, 0) # L2 never consulted + self.assertEqual(calls["n"], 2) # recomputed every call when L1 off - def test_reset_clears_l1(self): + def test_reset_recomputes(self): getter, calls = self._build_getter() getter() _L1_STORE.reset() getter() - self.assertEqual(calls["n"], 1) # reset drops L1, but L2 still serves - self.assertEqual(self.fake.gets, 2) # 2nd call re-reads L2 after reset + self.assertEqual(calls["n"], 2) # reset drops L1, so it recomputes def test_none_result_is_not_cached(self): getter, calls = self._build_getter(returns=None) self.assertIsNone(getter()) self.assertIsNone(getter()) self.assertEqual(calls["n"], 2) # None recomputed each call (never stored) - self.assertEqual(self.fake.sets, 0) - def test_invalidate_clears_l1_and_l2(self): + def test_invalidate_recomputes(self): getter, calls = self._build_getter(returns=7) self.assertEqual(getter(), 7) invalidate_dojo_settings_cache("k") getter() self.assertEqual(calls["n"], 2) # recomputed after invalidation - @override_settings(SETTINGS_CACHE_L2_TTL=-1) - def test_invalidate_skips_l2_when_disabled(self): - # With L2 off the backend may be unreachable (no Redis in unit tests); - # invalidation must not call cache.delete. - invalidate_dojo_settings_cache("k") - self.assertEqual(self.fake.deletes, 0) - class ModelDictRoundTripTest(DojoTestCase): diff --git a/unittests/test_system_settings.py b/unittests/test_system_settings.py index 6d8ba7b2793..4c37898d012 100644 --- a/unittests/test_system_settings.py +++ b/unittests/test_system_settings.py @@ -93,15 +93,15 @@ def test_post_request_initializes_form_with_finding_instance(self): self.assertIn(response.status_code, [200, 302]) -@override_settings(SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) +@override_settings(SETTINGS_CACHE_L1_TTL=30) class TestSystemSettingsMiddlewareIntegration(DojoTestCase): """ Integration tests for DojoSettingsManagerMiddleware + System_Settings_Manager. - Caching lives in dojo.caching (decorator); the middleware only resets the - request-scoped L1 tier and surfaces a load error. These tests pin L1 on / L2 - off via override_settings so they don't depend on the compose env. + Caching lives in dojo.caching (in-process L1 decorator); the middleware resets + the request-scoped L1 tier and surfaces a load error. These tests pin L1 on via + override_settings so they don't depend on the compose env. """ def setUp(self):