feat(cache): shared two-tier (L1+L2) read-through cache for singleton config

docker-compose.override.unit_tests.yml

@@ -22,6 +22,11 @@ services:
       DD_DATABASE_USER: ${DD_DATABASE_USER:-defectdojo}
       DD_DATABASE_PASSWORD: ${DD_DATABASE_PASSWORD:-defectdojo}
       DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite'
+      # No Redis/valkey in unit tests -> default django cache is LocMemCache.
+      DD_CACHE_URL: ''
+      # In-process singleton cache (dojo/caching.py) stays ON for deterministic
+      # assertNumQueries counts; reset per request (middleware) and per test.
+      DD_SETTINGS_CACHE_L1_TTL: '30'
       DD_JIRA_EXTRA_ISSUE_TYPES: 'Vulnerability' # Shouldn't trigger a migration error
   celerybeat: !reset
   celeryworker: !reset

docker-compose.override.unit_tests_cicd.yml

@@ -23,6 +23,12 @@ services:
       DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite'
       DD_JIRA_EXTRA_ISSUE_TYPES: 'Vulnerability' # Shouldn't trigger a migration error
       DD_V3_FEATURE_LOCATIONS: ${DD_V3_FEATURE_LOCATIONS:-False}
+      # No Redis/valkey in unit tests -> default django cache is LocMemCache.
+      DD_CACHE_URL: ''
+      # In-process singleton cache (dojo/caching.py) stays ON: a singleton is read
+      # once per request/test (deterministic assertNumQueries), reset per request
+      # (middleware) and per test (dojo_test_case setUp).
+      DD_SETTINGS_CACHE_L1_TTL: '30'
   celerybeat: !reset
   celeryworker: !reset
   initializer: !reset

docker-compose.yml

@@ -50,6 +50,7 @@ services:
       DD_ALLOWED_HOSTS: "${DD_ALLOWED_HOSTS:-*}"
       DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo}
       DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0}
+      DD_CACHE_URL: ${DD_CACHE_URL:-redis://valkey:6379/1}
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
       DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"
@@ -71,6 +72,7 @@ services:
     environment:
       DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo}
       DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0}
+      DD_CACHE_URL: ${DD_CACHE_URL:-redis://valkey:6379/1}
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
       DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"
@@ -91,6 +93,7 @@ services:
     environment:
       DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo}
       DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0}
+      DD_CACHE_URL: ${DD_CACHE_URL:-redis://valkey:6379/1}
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
       DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"

dojo/caching.py

@@ -0,0 +1,160 @@
+"""
+In-process read-through cache for global, low-cardinality singleton config.
+
+A single per-thread **L1** tier resolves a getter L1 → DB: a hit wins; a ``None``
+result means "not cached, compute it" and is never stored. This is deliberately
+simple — it is only for global, user-INDEPENDENT, signal-invalidated singletons
+(feature flags, system settings, and the like), never per-user or per-object data.
+
+There is intentionally **no shared/cross-process (L2) tier**: freshness is provided
+by resetting L1 at every request and task boundary (middleware + the Celery task
+base), so each request/task reads the singleton from the DB at most once and never
+serves a value cached during a prior request/task (e.g. a since-changed
+``System_Settings``). This keeps the design free of a Redis dependency, pickled
+model graphs, and cross-process invalidation — at the cost of one DB read per
+singleton per request/task. (The default ``django.core.cache`` backend may still be
+Redis for other uses; this module no longer reads or writes it.)
+
+Values are stored as plain dicts/scalars (see ``model_to_cache_dict`` /
+``cache_dict_to_model``), never pickled model instances.
+
+Configuration (Django setting, wired from env in ``settings.dist.py``):
+
+* ``SETTINGS_CACHE_L1_TTL`` — per-thread in-process freshness budget in seconds
+  (``-1`` disables the cache, making the decorator a pass-through). Keep it short.
+  L1 is reset at each request/task boundary, so it is effectively request/task
+  scoped.
+"""
+
+import threading
+import time
+from functools import wraps
+
+from django.conf import settings
+
+
+class _L1Store:
+
+    """
+    Per-thread in-process store, TTL-stamped. ``get`` returns the value or ``None``
+    (absent, expired, or L1 disabled via ``SETTINGS_CACHE_L1_TTL`` < 0).
+
+    Per-thread (not shared across threads) so it needs no locking, and is reset at
+    each request/task boundary (see ``reset``) — making it effectively request/task
+    scoped on a reused worker/uwsgi thread.
+    """
+
+    def __init__(self):
+        self._local = threading.local()
+
+    def _bucket(self):
+        bucket = getattr(self._local, "b", None)
+        if bucket is None:
+            bucket = self._local.b = {}
+        return bucket
+
+    def get(self, key):
+        if getattr(settings, "SETTINGS_CACHE_L1_TTL", 30) < 0:
+            return None
+        entry = self._bucket().get(key)
+        if entry is None:
+            return None
+        value, expiry = entry
+        if time.monotonic() >= expiry:
+            self._bucket().pop(key, None)
+            return None
+        return value
+
+    def set(self, key, value):
+        ttl = getattr(settings, "SETTINGS_CACHE_L1_TTL", 30)
+        if ttl < 0:
+            return
+        self._bucket()[key] = (value, time.monotonic() + ttl)
+
+    def invalidate(self, key):
+        # Only this thread; other threads/processes self-heal within the L1 TTL
+        # (and reset at their next request/task boundary).
+        self._bucket().pop(key, None)
+
+    def reset(self):
+        # Clear THIS thread's L1 bucket. Called at request/task boundaries so a
+        # reused worker/uwsgi thread never serves a value cached during a prior
+        # request or task (e.g. a since-changed System_Settings).
+        self._bucket().clear()
+
+    def clear(self):
+        # Test helper: drop this thread's entries.
+        self._local = threading.local()
+
+
+_L1_STORE = _L1Store()
+
+
+def dojo_settings_cache(*, key: str):
+    """
+    Read-through in-process (L1) cache for a fixed-key singleton getter.
+
+    Resolves L1 → wrapped function. A ``None`` result is treated as "no value" and
+    is not cached (so the next call retries). Becomes a pass-through when L1 is
+    disabled (``SETTINGS_CACHE_L1_TTL=-1``). Freshness across processes comes from
+    resetting L1 each request/task (see ``reset_l1_cache``), not a shared tier.
+    """
+
+    def decorator(fn):
+        @wraps(fn)
+        def wrapper(*args, **kwargs):
+            value = _L1_STORE.get(key)              # ---- L1 ----
+            if value is not None:
+                return value
+
+            value = fn(*args, **kwargs)             # ---- miss: compute ----
+            if value is not None:
+                _L1_STORE.set(key, value)
+            return value
+
+        return wrapper
+
+    return decorator
+
+
+def invalidate_dojo_settings_cache(key: str) -> None:
+    """
+    Drop a cached singleton from L1 (this thread).
+
+    With no shared tier, other threads/processes self-heal at their next
+    request/task boundary (L1 reset), so there is nothing cross-process to drop.
+    """
+    _L1_STORE.invalidate(key)
+
+
+def reset_l1_cache() -> None:
+    """
+    Reset the current thread's L1 tier.
+
+    Call at request/task boundaries (reused worker/uwsgi threads) so the
+    in-process L1 is effectively request/task-scoped and never serves a value
+    cached during a prior request or task.
+    """
+    _L1_STORE.reset()
+
+
+def model_to_cache_dict(instance) -> dict:
+    """
+    Flatten a model instance to a plain dict of concrete field values.
+
+    Keyed by ``attname`` (so a relation ``foo`` becomes ``foo_id``) and includes
+    the primary key. M2M and reverse relations are skipped. Storing this instead
+    of the model instance keeps the cache free of pickled model graphs; rebuild a
+    live instance with ``cache_dict_to_model``.
+    """
+    return {f.attname: f.value_from_object(instance) for f in instance._meta.concrete_fields}
+
+
+def cache_dict_to_model(model_cls, data: dict):
+    """
+    Rebuild an in-memory model instance from a ``model_to_cache_dict`` dict.
+
+    For read-only use; callers that persist changes must fetch a fresh DB instance
+    rather than saving a cache-derived one.
+    """
+    return model_cls(**data)

dojo/celery.py

@@ -28,6 +28,12 @@ def __call__(self, *args, **kwargs):
         """
         Restore user context in the celery worker via crum.impersonate.
 
+        Also resets the request/task-scoped L1 settings cache at the start of every
+        task (including eager): a prefork worker reuses its thread across tasks, so an
+        L1 value cached during a prior task (e.g. System_Settings) would otherwise be
+        served stale even after another process changed and saved it. The shared L2
+        tier is left intact, so a reset task re-reads each singleton once from L2.
+
         The apply_async method injects ``async_user_id`` into kwargs when a task
         is dispatched. Here we pop it, resolve to a user instance, and set it
         as the current user in thread-local storage so that all downstream
@@ -39,6 +45,9 @@ def __call__(self, *args, **kwargs):
         intact so that callers who already set a user (e.g. via
         crum.impersonate in tests or request middleware) are not disrupted.
         """
+        from dojo.caching import reset_l1_cache  # noqa: PLC0415
+        reset_l1_cache()
+
         if "async_user_id" not in kwargs:
             return super().__call__(*args, **kwargs)