fix(occurrences): remove all n+1 queries from occurrence API

ami/exports/format_types.py

@@ -44,6 +44,10 @@ def get_serializer_class(self):
         return get_export_serializer()
 
     def get_queryset(self):
+        # OccurrenceSerializer (the export base class) calls strict prefetch
+        # helpers in `_best_prediction` / `_best_identification` /
+        # `get_detection_images`. Apply detail prefetches so those helpers
+        # find detections + nested classifications in the cache.
         return (
             Occurrence.objects.valid()  # type: ignore[union-attr]  Custom manager method
             .filter(project=self.project)
@@ -55,6 +59,7 @@ def get_queryset(self):
             .with_timestamps()  # type: ignore[union-attr]  Custom queryset method
             .with_detections_count()
             .with_identifications()
+            .with_detail_prefetches()  # type: ignore[union-attr]  Custom queryset method
         )
 
     def export(self):

ami/main/api/serializers.py

@@ -1314,10 +1314,17 @@ class Meta:
 
 
 class OccurrenceListSerializer(DefaultSerializer):
+    # List responses render a single cover image per occurrence card; cap at 1
+    # to keep payload tight. Detail subclass overrides to None (unbounded).
+    # TODO: bound detail too once occurrence tracking lands — counts can reach
+    # thousands per occurrence and pagination is the right answer.
+    detection_images_limit: int | None = 1
+
     determination = CaptureTaxonSerializer(read_only=True)
     deployment = DeploymentNestedSerializer(read_only=True)
     event = EventNestedSerializer(read_only=True)
     # first_appearance = TaxonSourceImageNestedSerializer(read_only=True)
+    detection_images = serializers.SerializerMethodField()
     determination_details = serializers.SerializerMethodField()
     best_machine_prediction = serializers.SerializerMethodField()
     identifications = OccurrenceIdentificationSerializer(many=True, read_only=True)
@@ -1364,27 +1371,40 @@ class Meta:
             "updated_at",
         ]
 
-    def get_determination_details(self, obj: Occurrence):
-        # @TODO convert this to query methods to avoid N+1 queries.
-        # Currently at 100+ queries per page of 10 occurrences.
-        # Add a reusable method to the OccurrenceQuerySet class and call it from the ViewSet.
+    def get_detection_images(self, obj: Occurrence) -> list[str]:
+        from ami.main.models_future.occurrence import detection_image_urls_from_prefetch
 
-        context = self.context
+        return detection_image_urls_from_prefetch(obj, limit=self.detection_images_limit)
+
+    def _best_identification(self, obj: Occurrence) -> Identification | None:
+        from ami.main.models_future.occurrence import best_identification_from_prefetch
+
+        return best_identification_from_prefetch(obj)
+
+    def _best_prediction(self, obj: Occurrence):
+        from ami.main.models_future.occurrence import best_prediction_from_prefetch
 
+        return best_prediction_from_prefetch(obj)
+
+    def get_determination_details(self, obj: Occurrence):
+        context = self.context
         # Add this occurrence to the context so that the nested serializers can access it
         # the `parent` attribute is not available since we are manually instantiating the serializers
         context["occurrence"] = obj
 
         taxon = TaxonNestedSerializer(obj.determination, context=context).data if obj.determination else None
-        if obj.best_identification:
-            identification = OccurrenceIdentificationSerializer(obj.best_identification, context=context).data
+
+        best_ident = self._best_identification(obj)
+        if best_ident:
+            identification = OccurrenceIdentificationSerializer(best_ident, context=context).data
         else:
             identification = None
 
-        if identification or not obj.best_prediction:
+        if identification:
             prediction = None
         else:
-            prediction = ClassificationNestedSerializer(obj.best_prediction, context=context).data
+            best_pred = self._best_prediction(obj)
+            prediction = ClassificationNestedSerializer(best_pred, context=context).data if best_pred else None
 
         return dict(
             taxon=taxon,
@@ -1402,7 +1422,7 @@ def get_best_machine_prediction(self, obj: Occurrence) -> dict | None:
         context = self.context
         context["occurrence"] = obj
 
-        prediction = obj.best_prediction
+        prediction = self._best_prediction(obj)
         if not prediction:
             return None
 
@@ -1421,6 +1441,9 @@ def get_best_machine_prediction(self, obj: Occurrence) -> dict | None:
 
 
 class OccurrenceSerializer(OccurrenceListSerializer):
+    # Detail returns all detection_images (TODO: bound when tracking enabled).
+    detection_images_limit: int | None = None
+
     determination = CaptureTaxonSerializer(read_only=True)
     detections = DetectionNestedSerializer(many=True, read_only=True)
     identifications = OccurrenceIdentificationSerializer(many=True, read_only=True)

ami/main/api/views.py

@@ -1227,12 +1227,10 @@ def get_queryset(self) -> QuerySet["Occurrence"]:
         qs = qs.with_detections_count().with_timestamps()  # type: ignore
         qs = qs.with_identifications()  # type: ignore
         qs = qs.apply_default_filters(project, self.request)  # type: ignore
-        if self.action != "list":
-            qs = qs.prefetch_related(
-                Prefetch(
-                    "detections", queryset=Detection.objects.order_by("-timestamp").select_related("source_image")
-                )
-            )
+        if self.action == "list":
+            qs = qs.with_list_prefetches()  # type: ignore
+        else:
+            qs = qs.with_detail_prefetches()  # type: ignore
 
         return qs
 

ami/main/models.py

@@ -60,6 +60,11 @@
 # over non-terminal, then highest score, with pk as the deterministic tiebreaker.
 BEST_MACHINE_PREDICTION_ORDER: Final = ("-terminal", "-score", "-pk")
 
+# Ordering for "best identification" selection used by Occurrence.best_identification,
+# OccurrenceQuerySet.with_verification_info(), and best_identification_from_prefetch().
+# Most recent non-withdrawn identification wins, with pk as the deterministic tiebreaker.
+BEST_IDENTIFICATION_ORDER: Final = ("-created_at", "-pk")
+
 
 class TaxonRank(OrderedEnum):
     KINGDOM = "KINGDOM"
@@ -2932,6 +2937,21 @@ def with_identifications(self):
             "identifications__user",
         )
 
+    def with_list_prefetches(self):
+        """Add prefetches the list serializer needs (detection paths, classifications).
+
+        See `ami.main.models_future.occurrence` for the prefetch definitions.
+        """
+        from ami.main.models_future.occurrence import prefetches_for_list_serializer
+
+        return self.prefetch_related(*prefetches_for_list_serializer())
+
+    def with_detail_prefetches(self):
+        """Add prefetches the detail serializer needs (detections + source_image + classifications)."""
+        from ami.main.models_future.occurrence import prefetches_for_detail_serializer
+
+        return self.prefetch_related(*prefetches_for_detail_serializer())
+
     def with_best_detection(self):
         """
         Annotate the queryset with fields from the best detection.
@@ -3014,7 +3034,7 @@ def with_verification_info(self):
         """
         best_identification_subquery = Identification.objects.filter(
             occurrence=OuterRef("pk"), withdrawn=False
-        ).order_by("-created_at")
+        ).order_by(*BEST_IDENTIFICATION_ORDER)
 
         return self.annotate(
             verified_by_name=models.Subquery(best_identification_subquery.values("user__name")[:1]),
@@ -3240,7 +3260,11 @@ def best_identification(self):
 
         @TODO this could use a confidence level chosen manually by the users/experts.
         """
-        return Identification.objects.filter(occurrence=self, withdrawn=False).order_by("-created_at").first()
+        return (
+            Identification.objects.filter(occurrence=self, withdrawn=False)
+            .order_by(*BEST_IDENTIFICATION_ORDER)
+            .first()
+        )
 
     def get_determination_score(self) -> float | None:
         if not self.determination:

ami/main/models_future/occurrence.py

@@ -0,0 +1,168 @@
+"""
+Reusable Prefetch factories for Occurrence list/detail rendering.
+
+The serializer trusts the prefetch contract — the viewset is the single place
+that wires it up. Don't gate serializer methods on `_prefetched_objects_cache`
+membership; require the prefetch.
+
+Tracking issue: https://github.com/RolnickLab/antenna/issues/1271
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from django.db.models import Prefetch
+
+if TYPE_CHECKING:
+    from ami.main.models import Classification, Identification, Occurrence
+
+
+def _detections_prefetch(*, ordering: tuple[str, ...], with_source_image: bool) -> Prefetch:
+    from ami.main.models import Classification, Detection
+
+    qs = Detection.objects.prefetch_related(
+        Prefetch(
+            "classifications",
+            queryset=Classification.objects.select_related("taxon", "algorithm"),
+        )
+    ).order_by(*ordering)
+    if with_source_image:
+        qs = qs.select_related("source_image")
+    return Prefetch("detections", queryset=qs)
+
+
+def prefetch_detections_for_list() -> Prefetch:
+    """Detections + nested classifications, ordered for stable list image galleries."""
+    return _detections_prefetch(ordering=("frame_num", "timestamp"), with_source_image=False)
+
+
+def prefetch_detections_for_detail() -> Prefetch:
+    """Detections + nested classifications + source_image, ordered most-recent-first.
+
+    Detail responses serialize each detection via `DetectionNestedSerializer`,
+    which dereferences `source_image` (as `capture`).
+    """
+    return _detections_prefetch(ordering=("-timestamp",), with_source_image=True)
+
+
+def prefetches_for_list_serializer() -> list[Prefetch]:
+    return [prefetch_detections_for_list()]
+
+
+def prefetches_for_detail_serializer() -> list[Prefetch]:
+    return [prefetch_detections_for_detail()]
+
+
+def _require_prefetch(occurrence: Occurrence, *relations: str) -> None:
+    """Raise if any required relation is missing from the prefetch cache.
+
+    Strict contract: callers in serializer code must go through a viewset that
+    applied the corresponding prefetch. Silent slow-pathing (the serializer
+    triggering its own DB queries) is the N+1 we are fixing.
+
+    Caveat: only top-level relations are checked. Nested prefetch (e.g.
+    `detections__classifications`) is still required but not enforced here —
+    if a caller prefetches `detections` without nested `classifications`,
+    `best_prediction_from_prefetch` will silently re-introduce per-detection
+    queries. The list/detail prefetch factories pair them correctly; new
+    callers should use those factories rather than building Prefetch objects
+    by hand. Tighter enforcement is deferred to the django-zen-queries pass
+    (issue #1271 follow-up), which catches all per-row queries at the view
+    boundary regardless of depth.
+    """
+    cache = getattr(occurrence, "_prefetched_objects_cache", {})
+    missing = [r for r in relations if r not in cache]
+    if missing:
+        raise RuntimeError(
+            f"Occurrence {occurrence.pk} is missing prefetched relations {missing!r}. "
+            "Apply OccurrenceQuerySet.with_list_prefetches() / with_detail_prefetches() / "
+            "with_identifications() in the viewset's get_queryset()."
+        )
+
+
+def best_prediction_from_prefetch(occurrence: Occurrence) -> Classification | None:
+    """Pick the best machine prediction from a prefetched occurrence in Python.
+
+    Mirrors `Occurrence.best_prediction`, which calls `Occurrence.predictions()`
+    (per-algorithm max-score filtering) and then orders by `-terminal, -score`.
+
+    Null-score handling: classifications with `score=None` are skipped, which
+    matches the SQL behavior of `score__in=Subquery(...)` — `Max(score)` over
+    only-NULL rows returns NULL, and `score IN (NULL)` never matches in SQL.
+
+    Subtle SQL/Python difference (intentional): `Occurrence.predictions()` uses
+    `score__in=Subquery(per-algo maxes)`. If algo_A's max is 0.9 and algo_B has
+    a non-max classification at 0.9, the SQL form includes both; this Python
+    form keeps only per-algorithm maxima (algo_A's 0.9, not algo_B's). The
+    Python form is the stricter interpretation of the docstring intent
+    ("max score for each algorithm").
+
+    Strict: requires `detections` (and each detection's `classifications`) to be
+    prefetched. Raises if not.
+    """
+    _require_prefetch(occurrence, "detections")
+    classifications = [
+        c for det in occurrence.detections.all() for c in det.classifications.all() if c.score is not None
+    ]
+    if not classifications:
+        return None
+
+    max_score_per_algo: dict[object, float] = {}
+    for c in classifications:
+        existing = max_score_per_algo.get(c.algorithm_id)
+        if existing is None or c.score > existing:
+            max_score_per_algo[c.algorithm_id] = c.score
+
+    candidates = [c for c in classifications if c.score == max_score_per_algo[c.algorithm_id]]
+    if not candidates:
+        return None
+    candidates.sort(
+        key=lambda c: (
+            0 if getattr(c, "terminal", False) else 1,
+            -c.score,
+            -c.pk,
+        )
+    )
+    return candidates[0]
+
+
+def best_identification_from_prefetch(occurrence: Occurrence) -> Identification | None:
+    """Pick the most recent non-withdrawn identification from prefetched data.
+
+    Mirrors `Occurrence.best_identification`, which uses
+    `BEST_IDENTIFICATION_ORDER = ("-created_at", "-pk")`. `created_at=None` is
+    treated as lower than any real timestamp.
+
+    Strict: requires `identifications` to be prefetched (via
+    `OccurrenceQuerySet.with_identifications()`).
+    """
+    _require_prefetch(occurrence, "identifications")
+    best: Identification | None = None
+    best_key: tuple[bool, object, int] | None = None
+    for ident in occurrence.identifications.all():
+        if ident.withdrawn:
+            continue
+        ident_key: tuple[bool, object, int] = (ident.created_at is not None, ident.created_at, ident.pk)
+        if best_key is None or ident_key > best_key:
+            best = ident
+            best_key = ident_key
+    return best
+
+
+def detection_image_urls_from_prefetch(occurrence: Occurrence, limit: int | None = None) -> list[str]:
+    """Return media URLs for the prefetched detections (filtering out `path=None`).
+
+    Strict: requires `detections` to be prefetched. Pass `limit` to bound output —
+    list responses cap at a few cover images; detail responses currently return
+    all (TODO: bound when occurrence tracking lands; per-occurrence detection
+    counts can reach thousands).
+    """
+    _require_prefetch(occurrence, "detections")
+
+    from ami.main.models import get_media_url
+
+    detections = [det for det in occurrence.detections.all() if det.path]
+    if limit is not None:
+        detections = detections[:limit]
+    return [get_media_url(det.path) for det in detections]