fix(occurrences): remove all n+1 queries from occurrence API

ami/exports/format_types.py

@@ -44,6 +44,10 @@ def get_serializer_class(self):
         return get_export_serializer()
 
     def get_queryset(self):
+        # OccurrenceSerializer (the export base class) calls strict prefetch
+        # helpers in `_best_prediction` / `_best_identification` /
+        # `get_detection_images`. Apply detail prefetches so those helpers
+        # find detections + nested classifications in the cache.
         return (
             Occurrence.objects.valid()  # type: ignore[union-attr]  Custom manager method
             .filter(project=self.project)
@@ -55,6 +59,7 @@ def get_queryset(self):
             .with_timestamps()  # type: ignore[union-attr]  Custom queryset method
             .with_detections_count()
             .with_identifications()
+            .with_detail_prefetches()  # type: ignore[union-attr]  Custom queryset method
         )
 
     def export(self):

ami/main/api/serializers.py

@@ -591,7 +591,7 @@ def get_taxa(self, obj):
 class TaxonListSerializer(DefaultSerializer):
     # latest_detection = DetectionNestedSerializer(read_only=True)
     occurrences = serializers.SerializerMethodField()
-    parents = TaxonNestedSerializer(read_only=True)
+    parents = TaxonParentSerializer(many=True, read_only=True, source="parents_json")
     parent_id = serializers.PrimaryKeyRelatedField(queryset=Taxon.objects.all(), source="parent")
     tags = serializers.SerializerMethodField()
 
@@ -1314,10 +1314,14 @@ class Meta:
 
 
 class OccurrenceListSerializer(DefaultSerializer):
+    # List cards render one cover image; detail subclass raises this to 100.
+    detection_images_limit: int | None = 1
+
     determination = CaptureTaxonSerializer(read_only=True)
     deployment = DeploymentNestedSerializer(read_only=True)
     event = EventNestedSerializer(read_only=True)
     # first_appearance = TaxonSourceImageNestedSerializer(read_only=True)
+    detection_images = serializers.SerializerMethodField()
     determination_details = serializers.SerializerMethodField()
     best_machine_prediction = serializers.SerializerMethodField()
     identifications = OccurrenceIdentificationSerializer(many=True, read_only=True)
@@ -1364,27 +1368,32 @@ class Meta:
             "updated_at",
         ]
 
+    def get_detection_images(self, obj: Occurrence) -> list[str]:
+        from ami.main.models_future.occurrence import detection_image_urls_from_prefetch
+
+        return detection_image_urls_from_prefetch(obj, limit=self.detection_images_limit)
+
     def get_determination_details(self, obj: Occurrence):
-        # @TODO convert this to query methods to avoid N+1 queries.
-        # Currently at 100+ queries per page of 10 occurrences.
-        # Add a reusable method to the OccurrenceQuerySet class and call it from the ViewSet.
+        from ami.main.models_future.occurrence import best_identification_from_prefetch, best_prediction_from_prefetch
 
         context = self.context
-
         # Add this occurrence to the context so that the nested serializers can access it
         # the `parent` attribute is not available since we are manually instantiating the serializers
         context["occurrence"] = obj
 
         taxon = TaxonNestedSerializer(obj.determination, context=context).data if obj.determination else None
-        if obj.best_identification:
-            identification = OccurrenceIdentificationSerializer(obj.best_identification, context=context).data
+
+        best_ident = best_identification_from_prefetch(obj)
+        if best_ident:
+            identification = OccurrenceIdentificationSerializer(best_ident, context=context).data
         else:
             identification = None
 
-        if identification or not obj.best_prediction:
+        if identification:
             prediction = None
         else:
-            prediction = ClassificationNestedSerializer(obj.best_prediction, context=context).data
+            best_pred = best_prediction_from_prefetch(obj)
+            prediction = ClassificationNestedSerializer(best_pred, context=context).data if best_pred else None
 
         return dict(
             taxon=taxon,
@@ -1399,10 +1408,12 @@ def get_best_machine_prediction(self, obj: Occurrence) -> dict | None:
         Populated regardless of human verification status, so clients can always show
         the ML result alongside a human-set determination.
         """
+        from ami.main.models_future.occurrence import best_prediction_from_prefetch
+
         context = self.context
         context["occurrence"] = obj
 
-        prediction = obj.best_prediction
+        prediction = best_prediction_from_prefetch(obj)
         if not prediction:
             return None
 
@@ -1421,6 +1432,8 @@ def get_best_machine_prediction(self, obj: Occurrence) -> dict | None:
 
 
 class OccurrenceSerializer(OccurrenceListSerializer):
+    detection_images_limit: int | None = 100
+
     determination = CaptureTaxonSerializer(read_only=True)
     detections = DetectionNestedSerializer(many=True, read_only=True)
     identifications = OccurrenceIdentificationSerializer(many=True, read_only=True)

ami/main/api/views.py

@@ -1227,12 +1227,10 @@ def get_queryset(self) -> QuerySet["Occurrence"]:
         qs = qs.with_detections_count().with_timestamps()  # type: ignore
         qs = qs.with_identifications()  # type: ignore
         qs = qs.apply_default_filters(project, self.request)  # type: ignore
-        if self.action != "list":
-            qs = qs.prefetch_related(
-                Prefetch(
-                    "detections", queryset=Detection.objects.order_by("-timestamp").select_related("source_image")
-                )
-            )
+        if self.action == "list":
+            qs = qs.with_list_prefetches()  # type: ignore
+        else:
+            qs = qs.with_detail_prefetches()  # type: ignore
 
         return qs
 

ami/main/models.py

@@ -60,6 +60,11 @@
 # over non-terminal, then highest score, with pk as the deterministic tiebreaker.
 BEST_MACHINE_PREDICTION_ORDER: Final = ("-terminal", "-score", "-pk")
 
+# Ordering for "best identification" selection used by Occurrence.best_identification,
+# OccurrenceQuerySet.with_verification_info(), and best_identification_from_prefetch().
+# Most recent non-withdrawn identification wins, with pk as the deterministic tiebreaker.
+BEST_IDENTIFICATION_ORDER: Final = ("-created_at", "-pk")
+
 
 class TaxonRank(OrderedEnum):
     KINGDOM = "KINGDOM"
@@ -2932,6 +2937,18 @@ def with_identifications(self):
             "identifications__user",
         )
 
+    def with_list_prefetches(self):
+        """Add prefetches the list serializer needs (detection paths, classifications)."""
+        from ami.main.models_future.occurrence import prefetch_detections_for_list
+
+        return self.prefetch_related(prefetch_detections_for_list())
+
+    def with_detail_prefetches(self):
+        """Add prefetches the detail serializer needs (detections + source_image + classifications)."""
+        from ami.main.models_future.occurrence import prefetch_detections_for_detail
+
+        return self.prefetch_related(prefetch_detections_for_detail())
+
     def with_best_detection(self):
         """
         Annotate the queryset with fields from the best detection.
@@ -3014,7 +3031,7 @@ def with_verification_info(self):
         """
         best_identification_subquery = Identification.objects.filter(
             occurrence=OuterRef("pk"), withdrawn=False
-        ).order_by("-created_at")
+        ).order_by(*BEST_IDENTIFICATION_ORDER)
 
         return self.annotate(
             verified_by_name=models.Subquery(best_identification_subquery.values("user__name")[:1]),
@@ -3240,7 +3257,11 @@ def best_identification(self):
 
         @TODO this could use a confidence level chosen manually by the users/experts.
         """
-        return Identification.objects.filter(occurrence=self, withdrawn=False).order_by("-created_at").first()
+        return (
+            Identification.objects.filter(occurrence=self, withdrawn=False)
+            .order_by(*BEST_IDENTIFICATION_ORDER)
+            .first()
+        )
 
     def get_determination_score(self) -> float | None:
         if not self.determination:

ami/main/models_future/occurrence.py

@@ -0,0 +1,131 @@
+"""
+Reusable Prefetch factories for Occurrence list/detail rendering.
+
+The serializer trusts the prefetch contract — the viewset is the single place
+that wires it up. Don't gate serializer methods on `_prefetched_objects_cache`
+membership; require the prefetch.
+
+Tracking issue: https://github.com/RolnickLab/antenna/issues/1271
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from django.db.models import Prefetch
+
+if TYPE_CHECKING:
+    from ami.main.models import Classification, Identification, Occurrence
+
+
+def _detections_prefetch(*, ordering: tuple[str, ...], with_source_image: bool) -> Prefetch:
+    from ami.main.models import Classification, Detection
+
+    qs = Detection.objects.prefetch_related(
+        Prefetch(
+            "classifications",
+            queryset=Classification.objects.select_related("taxon", "algorithm"),
+        )
+    ).order_by(*ordering)
+    if with_source_image:
+        qs = qs.select_related("source_image")
+    return Prefetch("detections", queryset=qs)
+
+
+def prefetch_detections_for_list() -> Prefetch:
+    """Detections + nested classifications, ordered for stable list image galleries."""
+    return _detections_prefetch(ordering=("frame_num", "timestamp"), with_source_image=False)
+
+
+def prefetch_detections_for_detail() -> Prefetch:
+    """Detections + nested classifications + source_image, ordered most-recent-first.
+
+    Detail responses serialize each detection via `DetectionNestedSerializer`,
+    which dereferences `source_image` (as `capture`).
+    """
+    return _detections_prefetch(ordering=("-timestamp",), with_source_image=True)
+
+
+def _require_prefetch(occurrence: Occurrence, *relations: str) -> None:
+    """Raise if any required top-level relation is missing from the prefetch cache.
+
+    Only checks top-level relations; nested prefetch (e.g. `detections__classifications`)
+    is required by callers but not enforced here. Use the list/detail factories above
+    to keep the pairing correct. Tighter depth-checking is deferred to django-zen-queries
+    (#1271 follow-up).
+    """
+    cache = getattr(occurrence, "_prefetched_objects_cache", {})
+    missing = [r for r in relations if r not in cache]
+    if missing:
+        raise RuntimeError(
+            f"Occurrence {occurrence.pk} is missing prefetched relations {missing!r}. "
+            "Apply OccurrenceQuerySet.with_list_prefetches() / with_detail_prefetches() / "
+            "with_identifications() in the viewset's get_queryset()."
+        )
+
+
+def best_prediction_from_prefetch(occurrence: Occurrence) -> Classification | None:
+    """Pick the best machine prediction from a prefetched occurrence in Python.
+
+    Mirrors `Occurrence.best_prediction` (per-algorithm max-score, then `-terminal, -score`).
+    Skips `score=None` to match SQL semantics of `score__in=Subquery(...)`.
+
+    Strict: requires `detections` (and each detection's `classifications`) prefetched.
+    """
+    _require_prefetch(occurrence, "detections")
+    classifications = [
+        c for det in occurrence.detections.all() for c in det.classifications.all() if c.score is not None
+    ]
+    if not classifications:
+        return None
+
+    max_score_per_algo: dict[object, float] = {}
+    for c in classifications:
+        existing = max_score_per_algo.get(c.algorithm_id)
+        if existing is None or c.score > existing:
+            max_score_per_algo[c.algorithm_id] = c.score
+
+    candidates = [c for c in classifications if c.score == max_score_per_algo[c.algorithm_id]]
+    candidates.sort(
+        key=lambda c: (
+            0 if getattr(c, "terminal", False) else 1,
+            -c.score,
+            -c.pk,
+        )
+    )
+    return candidates[0]
+
+
+def best_identification_from_prefetch(occurrence: Occurrence) -> Identification | None:
+    """Pick the most recent non-withdrawn identification from prefetched data.
+
+    Mirrors `Occurrence.best_identification` (BEST_IDENTIFICATION_ORDER = -created_at, -pk).
+
+    Strict: requires `identifications` prefetched (via `OccurrenceQuerySet.with_identifications()`).
+    """
+    _require_prefetch(occurrence, "identifications")
+    best: Identification | None = None
+    best_key: tuple[bool, object, int] | None = None
+    for ident in occurrence.identifications.all():
+        if ident.withdrawn:
+            continue
+        ident_key: tuple[bool, object, int] = (ident.created_at is not None, ident.created_at, ident.pk)
+        if best_key is None or ident_key > best_key:
+            best = ident
+            best_key = ident_key
+    return best
+
+
+def detection_image_urls_from_prefetch(occurrence: Occurrence, limit: int | None = None) -> list[str]:
+    """Return media URLs for the prefetched detections (filtering out `path=None`).
+
+    Strict: requires `detections` prefetched. Pass `limit` to bound output.
+    """
+    _require_prefetch(occurrence, "detections")
+
+    from ami.main.models import get_media_url
+
+    detections = [det for det in occurrence.detections.all() if det.path]
+    if limit is not None:
+        detections = detections[:limit]
+    return [get_media_url(det.path) for det in detections]