poc: extend exists/2 to traverse embedded array attributes

lib/ash/expr/expr.ex

@@ -64,6 +64,7 @@ defmodule Ash.Expr do
              is_struct(value, Ash.Query.Exists) or
              is_struct(value, Ash.Query.Parent) or
              is_struct(value, Ash.Query.UpsertConflict) or
+             is_struct(value, Ash.Query.EmbeddedArrayElementField) or
              is_struct(value, Ash.CustomExpression) or
              (is_struct(value) and is_map_key(value, :__predicate__?)) do
     true

lib/ash/filter/filter.ex

@@ -1543,6 +1543,31 @@ defmodule Ash.Filter do
     add_ref_to_relevant_paths({rest, ref}, new_acc, new_trail)
   end
 
+  @doc false
+  # Rewrites references inside an `exists/2` predicate so that fields of the
+  # embedded resource currently being iterated are represented as
+  # `Ash.Query.EmbeddedArrayElementField` markers. SQL data layers translate
+  # these markers into per-element JSONB extractions (e.g.,
+  # `jsonb_extract_path_text(elem, 'field')::type`).
+  #
+  # Only refs whose `relationship_path` is empty and whose attribute belongs
+  # to `embedded_resource` are rewritten — outer-scope refs (e.g., via
+  # `parent/1`) are left intact.
+  def rewrite_for_embedded_array_scope(expr, embedded_resource) when is_atom(embedded_resource) do
+    map(expr, fn
+      %Ref{relationship_path: [], attribute: %Ash.Resource.Attribute{} = attr, resource: resource}
+      when resource == embedded_resource ->
+        %Ash.Query.EmbeddedArrayElementField{
+          field: attr.name,
+          type: attr.type,
+          constraints: attr.constraints || []
+        }
+
+      other ->
+        other
+    end)
+  end
+
   def map(%__MODULE__{expression: nil} = filter, _) do
     filter
   end
@@ -2163,8 +2188,19 @@ defmodule Ash.Filter do
   defp shortest_path_to_changed_data_layer(_resource, [], _acc), do: :error
 
   defp shortest_path_to_changed_data_layer(resource, [relationship | rest], acc) do
-    relationship = Ash.Resource.Info.relationship(resource, relationship)
+    case Ash.Resource.Info.relationship(resource, relationship) do
+      nil ->
+        # Segment is not a relationship (e.g., an embedded array attribute).
+        # Embedded arrays live in the same data layer as the parent, so they
+        # never change data layers; signal "no split needed."
+        :error
 
+      relationship ->
+        shortest_path_to_changed_data_layer_for_relationship(relationship, rest, acc, resource)
+    end
+  end
+
+  defp shortest_path_to_changed_data_layer_for_relationship(relationship, rest, acc, resource) do
     if relationship.type == :many_to_many do
       if Ash.DataLayer.data_layer_can?(resource, {:join, relationship.through}) do
         shortest_path_to_changed_data_layer(relationship.destination, rest, [
@@ -2909,6 +2945,42 @@ defmodule Ash.Filter do
     end
   end
 
+  # Like `related/2`, but a path segment may also be an attribute whose type is
+  # `{:array, EmbeddedResource}` (with `EmbeddedResource` declared as
+  # `use Ash.Resource, data_layer: :embedded`). In that case the next segment is
+  # resolved against the embedded resource. Used to power `exists/2` over
+  # embedded array attributes.
+  #
+  # Returns the destination resource, or `nil` if no segment matched.
+  defp related_or_embedded(context, segments) when not is_list(segments) do
+    related_or_embedded(context, [segments])
+  end
+
+  defp related_or_embedded(context, []), do: context.resource
+
+  defp related_or_embedded(context, [segment | rest]) do
+    case relationship(context, segment) do
+      %{destination: destination} ->
+        related_or_embedded(%{context | resource: destination}, rest)
+
+      nil ->
+        case embedded_array_attribute(context.resource, segment) do
+          nil -> nil
+          embedded_resource -> related_or_embedded(%{context | resource: embedded_resource}, rest)
+        end
+    end
+  end
+
+  defp embedded_array_attribute(resource, segment) do
+    with %{type: {:array, item_type}} <- Ash.Resource.Info.attribute(resource, segment),
+         true <- is_atom(item_type),
+         true <- Ash.Resource.Info.embedded?(item_type) do
+      item_type
+    else
+      _ -> nil
+    end
+  end
+
   defp parse_expression(%__MODULE__{expression: expression}, context),
     do: {:ok, move_to_relationship_path(expression, context[:relationship_path] || [])}
 
@@ -3036,7 +3108,7 @@ defmodule Ash.Filter do
 
     related =
       if related? do
-        related(context, at_path ++ path)
+        related_or_embedded(context, at_path ++ path)
       else
         resource
       end
@@ -4175,7 +4247,7 @@ defmodule Ash.Filter do
       when not is_nil(resource) do
     case Ash.Resource.Info.related(resource, relationship_path || []) do
       nil ->
-        {:error, "Invalid reference #{inspect(ref)}"}
+        {:error, invalid_reference_error(ref, resource, relationship_path || [])}
 
       related ->
         do_hydrate_refs(
@@ -4206,8 +4278,7 @@ defmodule Ash.Filter do
 
     case related(context, ref.relationship_path) do
       nil ->
-        {:error,
-         "Invalid reference #{inspect(ref)} at relationship_path #{inspect(ref.relationship_path)}"}
+        {:error, invalid_reference_error(ref, context.resource, ref.relationship_path)}
 
       related ->
         context = %{context | resource: related}
@@ -4624,10 +4695,27 @@ defmodule Ash.Filter do
         Ash.Resource.Info.related(context[:resource], expanded_at_path)
       end
 
-    expanded_path = expand_through_path_names(at_path_resource, path)
+    # `path` may start with an embedded-array attribute (Phase 1+). Only
+    # apply relationship `through`-expansion when the path is purely
+    # relationships; otherwise leave it as-is and resolve via the
+    # embedded-aware walker.
+    {expanded_path, new_resource} =
+      case unscoped_related_or_embedded(at_path_resource, path) do
+        nil ->
+          {path, nil}
+
+        resource ->
+          # If `path` is pure-relationship, normalize via through-expansion;
+          # otherwise keep as-is (embedded-array segments don't go through joins).
+          expanded_path =
+            try do
+              expand_through_path_names(at_path_resource, path)
+            rescue
+              Ash.Error.Query.NoSuchRelationship -> path
+            end
 
-    new_resource =
-      Ash.Resource.Info.related(context[:resource], expanded_at_path ++ expanded_path)
+          {expanded_path, resource}
+      end
 
     if new_resource do
       context = %{
@@ -4701,6 +4789,48 @@ defmodule Ash.Filter do
     {:ok, val}
   end
 
+  # Build a Ref-resolution error message. When a non-relationship path segment
+  # is an embedded array attribute, point the user at `exists/2` rather than
+  # leaving them with the generic "Invalid reference" string.
+  defp invalid_reference_error(ref, resource, relationship_path) do
+    embedded_array_segment =
+      Enum.find(relationship_path, fn segment ->
+        embedded_array_attribute(resource, segment) != nil
+      end)
+
+    if embedded_array_segment do
+      "Cannot reference fields through an embedded array attribute directly. " <>
+        "Use `exists/2` to filter over `#{embedded_array_segment}`, e.g. " <>
+        "`exists(#{embedded_array_segment}, ...)`. " <>
+        "Got: #{inspect(ref)}"
+    else
+      "Invalid reference #{inspect(ref)}"
+    end
+  end
+
+  # Variant of `related/2` for `do_hydrate_refs`, which previously used
+  # `Ash.Resource.Info.related/2` and therefore did not filter by `public?`.
+  # Recognizes `{:array, EmbeddedResource}` attributes in addition to
+  # relationships.
+  defp unscoped_related_or_embedded(resource, segments) when not is_list(segments) do
+    unscoped_related_or_embedded(resource, [segments])
+  end
+
+  defp unscoped_related_or_embedded(resource, []), do: resource
+
+  defp unscoped_related_or_embedded(resource, [segment | rest]) do
+    case Ash.Resource.Info.relationship(resource, segment) do
+      %{destination: destination} ->
+        unscoped_related_or_embedded(destination, rest)
+
+      nil ->
+        case embedded_array_attribute(resource, segment) do
+          nil -> nil
+          embedded_resource -> unscoped_related_or_embedded(embedded_resource, rest)
+        end
+    end
+  end
+
   defp combination_calc(first_combination, attribute) do
     with {:ok, calc} <- Map.fetch(first_combination.calculations, attribute) do
       case calc do

lib/ash/query/embedded_array_element_field.ex

@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: 2019 ash contributors <https://github.com/ash-project/ash/graphs/contributors>
+#
+# SPDX-License-Identifier: MIT
+
+defmodule Ash.Query.EmbeddedArrayElementField do
+  @moduledoc """
+  An internal expression node representing a reference to a field of the
+  *current element* being iterated inside an `exists/2` over an embedded
+  array attribute.
+
+  Not constructed by users. Produced by `Ash.Filter` as part of the
+  AST-rewrite step that prepares an `Ash.Query.Exists` whose first path
+  segment is an embedded-array attribute, so that downstream compilers
+  (e.g. AshPostgres) can translate the field reference into a
+  `jsonb_extract_path_text(elem.value, 'field')::type` style expression.
+  """
+
+  defstruct [:field, :type, :constraints]
+
+  @type t :: %__MODULE__{
+          field: atom(),
+          type: Ash.Type.t() | nil,
+          constraints: keyword()
+        }
+
+  defimpl Inspect do
+    import Inspect.Algebra
+
+    def inspect(%{field: field}, _opts) do
+      concat(["#elem.", to_string(field)])
+    end
+  end
+end

lib/ash/query/function/exists.ex

@@ -5,6 +5,41 @@
 defmodule Ash.Query.Exists do
   @moduledoc """
   Determines if a given related entity exists.
+
+  ## Path forms
+
+  The `path` argument is a dotted chain of segments. Each segment may be:
+
+    * a relationship — the usual case (`exists(comments, ...)`,
+      `exists(posts.comments, ...)`).
+    * an attribute whose type is `{:array, EmbeddedResource}`, where
+      `EmbeddedResource` is declared with `use Ash.Resource, data_layer: :embedded`.
+      Each element of the array is iterated and the predicate is evaluated
+      against it. Mixed paths (`exists(invoices.options, ...)`) compose
+      naturally — the relationship prefix is joined first, the embedded
+      array attribute is unnested afterwards.
+
+  ## `parent/1` semantics
+
+  Each `exists/2` call pushes the *calling* scope onto the parent stack
+  once, regardless of how many segments are in `path`. Inside the
+  predicate, `parent/1` therefore always refers to the resource the
+  `exists/2` call was made from, not to any intermediate scope along
+  the path.
+
+  > Note: `exists(a, exists(b, ...))` is auto-flattened by
+  > `Ash.Query.Exists.new/3` into `exists(a.b, ...)`, so explicit
+  > nesting does **not** create separate intermediate scopes for
+  > `parent/1` to reach.
+
+  ## Data layer support for embedded arrays
+
+  The embedded-array form requires the data layer to declare the
+  `{:exists, :embedded_array}` capability. AshPostgres supports it
+  (lowering to `jsonb_array_elements`). Other SQL data layers and any
+  in-memory layer with a generic record walker (e.g. `Ash.DataLayer.Ets`)
+  work as well. If the data layer does not support this form, a clear
+  error is raised at query build time.
   """
 
   defstruct [:path, :expr, :resource, at_path: [], related?: true, input?: false]

test/embedded_array/edge_cases_test.exs

@@ -0,0 +1,105 @@
+# SPDX-FileCopyrightText: 2019 ash contributors <https://github.com/ash-project/ash/graphs/contributors>
+#
+# SPDX-License-Identifier: MIT
+
+defmodule Ash.Test.EmbeddedArray.EdgeCasesTest do
+  @moduledoc """
+  Phase 5 — edge cases for `exists/2` over embedded array attributes.
+
+  Covers in-memory (ETS) behavior. The AshPostgres equivalents (where
+  semantics may differ for nil/empty arrays due to SQL NULL handling)
+  live in `ash_postgres/test/embedded_array_exists_test.exs`.
+
+  See `specs/embedded-array-exists.md`.
+  """
+  use ExUnit.Case, async: true
+
+  require Ash.Query
+
+  alias Ash.Test.EmbeddedArray.Estimate
+
+  describe "empty / nil arrays" do
+    setup do
+      [
+        empty:
+          Estimate
+          |> Ash.Changeset.for_create(:create, %{title: "empty", options: []})
+          |> Ash.create!(),
+        nil_options:
+          Estimate
+          |> Ash.Changeset.for_create(:create, %{title: "nil_options"})
+          |> Ash.create!(),
+        populated:
+          Estimate
+          |> Ash.Changeset.for_create(:create, %{
+            title: "populated",
+            options: [%{name: "x", total_amt: Decimal.new("10")}]
+          })
+          |> Ash.create!()
+      ]
+    end
+
+    test "exists/2 returns false for empty array", %{empty: empty} do
+      results =
+        Estimate
+        |> Ash.Query.filter(exists(options, total_amt > 0))
+        |> Ash.read!()
+        |> Enum.map(& &1.id)
+
+      refute empty.id in results
+    end
+
+    test "exists/2 returns false for nil array", %{nil_options: nil_opts} do
+      results =
+        Estimate
+        |> Ash.Query.filter(exists(options, total_amt > 0))
+        |> Ash.read!()
+        |> Enum.map(& &1.id)
+
+      refute nil_opts.id in results
+    end
+
+    test "not exists/2 returns true for empty and nil arrays", %{
+      empty: empty,
+      nil_options: nil_opts,
+      populated: populated
+    } do
+      results =
+        Estimate
+        |> Ash.Query.filter(not exists(options, total_amt > 0))
+        |> Ash.read!()
+        |> Enum.map(& &1.id)
+
+      assert empty.id in results
+      assert nil_opts.id in results
+      refute populated.id in results
+    end
+  end
+
+  describe "multiple references to the same field in one predicate" do
+    setup do
+      [
+        single_match:
+          Estimate
+          |> Ash.Changeset.for_create(:create, %{
+            title: "single_match",
+            options: [%{name: "matchy", total_amt: Decimal.new("50")}]
+          })
+          |> Ash.create!()
+      ]
+    end
+
+    test "same attribute referenced twice in the same predicate", %{
+      single_match: single_match
+    } do
+      results =
+        Estimate
+        |> Ash.Query.filter(exists(options, total_amt > 0 and total_amt < 1000))
+        |> Ash.read!()
+        |> Enum.map(& &1.id)
+
+      assert single_match.id in results
+    end
+  end
+
+end