experimental-inspect: Iterator __next__ not optional

guide/src/class/protocols.md

@@ -166,7 +166,7 @@ The given signatures should be interpreted as follows:
 Iterators can be defined using these methods:
 
 - `__iter__(<self>) -> object`
-- `__next__(<self>) -> Option<object> or IterNextOutput` ([see details](#returning-a-value-from-iteration))
+- `__next__(<self>) -> Option<object>` ([see details](#returning-a-value-from-iteration))
 
 Returning `None` from `__next__` indicates that that there are no further items.
 

newsfragments/6274.fixed.md

@@ -0,0 +1 @@
+`experimental-inspect`: `__next__` and `__anext__` returning `Option<T>` (or `PyResult<Option<T>>`) are now introspected as returning `T`, since `None` stops the iteration instead of being yielded.

pyo3-macros-backend/src/py_expr.rs

@@ -2,7 +2,7 @@
 
 use crate::utils::PyO3CratePath;
 use proc_macro2::TokenStream;
-use quote::quote;
+use quote::{format_ident, quote};
 use std::borrow::Cow;
 use syn::visit_mut::{visit_type_mut, VisitMut};
 use syn::{Expr, ExprLit, ExprPath, Lit, Type};
@@ -22,6 +22,10 @@ pub enum PyExpr {
     ArgumentType(Type),
     /// The Python type matching the given Rust type given as a function returned value
     ReturnType(Type),
+    /// The Python type `__next__` yields, without the `Option` meaning `StopIteration`
+    IterNextReturnType(Type),
+    /// The Python type `__anext__` yields, without the `Option` meaning `StopAsyncIteration`
+    AsyncIterNextReturnType(Type),
     /// The Python type matching the given Rust type
     Type(Type),
     /// A name
@@ -116,6 +120,20 @@ impl PyExpr {
         Self::ReturnType(clean_type(t, self_type))
     }
 
+    /// The type hint of the Rust type used as the output type of `__next__`
+    ///
+    /// If self_type is set, self_type will replace Self in the given type
+    pub fn from_iter_next_return_type(t: Type, self_type: Option<&Type>) -> Self {
+        Self::IterNextReturnType(clean_type(t, self_type))
+    }
+
+    /// The type hint of the Rust type used as the output type of `__anext__`
+    ///
+    /// If self_type is set, self_type will replace Self in the given type
+    pub fn from_async_iter_next_return_type(t: Type, self_type: Option<&Type>) -> Self {
+        Self::AsyncIterNextReturnType(clean_type(t, self_type))
+    }
+
     /// The type hint of the Rust type `PyTypeCheck` trait.
     ///
     /// If self_type is set, self_type will replace Self in the given type
@@ -228,6 +246,15 @@ impl PyExpr {
                     TYPE
                 }}
             }
+            Self::IterNextReturnType(t) => {
+                iter_next_output_type(pyo3_crate_path, t, "IterNextOutput", "IterNextTypeFallback")
+            }
+            Self::AsyncIterNextReturnType(t) => iter_next_output_type(
+                pyo3_crate_path,
+                t,
+                "AsyncIterNextOutput",
+                "AsyncIterNextTypeFallback",
+            ),
             Self::Type(t) => {
                 quote! { <#t as #pyo3_crate_path::type_object::PyTypeCheck>::TYPE_HINT }
             }
@@ -287,6 +314,27 @@ impl PyExpr {
     }
 }
 
+/// The type hint of what `__next__` / `__anext__` yields, read off the same wrapper the slot uses
+/// to convert the returned value so that the stub and the runtime agree on which return types say
+/// "iteration is over" with `None`.
+fn iter_next_output_type(
+    pyo3_crate_path: &PyO3CratePath,
+    t: &Type,
+    wrapper: &str,
+    fallback: &str,
+) -> TokenStream {
+    let wrapper = format_ident!("{wrapper}");
+    let fallback = format_ident!("{fallback}");
+    quote! {{
+        #[allow(
+            unused_imports,
+            reason = "the fallback trait is unused when the inherent const applies"
+        )]
+        use #pyo3_crate_path::impl_::pymethods::#fallback as _;
+        #pyo3_crate_path::impl_::pymethods::#wrapper::<#t>::OUTPUT_TYPE
+    }}
+}
+
 fn clean_type(mut t: Type, self_type: Option<&Type>) -> Type {
     if let Some(self_type) = self_type {
         replace_self(&mut t, self_type);

pyo3-macros-backend/src/pyimpl.rs

@@ -503,6 +503,15 @@ pub fn method_introspection_code(
         PyExpr::from_return_type(parse_quote!(#pyo3_path::PyClassGuard<Self>), Some(parent))
     } else {
         match spec.output.clone() {
+            // `__next__` and `__anext__` may say "iteration is over" with `None`, in which case
+            // that `Option` is not part of the Python-visible return type. This is about the value
+            // the slot returns, so it does not apply to the coroutine an `async fn` returns.
+            ReturnType::Type(_, t) if spec.asyncness.is_none() && name.as_str() == "__next__" => {
+                PyExpr::from_iter_next_return_type(*t, Some(parent))
+            }
+            ReturnType::Type(_, t) if spec.asyncness.is_none() && name.as_str() == "__anext__" => {
+                PyExpr::from_async_iter_next_return_type(*t, Some(parent))
+            }
             ReturnType::Type(_, t) => PyExpr::from_return_type(*t, Some(parent)),
             ReturnType::Default => PyExpr::none(),
         }

pyo3-macros-backend/src/pymethod.rs

@@ -1096,18 +1096,15 @@ pub const __RICHCMP__: SlotDef = SlotDef::new("Py_tp_richcompare", "richcmpfunc"
     .extract_error_mode(ExtractErrorMode::NotImplemented);
 const __GET__: SlotDef = SlotDef::new("Py_tp_descr_get", "descrgetfunc");
 const __ITER__: SlotDef = SlotDef::new("Py_tp_iter", "getiterfunc");
-const __NEXT__: SlotDef = SlotDef::new("Py_tp_iternext", "iternextfunc")
-    .return_specialized_conversion(
-        TokenGenerator(|_| quote! { IterBaseKind, IterOptionKind, IterResultOptionKind }),
-        TokenGenerator(|_| quote! { iter_tag }),
-    );
+const __NEXT__: SlotDef = SlotDef::new("Py_tp_iternext", "iternextfunc").return_iter_conversion(
+    TokenGenerator(|_| quote! { IterNextOutput }),
+    TokenGenerator(|_| quote! { IterNextConvertFallback }),
+);
 const __AWAIT__: SlotDef = SlotDef::new("Py_am_await", "unaryfunc");
 const __AITER__: SlotDef = SlotDef::new("Py_am_aiter", "unaryfunc");
-const __ANEXT__: SlotDef = SlotDef::new("Py_am_anext", "unaryfunc").return_specialized_conversion(
-    TokenGenerator(
-        |_| quote! { AsyncIterBaseKind, AsyncIterOptionKind, AsyncIterResultOptionKind },
-    ),
-    TokenGenerator(|_| quote! { async_iter_tag }),
+const __ANEXT__: SlotDef = SlotDef::new("Py_am_anext", "unaryfunc").return_iter_conversion(
+    TokenGenerator(|_| quote! { AsyncIterNextOutput }),
+    TokenGenerator(|_| quote! { AsyncIterNextConvertFallback }),
 );
 pub const __LEN__: SlotDef = SlotDef::new("Py_mp_length", "lenfunc");
 const __CONTAINS__: SlotDef = SlotDef::new("Py_sq_contains", "objobjproc");
@@ -1299,7 +1296,10 @@ fn extract_object(
 enum ReturnMode {
     ReturnSelf,
     Conversion(TokenGenerator),
-    SpecializedConversion(TokenGenerator, TokenGenerator),
+    /// `__next__` / `__anext__`: the return value goes through the wrapper named by the first
+    /// generator, whose inherent `convert` handles the return types saying "iteration is over"
+    /// with `None`, and whose fallback trait, named by the second, handles all the others.
+    IterConversion(TokenGenerator, TokenGenerator),
 }
 
 impl ReturnMode {
@@ -1313,13 +1313,17 @@ impl ReturnMode {
                     #pyo3_path::impl_::callback::convert(py, _result)
                 }
             }
-            ReturnMode::SpecializedConversion(traits, tag) => {
-                let traits = TokenGeneratorCtx(*traits, ctx);
-                let tag = TokenGeneratorCtx(*tag, ctx);
+            ReturnMode::IterConversion(wrapper, fallback) => {
+                let wrapper = TokenGeneratorCtx(*wrapper, ctx);
+                let fallback = TokenGeneratorCtx(*fallback, ctx);
                 quote! {
                     let _result = #call;
-                    use #pyo3_path::impl_::pymethods::{#traits};
-                    (&_result).#tag().convert(py, _result)
+                    #[allow(
+                        unused_imports,
+                        reason = "the fallback trait is unused when the inherent `convert` applies"
+                    )]
+                    use #pyo3_path::impl_::pymethods::#fallback as _;
+                    #pyo3_path::impl_::pymethods::#wrapper(_result).convert(py)
                 }
             }
             ReturnMode::ReturnSelf => quote! {
@@ -1434,12 +1438,12 @@ impl SlotDef {
         self
     }
 
-    const fn return_specialized_conversion(
+    const fn return_iter_conversion(
         mut self,
-        traits: TokenGenerator,
-        tag: TokenGenerator,
+        wrapper: TokenGenerator,
+        fallback: TokenGenerator,
     ) -> Self {
-        self.return_mode = Some(ReturnMode::SpecializedConversion(traits, tag));
+        self.return_mode = Some(ReturnMode::IterConversion(wrapper, fallback));
         self
     }
 

pytests/pyproject.toml

@@ -34,5 +34,6 @@ dev = [
     "pytest-asyncio>=0.21,<2",
     "pytest-benchmark>=3.4",
     "pytest>=7",
-    "typing_extensions>=4.0.0"
+    # 4.2 for `assert_type`
+    "typing_extensions>=4.2.0"
 ]

pytests/src/awaitable.rs

@@ -21,7 +21,7 @@ pub mod awaitable {
     #[pymethods]
     impl IterAwaitable {
         #[new]
-        fn new(result: Py<PyAny>) -> Self {
+        pub(crate) fn new(result: Py<PyAny>) -> Self {
             IterAwaitable {
                 result: Some(Ok(result)),
             }

pytests/src/pyclasses.rs

@@ -7,6 +7,8 @@ use pyo3::types::{PyComplex, PyType};
 #[cfg(not(any(Py_LIMITED_API, GraalPy)))]
 use pyo3::types::{PyDict, PyTuple};
 
+use crate::awaitable::awaitable::IterAwaitable;
+
 #[pyclass(from_py_object)]
 #[derive(Clone, Default)]
 pub struct EmptyClass {}
@@ -50,6 +52,92 @@ impl PyClassIter {
     }
 }
 
+/// This is for demonstrating how to stop iteration by returning `None` from __next__
+#[pyclass]
+#[derive(Default)]
+struct PyClassOptionIter {
+    count: usize,
+}
+
+#[pymethods]
+impl PyClassOptionIter {
+    #[new]
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
+        slf
+    }
+
+    fn __next__(&mut self) -> Option<usize> {
+        if self.count < 5 {
+            self.count += 1;
+            Some(self.count)
+        } else {
+            None
+        }
+    }
+}
+
+/// This is for demonstrating how to stop iteration by returning `None` from a fallible __next__
+#[pyclass]
+#[derive(Default)]
+struct PyClassResultOptionIter {
+    count: usize,
+}
+
+#[pymethods]
+impl PyClassResultOptionIter {
+    #[new]
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
+        slf
+    }
+
+    #[expect(clippy::unnecessary_wraps, reason = "covering the fallible signature")]
+    fn __next__(&mut self) -> PyResult<Option<usize>> {
+        if self.count < 5 {
+            self.count += 1;
+            Ok(Some(self.count))
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+/// This is for demonstrating how to stop iteration by returning `None` from __anext__
+#[pyclass]
+#[derive(Default)]
+struct PyClassOptionAsyncIter {
+    count: usize,
+}
+
+#[pymethods]
+impl PyClassOptionAsyncIter {
+    #[new]
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    fn __aiter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
+        slf
+    }
+
+    fn __anext__(&mut self, py: Python<'_>) -> PyResult<Option<IterAwaitable>> {
+        if self.count >= 5 {
+            return Ok(None);
+        }
+        self.count += 1;
+        // `__anext__` hands back an awaitable, which `async for` awaits for the next value.
+        let value = self.count.into_pyobject(py)?.into_any().unbind();
+        Ok(Some(IterAwaitable::new(value)))
+    }
+}
+
 #[pyclass]
 #[derive(Default)]
 struct PyClassThreadIter {
@@ -341,6 +429,7 @@ pub mod pyclasses {
     #[pymodule_export]
     use super::{
         map_a_class, AssertingBaseClass, ClassWithDecorators, ClassWithoutConstructor, EmptyClass,
-        Number, PlainObject, PyClassIter, PyClassThreadIter,
+        Number, PlainObject, PyClassIter, PyClassOptionAsyncIter, PyClassOptionIter,
+        PyClassResultOptionIter, PyClassThreadIter,
     };
 }

pytests/stubs/pyclasses.pyi

@@ -1,3 +1,4 @@
+from .awaitable import IterAwaitable
 from _typeshed import Incomplete
 from typing import Final, final
 
@@ -121,6 +122,33 @@ class PyClassIter:
         """
     def __next__(self, /) -> int: ...
 
+@final
+class PyClassOptionAsyncIter:
+    """
+    This is for demonstrating how to stop iteration by returning `None` from __anext__
+    """
+    def __aiter__(self, /) -> PyClassOptionAsyncIter: ...
+    def __anext__(self, /) -> IterAwaitable: ...
+    def __new__(cls, /) -> PyClassOptionAsyncIter: ...
+
+@final
+class PyClassOptionIter:
+    """
+    This is for demonstrating how to stop iteration by returning `None` from __next__
+    """
+    def __iter__(self, /) -> PyClassOptionIter: ...
+    def __new__(cls, /) -> PyClassOptionIter: ...
+    def __next__(self, /) -> int: ...
+
+@final
+class PyClassResultOptionIter:
+    """
+    This is for demonstrating how to stop iteration by returning `None` from a fallible __next__
+    """
+    def __iter__(self, /) -> PyClassResultOptionIter: ...
+    def __new__(cls, /) -> PyClassResultOptionIter: ...
+    def __next__(self, /) -> int: ...
+
 @final
 class PyClassThreadIter:
     def __new__(cls, /) -> PyClassThreadIter: ...