Replace a leaking StopIteration with RuntimeError

If the callable raises StopIteration (StopAsyncIteration in aiter()) which
does not match stop_exception, the consumer would mistake it for the end of
the iteration, or, in the asynchronous case, for the result of the await.
Replace it with RuntimeError, as PEP 479 and PEP 525 do for generators.

StopIteration is therefore no longer special: it stops the iteration only
because it is the default stop_exception.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: serhiy-storchaka

Doc/library/functions.rst

@@ -77,15 +77,18 @@ are always available.  They are listed here in alphabetical order.
    calls *callable* with no arguments and awaits the result
    for each call to its :meth:`~object.__anext__` method;
    if the awaited value is equal to *stop_value*,
-   or if the call raises :exc:`StopAsyncIteration` or an exception
-   matching *stop_exception*, :exc:`StopAsyncIteration` will be raised,
+   or if the call raises an exception matching *stop_exception*,
+   :exc:`StopAsyncIteration` will be raised,
    otherwise the value will be returned.
    The callable is only called when the result of :meth:`~object.__anext__`
    is awaited.
 
    *stop_exception* is an exception class or a tuple of exception classes.
    If *stop_value* is not specified,
    the iteration stops only when the callable raises an exception.
+   If the callable raises :exc:`StopAsyncIteration` which does not match
+   *stop_exception*, it is replaced with a :exc:`RuntimeError`,
+   as for asynchronous generators (see :pep:`525`).
 
    For example, reading fixed-size chunks from an asynchronous stream
    until the end of file is reached::
@@ -1191,13 +1194,15 @@ are always available.  They are listed here in alphabetical order.
    then the first argument must be a callable object.  The iterator created in this case
    will call *callable* with no arguments for each call to its
    :meth:`~iterator.__next__` method; if the value returned is equal to
-   *stop_value*, or if the call raises :exc:`StopIteration` or an exception
-   matching *stop_exception*, :exc:`StopIteration` will be raised, otherwise the
-   value will be returned.
+   *stop_value*, or if the call raises an exception matching *stop_exception*,
+   :exc:`StopIteration` will be raised, otherwise the value will be returned.
 
    *stop_exception* is an exception class or a tuple of exception classes.
    If *stop_value* is not specified,
    the iteration stops only when the callable raises an exception.
+   If the callable raises :exc:`StopIteration` which does not match
+   *stop_exception*, it is replaced with a :exc:`RuntimeError`,
+   as for generators (see :pep:`479`).
 
    See also :ref:`typeiter`.
 

Lib/test/test_asyncgen.py

@@ -835,9 +835,8 @@ async def spam():
             self.collect(aiter(spam, 100, stop_exception=LookupError)),
             [1, 2, 3, 4, 5])
 
-    def test_aiter_callable_stop_exception_redundant(self):
-        # StopAsyncIteration and an empty tuple stop the iteration in any
-        # case, so they are the same as no exception argument
+    def test_aiter_callable_stop_async_iteration(self):
+        # StopAsyncIteration is the default stop exception
         counter = self.make_counter()
         async def spam():
             value = await counter()
@@ -847,21 +846,38 @@ async def spam():
         self.assertEqual(
             self.collect(aiter(spam, stop_exception=StopAsyncIteration)),
             [1, 2, 3])
-        counter = self.make_counter()
-        self.assertEqual(self.collect(aiter(spam, stop_exception=())),
-                         [1, 2, 3])
 
-    def test_aiter_callable_stop_async_iteration(self):
-        # StopAsyncIteration stops the iteration even if other exception
-        # is specified
-        counter = self.make_counter()
+    def test_aiter_callable_leak_from_await(self):
+        # A StopAsyncIteration leaking from the await is replaced with
+        # RuntimeError (see PEP 525)
         async def spam():
-            value = await counter()
-            if value > 3:
-                raise StopAsyncIteration
-            return value
-        self.assertEqual(self.collect(aiter(spam, stop_exception=LookupError)),
-                         [1, 2, 3])
+            raise StopAsyncIteration
+        it = aiter(spam, 10, stop_exception=LookupError)
+        with self.assertRaisesRegex(RuntimeError,
+                                    'callable raised StopAsyncIteration') as cm:
+            self.loop.run_until_complete(anext(it))
+        self.assertIsInstance(cm.exception.__cause__, StopAsyncIteration)
+        # but if it matches stop_exception, it stops the iteration
+        it = aiter(spam, 10, stop_exception=(LookupError, StopAsyncIteration))
+        with self.assertRaises(StopAsyncIteration):
+            self.loop.run_until_complete(anext(it))
+
+    def test_aiter_callable_leak_from_call(self):
+        # StopIteration and StopAsyncIteration leaking from the call are
+        # replaced with RuntimeError (see PEP 525)
+        for exc in StopIteration, StopAsyncIteration:
+            with self.subTest(exc=exc):
+                def spam():
+                    raise exc
+                it = aiter(spam, 10, stop_exception=LookupError)
+                with self.assertRaisesRegex(
+                        RuntimeError, f'callable raised {exc.__name__}') as cm:
+                    self.loop.run_until_complete(anext(it))
+                self.assertIsInstance(cm.exception.__cause__, exc)
+                # but if it matches stop_exception, it stops the iteration
+                it = aiter(spam, 10, stop_exception=(LookupError, exc))
+                with self.assertRaises(StopAsyncIteration):
+                    self.loop.run_until_complete(anext(it))
 
     def test_aiter_callable_other_exception(self):
         async def spam():

Lib/test/test_iter.py

@@ -371,16 +371,18 @@ def test_iter_exception_and_stop(self):
         self.check_iterator(iter(CallableIterClass(), 200, stop_exception=IndexError),
                             list(range(101)))
 
-    # StopIteration stops the iteration even if other exception is specified
-    def test_iter_exception_stop_iteration(self):
-        def spam(state=[0]):
-            i = state[0]
-            if i == 10:
-                raise StopIteration
-            state[0] = i+1
-            return i
-        self.check_iterator(iter(spam, stop_exception=IndexError), list(range(10)),
-                            pickle=False)
+    # A leaking StopIteration is replaced with RuntimeError (see PEP 479)
+    def test_iter_exception_stop_iteration_leak(self):
+        def spam():
+            raise StopIteration
+        it = iter(spam, stop_exception=IndexError)
+        with self.assertRaisesRegex(RuntimeError,
+                                    'callable raised StopIteration') as cm:
+            next(it)
+        self.assertIsInstance(cm.exception.__cause__, StopIteration)
+        # but if it matches stop_exception, it stops the iteration
+        it = iter(spam, stop_exception=(IndexError, StopIteration))
+        self.assertRaises(StopIteration, next, it)
 
     # Other exceptions are propagated
     def test_iter_exception_not_matching(self):
@@ -395,22 +397,16 @@ def test_iter_exception_errors(self):
         self.assertRaises(TypeError, iter, len, stop_exception=(IndexError, 42))
         self.assertRaises(TypeError, iter, len, stop_exception=IndexError())
 
-    # StopIteration and an empty tuple stop the iteration in any case,
-    # so they are the same as no exception argument
-    def test_iter_exception_redundant(self):
-        def make_spam():
-            state = [0]
-            def spam():
-                if state[0] == 10:
-                    raise StopIteration
-                state[0] += 1
-                return state[0] - 1
-            return spam
-        for stop_exception in StopIteration, ():
-            with self.subTest(stop_exception=stop_exception):
-                self.check_iterator(
-                    iter(make_spam(), stop_exception=stop_exception),
-                    list(range(10)), pickle=False)
+    # StopIteration is the default stop exception
+    def test_iter_exception_stop_iteration(self):
+        def spam(state=[0]):
+            i = state[0]
+            if i == 10:
+                raise StopIteration
+            state[0] = i+1
+            return i
+        self.check_iterator(iter(spam, stop_exception=StopIteration),
+                            list(range(10)), pickle=False)
 
     def test_calliter_reduce(self):
         c = CallableIterClass()

Objects/iterobject.c

@@ -7,6 +7,7 @@
 #include "pycore_genobject.h"     // _PyCoro_GetAwaitableIter()
 #include "pycore_iterobject.h"    // _PyCallIter_NewEx()
 #include "pycore_object.h"        // _PyObject_GC_TRACK()
+#include "pycore_pyerrors.h"      // _PyErr_FormatFromCause()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 
 
@@ -187,10 +188,10 @@ PyTypeObject PySeqIter_Type = {
 
 typedef struct {
     PyObject_HEAD
-    /* All are set to NULL when the iterator is exhausted */
+    /* Both are set to NULL when the iterator is exhausted */
     PyObject *it_callable;
     PyObject *it_sentinel;  /* can be NULL */
-    PyObject *it_stop_exc;  /* not NULL if it_callable is not NULL */
+    PyObject *it_stop_exc;  /* never NULL */
 } calliterobject;
 
 PyObject *
@@ -264,14 +265,17 @@ calliter_iternext(PyObject *op)
         if (ok > 0) {
             Py_CLEAR(it->it_callable);
             Py_CLEAR(it->it_sentinel);
-            Py_CLEAR(it->it_stop_exc);
         }
     }
     else if (PyErr_ExceptionMatches(it->it_stop_exc)) {
         PyErr_Clear();
         Py_CLEAR(it->it_callable);
         Py_CLEAR(it->it_sentinel);
-        Py_CLEAR(it->it_stop_exc);
+    }
+    else if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
+        /* It would be mistaken for the end of the iteration (see PEP 479). */
+        _PyErr_FormatFromCause(PyExc_RuntimeError,
+                               "callable raised StopIteration");
     }
     Py_XDECREF(result);
     return NULL;
@@ -616,10 +620,10 @@ PyAnextAwaitable_New(PyObject *awaitable, PyObject *default_value)
 
 typedef struct {
     PyObject_HEAD
-    /* All are set to NULL when the iterator is exhausted */
+    /* Both are set to NULL when the iterator is exhausted */
     PyObject *it_callable;
     PyObject *it_sentinel;  /* can be NULL */
-    PyObject *it_stop_exc;  /* not NULL if it_callable is not NULL */
+    PyObject *it_stop_exc;  /* never NULL */
 } acalliterobject;
 
 #define acalliterobject_CAST(op)        ((acalliterobject *)(op))
@@ -660,7 +664,6 @@ acalliter_exhaust(acalliterobject *it)
 {
     Py_CLEAR(it->it_callable);
     Py_CLEAR(it->it_sentinel);
-    Py_CLEAR(it->it_stop_exc);
 }
 
 static void
@@ -783,6 +786,16 @@ acallawaitable_start(acallawaitableobject *aw)
             acalliter_exhaust(it);
             PyErr_SetNone(PyExc_StopAsyncIteration);
         }
+        else if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
+            /* It would be mistaken for the result of the await (PEP 525). */
+            _PyErr_FormatFromCause(PyExc_RuntimeError,
+                                   "callable raised StopIteration");
+        }
+        else if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) {
+            /* It would be mistaken for the end of the iteration (PEP 525). */
+            _PyErr_FormatFromCause(PyExc_RuntimeError,
+                                   "callable raised StopAsyncIteration");
+        }
         return -1;
     }
     aw->aw_wrapped = awaitable;
@@ -820,6 +833,11 @@ acallawaitable_handle_error(acallawaitableobject *aw)
         acalliter_exhaust(it);
         PyErr_SetNone(PyExc_StopAsyncIteration);
     }
+    else if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) {
+        /* It would be mistaken for the end of the iteration (see PEP 525). */
+        _PyErr_FormatFromCause(PyExc_RuntimeError,
+                               "callable raised StopAsyncIteration");
+    }
     return NULL;
 }
 

Python/bltinmodule.c

@@ -1908,13 +1908,13 @@ Get an iterator from an object.
 
 In the first form, the argument must supply its own iterator, or be a
 sequence.  In the second form, the callable is called until it returns
-the stop value or raises StopIteration or the specified exception.
+the stop value or raises the specified exception.
 [clinic start generated code]*/
 
 static PyObject *
 builtin_iter_impl(PyObject *module, PyObject *object, PyObject *stop_value,
                   PyObject *stop_exception)
-/*[clinic end generated code: output=eb9c9ae8f77bf400 input=d4eb3d19c8942790]*/
+/*[clinic end generated code: output=eb9c9ae8f77bf400 input=d3a2f767f29d9ae6]*/
 {
     if (stop_value == NULL && stop_exception == NULL) {
         return PyObject_GetIter(object);
@@ -1941,14 +1941,13 @@ aiter as builtin_aiter
 Return an AsyncIterator for an AsyncIterable object.
 
 In the second form, the callable is called and its result is awaited
-until it returns the stop value or raises StopAsyncIteration or the
-specified exception.
+until it returns the stop value or raises the specified exception.
 [clinic start generated code]*/
 
 static PyObject *
 builtin_aiter_impl(PyObject *module, PyObject *object, PyObject *stop_value,
                    PyObject *stop_exception)
-/*[clinic end generated code: output=2865edb3fbc45693 input=3eec4f0424a7ebac]*/
+/*[clinic end generated code: output=2865edb3fbc45693 input=2adb37d12adafd0c]*/
 {
     if (stop_value == NULL && stop_exception == NULL) {
         return PyObject_GetAIter(object);