Handle readline stopiteration

ShaharNaveh · ShaharNaveh · commit 30f54f6910fd · 2025-11-08T13:17:37.000+02:00
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
@@ -51,7 +51,6 @@ def check_tokenize(self, s, expected):
                          ["    ENCODING   'utf-8'       (0, 0) (0, 0)"] +
                          expected.rstrip().splitlines())
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_invalid_readline(self):
         def gen():
             yield "sdfosdg"
diff --git a/stdlib/src/tokenize.rs b/stdlib/src/tokenize.rs
@@ -5,7 +5,7 @@ mod _tokenize {
     use crate::{
         common::lock::PyRwLock,
         vm::{
-            Py, PyPayload, PyResult, VirtualMachine,
+            AsObject, Py, PyPayload, PyResult, VirtualMachine,
             builtins::{PyBytes, PyStr, PyStrRef, PyTypeRef},
             convert::ToPyObject,
             function::ArgCallable,
@@ -38,7 +38,12 @@ mod _tokenize {
             // we need to check if it's callable and raise a type error if it's not.
             let raw_line = match self.readline.invoke((), vm) {
                 Ok(v) => v,
-                Err(_) => return Ok(String::new()),
+                Err(err) => {
+                    if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) {
+                        return Ok(String::new());
+                    }
+                    return Err(err);
+                }
             };
             Ok(match &self.encoding {
                 Some(encoding) => {
