gh-67486: Say what is wrong when detect_encoding gets a text readline

Doc/library/tokenize.rst

@@ -109,7 +109,8 @@ function it uses to do this is available:
 
     It will call readline a maximum of twice, and return the encoding used
     (as a string) and a list of any lines (not decoded from bytes) it has read
-    in.
+    in. A :exc:`TypeError` is raised if readline returns a :class:`str`
+    instead of bytes.
 
     It detects the encoding from the presence of a UTF-8 BOM or an encoding
     cookie as specified in :pep:`263`. If both a BOM and a cookie are present,

Lib/test/test_tokenize.py

@@ -1693,6 +1693,34 @@ def test_false_encoding(self):
         self.assertEqual(encoding, 'utf-8')
         self.assertEqual(consumed_lines, [b'print("#coding=fake")'])
 
+    def test_readline_returning_str(self):
+        # gh-67486: readline must return bytes here, and saying so is more
+        # helpful than whatever the first operation on the line happens to
+        # complain about.
+        expected = 'readline callable returning bytes, but it returned str'
+
+        readline = self.get_readline(('print(something)\n',))
+        with self.assertRaisesRegex(TypeError, expected):
+            tokenize.detect_encoding(readline)
+
+        # The second line is read only when the first one is blank.
+        readline = self.get_readline((b'\n', 'print(something)\n'))
+        with self.assertRaisesRegex(TypeError, expected):
+            tokenize.detect_encoding(readline)
+
+        # The reported case went through tokenize(), which calls
+        # detect_encoding() itself.
+        with self.assertRaisesRegex(TypeError, expected):
+            list(tokenize.tokenize(StringIO('print(something)\n').readline))
+
+        # Only str is rejected. bytearray is not a documented input, but it
+        # has always worked, so a check for bytes exactly would break it.
+        line = bytearray(b'print(something)\n')
+        readline = self.get_readline((line,))
+        encoding, consumed_lines = tokenize.detect_encoding(readline)
+        self.assertEqual(encoding, 'utf-8')
+        self.assertEqual(consumed_lines, [line])
+
     @support.thread_unsafe
     def test_open(self):
         filename = os_helper.TESTFN + '.py'

Lib/tokenize.py

@@ -364,6 +364,7 @@ def detect_encoding(readline):
 
     It will call readline a maximum of twice, and return the encoding used
     (as a string) and a list of any lines (left as bytes) it has read in.
+    A TypeError is raised if readline returns a str instead of bytes.
 
     It detects the encoding from the presence of a utf-8 bom or an encoding
     cookie as specified in pep-0263.  If both a bom and a cookie are present,
@@ -382,9 +383,15 @@ def detect_encoding(readline):
     default = 'utf-8'
     def read_or_stop():
         try:
-            return readline()
+            line = readline()
         except StopIteration:
             return b''
+        if isinstance(line, str):
+            raise TypeError("detect_encoding() requires a readline callable "
+                            "returning bytes, but it returned str; read the "
+                            "source as bytes, or use "
+                            "tokenize.generate_tokens() to tokenize text")
+        return line
 
     def check(line, encoding):
         # Check if the line matches the encoding.

Misc/NEWS.d/next/Library/2026-09-24-22-58-45.gh-issue-67486.Ke_NQN.rst

@@ -0,0 +1,4 @@
+When *readline* returns a string instead of bytes,
+:func:`tokenize.detect_encoding` now raises a :exc:`TypeError` that says so.
+Previously the failure came later, from whichever operation touched the line
+first, and did not mention *readline*. Patch by Dmitry Voropaev.