From a5cdc2f37eb8dd3752ba51a186a5bde499fd9ef1 Mon Sep 17 00:00:00 2001 From: rui-zju <12245039@zju.edu.cn> Date: Fri, 9 Oct 2026 11:12:33 +0800 Subject: [PATCH] Bound numeric character reference conversion --- AUTHORS.rst | 1 + ChangeLog.rst | 6 ++++++ html2text/__init__.py | 10 ++++++++-- test/test_numeric_entities.py | 26 ++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 test/test_numeric_entities.py diff --git a/AUTHORS.rst b/AUTHORS.rst index 5b3ac10..42dd04d 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -3,6 +3,7 @@ The AUTHORS/Contributors are (and/or have been): * Aaron Swartz +* rui-zju * Yariv Barkan * Alex Musayev * Matěj Cepl diff --git a/ChangeLog.rst b/ChangeLog.rst index 2d96e3b..376387b 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,9 @@ +Unreleased +========== + +* Bound numeric character reference conversion to avoid Python's integer digit + limit while preserving valid references with many leading zeros. + 2025.4.15 ========= ---- diff --git a/html2text/__init__.py b/html2text/__init__.py index 621fa74..5802448 100644 --- a/html2text/__init__.py +++ b/html2text/__init__.py @@ -914,9 +914,15 @@ def handle_data(self, data: str, entity_char: bool = False) -> None: def charref(self, name: str) -> str: if name[0] in ["x", "X"]: - c = int(name[1:], 16) + digits, base = name[1:], 16 else: - c = int(name) + digits, base = name, 10 + + # Unicode code points need at most seven decimal (six hex) digits. + # Bound conversion before int() reaches Python's decimal digit limit, + # but keep valid references with arbitrarily many leading zeros. + digits = digits.lstrip("0") or "0" + c = int(digits, base) if len(digits) <= 7 else 0xFFFD if not 0 < c < 0x110000 or 0xD800 <= c < 0xE000: # invalid or surrogate c = 0xFFFD # REPLACEMENT CHARACTER diff --git a/test/test_numeric_entities.py b/test/test_numeric_entities.py new file mode 100644 index 0000000..f3b1978 --- /dev/null +++ b/test/test_numeric_entities.py @@ -0,0 +1,26 @@ +import pytest + +import html2text + + +@pytest.mark.parametrize( + "entity, expected", + [ + pytest.param("9" * 5000, "\ufffd", id="oversized-decimal"), + pytest.param("x" + "f" * 5000, "\ufffd", id="oversized-hex"), + pytest.param("0" * 5000 + "65", "A", id="decimal-leading-zeros"), + pytest.param("X" + "0" * 5000 + "41", "A", id="hex-leading-zeros"), + pytest.param("0" * 5000, "\ufffd", id="zero"), + pytest.param("1114112", "\ufffd", id="above-unicode-range"), + pytest.param("xD800", "\ufffd", id="surrogate"), + pytest.param("128", "\u20ac", id="windows-1252-replacement"), + ], +) +@pytest.mark.parametrize("unicode_snob", [False, True]) +def test_numeric_entities(entity, expected, unicode_snob): + converter = html2text.HTML2Text() + converter.unicode_snob = unicode_snob + + assert converter.handle("before &#" + entity + "; after") == ( + "before " + expected + " after\n\n" + )