Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AUTHORS.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
The AUTHORS/Contributors are (and/or have been):

* Aaron Swartz
* rui-zju <gh: Ychris12138>
* Yariv Barkan
* Alex Musayev
* Matěj Cepl
Expand Down
6 changes: 6 additions & 0 deletions ChangeLog.rst
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
Unreleased
==========

* Bound numeric character reference conversion to avoid Python's integer digit
limit while preserving valid references with many leading zeros.

2025.4.15
=========
----
Expand Down
10 changes: 8 additions & 2 deletions html2text/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -914,9 +914,15 @@ def handle_data(self, data: str, entity_char: bool = False) -> None:

def charref(self, name: str) -> str:
if name[0] in ["x", "X"]:
c = int(name[1:], 16)
digits, base = name[1:], 16
else:
c = int(name)
digits, base = name, 10

# Unicode code points need at most seven decimal (six hex) digits.
# Bound conversion before int() reaches Python's decimal digit limit,
# but keep valid references with arbitrarily many leading zeros.
digits = digits.lstrip("0") or "0"
c = int(digits, base) if len(digits) <= 7 else 0xFFFD

if not 0 < c < 0x110000 or 0xD800 <= c < 0xE000: # invalid or surrogate
c = 0xFFFD # REPLACEMENT CHARACTER
Expand Down
26 changes: 26 additions & 0 deletions test/test_numeric_entities.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import pytest

import html2text


@pytest.mark.parametrize(
"entity, expected",
[
pytest.param("9" * 5000, "\ufffd", id="oversized-decimal"),
pytest.param("x" + "f" * 5000, "\ufffd", id="oversized-hex"),
pytest.param("0" * 5000 + "65", "A", id="decimal-leading-zeros"),
pytest.param("X" + "0" * 5000 + "41", "A", id="hex-leading-zeros"),
pytest.param("0" * 5000, "\ufffd", id="zero"),
pytest.param("1114112", "\ufffd", id="above-unicode-range"),
pytest.param("xD800", "\ufffd", id="surrogate"),
pytest.param("128", "\u20ac", id="windows-1252-replacement"),
],
)
@pytest.mark.parametrize("unicode_snob", [False, True])
def test_numeric_entities(entity, expected, unicode_snob):
converter = html2text.HTML2Text()
converter.unicode_snob = unicode_snob

assert converter.handle("before &#" + entity + "; after") == (
"before " + expected + " after\n\n"
)