From 286eaf0fbbb78872315cd7a17e8bad43ef7dcae3 Mon Sep 17 00:00:00 2001 From: David PAVLOVSCHII Date: Wed, 29 Jul 2026 09:23:25 +0300 Subject: [PATCH 1/2] Read json data and write xml as UTF-8, whatever the locale Fixes #516 docxtpl opened the only two text files it handles without an explicit encoding, so both fell back to locale.getpreferredencoding(False) : - docxtpl/__main__.py : `python -m docxtpl` decoded the json data with the locale code page. Under cp1252 the UTF-8 bytes silently become mojibake in the generated docx, which is what #516 reports ; under cp936 or a C locale they raise UnicodeDecodeError instead. JSON is UTF-8 (RFC 8259), so the encoding is not a guess. "utf-8-sig" is used rather than "utf-8" to also skip the BOM that PowerShell and Windows editors write. A file that really is not UTF-8 is now reported like any other bad input instead of escaping the command line error handling. - docxtpl/template.py : write_xml() encoded the document xml with the same locale code page and raised UnicodeEncodeError as soon as the document contained a character the code page cannot represent (any CJK or Cyrillic text under cp1252, any non-ASCII text under a C locale). This is also why the bug looked unreproducible : on Linux and macOS the default encoding already is UTF-8. tests/utf8_locale.py re-runs itself in a child interpreter forced off UTF-8 (PYTHONUTF8=0 + C locale) so the regression reproduces on Linux and macOS too. Reverting either fix on its own makes it fail. --- docxtpl/__main__.py | 11 ++- docxtpl/template.py | 5 +- tests/templates/module_execute_utf8.json | 8 ++ tests/utf8_locale.py | 118 +++++++++++++++++++++++ 4 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 tests/templates/module_execute_utf8.json create mode 100644 tests/utf8_locale.py diff --git a/docxtpl/__main__.py b/docxtpl/__main__.py index 59cf049..46ed5fd 100644 --- a/docxtpl/__main__.py +++ b/docxtpl/__main__.py @@ -109,10 +109,19 @@ def validate_all_args(parsed_args): def get_json_data(json_path): - with open(json_path) as file: + # JSON is UTF-8 encoded (RFC 8259) : do not rely on the locale default + # encoding, it is not UTF-8 on Windows. "utf-8-sig" also skips the BOM + # that Windows editors and PowerShell put at the beginning of the file. + with open(json_path, encoding="utf-8-sig") as file: try: json_data = json.load(file) return json_data + except UnicodeDecodeError as e: + print( + "File {json_path} is not UTF-8 encoded : {e.reason} at byte {e.start}. " + "Please save it as UTF-8.".format(e=e, json_path=json_path) + ) + raise RuntimeError("Failed to get json data.") except json.JSONDecodeError as e: print( "There was an error on line {e.lineno}, column {e.colno} while trying " diff --git a/docxtpl/template.py b/docxtpl/template.py index f20280a..fed9a2d 100644 --- a/docxtpl/template.py +++ b/docxtpl/template.py @@ -79,7 +79,10 @@ def get_xml(self): return self.xml_to_string(self.docx._element.body) def write_xml(self, filename): - with open(filename, "w") as fh: + # XML defaults to UTF-8 : do not rely on the locale default encoding, + # it is not UTF-8 on Windows and raises UnicodeEncodeError as soon as + # the document contains a character it cannot represent. + with open(filename, "w", encoding="utf-8") as fh: fh.write(self.get_xml()) def patch_xml(self, src_xml): diff --git a/tests/templates/module_execute_utf8.json b/tests/templates/module_execute_utf8.json new file mode 100644 index 0000000..d6fd82a --- /dev/null +++ b/tests/templates/module_execute_utf8.json @@ -0,0 +1,8 @@ +{"json_dict_var" : {"json_dict_var":"successfully inserted"}, +"json_array_var": ["json","array","var","successfully", "inserted"], +"json_string_var":"äöü 世界 Привет", +"json_int_var":123, +"json_float_var":1.234, +"json_true_var":true, +"json_false_var":false, +"json_none_var":null} diff --git a/tests/utf8_locale.py b/tests/utf8_locale.py new file mode 100644 index 0000000..ab9644b --- /dev/null +++ b/tests/utf8_locale.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +""" +Regression test for issue #516 : docxtpl must read and write text files as +UTF-8 whatever the locale the interpreter runs under. + +This test re-runs itself in a child interpreter whose default encoding is NOT +UTF-8 (PYTHONUTF8=0 + "C" locale), which is what Windows users get with a +legacy code page. That is why the bug could not be reproduced on Linux/macOS, +where the default encoding already is UTF-8. + +Without an explicit encoding : + - `python -m docxtpl` decodes the json data with the locale code page, so + non-ASCII characters end up mangled in the generated docx (cp1252) or the + CLI dies with UnicodeDecodeError (cp936, C locale), + - DocxTemplate.write_xml() dies with UnicodeEncodeError as soon as the + document contains a character the code page cannot represent. +""" + +import locale +import os +import subprocess +import sys + +CHILD_ENV_FLAG = "DOCXTPL_NON_UTF8_LOCALE_CHILD" + +TEMPLATE_PATH = "templates/module_execute_tpl.docx" +JSON_PATH = "templates/module_execute_utf8.json" +XML_TEMPLATE_PATH = "templates/richtext_eastAsia_tpl.docx" +OUTPUT_FILENAME = "output/utf8_locale.docx" +XML_OUTPUT_FILENAME = "output/utf8_locale.xml" +LATIN1_JSON_FILENAME = "output/utf8_locale_latin1.json" + +# Same value as the one stored in templates/module_execute_utf8.json +EXPECTED_TEXT = "äöü 世界 Привет" + + +def rerun_with_non_utf8_locale(): + env = dict(os.environ) + env[CHILD_ENV_FLAG] = "1" + env["PYTHONUTF8"] = "0" # disable UTF-8 mode (PEP 540) + env["PYTHONCOERCECLOCALE"] = "0" # disable C locale coercion (PEP 538) + env["LC_ALL"] = "C" + env["LANG"] = "C" + # Give the bare file name : the child filesystem encoding is ASCII, so an + # accented character in the path would not survive as an argument. + return subprocess.call( + [sys.executable, os.path.basename(__file__)], + cwd=os.path.dirname(os.path.abspath(__file__)), + env=env, + ) + + +def check_json_data_is_read_as_utf8(): + if os.path.exists(OUTPUT_FILENAME): + os.unlink(OUTPUT_FILENAME) + + subprocess.check_call( + [sys.executable, "-m", "docxtpl"] + + [TEMPLATE_PATH, JSON_PATH, OUTPUT_FILENAME, "-o", "-q"] + ) + + from docx import Document + + text = "\n".join(p.text for p in Document(OUTPUT_FILENAME).paragraphs) + assert EXPECTED_TEXT in text, ( + "python -m docxtpl did not read the json data as UTF-8, got : %r" % text + ) + print(" --> %s has been generated with UTF-8 data." % OUTPUT_FILENAME) + + +def check_non_utf8_json_data_is_reported(): + # A json file that is not UTF-8 encoded must be reported like any other + # bad input, not crash the command line with a UnicodeDecodeError. + from docxtpl.__main__ import get_json_data + + with open(LATIN1_JSON_FILENAME, "wb") as fh: + fh.write('{"json_string_var": "caf\xe9"}'.encode("latin-1")) + + try: + get_json_data(LATIN1_JSON_FILENAME) + except RuntimeError: + print(" --> a non UTF-8 json file is reported as a normal error.") + else: + raise AssertionError("reading a non UTF-8 json file should have failed") + + +def check_xml_is_written_as_utf8(): + from docxtpl import DocxTemplate + + tpl = DocxTemplate(XML_TEMPLATE_PATH) + tpl.init_docx() + tpl.write_xml(XML_OUTPUT_FILENAME) + + with open(XML_OUTPUT_FILENAME, "rb") as fh: + written = fh.read().decode("utf-8") + assert written == tpl.get_xml(), "write_xml() did not write UTF-8" + print(" --> %s has been written as UTF-8." % XML_OUTPUT_FILENAME) + + +if __name__ == "__main__": + os.chdir(os.path.dirname(os.path.abspath(__file__))) + if not os.path.exists("output"): + os.mkdir("output") + if os.environ.get(CHILD_ENV_FLAG) != "1": + print( + "Re-running %s with a non UTF-8 locale ..." % os.path.basename(__file__), + flush=True, + ) + sys.exit(rerun_with_non_utf8_locale()) + if locale.getpreferredencoding(False).lower().replace("-", "") in ( + "utf8", + "cp65001", + ): + print(" --> skipped : the locale encoding already is UTF-8.") + sys.exit(0) + check_json_data_is_read_as_utf8() + check_non_utf8_json_data_is_reported() + check_xml_is_written_as_utf8() From 312499dd6b0d5606f4490bcc68b1f11d5eb48cc3 Mon Sep 17 00:00:00 2001 From: David PAVLOVSCHII Date: Thu, 30 Jul 2026 20:43:22 +0300 Subject: [PATCH 2/2] Resolve the tests directory before changing into it tests/utf8_locale.py computed os.path.dirname(os.path.abspath(__file__)) twice : once to chdir into tests, then again inside rerun_with_non_utf8_locale() to give the child its cwd. Up to Python 3.8 __file__ stays relative to the directory the interpreter was started from, so the second call resolved against the new directory and produced tests/tests. `PYTHONPATH=. python3.8 tests/utf8_locale.py` from the repository root died with FileNotFoundError before the child even started, while Python 3.9+ passed because __file__ is absolute there. Resolve the directory and the script name once, at import time, and reuse them for both os.chdir() and the child cwd, so neither depends on the current directory any more. The PYTHONPATH handed to the child had the same problem : the child runs in the tests directory, so a relative entry such as "." no longer pointed at the repository root and docxtpl was not importable. Make its entries absolute against the startup directory. --- tests/utf8_locale.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/utf8_locale.py b/tests/utf8_locale.py index ab9644b..fc258a6 100644 --- a/tests/utf8_locale.py +++ b/tests/utf8_locale.py @@ -23,6 +23,13 @@ CHILD_ENV_FLAG = "DOCXTPL_NON_UTF8_LOCALE_CHILD" +# Resolved once, at import time : until Python 3.9 __file__ is relative to the +# directory the interpreter was started from, so recomputing it after the +# os.chdir() below would append a second "tests" component. +TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +SCRIPT_NAME = os.path.basename(__file__) +STARTUP_DIR = os.getcwd() + TEMPLATE_PATH = "templates/module_execute_tpl.docx" JSON_PATH = "templates/module_execute_utf8.json" XML_TEMPLATE_PATH = "templates/richtext_eastAsia_tpl.docx" @@ -41,11 +48,20 @@ def rerun_with_non_utf8_locale(): env["PYTHONCOERCECLOCALE"] = "0" # disable C locale coercion (PEP 538) env["LC_ALL"] = "C" env["LANG"] = "C" + # PYTHONPATH entries are relative to the directory this script was started + # from, and the child runs in TESTS_DIR : make them absolute so that a + # `PYTHONPATH=. python tests/utf8_locale.py` still finds docxtpl. + python_path = env.get("PYTHONPATH") + if python_path: + env["PYTHONPATH"] = os.pathsep.join( + os.path.join(STARTUP_DIR, entry) if entry else STARTUP_DIR + for entry in python_path.split(os.pathsep) + ) # Give the bare file name : the child filesystem encoding is ASCII, so an # accented character in the path would not survive as an argument. return subprocess.call( - [sys.executable, os.path.basename(__file__)], - cwd=os.path.dirname(os.path.abspath(__file__)), + [sys.executable, SCRIPT_NAME], + cwd=TESTS_DIR, env=env, ) @@ -98,12 +114,12 @@ def check_xml_is_written_as_utf8(): if __name__ == "__main__": - os.chdir(os.path.dirname(os.path.abspath(__file__))) + os.chdir(TESTS_DIR) if not os.path.exists("output"): os.mkdir("output") if os.environ.get(CHILD_ENV_FLAG) != "1": print( - "Re-running %s with a non UTF-8 locale ..." % os.path.basename(__file__), + "Re-running %s with a non UTF-8 locale ..." % SCRIPT_NAME, flush=True, ) sys.exit(rerun_with_non_utf8_locale())