diff --git a/docxtpl/__main__.py b/docxtpl/__main__.py index 59cf049..46ed5fd 100644 --- a/docxtpl/__main__.py +++ b/docxtpl/__main__.py @@ -109,10 +109,19 @@ def validate_all_args(parsed_args): def get_json_data(json_path): - with open(json_path) as file: + # JSON is UTF-8 encoded (RFC 8259) : do not rely on the locale default + # encoding, it is not UTF-8 on Windows. "utf-8-sig" also skips the BOM + # that Windows editors and PowerShell put at the beginning of the file. + with open(json_path, encoding="utf-8-sig") as file: try: json_data = json.load(file) return json_data + except UnicodeDecodeError as e: + print( + "File {json_path} is not UTF-8 encoded : {e.reason} at byte {e.start}. " + "Please save it as UTF-8.".format(e=e, json_path=json_path) + ) + raise RuntimeError("Failed to get json data.") except json.JSONDecodeError as e: print( "There was an error on line {e.lineno}, column {e.colno} while trying " diff --git a/docxtpl/template.py b/docxtpl/template.py index f20280a..fed9a2d 100644 --- a/docxtpl/template.py +++ b/docxtpl/template.py @@ -79,7 +79,10 @@ def get_xml(self): return self.xml_to_string(self.docx._element.body) def write_xml(self, filename): - with open(filename, "w") as fh: + # XML defaults to UTF-8 : do not rely on the locale default encoding, + # it is not UTF-8 on Windows and raises UnicodeEncodeError as soon as + # the document contains a character it cannot represent. + with open(filename, "w", encoding="utf-8") as fh: fh.write(self.get_xml()) def patch_xml(self, src_xml): diff --git a/tests/templates/module_execute_utf8.json b/tests/templates/module_execute_utf8.json new file mode 100644 index 0000000..d6fd82a --- /dev/null +++ b/tests/templates/module_execute_utf8.json @@ -0,0 +1,8 @@ +{"json_dict_var" : {"json_dict_var":"successfully inserted"}, +"json_array_var": ["json","array","var","successfully", "inserted"], +"json_string_var":"äöü 世界 Привет", +"json_int_var":123, +"json_float_var":1.234, +"json_true_var":true, +"json_false_var":false, +"json_none_var":null} diff --git a/tests/utf8_locale.py b/tests/utf8_locale.py new file mode 100644 index 0000000..fc258a6 --- /dev/null +++ b/tests/utf8_locale.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +""" +Regression test for issue #516 : docxtpl must read and write text files as +UTF-8 whatever the locale the interpreter runs under. + +This test re-runs itself in a child interpreter whose default encoding is NOT +UTF-8 (PYTHONUTF8=0 + "C" locale), which is what Windows users get with a +legacy code page. That is why the bug could not be reproduced on Linux/macOS, +where the default encoding already is UTF-8. + +Without an explicit encoding : + - `python -m docxtpl` decodes the json data with the locale code page, so + non-ASCII characters end up mangled in the generated docx (cp1252) or the + CLI dies with UnicodeDecodeError (cp936, C locale), + - DocxTemplate.write_xml() dies with UnicodeEncodeError as soon as the + document contains a character the code page cannot represent. +""" + +import locale +import os +import subprocess +import sys + +CHILD_ENV_FLAG = "DOCXTPL_NON_UTF8_LOCALE_CHILD" + +# Resolved once, at import time : until Python 3.9 __file__ is relative to the +# directory the interpreter was started from, so recomputing it after the +# os.chdir() below would append a second "tests" component. +TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +SCRIPT_NAME = os.path.basename(__file__) +STARTUP_DIR = os.getcwd() + +TEMPLATE_PATH = "templates/module_execute_tpl.docx" +JSON_PATH = "templates/module_execute_utf8.json" +XML_TEMPLATE_PATH = "templates/richtext_eastAsia_tpl.docx" +OUTPUT_FILENAME = "output/utf8_locale.docx" +XML_OUTPUT_FILENAME = "output/utf8_locale.xml" +LATIN1_JSON_FILENAME = "output/utf8_locale_latin1.json" + +# Same value as the one stored in templates/module_execute_utf8.json +EXPECTED_TEXT = "äöü 世界 Привет" + + +def rerun_with_non_utf8_locale(): + env = dict(os.environ) + env[CHILD_ENV_FLAG] = "1" + env["PYTHONUTF8"] = "0" # disable UTF-8 mode (PEP 540) + env["PYTHONCOERCECLOCALE"] = "0" # disable C locale coercion (PEP 538) + env["LC_ALL"] = "C" + env["LANG"] = "C" + # PYTHONPATH entries are relative to the directory this script was started + # from, and the child runs in TESTS_DIR : make them absolute so that a + # `PYTHONPATH=. python tests/utf8_locale.py` still finds docxtpl. + python_path = env.get("PYTHONPATH") + if python_path: + env["PYTHONPATH"] = os.pathsep.join( + os.path.join(STARTUP_DIR, entry) if entry else STARTUP_DIR + for entry in python_path.split(os.pathsep) + ) + # Give the bare file name : the child filesystem encoding is ASCII, so an + # accented character in the path would not survive as an argument. + return subprocess.call( + [sys.executable, SCRIPT_NAME], + cwd=TESTS_DIR, + env=env, + ) + + +def check_json_data_is_read_as_utf8(): + if os.path.exists(OUTPUT_FILENAME): + os.unlink(OUTPUT_FILENAME) + + subprocess.check_call( + [sys.executable, "-m", "docxtpl"] + + [TEMPLATE_PATH, JSON_PATH, OUTPUT_FILENAME, "-o", "-q"] + ) + + from docx import Document + + text = "\n".join(p.text for p in Document(OUTPUT_FILENAME).paragraphs) + assert EXPECTED_TEXT in text, ( + "python -m docxtpl did not read the json data as UTF-8, got : %r" % text + ) + print(" --> %s has been generated with UTF-8 data." % OUTPUT_FILENAME) + + +def check_non_utf8_json_data_is_reported(): + # A json file that is not UTF-8 encoded must be reported like any other + # bad input, not crash the command line with a UnicodeDecodeError. + from docxtpl.__main__ import get_json_data + + with open(LATIN1_JSON_FILENAME, "wb") as fh: + fh.write('{"json_string_var": "caf\xe9"}'.encode("latin-1")) + + try: + get_json_data(LATIN1_JSON_FILENAME) + except RuntimeError: + print(" --> a non UTF-8 json file is reported as a normal error.") + else: + raise AssertionError("reading a non UTF-8 json file should have failed") + + +def check_xml_is_written_as_utf8(): + from docxtpl import DocxTemplate + + tpl = DocxTemplate(XML_TEMPLATE_PATH) + tpl.init_docx() + tpl.write_xml(XML_OUTPUT_FILENAME) + + with open(XML_OUTPUT_FILENAME, "rb") as fh: + written = fh.read().decode("utf-8") + assert written == tpl.get_xml(), "write_xml() did not write UTF-8" + print(" --> %s has been written as UTF-8." % XML_OUTPUT_FILENAME) + + +if __name__ == "__main__": + os.chdir(TESTS_DIR) + if not os.path.exists("output"): + os.mkdir("output") + if os.environ.get(CHILD_ENV_FLAG) != "1": + print( + "Re-running %s with a non UTF-8 locale ..." % SCRIPT_NAME, + flush=True, + ) + sys.exit(rerun_with_non_utf8_locale()) + if locale.getpreferredencoding(False).lower().replace("-", "") in ( + "utf8", + "cp65001", + ): + print(" --> skipped : the locale encoding already is UTF-8.") + sys.exit(0) + check_json_data_is_read_as_utf8() + check_non_utf8_json_data_is_reported() + check_xml_is_written_as_utf8()