Read JSON and write XML as UTF-8 regardless of locale

docxtpl/__main__.py

@@ -109,10 +109,19 @@ def validate_all_args(parsed_args):
 
 
 def get_json_data(json_path):
-    with open(json_path) as file:
+    # JSON is UTF-8 encoded (RFC 8259) : do not rely on the locale default
+    # encoding, it is not UTF-8 on Windows. "utf-8-sig" also skips the BOM
+    # that Windows editors and PowerShell put at the beginning of the file.
+    with open(json_path, encoding="utf-8-sig") as file:
         try:
             json_data = json.load(file)
             return json_data
+        except UnicodeDecodeError as e:
+            print(
+                "File {json_path} is not UTF-8 encoded : {e.reason} at byte {e.start}. "
+                "Please save it as UTF-8.".format(e=e, json_path=json_path)
+            )
+            raise RuntimeError("Failed to get json data.")
         except json.JSONDecodeError as e:
             print(
                 "There was an error on line {e.lineno}, column {e.colno} while trying "

docxtpl/template.py

@@ -79,7 +79,10 @@ def get_xml(self):
         return self.xml_to_string(self.docx._element.body)
 
     def write_xml(self, filename):
-        with open(filename, "w") as fh:
+        # XML defaults to UTF-8 : do not rely on the locale default encoding,
+        # it is not UTF-8 on Windows and raises UnicodeEncodeError as soon as
+        # the document contains a character it cannot represent.
+        with open(filename, "w", encoding="utf-8") as fh:
             fh.write(self.get_xml())
 
     def patch_xml(self, src_xml):

tests/templates/module_execute_utf8.json

@@ -0,0 +1,8 @@
+{"json_dict_var" : {"json_dict_var":"successfully inserted"},
+"json_array_var": ["json","array","var","successfully", "inserted"],
+"json_string_var":"äöü 世界 Привет",
+"json_int_var":123,
+"json_float_var":1.234,
+"json_true_var":true,
+"json_false_var":false,
+"json_none_var":null}

tests/utf8_locale.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+"""
+Regression test for issue #516 : docxtpl must read and write text files as
+UTF-8 whatever the locale the interpreter runs under.
+
+This test re-runs itself in a child interpreter whose default encoding is NOT
+UTF-8 (PYTHONUTF8=0 + "C" locale), which is what Windows users get with a
+legacy code page. That is why the bug could not be reproduced on Linux/macOS,
+where the default encoding already is UTF-8.
+
+Without an explicit encoding :
+  - `python -m docxtpl` decodes the json data with the locale code page, so
+    non-ASCII characters end up mangled in the generated docx (cp1252) or the
+    CLI dies with UnicodeDecodeError (cp936, C locale),
+  - DocxTemplate.write_xml() dies with UnicodeEncodeError as soon as the
+    document contains a character the code page cannot represent.
+"""
+
+import locale
+import os
+import subprocess
+import sys
+
+CHILD_ENV_FLAG = "DOCXTPL_NON_UTF8_LOCALE_CHILD"
+
+# Resolved once, at import time : until Python 3.9 __file__ is relative to the
+# directory the interpreter was started from, so recomputing it after the
+# os.chdir() below would append a second "tests" component.
+TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
+SCRIPT_NAME = os.path.basename(__file__)
+STARTUP_DIR = os.getcwd()
+
+TEMPLATE_PATH = "templates/module_execute_tpl.docx"
+JSON_PATH = "templates/module_execute_utf8.json"
+XML_TEMPLATE_PATH = "templates/richtext_eastAsia_tpl.docx"
+OUTPUT_FILENAME = "output/utf8_locale.docx"
+XML_OUTPUT_FILENAME = "output/utf8_locale.xml"
+LATIN1_JSON_FILENAME = "output/utf8_locale_latin1.json"
+
+# Same value as the one stored in templates/module_execute_utf8.json
+EXPECTED_TEXT = "äöü 世界 Привет"
+
+
+def rerun_with_non_utf8_locale():
+    env = dict(os.environ)
+    env[CHILD_ENV_FLAG] = "1"
+    env["PYTHONUTF8"] = "0"  # disable UTF-8 mode (PEP 540)
+    env["PYTHONCOERCECLOCALE"] = "0"  # disable C locale coercion (PEP 538)
+    env["LC_ALL"] = "C"
+    env["LANG"] = "C"
+    # PYTHONPATH entries are relative to the directory this script was started
+    # from, and the child runs in TESTS_DIR : make them absolute so that a
+    # `PYTHONPATH=. python tests/utf8_locale.py` still finds docxtpl.
+    python_path = env.get("PYTHONPATH")
+    if python_path:
+        env["PYTHONPATH"] = os.pathsep.join(
+            os.path.join(STARTUP_DIR, entry) if entry else STARTUP_DIR
+            for entry in python_path.split(os.pathsep)
+        )
+    # Give the bare file name : the child filesystem encoding is ASCII, so an
+    # accented character in the path would not survive as an argument.
+    return subprocess.call(
+        [sys.executable, SCRIPT_NAME],
+        cwd=TESTS_DIR,
+        env=env,
+    )
+
+
+def check_json_data_is_read_as_utf8():
+    if os.path.exists(OUTPUT_FILENAME):
+        os.unlink(OUTPUT_FILENAME)
+
+    subprocess.check_call(
+        [sys.executable, "-m", "docxtpl"]
+        + [TEMPLATE_PATH, JSON_PATH, OUTPUT_FILENAME, "-o", "-q"]
+    )
+
+    from docx import Document
+
+    text = "\n".join(p.text for p in Document(OUTPUT_FILENAME).paragraphs)
+    assert EXPECTED_TEXT in text, (
+        "python -m docxtpl did not read the json data as UTF-8, got : %r" % text
+    )
+    print("    --> %s has been generated with UTF-8 data." % OUTPUT_FILENAME)
+
+
+def check_non_utf8_json_data_is_reported():
+    # A json file that is not UTF-8 encoded must be reported like any other
+    # bad input, not crash the command line with a UnicodeDecodeError.
+    from docxtpl.__main__ import get_json_data
+
+    with open(LATIN1_JSON_FILENAME, "wb") as fh:
+        fh.write('{"json_string_var": "caf\xe9"}'.encode("latin-1"))
+
+    try:
+        get_json_data(LATIN1_JSON_FILENAME)
+    except RuntimeError:
+        print("    --> a non UTF-8 json file is reported as a normal error.")
+    else:
+        raise AssertionError("reading a non UTF-8 json file should have failed")
+
+
+def check_xml_is_written_as_utf8():
+    from docxtpl import DocxTemplate
+
+    tpl = DocxTemplate(XML_TEMPLATE_PATH)
+    tpl.init_docx()
+    tpl.write_xml(XML_OUTPUT_FILENAME)
+
+    with open(XML_OUTPUT_FILENAME, "rb") as fh:
+        written = fh.read().decode("utf-8")
+    assert written == tpl.get_xml(), "write_xml() did not write UTF-8"
+    print("    --> %s has been written as UTF-8." % XML_OUTPUT_FILENAME)
+
+
+if __name__ == "__main__":
+    os.chdir(TESTS_DIR)
+    if not os.path.exists("output"):
+        os.mkdir("output")
+    if os.environ.get(CHILD_ENV_FLAG) != "1":
+        print(
+            "Re-running %s with a non UTF-8 locale ..." % SCRIPT_NAME,
+            flush=True,
+        )
+        sys.exit(rerun_with_non_utf8_locale())
+    if locale.getpreferredencoding(False).lower().replace("-", "") in (
+        "utf8",
+        "cp65001",
+    ):
+        print("    --> skipped : the locale encoding already is UTF-8.")
+        sys.exit(0)
+    check_json_data_is_read_as_utf8()
+    check_non_utf8_json_data_is_reported()
+    check_xml_is_written_as_utf8()