From bbd5f70c3c17aa49a1b2189d37a4eae543869ac1 Mon Sep 17 00:00:00 2001 From: Nicola Urs Bruce Spieser <247831190+quantumnic@users.noreply.github.com> Date: Sun, 15 Mar 2026 11:49:49 +0100 Subject: [PATCH] fix: add explicit UTF-8 encoding to file operations for Windows compatibility (#1845) On Windows, Python's default encoding is the system locale (e.g., cp1252, gbk) rather than UTF-8. This causes UnicodeDecodeError when reading sdkconfig files, CMakeLists.txt, or JSON configs that contain non-ASCII characters (e.g., Chinese comments, UTF-8 BOM). Fix: Add explicit encoding='utf-8' to all text file open() calls in: - scripts/release.py (Path.open() for config.json, CMakeLists.txt, sdkconfig) - scripts/build_default_assets.py (io.open() for sdkconfig, open() for headers/configs) - scripts/versions.py (open() for info.json read/write) Relates to #1792 Co-authored-by: Nicola Spieser --- scripts/build_default_assets.py | 12 ++++++------ scripts/release.py | 10 +++++----- scripts/versions.py | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/build_default_assets.py b/scripts/build_default_assets.py index 9ca1464..90cee7d 100755 --- a/scripts/build_default_assets.py +++ b/scripts/build_default_assets.py @@ -424,7 +424,7 @@ def pack_assets_simple(target_path, include_path, out_file, assets_path, max_nam current_year = datetime.now().year asset_name = os.path.basename(assets_path) header_file_path = os.path.join(include_path, f'mmap_generate_{asset_name}.h') - with open(header_file_path, 'w') as output_header: + with open(header_file_path, 'w', encoding='utf-8') as output_header: output_header.write('/*\n') output_header.write(' * SPDX-FileCopyrightText: 2022-{} Espressif Systems (Shanghai) CO LTD\n'.format(current_year)) output_header.write(' *\n') @@ -463,7 +463,7 @@ def read_wakenet_from_sdkconfig(sdkconfig_path): return [] models = [] - with io.open(sdkconfig_path, "r") as f: + with io.open(sdkconfig_path, "r", encoding="utf-8") as f: for label in f: label = label.strip("\n") if 'CONFIG_SR_WN' in label and '#' not in label[0]: @@ -488,7 +488,7 @@ def read_multinet_from_sdkconfig(sdkconfig_path): print(f"Warning: sdkconfig file not found: {sdkconfig_path}") return [] - with io.open(sdkconfig_path, "r") as f: + with io.open(sdkconfig_path, "r", encoding="utf-8") as f: models_string = '' for label in f: label = label.strip("\n") @@ -549,7 +549,7 @@ def read_wake_word_type_from_sdkconfig(sdkconfig_path): 'wake_word_disabled': False } - with io.open(sdkconfig_path, "r") as f: + with io.open(sdkconfig_path, "r", encoding="utf-8") as f: for line in f: line = line.strip("\n") if line.startswith('#'): @@ -578,7 +578,7 @@ def read_custom_wake_word_from_sdkconfig(sdkconfig_path): return None config_values = {} - with io.open(sdkconfig_path, "r") as f: + with io.open(sdkconfig_path, "r", encoding="utf-8") as f: for line in f: line = line.strip("\n") if line.startswith('#') or '=' not in line: @@ -777,7 +777,7 @@ def build_assets_integrated(wakenet_model_paths, multinet_model_paths, text_font config_path = generate_config_json(temp_build_dir, assets_dir) # Load config and pack assets - with open(config_path, 'r') as f: + with open(config_path, 'r', encoding='utf-8') as f: config_data = json.load(f) # Use simplified packing function diff --git a/scripts/release.py b/scripts/release.py index 1989420..54ddb88 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -18,7 +18,7 @@ def get_board_type_from_compile_commands() -> Optional[str]: compile_file = Path("build/compile_commands.json") if not compile_file.exists(): return None - with compile_file.open() as f: + with compile_file.open(encoding='utf-8') as f: data = json.load(f) for item in data: if not item["file"].endswith("main.cc"): @@ -31,7 +31,7 @@ def get_board_type_from_compile_commands() -> Optional[str]: def get_project_version() -> Optional[str]: """Read set(PROJECT_VER "x.y.z") from root CMakeLists.txt""" - with Path("CMakeLists.txt").open() as f: + with Path("CMakeLists.txt").open(encoding='utf-8') as f: for line in f: if line.startswith("set(PROJECT_VER"): return line.split("\"")[1] @@ -87,7 +87,7 @@ def _collect_variants(config_filename: str = "config.json") -> list[dict[str, st board = board_dir.relative_to(_BOARDS_DIR).as_posix() try: - with cfg_path.open() as f: + with cfg_path.open(encoding='utf-8') as f: cfg = json.load(f) manufacturer = _get_manufacturer(cfg) @@ -232,7 +232,7 @@ def release(board_type: str, config_filename: str = "config.json", *, filter_nam project_version = get_project_version() print(f"Project Version: {project_version} ({cfg_path})") - with cfg_path.open() as f: + with cfg_path.open(encoding='utf-8') as f: cfg = json.load(f) target = cfg["target"] manufacturer = _get_manufacturer(cfg) @@ -279,7 +279,7 @@ def release(board_type: str, config_filename: str = "config.json", *, filter_nam sys.exit(1) # Append sdkconfig - with Path("sdkconfig").open("a") as f: + with Path("sdkconfig").open("a", encoding='utf-8') as f: f.write("\n") f.write("# Append by release.py\n") for append in sdkconfig_append: diff --git a/scripts/versions.py b/scripts/versions.py index ac9472c..a5a115c 100644 --- a/scripts/versions.py +++ b/scripts/versions.py @@ -236,11 +236,11 @@ def main(): target_dir = os.path.join("firmwares", tag) info["tag"] = tag info["url"] = os.path.join(os.environ['OSS_BUCKET_URL'], target_dir, "xiaozhi.bin") - open(info_path, "w").write(json.dumps(info, indent=4)) + open(info_path, "w", encoding="utf-8").write(json.dumps(info, indent=4)) # upload all file to oss upload_dir_to_oss(folder, target_dir) # read info.json - info = json.load(open(info_path)) + info = json.load(open(info_path, encoding="utf-8")) # post info.json to server post_info_to_server(info)