Windows: add a manifest to use UTF-8 code pages

As of Windows Version 1903 (May 2019 Update), we can use the ActiveCodePage property in the fusion manifest for unpackaged apps to force a process to use UTF-8 as the process code page. This method requires no changes to the source code. https://learn.microsoft.com/en-us/windows/apps/design/globalizing/use-utf8-code-page#set-a-process-code-page-to-utf-8
AOMediaCodec · Dec 23, 2023 · 94bfc29 · 94bfc29
1 parent d6d224a
commit 94bfc29
Show file tree

Hide file tree

Showing 9 changed files with 39 additions and 116 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -668,20 +668,21 @@ endif()
 
 if(AVIF_BUILD_APPS)
     add_executable(avifenc apps/avifenc.c)
+    if(WIN32 AND NOT MINGW)
+        target_sources(avifenc PRIVATE apps/utf8.manifest)
+    endif()
     if(AVIF_USE_CXX)
         set_target_properties(avifenc PROPERTIES LINKER_LANGUAGE "CXX")
     endif()
     target_link_libraries(avifenc avif_apps)
     add_executable(avifdec apps/avifdec.c)
+    if(WIN32 AND NOT MINGW)
+        target_sources(avifdec PRIVATE apps/utf8.manifest)
+    endif()
     if(AVIF_USE_CXX)
         set_target_properties(avifdec PROPERTIES LINKER_LANGUAGE "CXX")
     endif()
     target_link_libraries(avifdec avif_apps)
-    if(MINGW AND NOT "$ENV{MSYSTEM}" MATCHES "(MINGW32|MINGW64)")
-        # MINGW* are legacy environments that link against MSVCRT which does not support UTF8.
-        target_link_options(avifenc PRIVATE -municode)
-        target_link_options(avifdec PRIVATE -municode)
-    endif()
 
     if(NOT SKIP_INSTALL_APPS AND NOT SKIP_INSTALL_ALL)
         install(
@@ -709,6 +710,9 @@ if(AVIF_BUILD_APPS)
             )
 
             add_executable(avifgainmaputil "${AVIFGAINMAPUTIL_SRCS}")
+            if(WIN32 AND NOT MINGW)
+                target_sources(avifgainmaputil PRIVATE apps/utf8.manifest)
+            endif()
             set_target_properties(avifgainmaputil PROPERTIES LINKER_LANGUAGE "CXX")
             target_include_directories(avifgainmaputil PRIVATE apps/avifgainmaputil/)
             target_link_libraries(avifgainmaputil libargparse avif_apps)

diff --git a/apps/avifdec.c b/apps/avifdec.c
@@ -14,18 +14,12 @@
 #include <stdlib.h>
 #include <string.h>
 
-#if defined(_WIN32)
-#include <locale.h>
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#endif
-
 #define DEFAULT_JPEG_QUALITY 90
 
 #define NEXTARG()                                                     \
     if (((argIndex + 1) == argc) || (argv[argIndex + 1][0] == '-')) { \
         fprintf(stderr, "%s requires an argument.", arg);             \
-        goto cleanup;                                                 \
+        return 1;                                                     \
     }                                                                 \
     arg = argv[++argIndex]
 
@@ -59,7 +53,7 @@ static void syntax(void)
     avifPrintVersions();
 }
 
-MAIN()
+int main(int argc, char * argv[])
 {
     const char * inputFilename = NULL;
     const char * outputFilename = NULL;
@@ -77,15 +71,12 @@ MAIN()
     uint32_t frameIndex = 0;
     uint32_t imageSizeLimit = AVIF_DEFAULT_IMAGE_SIZE_LIMIT;
     uint32_t imageDimensionLimit = AVIF_DEFAULT_IMAGE_DIMENSION_LIMIT;
-    int returnCode = 1;
-    avifDecoder * decoder = NULL;
 
     if (argc < 2) {
         syntax();
         return 1;
     }
 
-    INIT_ARGV()
     int argIndex = 1;
     while (argIndex < argc) {
         const char * arg = argv[argIndex];
@@ -240,7 +231,7 @@ MAIN()
             return 1;
         }
 
-        decoder = avifDecoderCreate();
+        avifDecoder * decoder = avifDecoderCreate();
         if (!decoder) {
             fprintf(stderr, "Memory allocation failure\n");
             return 1;
@@ -318,7 +309,8 @@ MAIN()
            jobs,
            (jobs == 1) ? "" : "s");
 
-    decoder = avifDecoderCreate();
+    int returnCode = 1;
+    avifDecoder * decoder = avifDecoderCreate();
     if (!decoder) {
         fprintf(stderr, "Memory allocation failure\n");
         goto cleanup;
@@ -396,6 +388,5 @@ MAIN()
         }
         avifDecoderDestroy(decoder);
     }
-    FREE_ARGV()
     return returnCode;
 }
diff --git a/apps/avifenc.c b/apps/avifenc.c
@@ -18,11 +18,6 @@
 // for setmode()
 #include <fcntl.h>
 #include <io.h>
-#include <locale.h>
-#define WIN32_LEAN_AND_MEAN
-// Avoid the DEFAULT_QUALITY macro redefinition warning caused by including wingdi.h.
-#define NOGDI
-#include <windows.h>
 #endif
 
 #define NEXTARG()                                                     \
@@ -1396,13 +1391,13 @@ static avifBool avifEncodeImages(avifSettings * settings,
     return AVIF_TRUE;
 }
 
-MAIN()
+int main(int argc, char * argv[])
 {
     if (argc < 2) {
         syntaxShort();
         return 1;
     }
-    INIT_ARGV()
+
     const char * outputFilename = NULL;
 
     avifInput input;
@@ -2593,6 +2588,6 @@ MAIN()
         avifCodecSpecificOptionsFree(&file->settings.codecSpecificOptions);
     }
     free(input.files);
-    FREE_ARGV()
+
     return returnCode;
 }
diff --git a/apps/avifgainmaputil/avifgainmaputil.cc b/apps/avifgainmaputil/avifgainmaputil.cc
@@ -16,12 +16,6 @@
 #include "swapbase_command.h"
 #include "tonemap_command.h"
 
-#if defined(_WIN32)
-#include <locale.h>
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#endif
-
 namespace avif {
 namespace {
 
@@ -56,7 +50,7 @@ void PrintUsage(const std::vector<std::unique_ptr<ProgramCommand>>& commands) {
 }  // namespace
 }  // namespace avif
 
-MAIN() {
+int main(int argc, char** argv) {
   std::vector<std::unique_ptr<avif::ProgramCommand>> commands;
   commands.emplace_back(std::make_unique<avif::HelpCommand>());
   commands.emplace_back(std::make_unique<avif::CombineCommand>());
@@ -72,8 +66,6 @@ MAIN() {
     return 1;
   }
 
-  INIT_ARGV()
-
   const std::string command_name(argv[1]);
   if (command_name == "help") {
     if (argc >= 3) {

diff --git a/apps/shared/avifutil.h b/apps/shared/avifutil.h
@@ -10,67 +10,6 @@
 extern "C" {
 #endif
 
-// MAIN(), INIT_ARGV(), FREE_ARGV() for UTF8-aware command line parsing.
-// UCRT supports UTF8, contrary to MSVCRT.
-#if defined(_WIN32) && defined(_UCRT)
-#define MAIN() int wmain(int argc, wchar_t * wargv[])
-#else
-#define MAIN() int main(int argc, char * argv[])
-#endif
-
-#if defined(_WIN32) && defined(_UCRT)
-#ifdef __cplusplus
-#define INIT_ARGV()                                                                                           \
-    if (setlocale(LC_ALL, ".UTF8") == NULL) {                                                                 \
-        fprintf(stderr, "setlocale failed\n");                                                                \
-        return 1;                                                                                             \
-    }                                                                                                         \
-    std::vector<char> argvAllVector(1024 * argc);                                                             \
-    std::vector<char *> argvVector(argc);                                                                     \
-    char ** argv = argvVector.data();                                                                         \
-    for (int i = 0; i < argc; ++i) {                                                                          \
-        argvVector[i] = &argvAllVector[1024 * i];                                                             \
-        int rc = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, argv[i], 1024, NULL, NULL); \
-        if (rc == 0) {                                                                                        \
-            fprintf(stderr, "WideCharToMultiByte() failed\n");                                                \
-            return 1;                                                                                         \
-        }                                                                                                     \
-    }
-#else
-#define INIT_ARGV()                                                                                           \
-    char * argvAll = NULL;                                                                                    \
-    char ** argv = NULL;                                                                                      \
-    if (setlocale(LC_ALL, ".UTF8") == NULL) {                                                                 \
-        fprintf(stderr, "setlocale failed\n");                                                                \
-        return 1;                                                                                             \
-    }                                                                                                         \
-    argvAll = (char *)malloc(1024 * argc * sizeof(*argvAll));                                                 \
-    argv = (char **)malloc(argc * sizeof(*argv));                                                             \
-    if (argv == NULL || argvAll == NULL) {                                                                    \
-        FREE_ARGV()                                                                                           \
-        return 1;                                                                                             \
-    }                                                                                                         \
-    for (int i = 0; i < argc; ++i) {                                                                          \
-        argv[i] = argvAll + 1024 * i;                                                                         \
-        int rc = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, argv[i], 1024, NULL, NULL); \
-        if (rc == 0) {                                                                                        \
-            fprintf(stderr, "WideCharToMultiByte() failed\n");                                                \
-            FREE_ARGV()                                                                                       \
-            return 1;                                                                                         \
-        }                                                                                                     \
-    }
-#endif // __cplusplus
-#else
-#define INIT_ARGV()
-#endif
-#if defined(_WIN32) && defined(_UCRT) && !defined(__cplusplus)
-#define FREE_ARGV() \
-    free(argv);     \
-    free(argvAll);
-#else
-#define FREE_ARGV()
-#endif
-
 // The %z format specifier is not available in the old Windows CRT msvcrt,
 // hence the %I format specifier must be used instead to print out `size_t`.
 // The new Windows CRT UCRT, which is used by Visual Studio 2015 or later,

diff --git a/apps/utf8.manifest b/apps/utf8.manifest
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1">
+  <assemblyIdentity type="win32" name="..." version="6.0.0.0"/>
+  <application>
+    <windowsSettings>
+      <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
+    </windowsSettings>
+  </application>
+</assembly>
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -226,18 +226,17 @@ if(AVIF_BUILD_APPS)
     # When building apps, test the avifenc/avifdec.
     # 'are_images_equal' is used to make sure inputs/outputs are unchanged.
     add_executable(are_images_equal gtest/are_images_equal.cc)
+    if(WIN32 AND NOT MINGW)
+        target_sources(are_images_equal PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../apps/utf8.manifest)
+    endif()
     target_link_libraries(are_images_equal aviftest_helpers)
     add_test(NAME test_cmd COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/test_cmd.sh ${CMAKE_BINARY_DIR}
                                    ${CMAKE_CURRENT_SOURCE_DIR}/data
     )
     set_tests_properties(test_cmd PROPERTIES ENVIRONMENT "AVIF_TEST_UTF8=1")
     if(MINGW)
-        if("$ENV{MSYSTEM}" MATCHES "(MINGW32|MINGW64)")
-            # Those legacy environments link against MSVCRT which does not support UTF8.
-            set_tests_properties(test_cmd PROPERTIES ENVIRONMENT "AVIF_TEST_UTF8=0")
-        else()
-            target_link_options(are_images_equal PRIVATE -municode)
-        endif()
+        # Does not support manifest files.
+        set_tests_properties(test_cmd PROPERTIES ENVIRONMENT "AVIF_TEST_UTF8=0")
     endif()
     add_test(NAME test_cmd_animation COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/test_cmd_animation.sh ${CMAKE_BINARY_DIR}
                                              ${CMAKE_CURRENT_SOURCE_DIR}/data

diff --git a/tests/gtest/are_images_equal.cc b/tests/gtest/are_images_equal.cc
@@ -9,17 +9,9 @@
 #include "aviftest_helpers.h"
 #include "avifutil.h"
 
-#if defined(_WIN32)
-#include <locale.h>
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#endif
-
 using avif::ImagePtr;
 
-MAIN() {
-  INIT_ARGV()
-
+int main(int argc, char** argv) {
   if (argc != 4 && argc != 5) {
     std::cerr << "Wrong argument: " << argv[0]
               << " file1 file2 ignore_alpha_flag [psnr_threshold]" << std::endl;
@@ -79,7 +71,5 @@ MAIN() {
               << argv[2] << " are similar." << std::endl;
   }
 
-  FREE_ARGV()
-
   return 0;
 }
diff --git a/tests/test_cmd.sh b/tests/test_cmd.sh
@@ -60,7 +60,7 @@ OUT_MSG="avif_test_cmd_out_msg.txt"
 # Cleanup
 cleanup() {
   pushd ${TMP_DIR}
-    rm -- "${ENCODED_FILE}" "${ENCODED_FILE_WITH_DASH}" "${DECODED_FILE}" "${OUT_MSG}"
+    rm -f -- "${ENCODED_FILE}" "${ENCODED_FILE_WITH_DASH}" "${DECODED_FILE}" "${OUT_MSG}"
   popd
 }
 trap cleanup EXIT
@@ -75,7 +75,11 @@ pushd ${TMP_DIR}
     cp ${INPUT_Y4M} ${INPUT_UTF8_Y4M}
     "${AVIFENC}" -s 8 "${INPUT_UTF8_Y4M}" -o "${ENCODED_UTF8_FILE}"
     "${AVIFDEC}" "${ENCODED_UTF8_FILE}" "${DECODED_UTF8_FILE}"
-    "${ARE_IMAGES_EQUAL}" "${INPUT_UTF8_Y4M}" "${DECODED_UTF8_FILE}" 0 && exit 1
+    RET=0
+    "${ARE_IMAGES_EQUAL}" "${INPUT_UTF8_Y4M}" "${DECODED_UTF8_FILE}" 0 || RET=$?
+    if [ ${RET} -ne 1 ]; then
+      exit 1
+    fi
   fi
 
   # Argument parsing test with filenames starting with a dash.