Fix usage of utf-8 filenames for windows

the std::filesystem::path mechanism is reasonable, but needs to be told
it to look for a utf-8 string, and is not automatically that, and the
standards committee must have realized there were issues and change the
api in c++20

Signed-off-by: Kimball Thurston <[email protected]>

src/lib/OpenEXR/ImfStdIO.cpp

@@ -15,6 +15,10 @@
 #include <ImfStdIO.h>
 #include <errno.h>
 #include <filesystem>
+#if __cplusplus >= 202002L
+#    include <ranges>
+#    include <span>
+#endif
 
 using namespace std;
 #include "ImfNamespace.h"
@@ -27,15 +31,29 @@ namespace
 inline ifstream*
 make_ifstream (const char* filename)
 {
-    return new ifstream (std::filesystem::path (filename),
+#if __cplusplus >= 202002L
+    auto u8view = ranges::views::transform (span{filename, strlen(filename)},
+                                            [](char c) -> char8_t { return c; });
+    return new ifstream (filesystem::path (u8view.begin (), u8view.end ()),
                          ios_base::in | ios_base::binary);
+#else
+    return new ifstream (filesystem::u8path (filename),
+                         ios_base::in | ios_base::binary);
+#endif
 }
 
 inline ofstream*
 make_ofstream (const char* filename)
 {
-    return new ofstream (std::filesystem::path (filename),
+#if __cplusplus >= 202002L
+    auto u8view = ranges::views::transform (span{filename, strlen(filename)},
+                                            [](char c) -> char8_t { return c; });
+    return new ofstream (filesystem::path (u8view.begin (), u8view.end ()),
+                         ios_base::out | ios_base::binary);
+#else
+    return new ofstream (filesystem::u8path (filename),
                          ios_base::out | ios_base::binary);
+#endif
 }
 
 void

src/lib/OpenEXRCore/openexr_context.h

@@ -361,6 +361,13 @@ typedef struct _exr_context_initializer_v3
 /** @brief Check the magic number of the file and report
  * `EXR_ERR_SUCCESS` if the file appears to be a valid file (or at least
  * has the correct magic number and can be read).
+ *
+ * The filename is assumed to be a UTF-8 encoded
+ * filename, as is standard on current Unix filesystems. Under
+ * windows, this will be widened to be a wchar_t. If the user provides
+ * custom I/O routines, the responsibility passes to the user for that
+ * behavior.
+ *
  */
 EXR_EXPORT exr_result_t exr_test_file_header (
     const char* filename, const exr_context_initializer_t* ctxtdata);
@@ -380,6 +387,12 @@ EXR_EXPORT exr_result_t exr_finish (exr_context_t* ctxt);
  * previously opened a stream, file, or whatever and placed relevant
  * data in userdata to access that.
  *
+ * In the default case, the filename is assumed to be a UTF-8 encoded
+ * filename, as is standard on current Unix filesystems. Under
+ * windows, this will be widened to be a wchar_t. If the user provides
+ * custom I/O routines, the responsibility passes to the user for that
+ * behavior.
+ *
  * One notable attribute of the context is that once it has been
  * created and returned a successful code, it has parsed all the
  * header data. This is done as one step such that it is easier to
@@ -415,6 +428,12 @@ typedef enum exr_default_write_mode
  * a stream, file, or whatever and placed relevant data in userdata to
  * access that.
  *
+ * In the default case, the filename is assumed to be a UTF-8 encoded
+ * filename, as is standard on current Unix filesystems. Under
+ * windows, this will be widened to be a wchar_t. If the user provides
+ * custom I/O routines, the responsibility passes to the user for that
+ * behavior.
+ *
  * Multi-Threading: To avoid issues with creating multi-part EXR
  * files, the library approaches writing as a multi-step process, so
  * the same concurrent guarantees can not be made for writing a

src/test/OpenEXRCoreTest/CMakeLists.txt

@@ -98,6 +98,7 @@ define_openexrcore_tests(
  testStartWriteDeepScan
  testStartWriteTile
  testStartWriteDeepTile
+ testStartWriteUTF8
  testWriteAttrs
  testWriteScans
  testWriteTiles

src/test/OpenEXRCoreTest/main.cpp

@@ -187,6 +187,7 @@ main (int argc, char* argv[])
     TEST (testStartWriteTile, "core_write");
     TEST (testStartWriteDeepScan, "core_write");
     TEST (testStartWriteDeepTile, "core_write");
+    TEST (testStartWriteUTF8, "core_write");
     TEST (testWriteScans, "core_write");
     TEST (testWriteTiles, "core_write");
     TEST (testWriteMultiPart, "core_write");

src/test/OpenEXRCoreTest/write.cpp

@@ -1,6 +1,16 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright Contributors to the OpenEXR Project.
 
+#ifdef _WIN32
+// windows is very particular about when windows.h is included
+#include <windows.h>
+#include <fileapi.h>
+#include <inttypes.h>
+#include <strsafe.h>
+#else
+#include <unistd.h>
+#endif
+
 #include "write.h"
 
 #include "test_value.h"
@@ -10,6 +20,7 @@
 #include <float.h>
 #include <limits.h>
 #include <math.h>
+#include <stdio.h>
 #include <string.h>
 
 #include <iomanip>
@@ -1395,3 +1406,80 @@ testWriteMultiPart (const std::string& tempdir)
     EXRCORE_TEST_RVAL (exr_finish (&outf));
     remove (outfn.c_str ());
 }
+
+void
+testStartWriteUTF8 (const std::string& tempdir)
+{
+    exr_context_t outf;
+    // per google translate, image in Japanese
+    std::string   outfn = tempdir + "画像.exr";
+    int           partidx;
+
+    exr_context_initializer_t cinit = EXR_DEFAULT_CONTEXT_INITIALIZER;
+    cinit.error_handler_fn          = &err_cb;
+    cinit.zip_level = 3;
+    cinit.flags |= EXR_CONTEXT_FLAG_WRITE_LEGACY_HEADER;
+
+    exr_set_default_zip_compression_level (-1);
+
+    EXRCORE_TEST_RVAL (exr_start_write (
+        &outf, outfn.c_str (), EXR_WRITE_FILE_DIRECTLY, &cinit));
+    EXRCORE_TEST_RVAL (
+        exr_add_part (outf, "beauty", EXR_STORAGE_SCANLINE, &partidx));
+    EXRCORE_TEST (partidx == 0);
+    EXRCORE_TEST_RVAL (exr_get_count (outf, &partidx));
+    EXRCORE_TEST (partidx == 1);
+    partidx = 0;
+
+    int fw = 1;
+    int fh = 1;
+    exr_attr_box2i_t dataW = { {0, 0}, {0, 0} };
+
+    EXRCORE_TEST_RVAL (
+        exr_initialize_required_attr_simple (outf, partidx, fw, fh, EXR_COMPRESSION_NONE));
+    EXRCORE_TEST_RVAL (exr_set_data_window (outf, partidx, &dataW));
+
+    EXRCORE_TEST_RVAL (exr_add_channel (
+        outf, partidx, "h", EXR_PIXEL_HALF, EXR_PERCEPTUALLY_LOGARITHMIC, 1, 1));
+    EXRCORE_TEST_RVAL (exr_write_header (outf));
+
+    exr_chunk_info_t      cinfo;
+    exr_encode_pipeline_t encoder;
+
+    EXRCORE_TEST_RVAL (exr_write_scanline_chunk_info (outf, 0, 0, &cinfo));
+    EXRCORE_TEST_RVAL (
+        exr_encoding_initialize (outf, 0, &cinfo, &encoder));
+
+    uint16_t hval[] = { 0x1234, 0 };
+    for (int c = 0; c < encoder.channel_count; ++c)
+    {
+        encoder.channels[c].encode_from_ptr   = (const uint8_t *)hval;
+        encoder.channels[c].user_pixel_stride = 2;
+        encoder.channels[c].user_line_stride  = 2;
+    }
+    EXRCORE_TEST_RVAL (
+        exr_encoding_choose_default_routines (outf, 0, &encoder));
+    EXRCORE_TEST_RVAL (exr_encoding_run (outf, 0, &encoder));
+    EXRCORE_TEST_RVAL (exr_encoding_destroy (outf, &encoder));
+
+    EXRCORE_TEST_RVAL (exr_finish (&outf));
+#ifdef _WIN32
+    int      wcSize = 0, fnlen = 0;
+    wchar_t* wcFn = NULL;
+
+    fnlen  = (int) strlen (outfn.c_str ());
+    wcSize = MultiByteToWideChar (CP_UTF8, 0, outfn.c_str (), fnlen, NULL, 0);
+    wcFn   = (wchar_t*) malloc (sizeof (wchar_t) * (wcSize + 1));
+    if (wcFn)
+    {
+        MultiByteToWideChar (CP_UTF8, 0, outfn.c_str (), fnlen, wcFn, wcSize);
+        wcFn[wcSize] = 0;
+    }
+    EXRCORE_TEST ( _waccess (wcFn, 0) != -1 );
+    _wremove (wcFn);
+    free (wcFn);
+#else
+    EXRCORE_TEST ( access (outfn.c_str (), F_OK) != -1 );
+    remove (outfn.c_str ());
+#endif
+}

src/test/OpenEXRCoreTest/write.h

@@ -16,11 +16,13 @@ void testStartWriteScan (const std::string& tempdir);
 void testStartWriteTile (const std::string& tempdir);
 void testStartWriteDeepScan (const std::string& tempdir);
 void testStartWriteDeepTile (const std::string& tempdir);
+void testStartWriteUTF8 (const std::string& tempdir);
 
 void testUpdateMeta (const std::string& tempdir);
 
 void testWriteScans (const std::string& tempdir);
 void testWriteTiles (const std::string& tempdir);
 void testWriteMultiPart (const std::string& tempdir);
 
+
 #endif // OPENEXR_CORE_TEST_WRITE_H

src/test/OpenEXRTest/CMakeLists.txt

@@ -167,6 +167,7 @@ define_openexr_tests(
  testDeepTiledBasic
  testDwaLookups
  testExistingStreams
+ testExistingStreamsUTF8
  testFutureProofing
  testHeader
  testHuf

src/test/OpenEXRTest/main.cpp

@@ -202,6 +202,7 @@ main (int argc, char* argv[])
     TEST (testTiledLineOrder, "basic");
     TEST (testScanLineApi, "basic");
     TEST (testExistingStreams, "core");
+    TEST (testExistingStreamsUTF8, "core");
     TEST (testStandardAttributes, "core");
     TEST (testOptimized, "basic");
     TEST (testOptimizedInterleavePatterns, "basic");

src/test/OpenEXRTest/testExistingStreams.cpp

@@ -80,7 +80,8 @@ fillPixels2 (Array2D<Rgba>& pixels, int w, int h)
 
 //
 // class MMIFStream -- a memory-mapped implementation of
-// class IStream
+// class IStre
+// am
 //
 
 class MMIFStream : public OPENEXR_IMF_NAMESPACE::IStream
@@ -1074,3 +1075,78 @@ testExistingStreams (const std::string& tempDir)
         assert (false);
     }
 }
+
+void
+testExistingStreamsUTF8 (const std::string& tempDir)
+{
+
+    cout << "Testing reading and writing using existing streams" << endl;
+
+    const int W = 119;
+    const int H = 237;
+    Array2D<Rgba> p1 (H, W);
+
+    fillPixels1 (p1, W, H);
+
+    // per google translate, image in Japanese
+    std::string   outfn = tempDir + "画像.exr";
+
+    {
+        cout << "writing";
+#ifdef _WIN32
+        _wremove (WidenFilename (outfn.c_str ()).c_str ());
+#else
+        remove (outfn.c_str ());
+#endif
+        Header      header (
+            W,
+            H,
+            1,
+            IMATH_NAMESPACE::V2f (0, 0),
+            1,
+            INCREASING_Y,
+            NO_COMPRESSION);
+
+        RgbaOutputFile out (
+            outfn.c_str (),
+            header,
+            WRITE_RGBA);
+
+        out.setFrameBuffer (&p1[0][0], 1, W);
+        out.writePixels (H);
+    }
+
+    {
+        cout << ", reading";
+        RgbaInputFile in (outfn.c_str ());
+        const Box2i& dw = in.dataWindow ();
+        int          w  = dw.max.x - dw.min.x + 1;
+        int          h  = dw.max.y - dw.min.y + 1;
+        int          dx = dw.min.x;
+        int          dy = dw.min.y;
+
+        Array2D<Rgba> p2 (h, w);
+        in.setFrameBuffer (&p2[-dy][-dx], 1, w);
+        in.readPixels (dw.min.y, dw.max.y);
+
+        cout << ", comparing";
+        for (int y = 0; y < h; ++y)
+        {
+            for (int x = 0; x < w; ++x)
+            {
+                assert (p2[y][x].r == p1[y][x].r);
+                assert (p2[y][x].g == p1[y][x].g);
+                assert (p2[y][x].b == p1[y][x].b);
+                assert (p2[y][x].a == p1[y][x].a);
+            }
+        }
+    }
+
+    cout << endl;
+
+#ifdef _WIN32
+    _wremove (WidenFilename (outfn.c_str ()).c_str ());
+#else
+    remove (outfn.c_str ());
+#endif
+}

src/test/OpenEXRTest/testExistingStreams.h

@@ -6,3 +6,4 @@
 #include <string>
 
 void testExistingStreams (const std::string& tempDir);
+void testExistingStreamsUTF8 (const std::string& tempDir);