fix(python): Avoid exceptions when decoding non-UTF-8 data from string attrs

src/python/py_imagespec.cpp

@@ -159,7 +159,7 @@ declare_imagespec(py_module& m)
                                  int chan) { return spec.channelformat(chan); })
         .def("channel_name",
              [](const ImageSpec& spec, int chan) {
-                 return oiio_py::str(spec.channel_name(chan));
+                 return py_str_escaped(spec.channel_name(chan));
              })
         .def("channelindex",
              [](const ImageSpec& spec, const std::string& name) {
@@ -200,7 +200,7 @@ declare_imagespec(py_module& m)
             "get_string_attribute",
             [](const ImageSpec& spec, const std::string& name,
                const std::string& def) {
-                return oiio_py::str(spec.get_string_attribute(name, def));
+                return py_str_escaped(spec.get_string_attribute(name, def));
             },
             "name"_a, "defaultval"_a = "")
         .def(
@@ -233,7 +233,7 @@ declare_imagespec(py_module& m)
         .def_static(
             "metadata_val",
             [](const ParamValue& p, bool human) {
-                return oiio_py::str(ImageSpec::metadata_val(p, human));
+                return py_str_escaped(ImageSpec::metadata_val(p, human));
             },
             "param"_a, "human"_a = false)
         .def(
@@ -250,11 +250,11 @@ declare_imagespec(py_module& m)
                     verb = ImageSpec::SerialDetailed;
                 else if (Strutil::iequals(verbose, "detailedhuman"))
                     verb = ImageSpec::SerialDetailedHuman;
-                return oiio_py::str(spec.serialize(fmt, verb));
+                return py_str_escaped(spec.serialize(fmt, verb));
             },
             "format"_a = "text", "verbose"_a = "detailed")
         .def("to_xml",
-             [](const ImageSpec& spec) { return oiio_py::str(spec.to_xml()); })
+             [](const ImageSpec& spec) { return py_str_escaped(spec.to_xml()); })
         .def("from_xml",
              [](ImageSpec& self, const std::string& xml) {
                  self.from_xml(xml.c_str());

src/python/py_oiio.cpp

@@ -512,7 +512,7 @@ declare_global_attribute_functions(py_module& m)
     m.def(
         "get_string_attribute",
         [](const std::string& name, const std::string& def) {
-            return oiio_py::str(OIIO::get_string_attribute(name, def));
+            return py_str_escaped(OIIO::get_string_attribute(name, def));
         },
         "name"_a, "defaultval"_a = "");
     m.def("getattribute", &oiio_getattribute_typed, "name"_a,

src/python/py_oiio.h

@@ -469,6 +469,33 @@ py_to_stdvector(std::vector<T>& vals, const py::object& obj)
 }
 
 
+// Make a UTF-8 `py::str` from a string-like C++ type, using `'surrogateescape'`
+// error handling for any non-UTF-8 byte sequences. Compared to calling
+// `py::str` directly, this avoids raising a `UnicodeDecodeError` if the input
+// is not valid UTF-8.
+//
+// It is important to do this for string return values that come from
+// "untrusted" sources, such as being read from other files, so that we ensure
+// that we are returning valid UTF-8 str's to the calling Python so that
+// exceptions are not raised.
+inline py::str
+py_str_escaped(string_view value)
+{
+    py::handle py_str = PyUnicode_DecodeUTF8(value.data(), value.length(),
+                                             "surrogateescape");
+#if defined(OIIO_PY_BACKEND_NANOBIND)
+    if (!py_str) {
+        py::raise_python_error();
+    }
+    return py::steal<py::str>(py_str);
+#else
+    if (!py_str) {
+        throw py::error_already_set();
+    }
+    return py::reinterpret_steal<py::str>(py_str);
+#endif
+}
+
 
 template<typename T>
 inline py::tuple
@@ -477,6 +504,9 @@ C_to_tuple(cspan<T> vals)
     return oiio_py::make_tuple(vals.size(), [&](size_t i) {
         if constexpr (std::is_same_v<T, half>) {
             return py::cast(static_cast<float>(vals[i]));
+        } else if constexpr (std::is_same_v<std::decay_t<T>, std::string>
+                             || std::is_same_v<std::decay_t<T>, const char*>) {
+            return py_str_escaped(vals[i]);
         } else {
             return py::cast(vals[i]);
         }
@@ -500,6 +530,9 @@ C_to_tuple(const T* vals, size_t size)
     return oiio_py::make_tuple(size, [&](size_t i) {
         if constexpr (std::is_same_v<T, half>) {
             return py::cast(static_cast<float>(vals[i]));
+        } else if constexpr (std::is_same_v<std::decay_t<T>, std::string>
+                             || std::is_same_v<std::decay_t<T>, const char*>) {
+            return py_str_escaped(vals[i]);
         } else {
             return py::cast(vals[i]);
         }
@@ -540,6 +573,9 @@ C_to_val_or_tuple(const T* vals, TypeDesc type, int nvalues = 1)
     if (n == 1 && !type.arraylen) {
         if constexpr (std::is_same_v<T, half>) {
             return py::cast(static_cast<float>(vals[0]));
+        } else if constexpr (std::is_same_v<std::decay_t<T>, std::string>
+                             || std::is_same_v<std::decay_t<T>, const char*>) {
+            return py_str_escaped(vals[0]);
         } else {
             return py::cast(vals[0]);
         }

src/python/py_paramvalue.cpp

@@ -158,7 +158,7 @@ declare_paramvalue(py_module& m)
     py::class_<ParamValue>(m, "ParamValue")
         .OIIO_PY_PROP_RO("name",
                          [](const ParamValue& self) {
-                             return oiio_py::str(self.name().string());
+                             return py_str_escaped(self.name().string());
                          })
         .OIIO_PY_PROP_RO("type",
                          [](const ParamValue& self) { return self.type(); })