iv: tolerate partially-written EXR files

src/iv/imageviewer.cpp

@@ -1164,6 +1164,11 @@ ImageViewer::updateStatusBar()
     message = Strutil::fmt::format("({}/{}) : ", m_current_image + 1,
                                    (int)m_images.size());
     message += cur()->shortinfo();
+    if (cur()->partially_loaded()) {
+        message += "  [partially readable file: ";
+        message += cur()->partial_error();
+        message += "]";
+    }
     statusImgInfo->setText(message.c_str());
 
     message.clear();

src/iv/imageviewer.h

@@ -104,6 +104,16 @@ class IvImage final : public ImageBuf {
     ///
     bool image_valid() const { return m_image_valid; }
 
+    /// True if the last read only partially succeeded, i.e. the image
+    /// specification was readable but some of the pixel data could not be
+    /// read (for example, a file that was only partially written before a
+    /// renderer crashed).
+    bool partially_loaded() const { return m_partially_loaded; }
+
+    /// If the last read was only partial (see partially_loaded()), a
+    /// message describing the problem.
+    const std::string& partial_error() const { return m_partial_error; }
+
     /// Copies data from the read buffer to the secondary buffer, selecting the
     /// given channel:
     ///  -2 = luminance
@@ -139,6 +149,10 @@ class IvImage final : public ImageBuf {
     mutable std::string m_longinfo;
     bool m_image_valid;    ///< Image is valid and pixels can be read.
     bool m_auto_subimage;  ///< Automatically use subimages when zooming-in/out.
+    bool m_partially_loaded = false;   ///< Only some pixel data was readable.
+    std::string m_partial_error;       ///< Description of what was unreadable.
+    ImageSpec m_input_config;          ///< Copy of the input configuration spec
+    bool m_have_input_config = false;  ///< Was an input configuration given?
 };
 
 

src/iv/ivimage.cpp

@@ -6,6 +6,7 @@
 #include <iostream>
 
 #include "imageviewer.h"
+#include "ivutils.h"
 #include <OpenImageIO/imagecache.h>
 #include <OpenImageIO/strutil.h>
 
@@ -20,6 +21,10 @@ IvImage::IvImage(const std::string& filename, const ImageSpec* input_config)
     , m_image_valid(false)
     , m_auto_subimage(false)
 {
+    if (input_config) {
+        m_input_config      = *input_config;
+        m_have_input_config = true;
+    }
 }
 
 
@@ -77,6 +82,47 @@ IvImage::read_iv(int subimage, int miplevel, bool force, TypeDesc format,
                                        progress_callback,
                                        progress_callback_data);
 
+    m_partially_loaded = false;
+    m_partial_error.clear();
+    if (m_image_valid && storage() == ImageBuf::IMAGECACHE) {
+        // The image is backed by the ImageCache, which means that the
+        // pixel data has not been touched yet and read failures will only
+        // turn up later, awkwardly, as pixels are fetched for display.
+        // For a file that was only partially written (for example, an EXR
+        // that a renderer didn't finish), check now whether the pixel
+        // data is really all readable, and if not, fall through to the
+        // tolerant re-read below.
+        ImageSpec* config = m_have_input_config ? &m_input_config : nullptr;
+        m_image_valid = image_data_readable(name(), config, subimage, miplevel);
+    }
+    if (!m_image_valid) {
+        // The straightforward read failed. This can happen for a file that
+        // was only partially written -- for example, an EXR file from a
+        // renderer that crashed or was killed before it finished writing
+        // all of the pixel data. Rather than showing nothing at all, reopen
+        // the file with the "oiio:missingcolor" config attribute, which asks
+        // the OpenEXR reader to fill unreadable scanlines or tiles with the
+        // given color (black here) instead of failing the read. Readers
+        // without that support ignore the config, so the re-read simply
+        // fails the same way for them.
+        ImageSpec config;
+        if (m_have_input_config)
+            config = m_input_config;
+        config.attribute("oiio:missingcolor", "0");
+        reset(name(), 0, 0, {}, &config);
+        m_image_valid = ImageBuf::read(subimage, miplevel, force, format,
+                                       progress_callback,
+                                       progress_callback_data);
+        if (m_image_valid) {
+            // The read succeeded where the straightforward one failed, so
+            // part of the pixel data must have been unreadable -- remember
+            // that so the status bar can let the user know.
+            m_partially_loaded = true;
+            m_partial_error    = "partially readable file, showing the "
+                                 "readable portion";
+        }
+    }
+
     if (m_image_valid && secondary_data && spec().format == TypeDesc::UINT8) {
         m_corrected_image.reset(ImageSpec(spec().width, spec().height,
                                           std::min(spec().nchannels, 4),
@@ -388,8 +434,10 @@ IvImage::invalidate()
 {
     ustring filename(name());
     reset(filename.string());
-    m_thumbnail_valid = false;
-    m_image_valid     = false;
+    m_thumbnail_valid  = false;
+    m_image_valid      = false;
+    m_partially_loaded = false;
+    m_partial_error.clear();
     if (imagecache())
         imagecache()->invalidate(filename);
 }

src/iv/ivutils.h

@@ -5,6 +5,13 @@
 #ifndef OPENIMAGEIO_IV_UTILS_H
 #define OPENIMAGEIO_IV_UTILS_H
 
+#include <algorithm>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#include <OpenImageIO/imagebuf.h>
+#include <OpenImageIO/imageio.h>
 #include <OpenImageIO/oiioversion.h>
 
 OIIO_NAMESPACE_BEGIN
@@ -33,6 +40,56 @@ floor2f(float f)
     return powf(2.0f, floorf(logval));
 }
 
+
+/// Probe whether all of the pixel data of the image named `filename` is
+/// readable, without reading the whole image: attempt to read only the
+/// last scanline (for scanline files) or the last tile (for tiled files),
+/// which is the most likely region to be missing from a file that was
+/// only partially written. This is cheap enough to use as a check on
+/// files that were read through an ImageCache, where the pixel data is
+/// not touched until it is needed for display, at which point read
+/// failures are much less gracefully handled.
+///
+/// Returns true if the last scanline/tile could be read (and therefore it
+/// is likely that the entire pixel data block is intact), false if it
+/// could not (meaning that the file is probably truncated or otherwise
+/// partially written).
+inline bool
+image_data_readable(string_view filename, const ImageSpec* config, int subimage,
+                    int miplevel)
+{
+    auto in = ImageInput::open(filename, config);
+    if (!in)
+        return false;
+    if (!in->seek_subimage(subimage, miplevel)) {
+        in->close();
+        return false;
+    }
+    ImageSpec spec = in->spec(subimage, miplevel);
+    bool ok        = false;
+    if (spec.tile_width > 0) {
+        // Try to read the bottom-most, right-most tile.
+        int tx = spec.x
+                 + ((spec.width - 1) / spec.tile_width) * spec.tile_width;
+        int ty = spec.y
+                 + ((spec.height - 1) / spec.tile_height) * spec.tile_height;
+        int tz = spec.z
+                 + ((std::max(spec.depth - 1, 0))
+                    / std::max(spec.tile_depth, 1))
+                       * std::max(spec.tile_depth, 1);
+        std::vector<char> buf(spec.tile_bytes());
+        ok = in->read_tile(tx, ty, tz, spec.format, buf.data());
+    } else {
+        // Try to read the last scanline.
+        int y = spec.y + spec.height - 1;
+        std::vector<char> buf(spec.scanline_bytes());
+        ok = in->read_scanlines(subimage, miplevel, y, y + 1, spec.z, 0,
+                                spec.nchannels, TypeDesc::UNKNOWN, buf.data());
+    }
+    in->close();
+    return ok;
+}
+
 OIIO_NAMESPACE_END
 
 #endif  // OPENIMAGEIO_IV_UTILS_H

src/openexr.imageio/exr_pvt.h

@@ -260,6 +260,10 @@ class OpenEXRInput final : public ImageInput {
     ImageSpec spec_dimensions(int subimage, int miplevel) override;
     bool read_native_scanline(int subimage, int miplevel, int y, int z,
                               void* data) override;
+    // Unhide the base-class span-based read_native_scanlines overloads: our
+    // pointer-based overloads would otherwise hide them, which newer GCC
+    // versions flag as an error (-Woverloaded-virtual).
+    using ImageInput::read_native_scanlines;
     bool read_native_scanlines(int subimage, int miplevel, int ybegin, int yend,
                                int z, void* data) override;
     bool read_native_scanlines(int subimage, int miplevel, int ybegin, int yend,
@@ -374,10 +378,21 @@ class OpenEXRInput final : public ImageInput {
                            int chbegin, int chend, int cbegin, int cend,
                            size_t scanlinebytes, void* data);
 
+    // This is the real scanline reader; the externally-called
+    // read_native_scanlines overloads are wrappers around it that always use
+    // the chunk cache. The flag lets internal callers (the per-scanline
+    // retry, and read_cached_chunk's chunk decode) skip the chunk cache --
+    // without the skip, a failed cached chunk decode would re-enter the
+    // cache and recurse forever. See read_cached_chunk.
+    bool read_native_scanlines_impl(int subimage, int miplevel, int ybegin,
+                                    int yend, int z, int chbegin, int chend,
+                                    void* data, bool use_chunk_cache = true);
+
     bool read_native_scanlines_individually(int subimage, int miplevel,
                                             int ybegin, int yend, int z,
                                             int chbegin, int chend, void* data,
-                                            stride_t ystride);
+                                            stride_t ystride,
+                                            bool use_chunk_cache = true);
     bool read_native_tiles_individually(int subimage, int miplevel, int xbegin,
                                         int xend, int ybegin, int yend,
                                         int zbegin, int zend, int chbegin,

src/openexr.imageio/exrinput.cpp

@@ -1280,11 +1280,15 @@ OpenEXRInput::read_cached_chunk(int subimage, int miplevel, int ybegin,
                            ybegin, yend, scanlinebytes, data))
         return true;
 
-    // Cache miss. Decode the whole chunk. This recursive call asks for
-    // exactly one full chunk, so it won't come back here.
+    // Cache miss. Decode the whole chunk. The use_chunk_cache=false of
+    // this recursive call asks for exactly one full chunk, so it won't come
+    // back here -- this matters when the decode fails and tolerance is on:
+    // without the skip, the per-scanline retry would re-enter the chunk
+    // cache and recurse forever.
     default_init_vector<uint8_t> chunk(scanlinebytes * size_t(cend - cbegin));
-    if (!read_native_scanlines(subimage, miplevel, cbegin, cend, 0, chbegin,
-                               chend, chunk.data()))
+    if (!read_native_scanlines_impl(subimage, miplevel, cbegin, cend, 0,
+                                    chbegin, chend, chunk.data(),
+                                    /*use_chunk_cache=*/false))
         return false;
     memcpy(data, chunk.data() + scanlinebytes * size_t(ybegin - cbegin),
            scanlinebytes * size_t(yend - ybegin));
@@ -1294,10 +1298,25 @@ OpenEXRInput::read_cached_chunk(int subimage, int miplevel, int ybegin,
 
 
 
+// This is the externally-called read_native_scanlines, which is a wrapper
+// around the internal version (read_native_scanlines_impl) that takes an
+// additional parameter saying whether to use the chunk cache or not. When
+// called externally, we always do.
 bool
 OpenEXRInput::read_native_scanlines(int subimage, int miplevel, int ybegin,
                                     int yend, int z, int chbegin, int chend,
                                     void* data)
+{
+    return read_native_scanlines_impl(subimage, miplevel, ybegin, yend, z,
+                                      chbegin, chend, data,
+                                      /*use_chunk_cache=*/true);
+}
+
+bool
+OpenEXRInput::read_native_scanlines_impl(int subimage, int miplevel, int ybegin,
+                                         int yend, int z, int chbegin,
+                                         int chend, void* data,
+                                         bool use_chunk_cache)
 {
     lock_guard lock(*this);
     if (!seek_subimage(subimage, miplevel))
@@ -1340,8 +1359,8 @@ OpenEXRInput::read_native_scanlines(int subimage, int miplevel, int ybegin,
         size_t fullscanbytes  = size_t(m_spec.width) * fullpixelbytes;
         default_init_vector<uint8_t> scratch(fullscanbytes
                                              * size_t(yend - ybegin));
-        if (!read_native_scanlines(subimage, miplevel, ybegin, yend, z, 0,
-                                   m_spec.nchannels, scratch.data()))
+        if (!read_native_scanlines_impl(subimage, miplevel, ybegin, yend, z, 0,
+                                        m_spec.nchannels, scratch.data()))
             return false;
         size_t choff = m_spec.pixel_bytes(0, chbegin, true);
         for (int y = ybegin; y < yend; ++y) {
@@ -1364,7 +1383,7 @@ OpenEXRInput::read_native_scanlines(int subimage, int miplevel, int ybegin,
     // swath) at a time will ask for the rest of that chunk next. (The
     // library has a stash of its own, but it drops it every time we set the
     // frame buffer, which we must do on every read.)
-    if (part.scansperchunk > 1 && !part.luminance_chroma
+    if (part.scansperchunk > 1 && !part.luminance_chroma && use_chunk_cache
         && ybegin >= m_spec.y) {
         int ychunkstart = m_spec.y
                           + round_down_to_multiple(ybegin - m_spec.y,
@@ -1456,10 +1475,10 @@ OpenEXRInput::read_native_scanlines(int subimage, int miplevel, int ybegin,
             } else {
                 // Read of many tiles -- don't know which failed, so try
                 // again to read them all individually.
-                return read_native_scanlines_individually(subimage, miplevel,
-                                                          ybegin, yend, z,
-                                                          chbegin, chend, data,
-                                                          scanlinebytes);
+                return read_native_scanlines_individually(
+                    subimage, miplevel, ybegin, yend, z, chbegin, chend, data,
+                    scanlinebytes,
+                    /*use_chunk_cache=*/false);
             }
         } else {
             errorfmt("Failed OpenEXR read: {}", err);
@@ -1631,15 +1650,16 @@ bool
 OpenEXRInput::read_native_scanlines_individually(int subimage, int miplevel,
                                                  int ybegin, int yend, int z,
                                                  int chbegin, int chend,
-                                                 void* data, stride_t ystride)
+                                                 void* data, stride_t ystride,
+                                                 bool use_chunk_cache)
 {
     // Note: this is only called by read_native_scanlines, which still holds
     // the mutex, so it's safe to directly access m_spec.
     bool ok = true;
     for (int y = ybegin; y < yend; ++y) {
         char* d = (char*)data + (y - ybegin) * ystride;
-        ok &= read_native_scanlines(subimage, miplevel, y, y + 1, z, chbegin,
-                                    chend, d);
+        ok &= read_native_scanlines_impl(subimage, miplevel, y, y + 1, z,
+                                         chbegin, chend, d, use_chunk_cache);
     }
     return ok;
 }