Detect failed reads when a tree is missing from a DF

Author: autumn-mck

Framework/AnalysisSupport/src/DataInputDirector.cxx

@@ -30,12 +30,14 @@
 #include "TObjString.h"
 #include "TMap.h"
 #include "TFile.h"
+#include "TError.h"
 
 #include <arrow/dataset/file_base.h>
 #include <arrow/dataset/dataset.h>
 #include <uv.h>
 #include <exception>
 #include <memory>
+#include <string_view>
 
 #if __has_include(<TJAlienFile.h>)
 #include <TJAlienFile.h>
@@ -58,6 +60,42 @@ FileNameHolder makeFileNameHolder(std::string fileName)
   return holder;
 }
 
+namespace
+{
+// ROOT reports a failed key read (a folder's key list or the object itself) and carries on,
+// so the lookup returns nullptr, or an object built from whatever was in the buffer, and a
+// failed read looks like a missing object. Count the reports while the watch is alive.
+struct RootReadErrorWatch {
+  RootReadErrorWatch()
+    : mPrevious(SetErrorHandler(handler))
+  {
+    sPrevious = mPrevious;
+    sErrors = 0;
+  }
+  ~RootReadErrorWatch()
+  {
+    SetErrorHandler(mPrevious);
+  }
+  RootReadErrorWatch(RootReadErrorWatch const&) = delete;
+  RootReadErrorWatch& operator=(RootReadErrorWatch const&) = delete;
+
+  [[nodiscard]] bool failed() const { return sErrors > 0; }
+
+  static void handler(int level, Bool_t abort, char const* location, char const* msg)
+  {
+    if (level >= kError && location && std::string_view{location} == "TKey::ReadFile") {
+      ++sErrors;
+    }
+    (sPrevious ? sPrevious : DefaultErrorHandler)(level, abort, location, msg);
+  }
+
+  ErrorHandlerFunc_t mPrevious;
+  // Errors from other threads are passed on but not counted.
+  static inline thread_local int sErrors = 0;
+  static inline ErrorHandlerFunc_t sPrevious = nullptr;
+};
+} // namespace
+
 DataInputDescriptor::DataInputDescriptor(bool alienSupport, int level, DataInputDirectorContext& context)
   : mAlienSupport(alienSupport),
     mContext(context),
@@ -605,7 +643,19 @@ try {
 
   for (auto& capability : mFactory.capabilities) {
     auto objectPath = capability.lfn2objectPath(fullpath.path());
-    void* handle = capability.getHandle(rootFS, objectPath);
+    void* handle = nullptr;
+    bool readFailed = false;
+    try {
+      RootReadErrorWatch watch;
+      handle = capability.getHandle(rootFS, objectPath);
+      readFailed = watch.failed();
+    } catch (...) {
+      // Sizes from a partial read can be garbage, e.g. giving std::bad_alloc.
+      std::throw_with_nested(InvalidAODReadError(fmt::format(R"(Unable to look up "{}" in "{}")", objectPath, rootFS->GetFile()->GetName())));
+    }
+    if (readFailed) {
+      throw InvalidAODReadError(fmt::format(R"(Unable to read "{}" from "{}")", objectPath, rootFS->GetFile()->GetName()));
+    }
     if (handle) {
       format = capability.factory().format();
       creator = capability.factory().deferredOutputStreamer;