Exceptions in threads automatically handled

Author: gineshidalgo99

doc/release_notes.md

@@ -322,6 +322,7 @@ OpenPose Library - Release Notes
     30. Added pre-processing capabilities to Wrapper (WorkerType::PreProcessing), which will be run right after the image has been read.
     31. Removed boost::shared_ptr and caffe::Blob dependencies from the headers. No 3rdparty dependencies left on headers (except dim3 for CUDA).
     32. Added Array `poseNetOutput` to Datum so that user can introduce his custom network output.
+    33. OpenPose will never provoke a core dumped or crash. Exceptions in threads (`errorWorker()` instead of `error()`) lead to stopping the threads and reporting the error from the main thread, while exceptions in destructors (`errorDestructor()` instead of `error()`) are reported with std::cerr but not thrown as std::exceptions.
 2. Functions or parameters renamed:
     1. By default, python example `tutorial_developer/python_2_pose_from_heatmaps.py` was using 2 scales starting at -1x736, changed to 1 scale at -1x368.
     2. WrapperStructPose default parameters changed to match those of the OpenPose demo binary.

examples/tests/wrapperHandFromJsonTest.hpp

@@ -111,7 +111,7 @@ namespace op
         }
         catch (const std::exception& e)
         {
-            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
         }
     }
 

include/openpose/core/arrayCpuGpu.hpp

@@ -68,7 +68,7 @@ namespace op
 
         const T* cpu_data() const;
         void set_cpu_data(T* data);
-        // const int* gpu_shape() const; // Disabled because it produces compiler errors for some users
+        const int* gpu_shape() const;
         const T* gpu_data() const;
         void set_gpu_data(T* data);
         const T* cpu_diff() const;

include/openpose/thread/subThread.hpp

@@ -143,7 +143,7 @@ namespace op
         try
         {
             for (auto& tWorker : mTWorkers)
-                tWorker->initializationOnThread();
+                tWorker->initializationOnThreadNoException();
         }
         catch (const std::exception& e)
         {

include/openpose/thread/thread.hpp

@@ -96,7 +96,7 @@ namespace op
         }
         catch (const std::exception& e)
         {
-            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__);
         }
     }
 

include/openpose/thread/threadManager.hpp

@@ -235,6 +235,8 @@ namespace op
             for (auto& thread : mThreads)
                 thread->stopAndJoin();
             log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
+            checkWorkerErrors();
+            log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
         }
         catch (const std::exception& e)
         {
@@ -396,6 +398,9 @@ namespace op
         {
             if (!mThreadWorkerQueues.empty())
             {
+                // This avoids extra std::cout if errors occur on different threads
+                setMainThread();
+
                 // Check threads
                 checkAndCreateEmptyThreads();
 

include/openpose/thread/worker.hpp

@@ -13,7 +13,7 @@ namespace op
 
         virtual ~Worker();
 
-        virtual void initializationOnThread() = 0;
+        void initializationOnThreadNoException();
 
         bool checkAndWork(TDatums& tDatums);
 
@@ -35,6 +35,8 @@ namespace op
         }
 
     protected:
+        virtual void initializationOnThread() = 0;
+
         virtual void work(TDatums& tDatums) = 0;
 
     private:
@@ -62,12 +64,35 @@ namespace op
     {
     }
 
+    template<typename TDatums>
+    void Worker<TDatums>::initializationOnThreadNoException()
+    {
+        try
+        {
+            this->initializationOnThread();
+        }
+        catch (const std::exception& e)
+        {
+            this->stop();
+            errorWorker(e.what(), __LINE__, __FUNCTION__, __FILE__);
+        }
+    }
+
     template<typename TDatums>
     bool Worker<TDatums>::checkAndWork(TDatums& tDatums)
     {
-        if (mIsRunning)
-            work(tDatums);
-        return mIsRunning;
+        try
+        {
+            if (mIsRunning)
+                work(tDatums);
+            return mIsRunning;
+        }
+        catch (const std::exception& e)
+        {
+            this->stop();
+            errorWorker(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return false;
+        }
     }
 
     COMPILE_TEMPLATE_DATUM(Worker);

include/openpose/thread/workerConsumer.hpp

@@ -12,10 +12,7 @@ namespace op
     public:
         virtual ~WorkerConsumer();
 
-        inline void work(TDatums& tDatums)
-        {
-            workConsumer(tDatums);
-        }
+        void work(TDatums& tDatums);
 
     protected:
         virtual void workConsumer(const TDatums& tDatums) = 0;
@@ -34,6 +31,20 @@ namespace op
     {
     }
 
+    template<typename TDatums>
+    void WorkerConsumer<TDatums>::work(TDatums& tDatums)
+    {
+        try
+        {
+            workConsumer(tDatums);
+        }
+        catch (const std::exception& e)
+        {
+            this->stop();
+            errorWorker(e.what(), __LINE__, __FUNCTION__, __FILE__);
+        }
+    }
+
     COMPILE_TEMPLATE_DATUM(WorkerConsumer);
 }
 

include/openpose/thread/workerProducer.hpp

@@ -12,10 +12,7 @@ namespace op
     public:
         virtual ~WorkerProducer();
 
-        inline void work(TDatums& tDatums)
-        {
-            tDatums = std::move(workProducer());
-        }
+        void work(TDatums& tDatums);
 
     protected:
         virtual TDatums workProducer() = 0;
@@ -34,6 +31,20 @@ namespace op
     {
     }
 
+    template<typename TDatums>
+    void WorkerProducer<TDatums>::work(TDatums& tDatums)
+    {
+        try
+        {
+            tDatums = std::move(workProducer());
+        }
+        catch (const std::exception& e)
+        {
+            this->stop();
+            errorWorker(e.what(), __LINE__, __FUNCTION__, __FILE__);
+        }
+    }
+
     COMPILE_TEMPLATE_DATUM(WorkerProducer);
 }
 

include/openpose/utilities/errorAndLog.hpp

@@ -1,8 +1,6 @@
 #ifndef OPENPOSE_UTILITIES_ERROR_AND_LOG_HPP
 #define OPENPOSE_UTILITIES_ERROR_AND_LOG_HPP
 
-#include <atomic>
-#include <mutex>
 #include <sstream> // std::stringstream
 #include <string>
 #include <vector>
@@ -11,6 +9,14 @@
 
 namespace op
 {
+    OP_API void setMainThread();
+
+    OP_API std::string getThreadId();
+
+    OP_API bool getIfInMainThreadOrEmpty();
+
+    OP_API bool getIfNotInMainThreadOrEmpty();
+
     template<typename T>
     std::string tToString(const T& message)
     {
@@ -21,27 +27,65 @@ namespace op
         return oss.str();
     }
 
+    /**
+     * Differences between different kind of errrors:
+     *  - error() is a normal error in the code.
+     *  - errorWorker() is an error that occurred on a thread. Therefore, the machine will stop the threads, go back
+     *    to the main thread, and then throw the error.
+     *  - errorDestructor() is an error that occurred on a destructor. Exception on destructors provokes core dumped,
+     *    so we simply output an error message via std::cerr.
+     */
+
     // Error managment - How to use:
         // error(message, __LINE__, __FUNCTION__, __FILE__);
-    OP_API void error(const std::string& message, const int line = -1, const std::string& function = "",
-                      const std::string& file = "");
+    OP_API void error(
+        const std::string& message, const int line = -1, const std::string& function = "",
+        const std::string& file = "");
 
     template<typename T>
-    inline void error(const T& message, const int line = -1, const std::string& function = "",
-                      const std::string& file = "")
+    inline void error(
+        const T& message, const int line = -1, const std::string& function = "", const std::string& file = "")
     {
         error(tToString(message), line, function, file);
     }
 
+    // Worker error managment
+    OP_API void checkWorkerErrors();
+
+    OP_API void errorWorker(
+        const std::string& message, const int line = -1, const std::string& function = "",
+        const std::string& file = "");
+
+    template<typename T>
+    inline void errorWorker(
+        const T& message, const int line = -1, const std::string& function = "", const std::string& file = "")
+    {
+        errorWorker(tToString(message), line, function, file);
+    }
+
+    // Destructor error managment
+    OP_API void errorDestructor(
+        const std::string& message, const int line = -1, const std::string& function = "",
+        const std::string& file = "");
+
+    template<typename T>
+    inline void errorDestructor(
+        const T& message, const int line = -1, const std::string& function = "", const std::string& file = "")
+    {
+        errorDestructor(tToString(message), line, function, file);
+    }
+
     // Printing info - How to use:
         // It will print info if desiredPriority >= sPriorityThreshold
         // log(message, desiredPriority, __LINE__, __FUNCTION__, __FILE__);
-    OP_API void log(const std::string& message, const Priority priority = Priority::Max, const int line = -1,
-                    const std::string& function = "", const std::string& file = "");
+    OP_API void log(
+        const std::string& message, const Priority priority = Priority::Max, const int line = -1,
+        const std::string& function = "", const std::string& file = "");
 
     template<typename T>
-    inline void log(const T& message, const Priority priority = Priority::Max, const int line = -1,
-                    const std::string& function = "", const std::string& file = "")
+    inline void log(
+        const T& message, const Priority priority = Priority::Max, const int line = -1,
+        const std::string& function = "", const std::string& file = "")
     {
         log(tToString(message), priority, line, function, file);
     }
@@ -50,8 +94,9 @@ namespace op
         // It will print info if desiredPriority >= sPriorityThreshold
         // dLog(message, desiredPriority, __LINE__, __FUNCTION__, __FILE__);
     template<typename T>
-    inline void dLog(const T& message, const Priority priority = Priority::Max, const int line = -1,
-                     const std::string& function = "", const std::string& file = "")
+    inline void dLog(
+        const T& message, const Priority priority = Priority::Max, const int line = -1,
+        const std::string& function = "", const std::string& file = "")
     {
         #ifndef NDEBUG
             log(message, priority, line, function, file);