Add KDE depth unwrapping algorithms

This implements kernel density estimation based phase unwrapping
procedure. It shows improved depth imaging, especially for large depth and
outdoors scenes. The method was presented on ECCV 2016, see paper for more
information.

http://users.isy.liu.se/cvl/perfo/abstracts/jaremo16.html

The algorithms are added as OpenCL and CUDA processors. OpenCLKde and CudaKde
pipelines are also added as APIs.

Author: Felix

CMakeLists.txt

@@ -313,6 +313,7 @@ IF(ENABLE_OPENCL)
 
     LIST(APPEND SOURCES
       src/opencl_depth_packet_processor.cpp
+      src/opencl_kde_depth_packet_processor.cpp
     )
 
     LIST(APPEND LIBRARIES
@@ -321,6 +322,7 @@ IF(ENABLE_OPENCL)
 
     LIST(APPEND RESOURCES
       src/opencl_depth_packet_processor.cl
+      src/opencl_kde_depth_packet_processor.cl
     )
 
     # Major Linux distro stable releases have buggy OpenCL ICD loader.
@@ -359,6 +361,7 @@ IF(ENABLE_CUDA)
     ENDIF()
     CUDA_COMPILE(CUDA_OBJECTS
       src/cuda_depth_packet_processor.cu
+      src/cuda_kde_depth_packet_processor.cu
       OPTIONS ${CUDA_FLAGS}
     )
     SET(CMAKE_CXX_FLAGS "${OLD_CMAKE_CXX_FLAGS}")

README.md

@@ -20,6 +20,8 @@ Note: libfreenect2 does not do anything for either Kinect for Windows v1 or Kine
 
 If you are using libfreenect2 in an academic context, please cite our work using the following DOI: [![DOI](https://zenodo.org/badge/20096/OpenKinect/libfreenect2.svg)](https://zenodo.org/badge/latestdoi/20096/OpenKinect/libfreenect2)
 
+If you use the KDE depth unwrapping algorithm implemented in the library, please also cite this ECCV 2016 [paper](http://users.isy.liu.se/cvl/perfo/abstracts/jaremo16.html).
+
 This driver supports:
 * RGB image transfer
 * IR and depth image transfer

examples/Protonect.cpp

@@ -113,7 +113,7 @@ int main(int argc, char *argv[])
   std::string program_path(argv[0]);
   std::cerr << "Version: " << LIBFREENECT2_VERSION << std::endl;
   std::cerr << "Environment variables: LOGFILE=<protonect.log>" << std::endl;
-  std::cerr << "Usage: " << program_path << " [-gpu=<id>] [gl | cl | cuda | cpu] [<device serial>]" << std::endl;
+  std::cerr << "Usage: " << program_path << " [-gpu=<id>] [gl | cl | clkde | cuda | cudakde | cpu] [<device serial>]" << std::endl;
   std::cerr << "        [-noviewer] [-norgb | -nodepth] [-help] [-version]" << std::endl;
   std::cerr << "        [-frames <number of frames to process>]" << std::endl;
   std::cerr << "To pause and unpause: pkill -USR1 Protonect" << std::endl;
@@ -198,6 +198,15 @@ int main(int argc, char *argv[])
         pipeline = new libfreenect2::OpenCLPacketPipeline(deviceId);
 #else
       std::cout << "OpenCL pipeline is not supported!" << std::endl;
+#endif
+    }
+    else if(arg == "clkde")
+    {
+#ifdef LIBFREENECT2_WITH_OPENCL_SUPPORT
+      if(!pipeline)
+        pipeline = new libfreenect2::OpenCLKdePacketPipeline(deviceId);
+#else
+      std::cout << "OpenCL pipeline is not supported!" << std::endl;
 #endif
     }
     else if(arg == "cuda")
@@ -207,6 +216,15 @@ int main(int argc, char *argv[])
         pipeline = new libfreenect2::CudaPacketPipeline(deviceId);
 #else
       std::cout << "CUDA pipeline is not supported!" << std::endl;
+#endif
+    }
+    else if(arg == "cudakde")
+    {
+#ifdef LIBFREENECT2_WITH_CUDA_SUPPORT
+      if(!pipeline)
+        pipeline = new libfreenect2::CudaKdePacketPipeline(deviceId);
+#else
+      std::cout << "CUDA pipeline is not supported!" << std::endl;
 #endif
     }
     else if(arg.find_first_not_of("0123456789") == std::string::npos) //check if parameter could be a serial number

include/internal/libfreenect2/depth_packet_processor.h

@@ -90,6 +90,13 @@ class DepthPacketProcessor : public BaseDepthPacketProcessor
     float edge_avg_delta_threshold;
     float max_edge_count;
 
+    float kde_sigma_sqr;
+    float unwrapping_likelihood_scale;
+    float phase_confidence_scale;
+    float kde_threshold;
+    size_t kde_neigborhood_size;
+    size_t num_hyps;
+
     float min_depth;
     float max_depth;
 
@@ -184,6 +191,37 @@ class OpenCLDepthPacketProcessor : public DepthPacketProcessor
 private:
   OpenCLDepthPacketProcessorImpl *impl_;
 };
+
+/*
+ * The class below implement a depth packet processor using the phase unwrapping
+ * algorithm described in the paper "Efficient Phase Unwrapping using Kernel
+ * Density Estimation", ECCV 2016, Felix Järemo Lawin, Per-Erik Forssen and
+ * Hannes Ovren, see http://www.cvl.isy.liu.se/research/datasets/kinect2-dataset/.
+ */
+class OpenCLKdeDepthPacketProcessorImpl;
+
+/** Depth packet processor using OpenCL. */
+class OpenCLKdeDepthPacketProcessor : public DepthPacketProcessor
+{
+public:
+  OpenCLKdeDepthPacketProcessor(const int deviceId = -1);
+  virtual ~OpenCLKdeDepthPacketProcessor();
+  virtual void setConfiguration(const libfreenect2::DepthPacketProcessor::Config &config);
+
+  virtual void loadP0TablesFromCommandResponse(unsigned char* buffer, size_t buffer_length);
+
+  virtual void loadXZTables(const float *xtable, const float *ztable);
+  virtual void loadLookupTable(const short *lut);
+
+  virtual bool good();
+  virtual const char *name() { return "OpenCLKde"; }
+
+  virtual void process(const DepthPacket &packet);
+protected:
+  virtual Allocator *getAllocator();
+private:
+  OpenCLKdeDepthPacketProcessorImpl *impl_;
+};
 #endif // LIBFREENECT2_WITH_OPENCL_SUPPORT
 
 #ifdef LIBFREENECT2_WITH_CUDA_SUPPORT
@@ -210,6 +248,36 @@ class CudaDepthPacketProcessor : public DepthPacketProcessor
 private:
   CudaDepthPacketProcessorImpl *impl_;
 };
+
+/*
+ * The class below implement a depth packet processor using the phase unwrapping
+ * algorithm described in the paper "Efficient Phase Unwrapping using Kernel
+ * Density Estimation", ECCV 2016, Felix Järemo Lawin, Per-Erik Forssen and
+ * Hannes Ovren, see http://www.cvl.isy.liu.se/research/datasets/kinect2-dataset/.
+ */
+class CudaKdeDepthPacketProcessorImpl;
+
+class CudaKdeDepthPacketProcessor : public DepthPacketProcessor
+{
+public:
+  CudaKdeDepthPacketProcessor(const int deviceId = -1);
+  virtual ~CudaKdeDepthPacketProcessor();
+  virtual void setConfiguration(const libfreenect2::DepthPacketProcessor::Config &config);
+
+  virtual void loadP0TablesFromCommandResponse(unsigned char* buffer, size_t buffer_length);
+
+  virtual void loadXZTables(const float *xtable, const float *ztable);
+  virtual void loadLookupTable(const short *lut);
+
+  virtual bool good();
+  virtual const char *name() { return "CUDAKde"; }
+
+  virtual void process(const DepthPacket &packet);
+protected:
+  virtual Allocator *getAllocator();
+private:
+  CudaKdeDepthPacketProcessorImpl *impl_;
+};
 #endif // LIBFREENECT2_WITH_CUDA_SUPPORT
 
 class DumpDepthPacketProcessor : public DepthPacketProcessor

include/libfreenect2/packet_pipeline.h

@@ -113,6 +113,21 @@ class LIBFREENECT2_API OpenCLPacketPipeline : public PacketPipeline
   OpenCLPacketPipeline(const int deviceId = -1);
   virtual ~OpenCLPacketPipeline();
 };
+
+/*
+ * The class below implement a depth packet processor using the phase unwrapping
+ * algorithm described in the paper "Efficient Phase Unwrapping using Kernel
+ * Density Estimation", ECCV 2016, Felix Järemo Lawin, Per-Erik Forssen and
+ * Hannes Ovren, see http://www.cvl.isy.liu.se/research/datasets/kinect2-dataset/.
+ */
+class LIBFREENECT2_API OpenCLKdePacketPipeline : public PacketPipeline
+{
+protected:
+  const int deviceId;
+public:
+  OpenCLKdePacketPipeline(const int deviceId = -1);
+  virtual ~OpenCLKdePacketPipeline();
+};
 #endif // LIBFREENECT2_WITH_OPENCL_SUPPORT
 
 #ifdef LIBFREENECT2_WITH_CUDA_SUPPORT
@@ -124,6 +139,21 @@ class LIBFREENECT2_API CudaPacketPipeline : public PacketPipeline
   CudaPacketPipeline(const int deviceId = -1);
   virtual ~CudaPacketPipeline();
 };
+
+/*
+ * The class below implement a depth packet processor using the phase unwrapping
+ * algorithm described in the paper "Efficient Phase Unwrapping using Kernel
+ * Density Estimation", ECCV 2016, Felix Järemo Lawin, Per-Erik Forssen and
+ * Hannes Ovren, see http://www.cvl.isy.liu.se/research/datasets/kinect2-dataset/.
+ */
+class LIBFREENECT2_API CudaKdePacketPipeline : public PacketPipeline
+{
+protected:
+  const int deviceId;
+public:
+  CudaKdePacketPipeline(const int deviceId = -1);
+  virtual ~CudaKdePacketPipeline();
+};
 #endif // LIBFREENECT2_WITH_CUDA_SUPPORT
 
 ///@}