commit

Author: TarrySingh

deep-learning/Deep-Reinforcement-Learning-Complete-Collection/PyTorch-cpp/include/cpprl/algorithms/a2c.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include <torch/torch.h>
+
+#include "cpprl/algorithms/algorithm.h"
+
+namespace cpprl
+{
+class Policy;
+class ROlloutStorage;
+
+class A2C : public Algorithm
+{
+  private:
+    Policy &policy;
+    float value_loss_coef, entropy_coef, max_grad_norm;
+    std::unique_ptr<torch::optim::Optimizer> optimizer;
+
+  public:
+    A2C(Policy &policy,
+        float value_loss_coef,
+        float entropy_coef,
+        float learning_rate,
+        float epsilon = 1e-8,
+        float alpha = 0.99,
+        float max_grad_norm = 0.5);
+
+    std::vector<UpdateDatum> update(RolloutStorage &rollouts);
+};
+}

deep-learning/Deep-Reinforcement-Learning-Complete-Collection/PyTorch-cpp/include/cpprl/algorithms/algorithm.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "cpprl/storage.h"
+
+namespace cpprl
+{
+struct UpdateDatum
+{
+    std::string name;
+    float value;
+};
+
+class Algorithm
+{
+  public:
+    virtual ~Algorithm() = 0;
+
+    virtual std::vector<UpdateDatum> update(RolloutStorage &rollouts) = 0;
+};
+
+inline Algorithm::~Algorithm() {}
+}

deep-learning/Deep-Reinforcement-Learning-Complete-Collection/PyTorch-cpp/include/cpprl/algorithms/ppo.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include <torch/torch.h>
+
+#include "cpprl/algorithms/algorithm.h"
+
+namespace cpprl
+{
+class Policy;
+class ROlloutStorage;
+
+class PPO : public Algorithm
+{
+  private:
+    Policy &policy;
+    float clip_param, value_loss_coef, entropy_coef, max_grad_norm;
+    int num_epoch, num_mini_batch;
+    std::unique_ptr<torch::optim::Optimizer> optimizer;
+
+  public:
+    PPO(Policy &policy,
+        float clip_param,
+        int num_epoch,
+        int num_mini_batch,
+        float value_loss_coef,
+        float entropy_coef,
+        float learning_rate,
+        float epsilon = 1e-8,
+        float max_grad_norm = 0.5);
+
+    std::vector<UpdateDatum> update(RolloutStorage &rollouts);
+};
+}

deep-learning/Deep-Reinforcement-Learning-Complete-Collection/PyTorch-cpp/include/cpprl/cpprl.h

@@ -0,0 +1,15 @@
+#include "cpprl/algorithms/a2c.h"
+#include "cpprl/algorithms/algorithm.h"
+#include "cpprl/algorithms/ppo.h"
+#include "cpprl/distributions/distribution.h"
+#include "cpprl/distributions/categorical.h"
+#include "cpprl/generators/generator.h"
+#include "cpprl/generators/feed_forward_generator.h"
+#include "cpprl/model/cnn_base.h"
+#include "cpprl/model/mlp_base.h"
+#include "cpprl/model/model_utils.h"
+#include "cpprl/model/nn_base.h"
+#include "cpprl/model/output_layers.h"
+#include "cpprl/model/policy.h"
+#include "cpprl/spaces.h"
+#include "cpprl/storage.h"

deep-learning/Deep-Reinforcement-Learning-Complete-Collection/PyTorch-cpp/include/cpprl/distributions/categorical.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <c10/util/ArrayRef.h>
+#include <torch/torch.h>
+
+#include "cpprl/distributions/distribution.h"
+
+namespace cpprl
+{
+class Categorical : public Distribution
+{
+  private:
+    torch::Tensor probs;
+    torch::Tensor logits;
+    std::vector<long> batch_shape;
+    std::vector<long> event_shape;
+    torch::Tensor param;
+    int num_events;
+
+    std::vector<long> extended_shape(c10::ArrayRef<int64_t> sample_shape);
+
+  public:
+    Categorical(const torch::Tensor *probs, const torch::Tensor *logits);
+
+    torch::Tensor entropy();
+    torch::Tensor log_prob(torch::Tensor value);
+    torch::Tensor sample(c10::ArrayRef<int64_t> sample_shape = {});
+
+    inline torch::Tensor get_logits() { return logits; }
+    inline torch::Tensor get_probs() { return probs; }
+};
+}

deep-learning/Deep-Reinforcement-Learning-Complete-Collection/PyTorch-cpp/include/cpprl/distributions/distribution.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <torch/torch.h>
+
+namespace cpprl
+{
+class Distribution
+{
+  public:
+    virtual ~Distribution() = 0;
+
+    virtual torch::Tensor entropy() = 0;
+    virtual torch::Tensor get_logits() = 0;
+    virtual torch::Tensor get_probs() = 0;
+    virtual torch::Tensor log_prob(torch::Tensor value) = 0;
+    virtual torch::Tensor sample(c10::ArrayRef<int64_t> sample_shape = {}) = 0;
+};
+
+inline Distribution::~Distribution() {}
+}

deep-learning/Deep-Reinforcement-Learning-Complete-Collection/PyTorch-cpp/include/cpprl/generators/feed_forward_generator.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <torch/torch.h>
+
+#include "cpprl/generators/generator.h"
+
+namespace cpprl
+{
+class FeedForwardGenerator : public Generator
+{
+  private:
+    torch::Tensor observations, hidden_states, actions, value_predictions,
+        returns, masks, action_log_probs, advantages, indices;
+    int index;
+
+  public:
+    FeedForwardGenerator(int mini_batch_size,
+                         torch::Tensor observations,
+                         torch::Tensor hidden_states,
+                         torch::Tensor actions,
+                         torch::Tensor value_predictions,
+                         torch::Tensor returns,
+                         torch::Tensor masks,
+                         torch::Tensor action_log_probs,
+                         torch::Tensor advantages);
+
+    virtual bool done() const;
+    virtual MiniBatch next();
+};
+}

deep-learning/Deep-Reinforcement-Learning-Complete-Collection/PyTorch-cpp/include/cpprl/generators/generator.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <vector>
+
+#include <torch/torch.h>
+
+namespace cpprl
+{
+struct MiniBatch
+{
+    torch::Tensor observations, hidden_states, actions, value_predictions,
+        returns, masks, action_log_probs, advantages;
+
+    MiniBatch() {}
+    MiniBatch(torch::Tensor observations,
+              torch::Tensor hidden_states,
+              torch::Tensor actions,
+              torch::Tensor value_predictions,
+              torch::Tensor returns,
+              torch::Tensor masks,
+              torch::Tensor action_log_probs,
+              torch::Tensor advantages)
+        : observations(observations),
+          hidden_states(hidden_states),
+          actions(actions),
+          value_predictions(value_predictions),
+          returns(returns),
+          masks(masks),
+          action_log_probs(action_log_probs),
+          advantages(advantages) {}
+};
+
+class Generator
+{
+  public:
+    virtual ~Generator() = 0;
+
+    virtual bool done() const = 0;
+    virtual MiniBatch next() = 0;
+};
+
+inline Generator::~Generator() {}
+}

deep-learning/Deep-Reinforcement-Learning-Complete-Collection/PyTorch-cpp/include/cpprl/generators/recurrent_generator.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <torch/torch.h>
+
+#include "cpprl/generators/generator.h"
+
+namespace cpprl
+{
+class RecurrentGenerator : public Generator
+{
+  private:
+    torch::Tensor observations, hidden_states, actions, value_predictions,
+        returns, masks, action_log_probs, advantages, indices;
+    int index, num_envs_per_batch;
+
+  public:
+    RecurrentGenerator(int num_processes,
+                       int num_mini_batch,
+                       torch::Tensor observations,
+                       torch::Tensor hidden_states,
+                       torch::Tensor actions,
+                       torch::Tensor value_predictions,
+                       torch::Tensor returns,
+                       torch::Tensor masks,
+                       torch::Tensor action_log_probs,
+                       torch::Tensor advantages);
+
+    virtual bool done() const;
+    virtual MiniBatch next();
+};
+}

deep-learning/Deep-Reinforcement-Learning-Complete-Collection/PyTorch-cpp/include/cpprl/model/cnn_base.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <vector>
+
+#include <torch/torch.h>
+
+#include "cpprl/model/nn_base.h"
+
+using namespace torch;
+
+namespace cpprl
+{
+class CnnBase : public NNBase
+{
+  private:
+    nn::Sequential main;
+    nn::Sequential critic_linear;
+
+  public:
+    CnnBase(unsigned int num_inputs,
+            bool recurrent = false,
+            unsigned int hidden_size = 512);
+
+    std::vector<torch::Tensor> forward(torch::Tensor inputs,
+                                       torch::Tensor hxs,
+                                       torch::Tensor masks);
+};
+}

deep-learning/Deep-Reinforcement-Learning-Complete-Collection/PyTorch-cpp/include/cpprl/model/mlp_base.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <vector>
+
+#include <torch/torch.h>
+
+#include "cpprl/model/nn_base.h"
+
+using namespace torch;
+
+namespace cpprl
+{
+class MlpBase : public NNBase
+{
+  private:
+    nn::Sequential actor;
+    nn::Sequential critic;
+    nn::Linear critic_linear;
+
+  public:
+    MlpBase(unsigned int num_inputs,
+            bool recurrent = false,
+            unsigned int hidden_size = 64);
+
+    std::vector<torch::Tensor> forward(torch::Tensor inputs,
+                                       torch::Tensor hxs,
+                                       torch::Tensor masks);
+};
+}

deep-learning/Deep-Reinforcement-Learning-Complete-Collection/PyTorch-cpp/include/cpprl/model/model_utils.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <vector>
+
+#include <torch/torch.h>
+
+using namespace torch;
+
+namespace cpprl
+{
+struct FlattenImpl : nn::Module
+{
+    torch::Tensor forward(torch::Tensor x);
+};
+TORCH_MODULE(Flatten);
+
+void init_weights(torch::OrderedDict<std::string, torch::Tensor> parameters,
+                  double weight_gain,
+                  double bias_gain);
+}