DQN copy_model_parameters memory leak fixed, tensorboard summaries updated with cpu/mem usage

Author: Kismuz

DQN/Deep Q Learning Solution.ipynb

@@ -17,6 +17,7 @@
     "import os\n",
     "import random\n",
     "import sys\n",
+    "import psutil\n",
     "import tensorflow as tf\n",
     "\n",
     "if \"../\" not in sys.path:\n",
@@ -29,9 +30,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "env = gym.envs.make(\"Breakout-v0\")"
@@ -40,9 +39,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "# Atari Actions: 0 (noop), 1 (fire), 2 (left) and 3 (right) are valid actions\n",
@@ -86,9 +83,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "class Estimator():\n",
@@ -198,9 +193,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "# For Testing....\n",
@@ -234,30 +227,39 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false
+    "collapsed": true
    },
    "outputs": [],
    "source": [
-    "def copy_model_parameters(sess, estimator1, estimator2):\n",
+    "class ModelParametersCopier():\n",
     "    \"\"\"\n",
-    "    Copies the model parameters of one estimator to another.\n",
-    "\n",
-    "    Args:\n",
-    "      sess: Tensorflow session instance\n",
-    "      estimator1: Estimator to copy the paramters from\n",
-    "      estimator2: Estimator to copy the parameters to\n",
+    "    Copy model parameters of one estimator to another.\n",
     "    \"\"\"\n",
-    "    e1_params = [t for t in tf.trainable_variables() if t.name.startswith(estimator1.scope)]\n",
-    "    e1_params = sorted(e1_params, key=lambda v: v.name)\n",
-    "    e2_params = [t for t in tf.trainable_variables() if t.name.startswith(estimator2.scope)]\n",
-    "    e2_params = sorted(e2_params, key=lambda v: v.name)\n",
-    "\n",
-    "    update_ops = []\n",
-    "    for e1_v, e2_v in zip(e1_params, e2_params):\n",
-    "        op = e2_v.assign(e1_v)\n",
-    "        update_ops.append(op)\n",
-    "\n",
-    "    sess.run(update_ops)"
+    "    \n",
+    "    def __init__(self, estimator1, estimator2):\n",
+    "        \"\"\"\n",
+    "        Defines copy-work operation graph.  \n",
+    "        Args:\n",
+    "          estimator1: Estimator to copy the paramters from\n",
+    "          estimator2: Estimator to copy the parameters to\n",
+    "        \"\"\"\n",
+    "        e1_params = [t for t in tf.trainable_variables() if t.name.startswith(estimator1.scope)]\n",
+    "        e1_params = sorted(e1_params, key=lambda v: v.name)\n",
+    "        e2_params = [t for t in tf.trainable_variables() if t.name.startswith(estimator2.scope)]\n",
+    "        e2_params = sorted(e2_params, key=lambda v: v.name)\n",
+    "\n",
+    "        self.update_ops = []\n",
+    "        for e1_v, e2_v in zip(e1_params, e2_params):\n",
+    "            op = e2_v.assign(e1_v)\n",
+    "            self.update_ops.append(op)\n",
+    "            \n",
+    "    def make(self, sess):\n",
+    "        \"\"\"\n",
+    "        Makes copy.\n",
+    "        Args:\n",
+    "            sess: Tensorflow session instance\n",
+    "        \"\"\"\n",
+    "        sess.run(self.update_ops)"
    ]
   },
   {
@@ -293,9 +295,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "def deep_q_learning(sess,\n",
@@ -347,11 +347,17 @@
     "\n",
     "    # The replay memory\n",
     "    replay_memory = []\n",
+    "    \n",
+    "    # Make model copier object\n",
+    "    estimator_copy = ModelParametersCopier(q_estimator, target_estimator)\n",
     "\n",
     "    # Keeps track of useful statistics\n",
     "    stats = plotting.EpisodeStats(\n",
     "        episode_lengths=np.zeros(num_episodes),\n",
     "        episode_rewards=np.zeros(num_episodes))\n",
+    "    \n",
+    "    # For 'system/' summaries, usefull to check if currrent process looks healthy\n",
+    "    current_process = psutil.Process()\n",
     "\n",
     "    # Create directories for checkpoints and summaries\n",
     "    checkpoint_dir = os.path.join(experiment_dir, \"checkpoints\")\n",
@@ -422,14 +428,9 @@
     "            # Epsilon for this time step\n",
     "            epsilon = epsilons[min(total_t, epsilon_decay_steps-1)]\n",
     "\n",
-    "            # Add epsilon to Tensorboard\n",
-    "            episode_summary = tf.Summary()\n",
-    "            episode_summary.value.add(simple_value=epsilon, tag=\"epsilon\")\n",
-    "            q_estimator.summary_writer.add_summary(episode_summary, total_t)\n",
-    "\n",
     "            # Maybe update the target estimator\n",
     "            if total_t % update_target_estimator_every == 0:\n",
-    "                copy_model_parameters(sess, q_estimator, target_estimator)\n",
+    "                estimator_copy.make(sess)\n",
     "                print(\"\\nCopied model parameters to target network.\")\n",
     "\n",
     "            # Print out which step we're on, useful for debugging.\n",
@@ -475,11 +476,14 @@
     "\n",
     "        # Add summaries to tensorboard\n",
     "        episode_summary = tf.Summary()\n",
-    "        episode_summary.value.add(simple_value=stats.episode_rewards[i_episode], node_name=\"episode_reward\", tag=\"episode_reward\")\n",
-    "        episode_summary.value.add(simple_value=stats.episode_lengths[i_episode], node_name=\"episode_length\", tag=\"episode_length\")\n",
-    "        q_estimator.summary_writer.add_summary(episode_summary, total_t)\n",
+    "        episode_summary.value.add(simple_value=epsilon, tag=\"episode/epsilon\")\n",
+    "        episode_summary.value.add(simple_value=stats.episode_rewards[i_episode], tag=\"episode/reward\")\n",
+    "        episode_summary.value.add(simple_value=stats.episode_lengths[i_episode], tag=\"episode/length\")\n",
+    "        episode_summary.value.add(simple_value=current_process.cpu_percent(), tag=\"system/cpu_usage_percent\")\n",
+    "        episode_summary.value.add(simple_value=current_process.memory_percent(memtype=\"vms\"), tag=\"system/v_memeory_usage_percent\")\n",
+    "        q_estimator.summary_writer.add_summary(episode_summary, i_episode)\n",
     "        q_estimator.summary_writer.flush()\n",
-    "\n",
+    "        \n",
     "        yield total_t, plotting.EpisodeStats(\n",
     "            episode_lengths=stats.episode_lengths[:i_episode+1],\n",
     "            episode_rewards=stats.episode_rewards[:i_episode+1])\n",
@@ -490,9 +494,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "tf.reset_default_graph()\n",
@@ -504,7 +506,7 @@
     "global_step = tf.Variable(0, name='global_step', trainable=False)\n",
     "    \n",
     "# Create estimators\n",
-    "q_estimator = Estimator(scope=\"q\", summaries_dir=experiment_dir)\n",
+    "q_estimator = Estimator(scope=\"q_estimator\", summaries_dir=experiment_dir)\n",
     "target_estimator = Estimator(scope=\"target_q\")\n",
     "\n",
     "# State processor\n",
@@ -531,6 +533,24 @@
     "\n",
     "        print(\"\\nEpisode Reward: {}\".format(stats.episode_rewards[-1]))"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -549,9 +569,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.4.3"
+   "version": "3.6.0"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 0
+ "nbformat_minor": 1
 }