Merge pull request dennybritz#118 from praveen-palanisamy/master

Fixes for issues and for compatibility with TensorFlow v 1.0+

Author: dennybritz

DQN/Deep Q Learning.ipynb

@@ -11,6 +11,7 @@
     "%matplotlib inline\n",
     "\n",
     "import gym\n",
+    "from gym.wrappers import Monitor\n",
     "import itertools\n",
     "import numpy as np\n",
     "import os\n",
@@ -67,7 +68,7 @@
     "            self.output = tf.image.rgb_to_grayscale(self.input_state)\n",
     "            self.output = tf.image.crop_to_bounding_box(self.output, 34, 0, 160, 160)\n",
     "            self.output = tf.image.resize_images(\n",
-    "                self.output, 84, 84, method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n",
+    "                self.output, [84, 84], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n",
     "            self.output = tf.squeeze(self.output)\n",
     "\n",
     "    def process(self, sess, state):\n",
@@ -107,7 +108,7 @@
     "                summary_dir = os.path.join(summaries_dir, \"summaries_{}\".format(scope))\n",
     "                if not os.path.exists(summary_dir):\n",
     "                    os.makedirs(summary_dir)\n",
-    "                self.summary_writer = tf.train.SummaryWriter(summary_dir)\n",
+    "                self.summary_writer = tf.summary.FileWriter(summary_dir)\n",
     "\n",
     "    def _build_model(self):\n",
     "        \"\"\"\n",
@@ -151,11 +152,11 @@
     "        self.train_op = self.optimizer.minimize(self.loss, global_step=tf.contrib.framework.get_global_step())\n",
     "\n",
     "        # Summaries for Tensorboard\n",
-    "        self.summaries = tf.merge_summary([\n",
-    "            tf.scalar_summary(\"loss\", self.loss),\n",
-    "            tf.histogram_summary(\"loss_hist\", self.losses),\n",
-    "            tf.histogram_summary(\"q_values_hist\", self.predictions),\n",
-    "            tf.scalar_summary(\"max_q_value\", tf.reduce_max(self.predictions))\n",
+    "        self.summaries = tf.summary.merge([\n",
+    "            tf.summary.scalar(\"loss\", self.loss),\n",
+    "            tf.summary.histogram(\"loss_hist\", self.losses),\n",
+    "            tf.summary.histogram(\"q_values_hist\", self.predictions),\n",
+    "            tf.summary.scalar(\"max_q_value\", tf.reduce_max(self.predictions))\n",
     "        ])\n",
     "\n",
     "\n",
@@ -212,7 +213,7 @@
     "sp = StateProcessor()\n",
     "\n",
     "with tf.Session() as sess:\n",
-    "    sess.run(tf.initialize_all_variables())\n",
+    "    sess.run(tf.global_variables_initializer())\n",
     "    \n",
     "    # Example observation batch\n",
     "    observation = env.reset()\n",
@@ -391,9 +392,10 @@
     "        pass\n",
     "\n",
     "    # Record videos\n",
-    "    env.monitor.start(monitor_path,\n",
-    "                      resume=True,\n",
-    "                      video_callable=lambda count: count % record_video_every == 0)\n",
+    "    env= Monitor(env,\n",
+    "                 directory=monitor_path,\n",
+    "                 resume=True,\n",
+    "                 video_callable=lambda count: count % record_video_every == 0)\n",
     "\n",
     "    for i_episode in range(num_episodes):\n",
     "\n",
@@ -526,7 +528,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.1"
+   "version": "3.6.0"
   }
  },
  "nbformat": 4,

DQN/dqn.py

@@ -1,4 +1,5 @@
 import gym
+from gym.wrappers import Monitor
 import itertools
 import numpy as np
 import os
@@ -28,7 +29,7 @@ def __init__(self):
             self.output = tf.image.rgb_to_grayscale(self.input_state)
             self.output = tf.image.crop_to_bounding_box(self.output, 34, 0, 160, 160)
             self.output = tf.image.resize_images(
-                self.output, 84, 84, method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
+                self.output, [84, 84], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
             self.output = tf.squeeze(self.output)
 
     def process(self, sess, state):
@@ -59,7 +60,7 @@ def __init__(self, scope="estimator", summaries_dir=None):
                 summary_dir = os.path.join(summaries_dir, "summaries_{}".format(scope))
                 if not os.path.exists(summary_dir):
                     os.makedirs(summary_dir)
-                self.summary_writer = tf.train.SummaryWriter(summary_dir)
+                self.summary_writer = tf.summary.FileWriter(summary_dir)
 
     def _build_model(self):
         """
@@ -103,11 +104,11 @@ def _build_model(self):
         self.train_op = self.optimizer.minimize(self.loss, global_step=tf.contrib.framework.get_global_step())
 
         # Summaries for Tensorboard
-        self.summaries = tf.merge_summary([
-            tf.scalar_summary("loss", self.loss),
-            tf.histogram_summary("loss_hist", self.losses),
-            tf.histogram_summary("q_values_hist", self.predictions),
-            tf.scalar_summary("max_q_value", tf.reduce_max(self.predictions))
+        self.summaries = tf.summary.merge([
+            tf.summary.scalar("loss", self.loss),
+            tf.summary.histogram("loss_hist", self.losses),
+            tf.summary.histogram("q_values_hist", self.predictions),
+            tf.summary.scalar("max_q_value", tf.reduce_max(self.predictions))
         ])
 
 
@@ -292,9 +293,11 @@ def deep_q_learning(sess,
             state = next_state
 
     # Record videos
-    env.monitor.start(monitor_path,
-                      resume=True,
-                      video_callable=lambda count: count % record_video_every == 0)
+    # Use the gym env Monitor wrapper
+    env = Monitor(env,
+                  directory=monitor_path,
+                  resume=True,
+                  video_callable=lambda count: count % record_video_every ==0)
 
     for i_episode in range(num_episodes):
 
@@ -398,7 +401,7 @@ def deep_q_learning(sess,
 state_processor = StateProcessor()
 
 with tf.Session() as sess:
-    sess.run(tf.initialize_all_variables())
+    sess.run(tf.global_variables_initializer())
     for t, stats in deep_q_learning(sess,
                                     env,
                                     q_estimator=q_estimator,