Merge pull request dennybritz#134 from keithmgould/master

update value estimator only after calculating advantage

Author: dennybritz

PolicyGradient/CliffWalk REINFORCE with Baseline Solution.ipynb

@@ -196,11 +196,11 @@
     "        for t, transition in enumerate(episode):\n",
     "            # The return after this timestep\n",
     "            total_return = sum(discount_factor**i * t.reward for i, t in enumerate(episode[t:]))\n",
-    "            # Update our value estimator\n",
-    "            estimator_value.update(transition.state, total_return)\n",
     "            # Calculate baseline/advantage\n",
     "            baseline_value = estimator_value.predict(transition.state)            \n",
     "            advantage = total_return - baseline_value\n",
+    "            # Update our value estimator\n",
+    "            estimator_value.update(transition.state, total_return)\n",
     "            # Update our policy estimator\n",
     "            estimator_policy.update(transition.state, advantage, transition.action)\n",
     "    \n",