add discount factor when compute G

Author: rianrajagede

MC/MC Control with Epsilon-Greedy Policies Solution.ipynb

@@ -133,7 +133,7 @@
     "            first_occurence_idx = next(i for i,x in enumerate(episode)\n",
     "                                       if x[0] == state and x[1] == action)\n",
     "            # Sum up all rewards since the first occurance\n",
-    "            G = sum([x[2] for x in episode[first_occurence_idx:]])\n",
+    "            G = sum([x[2]*(discount_factor**i) for i,x in enumerate(episode[first_occurence_idx:])])\n",
     "            # Calculate average return for this state over all sampled episodes\n",
     "            returns_sum[sa_pair] += G\n",
     "            returns_count[sa_pair] += 1.0\n",
@@ -213,21 +213,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 2",
    "language": "python",
-   "name": "python3"
+   "name": "python2"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 2
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.5.1"
+   "pygments_lexer": "ipython2",
+   "version": "2.7.11"
   }
  },
  "nbformat": 4,

MC/MC Control with Epsilon-Greedy Policies.ipynb

@@ -148,21 +148,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 2",
    "language": "python",
-   "name": "python3"
+   "name": "python2"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 2
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.5.1"
+   "pygments_lexer": "ipython2",
+   "version": "2.7.11"
   }
  },
  "nbformat": 4,

MC/MC Prediction Solution.ipynb

@@ -118,21 +118,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 2",
    "language": "python",
-   "name": "python3"
+   "name": "python2"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 2
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.5.1"
+   "pygments_lexer": "ipython2",
+   "version": "2.7.11"
   }
  },
  "nbformat": 4,