Update notebook and saved runs.

Author: Jun Gong

ray-rllib/odsc_west_workshop_2022/anyscale_recsys_tutorial.ipynb

@@ -279,7 +279,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# TODO (exercise): code here"
+    "# Let's checkout the reward space\n",
+    "obs = env.reset()\n",
+    "rewards = []\n",
+    "done = False\n",
+    "while not done:\n",
+    "    action = # TODO (exercise): code here\n",
+    "    obs, reward, done, info = env.step(action)\n",
+    "    rewards.append(reward)"
    ]
   },
   {
@@ -608,8 +615,8 @@
     "    DQN,\n",
     "    param_space=bandit_config_offline.to_dict(),\n",
     "    run_config=air.RunConfig(\n",
-    "        local_dir=\"./results_notebook/offline_bandit/\",\n",
-    "        stop={\"training_iteration\": 100},\n",
+    "        local_dir=\"./results_notebook/offline_bandits/\",\n",
+    "        stop={\"training_iteration\": 1000},\n",
     "    )\n",
     ")\n",
     "offline_bandit_results = bandit_tuner.fit()"
@@ -629,7 +636,7 @@
    "outputs": [],
    "source": [
     "print('Mean Bandit Episode reward:')\n",
-    "offline_bandit_results[0].metrics['evaluation']['episode_reward_mean']"
+    "offline_bandits_results[0].metrics['evaluation']['episode_reward_mean']"
    ]
   },
   {
@@ -683,10 +690,10 @@
     "    param_space=dqn_config_offline.to_dict(),\n",
     "    run_config=air.RunConfig(\n",
     "        local_dir=\"./results_notebook/offline_rl/\",\n",
-    "        stop={\"training_iteration\": 100},\n",
+    "        stop={\"training_iteration\": 30},\n",
     "    )\n",
     ")\n",
-    "offline_dqn_results = dqn_tuner.fit()"
+    "offline_rl_results = dqn_tuner.fit()"
    ]
   },
   {
@@ -696,7 +703,7 @@
    "outputs": [],
    "source": [
     "print('Mean DQN Episode reward:')\n",
-    "offline_dqn_results[0].metrics['evaluation']['episode_reward_mean']"
+    "offline_rl_results[0].metrics['evaluation']['episode_reward_mean']"
    ]
   },
   {
@@ -705,8 +712,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import pandas as pd\n",
+    "\n",
     "# plot the results and compare to baselines\n",
-    "offline_dqn_df = pd.read_csv(\"saved_runs/dqn_offline/random_data/progress.csv\")"
+    "offline_rl_df = pd.read_csv(\"saved_runs/offline_rl/progress.csv\")\n",
+    "offline_bandits_df = pd.read_csv(\"saved_runs/offline_bandits/progress.csv\")"
    ]
   },
   {
@@ -716,9 +726,8 @@
    "outputs": [],
    "source": [
     "\n",
-    "sns.lineplot(data=offline_dqn_df, x=\"training_iteration\", y=\"evaluation/episode_reward_mean\", label=\"Offline_DQN\")\n",
-    "sns.lineplot(data=dqn_df, x=\"training_iteration\", y=\"episode_reward_mean\", label=\"Online_DQN\")\n",
-    "sns.lineplot(data=bandit_df, x=\"training_iteration\", y=\"episode_reward_mean\", label=\"Bandits\")\n",
+    "sns.lineplot(data=offline_rl_df, x=\"training_iteration\", y=\"evaluation/episode_reward_mean\", label=\"Offline_DQN\")\n",
+    "sns.lineplot(data=offline_bandits_df, x=\"training_iteration\", y=\"evaluation/episode_reward_mean\", label=\"Offline_Bandits\")\n",
     "plt.axhline(random_baseline, color=\"red\", linestyle='--', label=\"random baseline\")\n",
     "plt.legend()\n",
     "plt.title('Offline RL vs. Baselines training performance')"