Merged commit includes the following changes:

374584039  by Waymo Research:

    Internal fixes.

--
374581701  by Waymo Research:

    Improve comments.

--
374551331  by Waymo Research:

    Improve comments in colab.

--

PiperOrigin-RevId: 374584039

Author: Waymo Research

tutorial/BUILD

@@ -29,8 +29,6 @@ py_binary(
     deps = [
         "//waymo_open_dataset:dataset_proto_py_pb2",
         "//waymo_open_dataset:label_proto_py_pb2",
-        "//waymo_open_dataset/metrics/ops:gen_metrics_ops",
-        "//waymo_open_dataset/metrics/ops:motion_metrics_ops",
         "//waymo_open_dataset/metrics/ops:py_metrics_ops",
         "//waymo_open_dataset/metrics/python:config_util_py",
         "//waymo_open_dataset/protos:breakdown_proto_py_pb2",

tutorial/tutorial_motion.ipynb

@@ -44,19 +44,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 1,
+      "execution_count": null,
       "metadata": {
-        "executionInfo": {
-          "elapsed": 401,
-          "status": "ok",
-          "timestamp": 1620951571164,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 420
-        },
         "id": "xdEcN6WilcBn"
       },
       "outputs": [],
@@ -72,19 +61,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 2,
+      "execution_count": null,
       "metadata": {
-        "executionInfo": {
-          "elapsed": 2591,
-          "status": "ok",
-          "timestamp": 1620951573762,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 420
-        },
         "id": "M5gzSlBTlTiS"
       },
       "outputs": [],
@@ -255,19 +233,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 3,
+      "execution_count": null,
       "metadata": {
-        "executionInfo": {
-          "elapsed": 81,
-          "status": "ok",
-          "timestamp": 1620951573865,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 420
-        },
         "id": "TpEZq1EMtXV9"
       },
       "outputs": [],
@@ -288,19 +255,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 7,
+      "execution_count": null,
       "metadata": {
-        "executionInfo": {
-          "elapsed": 6331,
-          "status": "ok",
-          "timestamp": 1620939490073,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 420
-        },
         "id": "utTE9Mtgx3Fq"
       },
       "outputs": [],
@@ -722,6 +678,7 @@
         "\n",
         "  def result(self):\n",
         "    # [batch_size, num_preds, 1, 1, steps, 2].\n",
+        "    # The ones indicate top_k = 1, num_agents_per_joint_prediction = 1.\n",
         "    prediction_trajectory = tf.concat(self._prediction_trajectory, 0)\n",
         "    # [batch_size, num_preds, 1].\n",
         "    prediction_score = tf.concat(self._prediction_score, 0)\n",
@@ -793,7 +750,8 @@
         "  optimizer.apply_gradients(zip(grads, model.trainable_weights))\n",
         "\n",
         "  # [batch_size, num_agents, steps, 2] -\u003e\n",
-        "  # [batch_size, num_agents, 1, 1, steps, 2]\n",
+        "  # [batch_size, num_agents, 1, 1, steps, 2].\n",
+        "  # The added dimensions are top_k = 1, num_agents_per_joint_prediction = 1.\n",
         "  pred_trajectory = pred_trajectory[:, :, tf.newaxis, tf.newaxis]\n",
         "\n",
         "  # Fake the score since this model does not generate any score per predicted\n",

waymo_open_dataset/latency/compare_objects_file_to_submission_main.cc

@@ -20,10 +20,9 @@ limitations under the License.
 #include <iostream>
 #include <memory>
 #include <numeric>
-#include <vector>
 #include <tuple>
+#include <vector>
 
-#include "base/init_google.h"
 #include <glog/logging.h>
 #include "absl/flags/flag.h"
 #include "absl/flags/parse.h"
@@ -207,12 +206,11 @@ int Compute(const std::string& latency_result_filename,
     }
 
     if (unmatched_detections > 0.05 * num_detections) {
-      LOG(ERROR).NoPrefix()
-          << "Latency evaluator results did not match submission "
-          << "proto for " << print_key(example_key) << std::endl
-          << unmatched_detections << " detections out of " << num_detections
-          << " did not match. This exceeds our "
-          << "cut-off of 5% of detections being unmatched.";
+      LOG(FATAL) << "Latency evaluator results did not match submission "
+                 << "proto for " << print_key(example_key) << std::endl
+                 << unmatched_detections << " detections out of "
+                 << num_detections << " did not match. This exceeds our "
+                 << "cut-off of 5% of detections being unmatched.";
       return 1;
     }
 
@@ -228,7 +226,6 @@ int Compute(const std::string& latency_result_filename,
 }  // namespace waymo
 
 int main(int argc, char* argv[]) {
-  InitGoogle(argv[0], &argc, &argv, false);
   absl::ParseCommandLine(argc, argv);
 
   const std::string latency_result_filename =

waymo_open_dataset/metrics/ops/metrics_ops.cc

@@ -103,17 +103,23 @@ REGISTER_OP("MotionMetrics")
 Computes motion metrics.
 
 - Notations:
-- B: batch size. Each batch should contain 1 scenario.
+- B: batch size. Each batch should contain exactly 1 scenario.
 - M: Number of joint prediction groups to predict per scenario.
+- K: top_K predictions per joint prediction.
 - N: number of agents in a joint prediction. 1 if mutual independence is
     assumed between agents.
-- K: top_K predictions per joint prediction.
 - A: number of agents in the groundtruth.
 - TP: number of steps to evaluate on. Matches len(config.step_measurement).
 - TG: number of steps in the groundtruth track. Matches
     config.track_history_samples + 1 + config.future_history_samples.
 - BR: number of breakdowns.
 
+For the marginal (mutually-independent) prediction, M is the number of agents to
+predict for, N is 1.
+
+For joint predictions of multiple agents, M is the number of joint predictions,
+and N is the number of agents per joint prediction.
+
 prediction_trajectory: [B, M, K, N, TP, 2]. Predicted trajectories.
   The inner-most dimensions are [x, y].
 prediction_score: [B, M, K]. Scores per joint prediction.

waymo_open_dataset/metrics/ops/utils.h

@@ -87,14 +87,20 @@ ParseObjectGroupedBySequenceFromTensors(
 //
 // - B: batch size. Each batch should contain 1 scenario.
 // - M: Number of joint prediction groups to predict per scenario.
+// - K: top_K predictions per joint prediction.
 // - N: number of agents in a joint prediction. 1 if mutual independence is
 //     assumed between agents.
-// - K: top_K predictions per joint prediction.
 // - A: number of agents in the groundtruth.
 // - TP: number of steps to evaluate on. Matches len(config.step_measurement).
 // - TG: number of steps in the groundtruth track. Matches
 //     config.track_history_samples + 1 + config.future_history_samples.
 //
+// For the marginal (mutually-independent) prediction, M is the number of agents
+// to predict for, N is 1.
+//
+// For joint predictions of multiple agents, M is the number of joint
+// predictions, and N is the number of agents per joint prediction.
+//
 // pred_trajectory: [B, M, K, N, TP, 2]. Predicted trajectories.
 //   The inner-most dimensions are [x, y].
 // pred_score: [B, M, K]. Scores per joint prediction.