Send additional 25th and 99th percentile for feature value metrics (feast-dev#511)

Author: feast-ci-bot

ingestion/src/main/java/feast/ingestion/options/ImportOptions.java

@@ -65,8 +65,7 @@ public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions,
    */
   void setDeadLetterTableSpec(String deadLetterTableSpec);
 
-  // TODO: expound
-  @Description("MetricsAccumulator exporter type to instantiate.")
+  @Description("MetricsAccumulator exporter type to instantiate. Supported type: statsd")
   @Default.String("none")
   String getMetricsExporterType();
 
@@ -86,10 +85,11 @@ public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions,
   void setStatsdPort(int StatsdPort);
 
   @Description(
-      "Fixed window size in seconds (default 30) to apply before aggregation of numerical value of features"
-          + "and writing the aggregated value to StatsD. Refer to feast.ingestion.transform.metrics.WriteFeatureValueMetricsDoFn"
-          + "for details on the metric names and types.")
-  @Default.Integer(30)
+      "Fixed window size in seconds (default 60) to apply before aggregating the numerical value of "
+          + "features and exporting the aggregated values as metrics. Refer to "
+          + "feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java"
+          + "for the metric nameas and types used.")
+  @Default.Integer(60)
   int getWindowSizeInSecForFeatureValueMetric();
 
   void setWindowSizeInSecForFeatureValueMetric(int seconds);

ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java

@@ -90,9 +90,11 @@ abstract static class Builder {
   public static String GAUGE_NAME_FEATURE_VALUE_MIN = "feature_value_min";
   public static String GAUGE_NAME_FEATURE_VALUE_MAX = "feature_value_max";
   public static String GAUGE_NAME_FEATURE_VALUE_MEAN = "feature_value_mean";
+  public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_25 = "feature_value_percentile_25";
   public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_50 = "feature_value_percentile_50";
   public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_90 = "feature_value_percentile_90";
   public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_95 = "feature_value_percentile_95";
+  public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_99 = "feature_value_percentile_99";
 
   @Setup
   public void setup() {
@@ -205,6 +207,12 @@ public void processElement(
         values[i] = valueList.get(i);
       }
 
+      double p25 = new Percentile().evaluate(values, 25);
+      if (p25 < 0) {
+        statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_25, 0, tags);
+      }
+      statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_25, p25, tags);
+
       double p50 = new Percentile().evaluate(values, 50);
       if (p50 < 0) {
         statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_50, 0, tags);
@@ -222,6 +230,12 @@ public void processElement(
         statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_95, 0, tags);
       }
       statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_95, p95, tags);
+
+      double p99 = new Percentile().evaluate(values, 99);
+      if (p99 < 0) {
+        statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_99, 0, tags);
+      }
+      statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_99, p99, tags);
     }
   }