Merge branch 'feastmaster'

Author: Lavkesh Lahngir

.prow/scripts/test-end-to-end-batch.sh

@@ -255,6 +255,9 @@ if [[ ${TEST_EXIT_CODE} != 0 ]]; then
   echo "[DEBUG] Printing logs"
   ls -ltrh /var/log/feast*
   cat /var/log/feast-serving-warehouse.log /var/log/feast-core.log
+
+  echo "[DEBUG] Printing Python packages list"
+  pip list
 fi
 
 cd ${ORIGINAL_DIR}

.prow/scripts/test-end-to-end.sh

@@ -67,24 +67,24 @@ tail -n10 /var/log/kafka.log
 kafkacat -b localhost:9092 -L
 
 if [[ ${SKIP_BUILD_JARS} != "true" ]]; then
-  echo "
-  ============================================================
-  Building jars for Feast
-  ============================================================
-  "
+echo "
+============================================================
+Building jars for Feast
+============================================================
+"
 
-  .prow/scripts/download-maven-cache.sh \
-      --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
-      --output-dir /root/
+.prow/scripts/download-maven-cache.sh \
+    --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
+    --output-dir /root/
 
-  # Build jars for Feast
-  mvn --quiet --batch-mode --define skipTests=true clean package
+# Build jars for Feast
+mvn --quiet --batch-mode --define skipTests=true clean package
 
-  ls -lh core/target/*jar
-  ls -lh serving/target/*jar
-  else
-    echo "[DEBUG] Skipping building jars"
-  fi
+ls -lh core/target/*jar
+ls -lh serving/target/*jar
+else
+  echo "[DEBUG] Skipping building jars"
+fi
 
 echo "
 ============================================================
@@ -229,6 +229,9 @@ if [[ ${TEST_EXIT_CODE} != 0 ]]; then
   echo "[DEBUG] Printing logs"
   ls -ltrh /var/log/feast*
   cat /var/log/feast-serving-online.log /var/log/feast-core.log
+
+  echo "[DEBUG] Printing Python packages list"
+  pip list
 fi
 
 cd ${ORIGINAL_DIR}

ingestion/pom.xml

@@ -248,5 +248,12 @@
       <version>2.8.1</version>
     </dependency>
 
+    <!-- For calculation of percentiles in feature values -->
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-math3</artifactId>
+      <version>3.6.1</version>
+    </dependency>
+
   </dependencies>
 </project>

ingestion/src/main/java/feast/ingestion/options/ImportOptions.java

@@ -26,6 +26,7 @@
 
 /** Options passed to Beam to influence the job's execution environment */
 public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions, DirectOptions {
+
   @Required
   @Description(
       "JSON string representation of the FeatureSet that the import job will process, in BZip2 binary format."
@@ -83,4 +84,13 @@ public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions,
   int getStatsdPort();
 
   void setStatsdPort(int StatsdPort);
+
+  @Description(
+      "Fixed window size in seconds (default 30) to apply before aggregation of numerical value of features"
+          + "and writing the aggregated value to StatsD. Refer to feast.ingestion.transform.metrics.WriteFeatureValueMetricsDoFn"
+          + "for details on the metric names and types.")
+  @Default.Integer(30)
+  int getWindowSizeInSecForFeatureValueMetric();
+
+  void setWindowSizeInSecForFeatureValueMetric(int seconds);
 }