Add timeout to command, fix command path during install

Author: Etienne Tremel

cmd/monitor_elasticsearch.go

@@ -25,7 +25,7 @@ query string.
 
 Usage with Lucene query:
 
-	$ helm monitor elasticsearch frontend 'status:500 AND kubernetes.labels.app:app'
+	$ helm monitor elasticsearch frontend 'status:500 AND kubernetes.labels.app:app AND version:2.0.0'
 
 Usage with query DSL file:
 
@@ -85,7 +85,7 @@ func newMonitorElasticSearchCmd(client helm.Interface, out io.Writer) *cobra.Com
 	f.BoolVar(&elasticSearchMonitor.dryRun, "dry-run", false, "simulate a monitoring")
 	f.Int64Var(&elasticSearchMonitor.timeout, "timeout", 300, "time in seconds to wait before assuming a monitoring action is successfull")
 	f.Int64Var(&elasticSearchMonitor.rollbackTimeout, "rollback-timeout", 300, "time in seconds to wait for any individual Kubernetes operation during the rollback (like Jobs for hooks)")
-	f.Int64Var(&elasticSearchMonitor.interval, "interval", 10, "time in seconds between each PromQL query")
+	f.Int64Var(&elasticSearchMonitor.interval, "interval", 10, "time in seconds between each query")
 	f.BoolVar(&elasticSearchMonitor.wait, "wait", false, "if set, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment are in a ready state before marking a rollback as successful. It will wait for as long as --rollback-timeout")
 	f.BoolVar(&elasticSearchMonitor.force, "force", false, "force resource update through delete/recreate if needed")
 	f.BoolVar(&elasticSearchMonitor.disableHooks, "no-hooks", false, "prevent hooks from running during rollback")
@@ -129,6 +129,12 @@ func (m *monitorElasticSearchCmd) run() error {
 
 	ticker := time.NewTicker(time.Second * time.Duration(m.interval))
 
+	go func() {
+		time.Sleep(time.Second * time.Duration(m.timeout))
+		fmt.Fprintf(m.out, "No results after %d second(s)\n", m.timeout)
+		close(quit)
+	}()
+
 	for {
 		select {
 		case <-ticker.C:
@@ -154,7 +160,10 @@ func (m *monitorElasticSearchCmd) run() error {
 			}
 
 			if response.Count > 0 {
-				fmt.Fprintf(m.out, "Rolling back\n")
+				ticker.Stop()
+
+				fmt.Fprintf(m.out, "Failure detected, rolling back...\n")
+
 				_, err := m.client.RollbackRelease(
 					m.name,
 					helm.RollbackDryRun(m.dryRun),
@@ -170,12 +179,11 @@ func (m *monitorElasticSearchCmd) run() error {
 				}
 
 				fmt.Fprintf(m.out, "Successfully rolled back to previous revision!\n")
-				break
+				return nil
 			}
 
 		case <-quit:
 			ticker.Stop()
-			fmt.Fprintf(m.out, "No results after %d...\n", m.timeout)
 			return nil
 		}
 	}

cmd/monitor_prometheus.go

@@ -80,7 +80,7 @@ func newMonitorPrometheusCmd(client helm.Interface, out io.Writer) *cobra.Comman
 	f.BoolVar(&prometheusMonitor.dryRun, "dry-run", false, "simulate a monitoring")
 	f.Int64Var(&prometheusMonitor.timeout, "timeout", 300, "time in seconds to wait before assuming a monitoring action is successfull")
 	f.Int64Var(&prometheusMonitor.rollbackTimeout, "rollback-timeout", 300, "time in seconds to wait for any individual Kubernetes operation during the rollback (like Jobs for hooks)")
-	f.Int64Var(&prometheusMonitor.interval, "interval", 10, "time in seconds between each PromQL query")
+	f.Int64Var(&prometheusMonitor.interval, "interval", 10, "time in seconds between each query")
 	f.BoolVar(&prometheusMonitor.wait, "wait", false, "if set, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment are in a ready state before marking a rollback as successful. It will wait for as long as --rollback-timeout")
 	f.BoolVar(&prometheusMonitor.force, "force", false, "force resource update through delete/recreate if needed")
 	f.BoolVar(&prometheusMonitor.disableHooks, "no-hooks", false, "prevent hooks from running during rollback")
@@ -113,6 +113,12 @@ func (m *monitorPrometheusCmd) run() error {
 
 	ticker := time.NewTicker(time.Second * time.Duration(m.interval))
 
+	go func() {
+		time.Sleep(time.Second * time.Duration(m.timeout))
+		fmt.Fprintf(m.out, "No results after %d second(s)\n", m.timeout)
+		close(quit)
+	}()
+
 	for {
 		select {
 		case <-ticker.C:
@@ -136,7 +142,10 @@ func (m *monitorPrometheusCmd) run() error {
 			}
 
 			if len(response.Data.Result) > 0 {
-				fmt.Fprintf(m.out, "Rolling back\n")
+				ticker.Stop()
+
+				fmt.Fprintf(m.out, "Failure detected, rolling back...\n")
+
 				_, err := m.client.RollbackRelease(
 					m.name,
 					helm.RollbackDryRun(m.dryRun),
@@ -152,12 +161,11 @@ func (m *monitorPrometheusCmd) run() error {
 				}
 
 				fmt.Fprintf(m.out, "Successfully rolled back to previous revision!\n")
-				break
+				return nil
 			}
 
 		case <-quit:
 			ticker.Stop()
-			fmt.Fprintf(m.out, "No results after %d...\n", m.timeout)
 			return nil
 		}
 	}

install.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-current_version=$(sed -n -e 's/version:[ "]*\([^"]*\).*/\1/p' plugin.yaml)
+current_version=$(sed -n -e 's/version:[ "]*\([^"]*\).*/\1/p' $(dirname $0)/plugin.yaml)
 HELM_MONITOR_VERSION=${HELM_MONITOR_VERSION:-$current_version}
 
 dir=${HELM_PLUGIN_DIR:-"$(helm home)/plugins/helm-monitor"}

plugin.yaml

@@ -1,9 +1,10 @@
 ---
 name: "monitor"
 version: "0.1.0"
-usage: "monitor and rollback based on metrics"
+usage: "monitor and rollback in case of failure based on metrics or logs"
 description: |-
-  Monitor a release, rollback to a previous version based on a query
+  Query at a given interval a Prometheus or ElasticSearch instance, a rollback
+  of the release is initiated if the number of item from the result is positive.
 ignoreFlags: false
 useTunnel: true
 command: "$HELM_PLUGIN_DIR/helm-monitor"