remove specific error, wrap more stuff in retry

Author: maratoid

zerg/lib/zerg/version.rb

@@ -22,5 +22,5 @@
 #++
 
 module Zerg
-  VERSION = "0.0.18"
+  VERSION = "0.0.19"
 end

zerg/zerg.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |s|
   s.add_dependency "thor"
   s.add_dependency "highline"
   s.add_dependency "zergrush_vagrant", ">= 0.0.4"
-  s.add_dependency "zergrush_cf", ">= 0.0.11"
+  s.add_dependency "zergrush_cf", ">= 0.0.12"
 
   s.files        = `git ls-files`.split("\n")
   s.executables  = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact

zerg_plugins/zergrush_cf/lib/zergrush_cf/init.rb

@@ -111,19 +111,24 @@ def rush hive_location, task_name, task_hash, debug
 
 
         # get the event collection and initial info
-        outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
-        while outputs_info == nil do
-            sleep 1 
+        outputs_info = nil
+        with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20) {
             outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
+        }
+        while outputs_info == nil do
+            sleep 3
+            with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20) { 
+                outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
+            }
         end
 
         events = nil
-        with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::Errors::Error) {
+        with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20) {
             events = cf.describe_stack_events(stack_name).body['StackEvents']
         }
         while events == nil do
             sleep 3
-            with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::Errors::Error) {
+            with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20) {
                 events = cf.describe_stack_events(stack_name).body['StackEvents']
             }
         end
@@ -134,10 +139,10 @@ def rush hive_location, task_name, task_hash, debug
             logRabbitEvents(events.first(events.length - event_counter), rabbit_objects, eval_params(task_hash["vm"]["driver"]["driveroptions"][0]["rabbit"])) 
             event_counter = events.length
 
-            with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::Errors::Error) {
+            with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20) {
                 events = cf.describe_stack_events(stack_name).body['StackEvents']
+                outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
             }
-            outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
             if outputs_info.body["Stacks"][0]["StackStatus"] == "CREATE_COMPLETE"
                 logEvents(events.first(events.length - event_counter))
                 logRabbitEvents(events.first(events.length - event_counter), rabbit_objects, eval_params(task_hash["vm"]["driver"]["driveroptions"][0]["rabbit"])) 
@@ -203,23 +208,25 @@ def clean hive_location, task_name, task_hash, debug
          # get the event collection and initial info
         outputs_info = nil
         while outputs_info == nil do
-            sleep 1 
-            begin
-                outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
-            rescue Fog::AWS::CloudFormation::NotFound
-                rabbit_objects[:connection].close unless rabbit_objects == nil
-                return 0
-            end
+            sleep 3
+            outputs_info = cf.describe_stacks({ 'StackName' => stack_name }) 
+                begin
+                    outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
+                rescue Fog::AWS::CloudFormation::NotFound
+                    rabbit_objects[:connection].close unless rabbit_objects == nil
+                    return 0
+                end
+            }
         end
 
         events = nil
-        with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::Errors::Error) {
+        with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20) {
             events = cf.describe_stack_events(stack_name).body['StackEvents']
         }
         while events == nil do
             sleep 3
             begin
-                with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::Errors::Error) {
+                with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20) {
                     events = cf.describe_stack_events(stack_name).body['StackEvents']
                 }
             rescue Fog::AWS::CloudFormation::NotFound
@@ -234,17 +241,17 @@ def clean hive_location, task_name, task_hash, debug
             logRabbitEvents(events.first(events.length - event_counter), rabbit_objects, eval_params(task_hash["vm"]["driver"]["driveroptions"][0]["rabbit"])) 
 
             event_counter = events.length
-            begin
-                with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::Errors::Error) {
+            with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20) {
+                begin
                     events = cf.describe_stack_events(stack_name).body['StackEvents']
-                }
-                outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
-            rescue Fog::AWS::CloudFormation::NotFound
-                logEvents(events.first(events.length - event_counter))
-                logRabbitEvents(events.first(events.length - event_counter), rabbit_objects, eval_params(task_hash["vm"]["driver"]["driveroptions"][0]["rabbit"])) 
-                rabbit_objects[:connection].close unless rabbit_objects == nil
-                return 0
-            end
+                    outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
+                rescue Fog::AWS::CloudFormation::NotFound
+                    logEvents(events.first(events.length - event_counter))
+                    logRabbitEvents(events.first(events.length - event_counter), rabbit_objects, eval_params(task_hash["vm"]["driver"]["driveroptions"][0]["rabbit"])) 
+                    rabbit_objects[:connection].close unless rabbit_objects == nil
+                    return 0
+                end
+            }
         end
 
         # log remaining events for error case

zerg_plugins/zergrush_cf/lib/zergrush_cf/version.rb

@@ -22,5 +22,5 @@
 #++
 
 module ZergrushCF
-  VERSION = "0.0.11"
+  VERSION = "0.0.12"
 end

zerg_plugins/zergrush_cf/zergrush_cf.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency "bundler", ">= 1.0.0"
   s.add_development_dependency "rake"
-  s.add_development_dependency "zergrush", ">= 0.0.18"
+  s.add_development_dependency "zergrush", ">= 0.0.19"
 
   s.add_dependency "fog", ">=1.20.0"
   s.add_dependency "bunny", ">=1.2.1"