- Change the RockQueue's Drain() queue to write unhandled exceptions to the Rock Log instead of polluting the ExceptionLog and retry register controllers transaction 5 times before giving up.

Author: nairdo

Rock.Rest/Swagger/SwaggerV2CacheBuilder.cs

@@ -22,9 +22,13 @@
 using System.Web.Http;
 using System.Web.Http.Routing;
 
+using Microsoft.Extensions.Logging;
+
+using Rock.Logging;
 using Rock.Model;
 using Rock.Utility;
 
+[assembly: Rock.Logging.RockLoggingCategory( "Rock.Rest.Swagger" )]
 namespace Rock.Rest.Swagger
 {
     /// <summary>
@@ -76,7 +80,8 @@ public void OnStartup()
                 }
                 catch ( Exception ex )
                 {
-                    ExceptionLogService.LogException( new Exception( "Failed to initialize the API documentation during startup.", ex ) );
+                    var logger = RockLogger.LoggerFactory.CreateLogger( "Rock.Rest.Swagger" );
+                    logger.LogWarning( ex, "Failed to initialize the API documentation during startup." );
                 }
             } );
         }

Rock/Model/CMS/RestController/RestControllerService.WebForms.cs

@@ -19,6 +19,7 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Reflection;
+using System.Threading;
 using System.Web.Http;
 using System.Web.Http.Controllers;
 
@@ -59,29 +60,71 @@ public override string ToString()
         }
 
         /// <summary>
-        /// Registers the controllers.
+        /// Registers the controllers; retrying up to 5 times if an InvalidOperationException
+        /// (Collection was modified; enumeration operation may not execute) occurs.
         /// </summary>
         public static void RegisterControllers()
         {
             /*
-             * 05/13/2022 MDP/DMV
-             * 
-             * In addition to the 12/19/2019 BJW note, we also added a RockGuid attribute to 
-             * controllers and methods (except for inherited methods). This will prevent
-             * loosing security on methods that have changed their signature. 
-             * 
-             * 
-             * 12/19/2019 BJW
-             *
-             * There was an issue with the SecuredAttribute not calculating API ID the same as was being calculated here.
-             * This caused the secured attribute to sometimes not find the RestAction record and thus not find the
-             * appropriate permissions (Auth table). The new method "GetApiId" is used in both places as a standardized
-             * API ID generator to ensure that this does not occur. The following code has also been modified to gracefully
-             * update any old style API IDs and update them to the new format without losing any foreign key associations, such
-             * as permissions.
-             *
-             * See task for detailed background: https://app.asana.com/0/474497188512037/1150703513867003/f
-             */
+                 6/2/2025 - NA
+
+                 IApiExplorer.ApiDescriptions returns a Collection<ApiDescription> that can be lazily populated
+                 by ApiExplorer at access time. If other threads are also accessing or modifying routes/controllers
+                 during this time (e.g., app initialization, concurrent requests), we can hit this concurrency
+                 exception even when just reading the .Count property.
+
+                 Reason: Try 5 times before giving up.
+            */
+
+            int maxAttempts = 5;
+            int baseDelayMs = 3000; // 3 seconds base
+
+            for ( int attempt = 1; attempt <= maxAttempts ; attempt++ )
+            {
+                try
+                {
+                    RegisterControllersInternal();
+                    break; // success, exit loop
+                }
+                catch ( InvalidOperationException )
+                {
+                    if ( attempt == maxAttempts )
+                    {
+                        throw; // fail after final attempt
+                    }
+
+                    // small delay
+                    Thread.Sleep( baseDelayMs * attempt );
+                }
+            }
+        }
+
+        /// <summary>
+        /// Registers the controllers.
+        /// </summary>
+        private static void RegisterControllersInternal()
+        {
+            /*
+                 5/13/2022 - MDP/DMV
+
+                 In addition to the 12/19/2019 BJW note, we also added a RockGuid attribute to 
+                 controllers and methods (except for inherited methods). This will prevent
+                 loosing security on methods that have changed their signature. 
+
+
+                 12/19/2019 - BJW
+
+                 There was an issue with the SecuredAttribute not calculating API ID the same as was being calculated here.
+                 This caused the secured attribute to sometimes not find the RestAction record and thus not find the
+                 appropriate permissions (Auth table). The new method "GetApiId" is used in both places as a standardized
+                 API ID generator to ensure that this does not occur. The following code has also been modified to gracefully
+                 update any old style API IDs and update them to the new format without losing any foreign key associations, such
+                 as permissions.
+
+                 See task for detailed background: https://app.asana.com/0/474497188512037/1150703513867003/f
+
+                 Reason: Preserve security settings when method signatures change and ensure consistent API ID generation.
+            */
 
             // Controller Class Name => New Format Id => Old Format Id
             var controllerApiIdMap = new Dictionary<string, Dictionary<string, string>>();

Rock/Properties/AssemblyInfo.cs

@@ -54,6 +54,8 @@
 [assembly: Rock.Logging.RockLoggingCategory( "Rock.Financial" )]
 [assembly: Rock.Logging.RockLoggingCategory( "Rock.Lava" )]
 [assembly: Rock.Logging.RockLoggingCategory( "Rock.Model" )]
+[assembly: Rock.Logging.RockLoggingCategory( "Rock.Transactions" )]
+[assembly: Rock.Logging.RockLoggingCategory( "Rock.Transactions.RockQueue" )]
 [assembly: Rock.Logging.RockLoggingCategory( "Rock.Web" )]
 [assembly: Rock.Logging.RockLoggingCategory( "RockWeb.Blocks" )]
 [assembly: Rock.Logging.RockLoggingCategory( "RockWeb.App_Code.TwilioDefaultResponseAsync" )]

Rock/Transactions/RegisterControllersTransaction.cs

@@ -20,14 +20,16 @@
 namespace Rock.Transactions
 {
     /*
-    05-02-2022 MDP
-    We changed this back to use Rock.Transactions.RegisterControllersTransaction instead.
-    This is because RestControllerService.RegisterControllers has a dependancy on having
-    all the Routes configured. The Bus message approach (Rock.Tasks.RegisterRestControllers)
-    would sometimes result in RegisterControllers happening before Routes would be configured,
-    which would cause RestController and RestAction records to get deleted.
+         5/2/2022 - MDP
 
-     */
+         We changed this back to use Rock.Transactions.RegisterControllersTransaction instead.
+         This is because RestControllerService.RegisterControllers has a dependency on having
+         all the Routes configured. The Bus message approach (Rock.Tasks.RegisterRestControllers)
+         would sometimes result in RegisterControllers happening before Routes would be configured,
+         which would cause RestController and RestAction records to get deleted.
+
+         Reason: RestControllerService.RegisterControllers requires Routes to be configured first.
+    */
 
     /// <summary>
     /// Registers controllers

Rock/Transactions/RockQueue.cs

@@ -239,7 +239,7 @@ public static void Drain( Action<Exception> errorHandler )
                 }
                 catch ( Exception ex )
                 {
-                    errorHandler( new Exception( $"Exception in RockQueue.Drain(): {transaction.GetType().Name}", ex ) );
+                    errorHandler( new Exception( $"Unhandled exception in RockQueue.Drain(): {transaction.GetType().Name} {transaction.ToJson()}", ex ) );
                 }
             }
         }

RockWeb/App_Code/Global.asax.cs

@@ -27,13 +27,14 @@
 using System.Web.Http;
 using System.Web.Optimization;
 using System.Web.Routing;
-
+using Microsoft.Extensions.Logging;
 using Rock;
 using Rock.Blocks;
 using Rock.Communication;
 using Rock.Configuration;
 using Rock.Data;
 using Rock.Enums.Cms;
+using Rock.Logging;
 using Rock.Model;
 using Rock.Observability;
 using Rock.Security;
@@ -43,6 +44,7 @@
 using Rock.Web.UI;
 using Rock.WebStartup;
 
+[assembly: Rock.Logging.RockLoggingCategory( "RockWeb.Global" )]
 namespace RockWeb
 {
     /// <summary>
@@ -235,7 +237,7 @@ private void ApplicationStartupStage2()
 
                 SetError66();
                 var startupException = new RockStartupException( "Error occurred during application startup", ex );
-                LogError( startupException, null );
+                LogException( startupException, null );
                 throw startupException;
             }
 
@@ -358,7 +360,7 @@ private static void StartEnsureChromeEngineThread()
                 }
                 catch ( Exception ex )
                 {
-                    LogError( ex, null );
+                    LogException( ex, null );
                 }
             } ).Start();
         }
@@ -1194,17 +1196,42 @@ public static void DrainTransactionQueue()
             if ( !Global.QueueInUse )
             {
                 Global.QueueInUse = true;
-                RockQueue.Drain( ( ex ) => LogError( ex, null ) );
+                RockQueue.Drain( ( ex ) => WriteErrorToRockLog( ex, "Rock.Transactions", null ) );
                 Global.QueueInUse = false;
             }
         }
 
+        /// <summary>
+        /// A handler for Rock Logging exception messages via Rock Logger.
+        /// If a message is provided, it will log that message otherwise
+        /// it will log the exception message.
+        /// </summary>
+        /// <param name="ex"></param>
+        private static void WriteErrorToRockLog( Exception ex, string loggerCategory, string message )
+        {
+            if ( string.IsNullOrWhiteSpace( loggerCategory ) )
+            {
+                loggerCategory = "RockWeb.Global";
+            }
+
+            var logger = RockLogger.LoggerFactory.CreateLogger( loggerCategory );
+
+            if ( !string.IsNullOrWhiteSpace( message ) )
+            {
+                logger.LogError( ex, message );
+            }
+            else
+            {
+                logger.LogError( ex.Message );
+            }
+        }
+
         /// <summary>
         /// Logs the error to database
         /// </summary>
         /// <param name="ex">The ex.</param>
         /// <param name="context">The context.</param>
-        private static void LogError( Exception ex, HttpContext context )
+        private static void LogException( Exception ex, HttpContext context )
         {
             int? pageId;
             int? siteId;
@@ -1278,7 +1305,7 @@ public static void CacheItemRemoved( string k, object v, CacheItemRemovedReason
             }
             catch ( Exception ex )
             {
-                LogError( ex, null );
+                LogException( ex, null );
             }
         }
 
@@ -1308,7 +1335,7 @@ Rock.SystemKey.SystemSetting.ENABLE_KEEP_ALIVE can be enabled in Rock's System S
                 }
                 catch ( Exception ex )
                 {
-                    LogError( new Exception( "Error doing KeepAlive request.", ex ), null );
+                    LogException( new Exception( "Error doing KeepAlive request.", ex ), null );
                 }
             }
         }