Updating SyncTriggers to include host config (#9898)

mathewc · web-flow · commit 3df570cbe8a6 · 2024-03-19T19:50:11.000-07:00
diff --git a/src/WebJobs.Script.WebHost/Controllers/HostController.cs b/src/WebJobs.Script.WebHost/Controllers/HostController.cs
@@ -16,8 +16,10 @@
 using Microsoft.Azure.WebJobs.Host;
 using Microsoft.Azure.WebJobs.Host.Executors;
 using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.ExtensionBundle;
 using Microsoft.Azure.WebJobs.Script.Scale;
+using Microsoft.Azure.WebJobs.Script.WebHost.Extensions;
 using Microsoft.Azure.WebJobs.Script.WebHost.Filters;
 using Microsoft.Azure.WebJobs.Script.WebHost.Management;
 using Microsoft.Azure.WebJobs.Script.WebHost.Models;
@@ -44,6 +46,7 @@ public class HostController : Controller
         private readonly IScriptHostManager _scriptHostManager;
         private readonly IFunctionsSyncManager _functionsSyncManager;
         private readonly HostPerformanceManager _performanceManager;
+        private readonly IMetricsLogger _metricsLogger;
         private static readonly SemaphoreSlim _drainSemaphore = new SemaphoreSlim(1, 1);
         private static readonly SemaphoreSlim _resumeSemaphore = new SemaphoreSlim(1, 1);
 
@@ -52,14 +55,16 @@ public HostController(IOptions<ScriptApplicationHostOptions> applicationHostOpti
             IEnvironment environment,
             IScriptHostManager scriptHostManager,
             IFunctionsSyncManager functionsSyncManager,
-            HostPerformanceManager performanceManager)
+            HostPerformanceManager performanceManager,
+            IMetricsLogger metricsLogger)
         {
             _applicationHostOptions = applicationHostOptions;
             _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryHostController);
             _environment = environment;
             _scriptHostManager = scriptHostManager;
             _functionsSyncManager = functionsSyncManager;
             _performanceManager = performanceManager;
+            _metricsLogger = metricsLogger;
         }
 
         [HttpGet]
@@ -388,6 +393,22 @@ public IActionResult LaunchDebugger()
         [Authorize(Policy = PolicyNames.AdminAuthLevelOrInternal)]
         public async Task<IActionResult> SyncTriggers()
         {
+            _metricsLogger.LogEvent(MetricEventNames.SyncTriggersInvoked);
+
+            // TEMP: Collecting temporary metrics on the host state during SyncTrigger requests
+            if (!_scriptHostManager.HostIsInitialized())
+            {
+                _metricsLogger.LogEvent(MetricEventNames.SyncTriggersHostNotInitialized);
+            }
+
+            // TODO: We plan on making SyncTriggers RequiresRunningHost across the board,
+            // but for now only for Flex.
+            // https://github.com/Azure/azure-functions-host/issues/9904
+            if (_environment.IsFlexConsumptionSku())
+            {
+                await HttpContext.WaitForRunningHostAsync(_scriptHostManager, _applicationHostOptions.Value);
+            }
+
             var result = await _functionsSyncManager.TrySyncTriggersAsync();
 
             // Return a dummy body to make it valid in ARM template action evaluation
diff --git a/src/WebJobs.Script.WebHost/Extensions/HttpContextExtensions.cs b/src/WebJobs.Script.WebHost/Extensions/HttpContextExtensions.cs
@@ -2,14 +2,41 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System.IO;
+using System.Net;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc.Filters;
 using Microsoft.Azure.WebJobs.Script.Extensions;
 
 namespace Microsoft.Azure.WebJobs.Script.WebHost.Extensions
 {
     internal static class HttpContextExtensions
     {
+        public static async Task WaitForRunningHostAsync(this HttpContext httpContext, IScriptHostManager hostManager, ScriptApplicationHostOptions applicationHostOptions, int timeoutSeconds = ScriptConstants.HostTimeoutSeconds, int pollingIntervalMilliseconds = ScriptConstants.HostPollingIntervalMilliseconds, ActionExecutionDelegate next = null)
+        {
+            if (hostManager.State == ScriptHostState.Offline)
+            {
+                await httpContext.SetOfflineResponseAsync(applicationHostOptions.ScriptPath);
+            }
+            else
+            {
+                // If the host is not ready, we'll wait a bit for it to initialize.
+                // This might happen if http requests come in while the host is starting
+                // up for the first time, or if it is restarting.
+                bool hostReady = await hostManager.DelayUntilHostReady(timeoutSeconds, pollingIntervalMilliseconds);
+
+                if (!hostReady)
+                {
+                    throw new HttpException(HttpStatusCode.ServiceUnavailable, "Function host is not running.");
+                }
+
+                if (next != null)
+                {
+                    await next();
+                }
+            }
+        }
+
         public static async Task SetOfflineResponseAsync(this HttpContext httpContext, string scriptPath)
         {
             httpContext.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
diff --git a/src/WebJobs.Script.WebHost/Extensions/ScriptHostManagerExtensions.cs b/src/WebJobs.Script.WebHost/Extensions/ScriptHostManagerExtensions.cs
@@ -1,9 +1,6 @@
 ﻿// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
-using System;
-using System.Collections.Generic;
-using System.Text;
 using System.Threading.Tasks;
 
 namespace Microsoft.Azure.WebJobs.Script
@@ -12,22 +9,27 @@ public static class ScriptHostManagerExtensions
     {
         public static async Task<bool> DelayUntilHostReady(this IScriptHostManager hostManager, int timeoutSeconds = ScriptConstants.HostTimeoutSeconds, int pollingIntervalMilliseconds = ScriptConstants.HostPollingIntervalMilliseconds)
         {
-            if (CanInvoke(hostManager))
+            if (HostIsInitialized(hostManager))
             {
                 return true;
             }
             else
             {
                 await Utility.DelayAsync(timeoutSeconds, pollingIntervalMilliseconds, () =>
                 {
-                    return !CanInvoke(hostManager) && hostManager.State != ScriptHostState.Error;
+                    return !HostIsInitialized(hostManager) && hostManager.State != ScriptHostState.Error;
                 });
 
-                return CanInvoke(hostManager);
+                return HostIsInitialized(hostManager);
             }
         }
 
-        public static bool CanInvoke(this IScriptHostManager hostManager)
+        /// <summary>
+        /// Gets a value indicating whether the host in an initialized or running state.
+        /// </summary>
+        /// <param name="hostManager">The <see cref="IScriptHostManager"/> to check.</param>
+        /// <returns>True if the host is initialized or running, false otherwise.</returns>
+        public static bool HostIsInitialized(this IScriptHostManager hostManager)
         {
             return hostManager.State == ScriptHostState.Running || hostManager.State == ScriptHostState.Initialized;
         }
diff --git a/src/WebJobs.Script.WebHost/Filters/RequiresRunningHostAttribute.cs b/src/WebJobs.Script.WebHost/Filters/RequiresRunningHostAttribute.cs
@@ -2,7 +2,6 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
-using System.Net;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Mvc.Filters;
 using Microsoft.Azure.WebJobs.Script.WebHost.Extensions;
@@ -64,24 +63,7 @@ public RunningHostCheckAttribute(IScriptHostManager hostManager, IOptionsMonitor
 
             public override async Task OnActionExecutionAsync(ActionExecutingContext actionContext, ActionExecutionDelegate next)
             {
-                if (_hostManager.State == ScriptHostState.Offline)
-                {
-                    await actionContext.HttpContext.SetOfflineResponseAsync(_applicationHostOptions.CurrentValue.ScriptPath);
-                }
-                else
-                {
-                    // If the host is not ready, we'll wait a bit for it to initialize.
-                    // This might happen if http requests come in while the host is starting
-                    // up for the first time, or if it is restarting.
-                    bool hostReady = await _hostManager.DelayUntilHostReady(TimeoutSeconds, PollingIntervalMilliseconds);
-
-                    if (!hostReady)
-                    {
-                        throw new HttpException(HttpStatusCode.ServiceUnavailable, "Function host is not running.");
-                    }
-
-                    await next();
-                }
+                await actionContext.HttpContext.WaitForRunningHostAsync(_hostManager, _applicationHostOptions.CurrentValue, TimeoutSeconds, PollingIntervalMilliseconds, next);
             }
         }
     }
diff --git a/src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs b/src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs
@@ -14,6 +14,7 @@
 using Azure.Storage.Blobs;
 using Microsoft.Azure.WebJobs.Host.Executors;
 using Microsoft.Azure.WebJobs.Host.Storage;
+using Microsoft.Azure.WebJobs.Logging;
 using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Models;
@@ -63,10 +64,11 @@ public class FunctionsSyncManager : IFunctionsSyncManager, IDisposable
         private readonly SemaphoreSlim _syncSemaphore = new SemaphoreSlim(1, 1);
         private readonly IAzureBlobStorageProvider _azureBlobStorageProvider;
         private readonly IOptions<FunctionsHostingConfigOptions> _hostingConfigOptions;
+        private readonly IScriptHostManager _scriptHostManager;
 
         private BlobClient _hashBlobClient;
 
-        public FunctionsSyncManager(IHostIdProvider hostIdProvider, IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, ILogger<FunctionsSyncManager> logger, IHttpClientFactory httpClientFactory, ISecretManagerProvider secretManagerProvider, IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment, HostNameProvider hostNameProvider, IFunctionMetadataManager functionMetadataManager, IAzureBlobStorageProvider azureBlobStorageProvider, IOptions<FunctionsHostingConfigOptions> functionsHostingConfigOptions)
+        public FunctionsSyncManager(IHostIdProvider hostIdProvider, IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, ILogger<FunctionsSyncManager> logger, IHttpClientFactory httpClientFactory, ISecretManagerProvider secretManagerProvider, IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment, HostNameProvider hostNameProvider, IFunctionMetadataManager functionMetadataManager, IAzureBlobStorageProvider azureBlobStorageProvider, IOptions<FunctionsHostingConfigOptions> functionsHostingConfigOptions, IScriptHostManager scriptHostManager)
         {
             _applicationHostOptions = applicationHostOptions;
             _logger = logger;
@@ -79,6 +81,7 @@ public FunctionsSyncManager(IHostIdProvider hostIdProvider, IOptionsMonitor<Scri
             _functionMetadataManager = functionMetadataManager;
             _azureBlobStorageProvider = azureBlobStorageProvider;
             _hostingConfigOptions = functionsHostingConfigOptions;
+            _scriptHostManager = scriptHostManager;
         }
 
         internal bool ArmCacheEnabled
@@ -313,9 +316,22 @@ public async Task<SyncTriggersPayload> GetSyncTriggersPayload()
             var triggersArray = new JArray(triggers);
             int count = triggersArray.Count;
 
+            JObject hostConfig = null;
+            if (_environment.IsFlexConsumptionSku())
+            {
+                // TODO: Only adding host configuration for Flex. Once SyncTriggers is marked RequiresRunningHost,
+                // we'll enable across the board.
+                // https://github.com/Azure/azure-functions-host/issues/9904
+                // If an active host is running, add host configuration.
+                if (Utility.TryGetHostService<IHostOptionsProvider>(_scriptHostManager, out IHostOptionsProvider hostOptionsProvider))
+                {
+                    hostConfig = hostOptionsProvider.GetOptions();
+                }
+            }
+
             // Form the base minimal result
             string hostId = await _hostIdProvider.GetHostIdAsync(CancellationToken.None);
-            JObject result = GetMinimalPayload(hostId, triggersArray);
+            JObject result = GetMinimalPayload(hostId, triggersArray, hostConfig);
 
             if (!ArmCacheEnabled)
             {
@@ -396,7 +412,7 @@ public async Task<SyncTriggersPayload> GetSyncTriggersPayload()
                 // If we're over limit, revert to the minimal triggers format.
                 _logger.LogWarning($"SyncTriggers payload of length '{json.Length}' exceeds max length of '{ScriptConstants.MaxTriggersStringLength}'. Reverting to minimal format.");
 
-                var minimalResult = GetMinimalPayload(hostId, triggersArray);
+                var minimalResult = GetMinimalPayload(hostId, triggersArray, hostConfig);
                 json = JsonConvert.SerializeObject(minimalResult);
             }
 
@@ -407,14 +423,21 @@ public async Task<SyncTriggersPayload> GetSyncTriggersPayload()
             };
         }
 
-        private JObject GetMinimalPayload(string hostId, JArray triggersArray)
+        private JObject GetMinimalPayload(string hostId, JArray triggersArray, JObject hostConfig)
         {
             // When the HostId is sent, ScaleController will use it directly rather than compute it itself.
-            return new JObject
+            var result = new JObject
             {
                 { "triggers", triggersArray },
                 { "hostId", hostId }
             };
+
+            if (hostConfig != null)
+            {
+                result.Add("hostConfig", Sanitizer.Sanitize(hostConfig));
+            }
+
+            return result;
         }
 
         internal static async Task<JObject> GetHostJsonExtensionsAsync(IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, ILogger logger)
diff --git a/src/WebJobs.Script.WebHost/Middleware/HostAvailabilityCheckMiddleware.cs b/src/WebJobs.Script.WebHost/Middleware/HostAvailabilityCheckMiddleware.cs
@@ -29,7 +29,7 @@ public Task Invoke(HttpContext httpContext)
         {
             if (_scriptHostManager.State != ScriptHostState.Offline)
             {
-                if (!_scriptHostManager.CanInvoke())
+                if (!_scriptHostManager.HostIsInitialized())
                 {
                     // If we're not ready, take the slower/more expensive route and await being ready
                     return InvokeAwaitingHost(httpContext, _next, _logger, _scriptHostManager);
diff --git a/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs b/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs
@@ -54,8 +54,6 @@ public class HostJsonFileConfigurationProvider : ConfigurationProvider
                 "customHandler", "httpWorker", "extensions", "concurrency", ConfigurationSectionNames.SendCanceledInvocationsToWorker
             };
 
-            private static readonly string[] CredentialNameFragments = new[] { "password", "pwd", "key", "secret", "token", "sas" };
-
             private readonly HostJsonFileConfigurationSource _configurationSource;
             private readonly Stack<string> _path;
             private readonly ILogger _logger;
@@ -147,7 +145,10 @@ private JObject LoadHostConfigurationFile()
                     string hostFilePath = Path.Combine(options.ScriptPath, ScriptConstants.HostMetadataFileName);
                     JObject hostConfigObject = LoadHostConfig(hostFilePath);
                     hostConfigObject = InitializeHostConfig(hostFilePath, hostConfigObject);
-                    string sanitizedJson = SanitizeHostJson(hostConfigObject);
+
+                    Func<string, bool> selector = (name) => WellKnownHostJsonProperties.Contains(name);
+                    var sanitizedHostConfigObject = Sanitizer.Sanitize(hostConfigObject, selector);
+                    string sanitizedJson = sanitizedHostConfigObject.ToString();
 
                     _logger.HostConfigApplied();
 
@@ -283,73 +284,6 @@ private JObject TryAddBundleConfiguration(JObject content, string bundleId, stri
                 }
                 return content;
             }
-
-            internal static string SanitizeHostJson(JObject hostJsonObject)
-            {
-                static bool IsPotentialCredential(string name)
-                {
-                    foreach (string fragment in CredentialNameFragments)
-                    {
-                        if (name.Contains(fragment, StringComparison.OrdinalIgnoreCase))
-                        {
-                            return true;
-                        }
-                    }
-
-                    return false;
-                }
-
-                static JToken Sanitize(JToken token)
-                {
-                    if (token is JObject obj)
-                    {
-                        JObject sanitized = new JObject();
-                        foreach (var prop in obj)
-                        {
-                            if (IsPotentialCredential(prop.Key))
-                            {
-                                sanitized[prop.Key] = Sanitizer.SecretReplacement;
-                            }
-                            else
-                            {
-                                sanitized[prop.Key] = Sanitize(prop.Value);
-                            }
-                        }
-
-                        return sanitized;
-                    }
-
-                    if (token is JArray arr)
-                    {
-                        JArray sanitized = new JArray();
-                        foreach (var value in arr)
-                        {
-                            sanitized.Add(Sanitize(value));
-                        }
-
-                        return sanitized;
-                    }
-
-                    if (token.Type == JTokenType.String)
-                    {
-                        return Sanitizer.Sanitize(token.ToString());
-                    }
-
-                    return token;
-                }
-
-                JObject sanitizedObject = new JObject();
-                foreach (var propName in WellKnownHostJsonProperties)
-                {
-                    var propValue = hostJsonObject[propName];
-                    if (propValue != null)
-                    {
-                        sanitizedObject[propName] = Sanitize(propValue);
-                    }
-                }
-
-                return sanitizedObject.ToString();
-            }
         }
     }
 }
diff --git a/src/WebJobs.Script/Diagnostics/MetricEventNames.cs b/src/WebJobs.Script/Diagnostics/MetricEventNames.cs
@@ -20,6 +20,8 @@ public static class MetricEventNames
         public const string IdentifiableSecretLoaded = "host.secrets.identifiable";
         public const string HISStrictModeEnabled = "host.hismode.strict";
         public const string HISStrictModeWarn = "host.hismode.warn";
+        public const string SyncTriggersInvoked = "host.synctriggers.invoke";
+        public const string SyncTriggersHostNotInitialized = "host.synctriggers.hostnotinitialized";
 
         // Script host level events
         public const string ScriptHostManagerBuildScriptHost = "scripthostmanager.buildscripthost.latency";
diff --git a/src/WebJobs.Script/Sanitizer.cs b/src/WebJobs.Script/Sanitizer.cs
diff --git a/test/WebJobs.Script.Tests.Integration/Management/FunctionsSyncManagerTests.cs b/test/WebJobs.Script.Tests.Integration/Management/FunctionsSyncManagerTests.cs
diff --git a/test/WebJobs.Script.Tests/Controllers/Admin/HostControllerTests.cs b/test/WebJobs.Script.Tests/Controllers/Admin/HostControllerTests.cs
diff --git a/test/WebJobs.Script.Tests/Managment/WebFunctionsManagerTests.cs b/test/WebJobs.Script.Tests/Managment/WebFunctionsManagerTests.cs

Original file line number	Diff line number	Diff line change
`@@ -1,9 +1,6 @@`
`1`	`1`	`// Copyright (c) .NET Foundation. All rights reserved.`
`2`	`2`	`// Licensed under the MIT License. See License.txt in the project root for license information.`
`3`	`3`
`4`		`-using System;`
`5`		`-using System.Collections.Generic;`
`6`		`-using System.Text;`
`7`	`4`	`using System.Threading.Tasks;`
`8`	`5`
`9`	`6`	`namespace Microsoft.Azure.WebJobs.Script`
`@@ -12,22 +9,27 @@ public static class ScriptHostManagerExtensions`
`12`	`9`	`{`
`13`	`10`	`public static async Task<bool> DelayUntilHostReady(this IScriptHostManager hostManager, int timeoutSeconds = ScriptConstants.HostTimeoutSeconds, int pollingIntervalMilliseconds = ScriptConstants.HostPollingIntervalMilliseconds)`
`14`	`11`	`{`
`15`		`- if (CanInvoke(hostManager))`
	`12`	`+ if (HostIsInitialized(hostManager))`
`16`	`13`	`{`
`17`	`14`	`return true;`
`18`	`15`	`}`
`19`	`16`	`else`
`20`	`17`	`{`
`21`	`18`	`await Utility.DelayAsync(timeoutSeconds, pollingIntervalMilliseconds, () =>`
`22`	`19`	`{`
`23`		`- return !CanInvoke(hostManager) && hostManager.State != ScriptHostState.Error;`
	`20`	`+ return !HostIsInitialized(hostManager) && hostManager.State != ScriptHostState.Error;`
`24`	`21`	`});`
`25`	`22`
`26`		`- return CanInvoke(hostManager);`
	`23`	`+ return HostIsInitialized(hostManager);`
`27`	`24`	`}`
`28`	`25`	`}`
`29`	`26`
`30`		`- public static bool CanInvoke(this IScriptHostManager hostManager)`
	`27`	`+ /// <summary>`
	`28`	`+ /// Gets a value indicating whether the host in an initialized or running state.`
	`29`	`+ /// </summary>`
	`30`	`+ /// <param name="hostManager">The <see cref="IScriptHostManager"/> to check.</param>`
	`31`	`+ /// <returns>True if the host is initialized or running, false otherwise.</returns>`
	`32`	`+ public static bool HostIsInitialized(this IScriptHostManager hostManager)`
`31`	`33`	`{`
`32`	`34`	`return hostManager.State == ScriptHostState.Running \|\| hostManager.State == ScriptHostState.Initialized;`
`33`	`35`	`}`
Original file line number	Diff line number	Diff line change
`@@ -29,7 +29,7 @@ public Task Invoke(HttpContext httpContext)`
`29`	`29`	`{`
`30`	`30`	`if (_scriptHostManager.State != ScriptHostState.Offline)`
`31`	`31`	`{`
`32`		`- if (!_scriptHostManager.CanInvoke())`
	`32`	`+ if (!_scriptHostManager.HostIsInitialized())`
`33`	`33`	`{`
`34`	`34`	`// If we're not ready, take the slower/more expensive route and await being ready`
`35`	`35`	`return InvokeAwaitingHost(httpContext, _next, _logger, _scriptHostManager);`