From e8d744ecd405b97e317e1b1716db9b35934186cb Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Thu, 12 Mar 2026 14:20:36 -0700 Subject: [PATCH 1/6] try fix race condntion in cleanup --- src/Config/ConfigFileWatcher.cs | 26 ++++++++++++++++++- src/Config/FileSystemRuntimeConfigLoader.cs | 19 +++++++++++++- .../Configuration/ConfigurationTests.cs | 18 ++++++++++++- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/Config/ConfigFileWatcher.cs b/src/Config/ConfigFileWatcher.cs index 288a95e3d7..e596bc7701 100644 --- a/src/Config/ConfigFileWatcher.cs +++ b/src/Config/ConfigFileWatcher.cs @@ -20,8 +20,10 @@ namespace Azure.DataApiBuilder.Config; /// /// /// -public class ConfigFileWatcher +public class ConfigFileWatcher : IDisposable { + private bool _disposed; + /// /// Watches a specific file for modifications and alerts /// this class when a change is detected. @@ -120,4 +122,26 @@ private void OnConfigFileChange(object sender, FileSystemEventArgs e) Console.WriteLine("Unable to hot reload configuration file due to " + ex.Message); } } + + /// + /// Disposes the file watcher and unsubscribes from events to release + /// file handles and prevent further file change notifications. + /// + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + + if (_fileWatcher is not null) + { + _fileWatcher.EnableRaisingEvents = false; + _fileWatcher.Changed -= OnConfigFileChange; + _fileWatcher.Dispose(); + _fileWatcher = null; + } + } } diff --git a/src/Config/FileSystemRuntimeConfigLoader.cs b/src/Config/FileSystemRuntimeConfigLoader.cs index 70b36d8294..cee42cf341 100644 --- a/src/Config/FileSystemRuntimeConfigLoader.cs +++ b/src/Config/FileSystemRuntimeConfigLoader.cs @@ -30,8 +30,9 @@ namespace Azure.DataApiBuilder.Config; /// which allows for mocking of the file system in tests, providing a way to run the test /// in isolation of other tests or the actual file system. /// -public class FileSystemRuntimeConfigLoader : RuntimeConfigLoader +public class FileSystemRuntimeConfigLoader : RuntimeConfigLoader, IDisposable { + private bool _disposed; /// /// This stores either the default config name e.g. dab-config.json /// or user provided config file which could be a relative file path, @@ -91,6 +92,22 @@ public FileSystemRuntimeConfigLoader( _isCliLoader = isCliLoader; } + /// + /// Disposes the config file watcher to release file handles and stop + /// monitoring the config file for changes. + /// + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _configFileWatcher?.Dispose(); + _configFileWatcher = null; + } + /// /// Get the directory name of the config file and /// return as a string. diff --git a/src/Service.Tests/Configuration/ConfigurationTests.cs b/src/Service.Tests/Configuration/ConfigurationTests.cs index 0ef9b67a4b..b503bc9f23 100644 --- a/src/Service.Tests/Configuration/ConfigurationTests.cs +++ b/src/Service.Tests/Configuration/ConfigurationTests.cs @@ -707,9 +707,25 @@ type Moon { [TestCleanup] public void CleanupAfterEachTest() { + // Retry file deletion with exponential back-off to handle cases where a + // file watcher or hot-reload process may still hold a handle on the file. if (File.Exists(CUSTOM_CONFIG_FILENAME)) { - File.Delete(CUSTOM_CONFIG_FILENAME); + int retryCount = 0; + const int maxRetries = 3; + while (true) + { + try + { + File.Delete(CUSTOM_CONFIG_FILENAME); + break; + } + catch (IOException) when (retryCount < maxRetries) + { + retryCount++; + Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retryCount))); + } + } } TestHelper.UnsetAllDABEnvironmentVariables(); From 70397af4a53afca53d3cac79b0644b16f4c64288 Mon Sep 17 00:00:00 2001 From: aaronburtle <93220300+aaronburtle@users.noreply.github.com> Date: Thu, 12 Mar 2026 19:32:01 -0700 Subject: [PATCH 2/6] Update src/Config/FileSystemRuntimeConfigLoader.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/Config/FileSystemRuntimeConfigLoader.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Config/FileSystemRuntimeConfigLoader.cs b/src/Config/FileSystemRuntimeConfigLoader.cs index cee42cf341..2f4dff28c4 100644 --- a/src/Config/FileSystemRuntimeConfigLoader.cs +++ b/src/Config/FileSystemRuntimeConfigLoader.cs @@ -104,8 +104,13 @@ public void Dispose() } _disposed = true; - _configFileWatcher?.Dispose(); - _configFileWatcher = null; + + if (_configFileWatcher is not null) + { + _configFileWatcher.NewFileContentsDetected -= OnNewFileContentsDetected; + _configFileWatcher.Dispose(); + _configFileWatcher = null; + } } /// From b4b097975350a2e9f564693d342cd1e9a930b348 Mon Sep 17 00:00:00 2001 From: aaronburtle <93220300+aaronburtle@users.noreply.github.com> Date: Thu, 12 Mar 2026 19:33:20 -0700 Subject: [PATCH 3/6] Update src/Config/ConfigFileWatcher.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/Config/ConfigFileWatcher.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Config/ConfigFileWatcher.cs b/src/Config/ConfigFileWatcher.cs index e596bc7701..e1afb39838 100644 --- a/src/Config/ConfigFileWatcher.cs +++ b/src/Config/ConfigFileWatcher.cs @@ -141,7 +141,6 @@ public void Dispose() _fileWatcher.EnableRaisingEvents = false; _fileWatcher.Changed -= OnConfigFileChange; _fileWatcher.Dispose(); - _fileWatcher = null; } } } From aa0a5a12a4ab11d9e55f68f27630ac64e4d1bb17 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Thu, 12 Mar 2026 23:24:07 -0700 Subject: [PATCH 4/6] add some logging for clarity --- src/Service.Tests/Configuration/ConfigurationTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Service.Tests/Configuration/ConfigurationTests.cs b/src/Service.Tests/Configuration/ConfigurationTests.cs index 094a2a556e..e328ddca84 100644 --- a/src/Service.Tests/Configuration/ConfigurationTests.cs +++ b/src/Service.Tests/Configuration/ConfigurationTests.cs @@ -723,9 +723,10 @@ public void CleanupAfterEachTest() File.Delete(CUSTOM_CONFIG_FILENAME); break; } - catch (IOException) when (retryCount < maxRetries) + catch (IOException ex) when (retryCount < maxRetries) { retryCount++; + Console.WriteLine($"CleanupAfterEachTest: Retry {retryCount}/{maxRetries} deleting {CUSTOM_CONFIG_FILENAME}. {ex.Message}"); Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retryCount))); } } From e353294450148ee09c0ec111fc0ec3dae877f349 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Tue, 17 Mar 2026 13:51:06 -0700 Subject: [PATCH 5/6] add using statement and register differently for disposal --- .../Configuration/ConfigurationTests.cs | 44 ++++++++++--------- src/Service/Startup.cs | 2 +- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/Service.Tests/Configuration/ConfigurationTests.cs b/src/Service.Tests/Configuration/ConfigurationTests.cs index 26a0313e59..f162a0328b 100644 --- a/src/Service.Tests/Configuration/ConfigurationTests.cs +++ b/src/Service.Tests/Configuration/ConfigurationTests.cs @@ -750,7 +750,7 @@ public async Task TestNoConfigReturnsServiceUnavailable( string[] args, bool isUpdateableRuntimeConfig) { - TestServer server; + TestServer server = null; try { @@ -775,6 +775,10 @@ public async Task TestNoConfigReturnsServiceUnavailable( $"Could not initialize the engine with the runtime config file: {DEFAULT_CONFIG_FILE_NAME}", e.Message); } + finally + { + server?.Dispose(); + } } /// @@ -1037,7 +1041,7 @@ public void TestConnectionStringIsCorrectlyUpdatedWithApplicationName( [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestConflictAlreadySetConfiguration(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint); @@ -1056,7 +1060,7 @@ public async Task TestConflictLocalConfiguration(string configurationEndpoint) { Environment.SetEnvironmentVariable (ASP_NET_CORE_ENVIRONMENT_VAR_NAME, COSMOS_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); ValidateCosmosDbSetup(server); @@ -1073,7 +1077,7 @@ public async Task TestConflictLocalConfiguration(string configurationEndpoint) [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestSettingConfigurations(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint); @@ -1088,7 +1092,7 @@ public async Task TestSettingConfigurations(string configurationEndpoint) [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestInvalidConfigurationAtRuntime(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint, "invalidString"); @@ -1103,7 +1107,7 @@ public async Task TestInvalidConfigurationAtRuntime(string configurationEndpoint [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestSettingFailureConfigurations(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint); @@ -1125,7 +1129,7 @@ public async Task TestSettingFailureConfigurations(string configurationEndpoint) [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestLongRunningConfigUpdatedHandlerConfigurations(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint); @@ -1164,7 +1168,7 @@ public async Task TestLongRunningConfigUpdatedHandlerConfigurations(string confi [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestSqlSettingPostStartupConfigurations(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); RuntimeConfig configuration = AuthorizationHelpers.InitRuntimeConfig( @@ -1243,7 +1247,7 @@ public async Task TestSqlSettingPostStartupConfigurations(string configurationEn [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestValidMultiSourceRunTimePostStartupConfigurations(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); RuntimeConfig config = AuthorizationHelpers.InitRuntimeConfig( @@ -1275,7 +1279,7 @@ public async Task TestValidMultiSourceRunTimePostStartupConfigurations(string co public void TestLoadingLocalCosmosSettings() { Environment.SetEnvironmentVariable(ASP_NET_CORE_ENVIRONMENT_VAR_NAME, COSMOS_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); ValidateCosmosDbSetup(server); } @@ -1285,7 +1289,7 @@ public void TestLoadingLocalCosmosSettings() [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestLoadingAccessTokenForCosmosClient(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint, null, true); @@ -1303,7 +1307,7 @@ public async Task TestLoadingAccessTokenForCosmosClient(string configurationEndp public void TestLoadingLocalMsSqlSettings() { Environment.SetEnvironmentVariable(ASP_NET_CORE_ENVIRONMENT_VAR_NAME, MSSQL_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); QueryEngineFactory queryEngineFactory = (QueryEngineFactory)server.Services.GetService(typeof(IQueryEngineFactory)); Assert.IsInstanceOfType(queryEngineFactory.GetQueryEngine(DatabaseType.MSSQL), typeof(SqlQueryEngine)); @@ -1323,7 +1327,7 @@ public void TestLoadingLocalMsSqlSettings() public void TestLoadingLocalPostgresSettings() { Environment.SetEnvironmentVariable(ASP_NET_CORE_ENVIRONMENT_VAR_NAME, POSTGRESQL_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); QueryEngineFactory queryEngineFactory = (QueryEngineFactory)server.Services.GetService(typeof(IQueryEngineFactory)); Assert.IsInstanceOfType(queryEngineFactory.GetQueryEngine(DatabaseType.PostgreSQL), typeof(SqlQueryEngine)); @@ -1343,7 +1347,7 @@ public void TestLoadingLocalPostgresSettings() public void TestLoadingLocalMySqlSettings() { Environment.SetEnvironmentVariable(ASP_NET_CORE_ENVIRONMENT_VAR_NAME, MYSQL_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); QueryEngineFactory queryEngineFactory = (QueryEngineFactory)server.Services.GetService(typeof(IQueryEngineFactory)); Assert.IsInstanceOfType(queryEngineFactory.GetQueryEngine(DatabaseType.MySQL), typeof(SqlQueryEngine)); @@ -1365,7 +1369,7 @@ public void TestLoadingLocalMySqlSettings() public async Task TestOverridingLocalSettingsFails(string configurationEndpoint) { Environment.SetEnvironmentVariable(ASP_NET_CORE_ENVIRONMENT_VAR_NAME, COSMOS_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); HttpClient client = server.CreateClient(); JsonContent config = GetJsonContentForCosmosConfigRequest(configurationEndpoint); @@ -1379,7 +1383,7 @@ public async Task TestOverridingLocalSettingsFails(string configurationEndpoint) [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestSettingConfigurationCreatesCorrectClasses(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient client = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint); @@ -1489,7 +1493,7 @@ public void TestCommandLineConfigurationProvider() $"{COSMOS_ENVIRONMENT}{CONFIG_EXTENSION}" }; - TestServer server = new(Program.CreateWebHostBuilder(args)); + using TestServer server = new(Program.CreateWebHostBuilder(args)); ValidateCosmosDbSetup(server); } @@ -1506,7 +1510,7 @@ public void TestRuntimeEnvironmentVariable() Environment.SetEnvironmentVariable( RUNTIME_ENVIRONMENT_VAR_NAME, COSMOS_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); ValidateCosmosDbSetup(server); } @@ -2245,7 +2249,7 @@ public void TestConnectionStringEnvVarHasHighestPrecedence() try { - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); _ = server.Services.GetService(typeof(CosmosClientProvider)) as CosmosClientProvider; Assert.Fail($"{RUNTIME_ENV_CONNECTION_STRING} is not given highest precedence"); } @@ -4321,7 +4325,7 @@ private static void ValidateLogLevelFilters(LogLevel logLevel, string loggingFil // Start a new server with the custom log level to ensure the // instantiation of the valid log level filters works as expected. - TestServer server = new(Program.CreateWebHostBuilder(args)); + using TestServer server = new(Program.CreateWebHostBuilder(args)); RuntimeConfigProvider runtimeConfigProvider = server.Services.GetService(); // RuntimeConfig with instantiated log level filters. diff --git a/src/Service/Startup.cs b/src/Service/Startup.cs index 12cbd18ab0..f084185929 100644 --- a/src/Service/Startup.cs +++ b/src/Service/Startup.cs @@ -119,7 +119,7 @@ public void ConfigureServices(IServiceCollection services) _configProvider = configProvider; services.AddSingleton(fileSystem); - services.AddSingleton(configLoader); + services.AddSingleton(sp => configLoader); services.AddSingleton(configProvider); bool runtimeConfigAvailable = configProvider.TryGetConfig(out RuntimeConfig? runtimeConfig); From 0f3db562f2fcbb21dc9870b5ad22a0c93694a28e Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Tue, 17 Mar 2026 16:11:40 -0700 Subject: [PATCH 6/6] hot-reload race condition fixed --- .../HotReload/ConfigurationHotReloadTests.cs | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs index 0f9ff6c1b8..18c19cbd7a 100644 --- a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs +++ b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs @@ -666,13 +666,33 @@ await WaitForConditionAsync( RuntimeConfig updatedRuntimeConfig = _configProvider.GetConfig(); MsSqlOptions actualSessionContext = updatedRuntimeConfig.DataSource.GetTypedOptions(); - JsonElement reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( - _testClient, - _configProvider, - GQL_QUERY_NAME, - GQL_QUERY); + + // Retry GraphQL request because metadata re-initialization happens asynchronously + // after the "Validated hot-reloaded configuration file" message. The metadata provider + // factory clears and re-initializes providers on the hot-reload thread, so requests + // arriving before that completes will fail with "Initialization of metadata incomplete." + JsonElement reloadGQLContents = default; + bool querySucceeded = false; + for (int attempt = 1; attempt <= 10; attempt++) + { + reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( + _testClient, + _configProvider, + GQL_QUERY_NAME, + GQL_QUERY); + + if (reloadGQLContents.ValueKind == JsonValueKind.Object && + reloadGQLContents.TryGetProperty("items", out _)) + { + querySucceeded = true; + break; + } + + await Task.Delay(1000); + } // Assert + Assert.IsTrue(querySucceeded, "GraphQL query did not return valid results after hot-reload. Metadata initialization may not have completed."); Assert.AreNotEqual(previousSessionContext, actualSessionContext); Assert.AreEqual(false, actualSessionContext.SetSessionContext); SqlTestHelper.PerformTestEqualJsonStrings(_bookDBOContents, reloadGQLContents.GetProperty("items").ToString());