diff --git a/src/Config/ConfigFileWatcher.cs b/src/Config/ConfigFileWatcher.cs index 288a95e3d7..e1afb39838 100644 --- a/src/Config/ConfigFileWatcher.cs +++ b/src/Config/ConfigFileWatcher.cs @@ -20,8 +20,10 @@ namespace Azure.DataApiBuilder.Config; /// /// /// -public class ConfigFileWatcher +public class ConfigFileWatcher : IDisposable { + private bool _disposed; + /// /// Watches a specific file for modifications and alerts /// this class when a change is detected. @@ -120,4 +122,25 @@ private void OnConfigFileChange(object sender, FileSystemEventArgs e) Console.WriteLine("Unable to hot reload configuration file due to " + ex.Message); } } + + /// + /// Disposes the file watcher and unsubscribes from events to release + /// file handles and prevent further file change notifications. + /// + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + + if (_fileWatcher is not null) + { + _fileWatcher.EnableRaisingEvents = false; + _fileWatcher.Changed -= OnConfigFileChange; + _fileWatcher.Dispose(); + } + } } diff --git a/src/Config/FileSystemRuntimeConfigLoader.cs b/src/Config/FileSystemRuntimeConfigLoader.cs index ecefd6a9c2..7b888a82bf 100644 --- a/src/Config/FileSystemRuntimeConfigLoader.cs +++ b/src/Config/FileSystemRuntimeConfigLoader.cs @@ -30,8 +30,9 @@ namespace Azure.DataApiBuilder.Config; /// which allows for mocking of the file system in tests, providing a way to run the test /// in isolation of other tests or the actual file system. /// -public class FileSystemRuntimeConfigLoader : RuntimeConfigLoader +public class FileSystemRuntimeConfigLoader : RuntimeConfigLoader, IDisposable { + private bool _disposed; /// /// This stores either the default config name e.g. dab-config.json /// or user provided config file which could be a relative file path, @@ -102,6 +103,27 @@ public FileSystemRuntimeConfigLoader( _logBuffer = logBuffer; } + /// + /// Disposes the config file watcher to release file handles and stop + /// monitoring the config file for changes. + /// + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + + if (_configFileWatcher is not null) + { + _configFileWatcher.NewFileContentsDetected -= OnNewFileContentsDetected; + _configFileWatcher.Dispose(); + _configFileWatcher = null; + } + } + /// /// Get the directory name of the config file and /// return as a string. diff --git a/src/Service.Tests/Configuration/ConfigurationTests.cs b/src/Service.Tests/Configuration/ConfigurationTests.cs index 53a97ae722..6dac0266bd 100644 --- a/src/Service.Tests/Configuration/ConfigurationTests.cs +++ b/src/Service.Tests/Configuration/ConfigurationTests.cs @@ -710,9 +710,26 @@ type Moon { [TestCleanup] public void CleanupAfterEachTest() { + // Retry file deletion with exponential back-off to handle cases where a + // file watcher or hot-reload process may still hold a handle on the file. if (File.Exists(CUSTOM_CONFIG_FILENAME)) { - File.Delete(CUSTOM_CONFIG_FILENAME); + int retryCount = 0; + const int maxRetries = 3; + while (true) + { + try + { + File.Delete(CUSTOM_CONFIG_FILENAME); + break; + } + catch (IOException ex) when (retryCount < maxRetries) + { + retryCount++; + Console.WriteLine($"CleanupAfterEachTest: Retry {retryCount}/{maxRetries} deleting {CUSTOM_CONFIG_FILENAME}. {ex.Message}"); + Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retryCount))); + } + } } TestHelper.UnsetAllDABEnvironmentVariables(); @@ -733,7 +750,7 @@ public async Task TestNoConfigReturnsServiceUnavailable( string[] args, bool isUpdateableRuntimeConfig) { - TestServer server; + TestServer server = null; try { @@ -758,6 +775,10 @@ public async Task TestNoConfigReturnsServiceUnavailable( $"Could not initialize the engine with the runtime config file: {DEFAULT_CONFIG_FILE_NAME}", e.Message); } + finally + { + server?.Dispose(); + } } /// @@ -1020,7 +1041,7 @@ public void TestConnectionStringIsCorrectlyUpdatedWithApplicationName( [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestConflictAlreadySetConfiguration(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint); @@ -1039,7 +1060,7 @@ public async Task TestConflictLocalConfiguration(string configurationEndpoint) { Environment.SetEnvironmentVariable (ASP_NET_CORE_ENVIRONMENT_VAR_NAME, COSMOS_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); ValidateCosmosDbSetup(server); @@ -1056,7 +1077,7 @@ public async Task TestConflictLocalConfiguration(string configurationEndpoint) [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestSettingConfigurations(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint); @@ -1071,7 +1092,7 @@ public async Task TestSettingConfigurations(string configurationEndpoint) [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestInvalidConfigurationAtRuntime(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint, "invalidString"); @@ -1086,7 +1107,7 @@ public async Task TestInvalidConfigurationAtRuntime(string configurationEndpoint [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestSettingFailureConfigurations(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint); @@ -1108,7 +1129,7 @@ public async Task TestSettingFailureConfigurations(string configurationEndpoint) [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestLongRunningConfigUpdatedHandlerConfigurations(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint); @@ -1147,7 +1168,7 @@ public async Task TestLongRunningConfigUpdatedHandlerConfigurations(string confi [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestSqlSettingPostStartupConfigurations(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); RuntimeConfig configuration = AuthorizationHelpers.InitRuntimeConfig( @@ -1226,7 +1247,7 @@ public async Task TestSqlSettingPostStartupConfigurations(string configurationEn [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestValidMultiSourceRunTimePostStartupConfigurations(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); RuntimeConfig config = AuthorizationHelpers.InitRuntimeConfig( @@ -1258,7 +1279,7 @@ public async Task TestValidMultiSourceRunTimePostStartupConfigurations(string co public void TestLoadingLocalCosmosSettings() { Environment.SetEnvironmentVariable(ASP_NET_CORE_ENVIRONMENT_VAR_NAME, COSMOS_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); ValidateCosmosDbSetup(server); } @@ -1268,7 +1289,7 @@ public void TestLoadingLocalCosmosSettings() [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestLoadingAccessTokenForCosmosClient(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient httpClient = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint, null, true); @@ -1286,7 +1307,7 @@ public async Task TestLoadingAccessTokenForCosmosClient(string configurationEndp public void TestLoadingLocalMsSqlSettings() { Environment.SetEnvironmentVariable(ASP_NET_CORE_ENVIRONMENT_VAR_NAME, MSSQL_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); QueryEngineFactory queryEngineFactory = (QueryEngineFactory)server.Services.GetService(typeof(IQueryEngineFactory)); Assert.IsInstanceOfType(queryEngineFactory.GetQueryEngine(DatabaseType.MSSQL), typeof(SqlQueryEngine)); @@ -1306,7 +1327,7 @@ public void TestLoadingLocalMsSqlSettings() public void TestLoadingLocalPostgresSettings() { Environment.SetEnvironmentVariable(ASP_NET_CORE_ENVIRONMENT_VAR_NAME, POSTGRESQL_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); QueryEngineFactory queryEngineFactory = (QueryEngineFactory)server.Services.GetService(typeof(IQueryEngineFactory)); Assert.IsInstanceOfType(queryEngineFactory.GetQueryEngine(DatabaseType.PostgreSQL), typeof(SqlQueryEngine)); @@ -1326,7 +1347,7 @@ public void TestLoadingLocalPostgresSettings() public void TestLoadingLocalMySqlSettings() { Environment.SetEnvironmentVariable(ASP_NET_CORE_ENVIRONMENT_VAR_NAME, MYSQL_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); QueryEngineFactory queryEngineFactory = (QueryEngineFactory)server.Services.GetService(typeof(IQueryEngineFactory)); Assert.IsInstanceOfType(queryEngineFactory.GetQueryEngine(DatabaseType.MySQL), typeof(SqlQueryEngine)); @@ -1348,7 +1369,7 @@ public void TestLoadingLocalMySqlSettings() public async Task TestOverridingLocalSettingsFails(string configurationEndpoint) { Environment.SetEnvironmentVariable(ASP_NET_CORE_ENVIRONMENT_VAR_NAME, COSMOS_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); HttpClient client = server.CreateClient(); JsonContent config = GetJsonContentForCosmosConfigRequest(configurationEndpoint); @@ -1362,7 +1383,7 @@ public async Task TestOverridingLocalSettingsFails(string configurationEndpoint) [DataRow(CONFIGURATION_ENDPOINT_V2)] public async Task TestSettingConfigurationCreatesCorrectClasses(string configurationEndpoint) { - TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty())); HttpClient client = server.CreateClient(); JsonContent content = GetJsonContentForCosmosConfigRequest(configurationEndpoint); @@ -1472,7 +1493,7 @@ public void TestCommandLineConfigurationProvider() $"{COSMOS_ENVIRONMENT}{CONFIG_EXTENSION}" }; - TestServer server = new(Program.CreateWebHostBuilder(args)); + using TestServer server = new(Program.CreateWebHostBuilder(args)); ValidateCosmosDbSetup(server); } @@ -1489,7 +1510,7 @@ public void TestRuntimeEnvironmentVariable() Environment.SetEnvironmentVariable( RUNTIME_ENVIRONMENT_VAR_NAME, COSMOS_ENVIRONMENT); - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); ValidateCosmosDbSetup(server); } @@ -2230,7 +2251,7 @@ public void TestConnectionStringEnvVarHasHighestPrecedence() try { - TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); + using TestServer server = new(Program.CreateWebHostBuilder(Array.Empty())); _ = server.Services.GetService(typeof(CosmosClientProvider)) as CosmosClientProvider; Assert.Fail($"{RUNTIME_ENV_CONNECTION_STRING} is not given highest precedence"); } @@ -4306,7 +4327,7 @@ private static void ValidateLogLevelFilters(LogLevel logLevel, string loggingFil // Start a new server with the custom log level to ensure the // instantiation of the valid log level filters works as expected. - TestServer server = new(Program.CreateWebHostBuilder(args)); + using TestServer server = new(Program.CreateWebHostBuilder(args)); RuntimeConfigProvider runtimeConfigProvider = server.Services.GetService(); // RuntimeConfig with instantiated log level filters. diff --git a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs index 0f9ff6c1b8..18c19cbd7a 100644 --- a/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs +++ b/src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs @@ -666,13 +666,33 @@ await WaitForConditionAsync( RuntimeConfig updatedRuntimeConfig = _configProvider.GetConfig(); MsSqlOptions actualSessionContext = updatedRuntimeConfig.DataSource.GetTypedOptions(); - JsonElement reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( - _testClient, - _configProvider, - GQL_QUERY_NAME, - GQL_QUERY); + + // Retry GraphQL request because metadata re-initialization happens asynchronously + // after the "Validated hot-reloaded configuration file" message. The metadata provider + // factory clears and re-initializes providers on the hot-reload thread, so requests + // arriving before that completes will fail with "Initialization of metadata incomplete." + JsonElement reloadGQLContents = default; + bool querySucceeded = false; + for (int attempt = 1; attempt <= 10; attempt++) + { + reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync( + _testClient, + _configProvider, + GQL_QUERY_NAME, + GQL_QUERY); + + if (reloadGQLContents.ValueKind == JsonValueKind.Object && + reloadGQLContents.TryGetProperty("items", out _)) + { + querySucceeded = true; + break; + } + + await Task.Delay(1000); + } // Assert + Assert.IsTrue(querySucceeded, "GraphQL query did not return valid results after hot-reload. Metadata initialization may not have completed."); Assert.AreNotEqual(previousSessionContext, actualSessionContext); Assert.AreEqual(false, actualSessionContext.SetSessionContext); SqlTestHelper.PerformTestEqualJsonStrings(_bookDBOContents, reloadGQLContents.GetProperty("items").ToString()); diff --git a/src/Service/Startup.cs b/src/Service/Startup.cs index 12cbd18ab0..f084185929 100644 --- a/src/Service/Startup.cs +++ b/src/Service/Startup.cs @@ -119,7 +119,7 @@ public void ConfigureServices(IServiceCollection services) _configProvider = configProvider; services.AddSingleton(fileSystem); - services.AddSingleton(configLoader); + services.AddSingleton(sp => configLoader); services.AddSingleton(configProvider); bool runtimeConfigAvailable = configProvider.TryGetConfig(out RuntimeConfig? runtimeConfig);