Skip to content

Commit f8373b8

Browse files
rohkhannayush3797aaronpowellseantleonard
authored
Runtime config changes for multiple database support (#1639)
## Why make this change? 1. In this pr, RuntimeConfig object is extended to support multiple database scenario. Along with this all the class objects are respectively extended to be able to take multiple database input. 2. Consequently, there are downstream changes done in the different executors. Currently each executor will use the default datasource to run their queries. 3. In subsequent pr's through Named dependency injection, the executors will be able to choose which db to act against. 4. Please see RFC: #1638 ## What is this change? - This change is meant to be part of a series of changes that will enable multiple database support on data api builder. - This first change ensures compatibility of runtimeconfig with multiple database approach. - This change does not unlock any new scenario as of now. (there will be further pr's that will build on this change to fully support multiple database scenario). It just extends runtimeconfig while ensuring that the single datasource scenario works as expected. ## How was this tested? - Existing integration tests cover all execution paths to test for backword compatibility. - Ran dab locally as well to test that it works. - Existing unit tests cover all execution paths to test for backword compatibility. --------- Co-authored-by: Ayush Agarwal <[email protected]> Co-authored-by: Aaron Powell <[email protected]> Co-authored-by: Sean Leonard <[email protected]>
1 parent 39a014b commit f8373b8

23 files changed

Lines changed: 636 additions & 227 deletions

‎src/Config/DataApiBuilderException.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,11 @@ public enum SubStatusCodes
9797
/// <summary>
9898
/// Global REST endpoint disabled in runtime configuration.
9999
/// </summary>
100-
GlobalRestEndpointDisabled
100+
GlobalRestEndpointDisabled,
101+
/// <summary>
102+
/// DataSource not found for multiple db scenario.
103+
/// </summary>
104+
DataSourceNotFound,
101105
}
102106

103107
public HttpStatusCode StatusCode { get; }

‎src/Config/ObjectModel/RuntimeConfig.cs‎

Lines changed: 198 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,214 @@
11
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT License.
33

4+
using System.Net;
45
using System.Text.Json;
56
using System.Text.Json.Serialization;
7+
using Azure.DataApiBuilder.Service.Exceptions;
68

79
namespace Azure.DataApiBuilder.Config.ObjectModel;
810

9-
public record RuntimeConfig(
10-
[property: JsonPropertyName("$schema")] string Schema,
11-
DataSource DataSource,
12-
RuntimeOptions Runtime,
13-
RuntimeEntities Entities)
11+
public record RuntimeConfig
1412
{
13+
[JsonPropertyName("$schema")]
14+
public string Schema { get; init; }
15+
16+
public DataSource DataSource { get; init; }
17+
18+
public RuntimeOptions Runtime { get; init; }
19+
20+
public RuntimeEntities Entities { get; init; }
21+
22+
private string _defaultDataSourceName;
23+
24+
private Dictionary<string, DataSource> _dataSourceNameToDataSource;
25+
26+
private Dictionary<string, string> _entityNameToDataSourceName;
27+
28+
/// <summary>
29+
/// List of all datasources.
30+
/// </summary>
31+
/// <returns>List of datasources</returns>
32+
public IEnumerable<DataSource> ListAllDataSources()
33+
{
34+
return _dataSourceNameToDataSource.Values;
35+
}
36+
37+
/// <summary>
38+
/// Get Iterator to iterate over dictionary.
39+
/// </summary>
40+
public IEnumerable<KeyValuePair<string, DataSource>> GetDataSourceNamesToDataSourcesIterator()
41+
{
42+
return _dataSourceNameToDataSource.AsEnumerable();
43+
}
44+
45+
/// <summary>
46+
/// Constructor for runtimeConfig.
47+
/// </summary>
48+
/// <param name="Schema">schema.</param>
49+
/// <param name="DataSource">Default datasource.</param>
50+
/// <param name="Runtime">Runtime settings.</param>
51+
/// <param name="Entities">Entities</param>
52+
[JsonConstructor]
53+
public RuntimeConfig(string Schema, DataSource DataSource, RuntimeOptions Runtime, RuntimeEntities Entities)
54+
{
55+
this.Schema = Schema;
56+
this.DataSource = DataSource;
57+
this.Runtime = Runtime;
58+
this.Entities = Entities;
59+
this._dataSourceNameToDataSource = new Dictionary<string, DataSource>();
60+
this._defaultDataSourceName = Guid.NewGuid().ToString();
61+
this._dataSourceNameToDataSource.Add(this._defaultDataSourceName, this.DataSource);
62+
63+
this._entityNameToDataSourceName = new Dictionary<string, string>();
64+
foreach (KeyValuePair<string, Entity> entity in Entities)
65+
{
66+
_entityNameToDataSourceName.TryAdd(entity.Key, this._defaultDataSourceName);
67+
}
68+
69+
}
70+
71+
/// <summary>
72+
/// Gets the DataSource corresponding to the datasourceName.
73+
/// </summary>
74+
/// <param name="dataSourceName">Name of datasource.</param>
75+
/// <returns>DataSource object.</returns>
76+
/// <exception cref="DataApiBuilderException">Not found exception if key is not found.</exception>
77+
public DataSource GetDataSourceFromDataSourceName(string dataSourceName)
78+
{
79+
CheckDataSourceNamePresent(dataSourceName);
80+
return _dataSourceNameToDataSource[dataSourceName];
81+
}
82+
83+
/// <summary>
84+
/// Tries to add the datasource to the DataSourceNameToDataSource dictionary.
85+
/// </summary>
86+
/// <param name="dataSourceName">dataSourceName.</param>
87+
/// <param name="dataSource">dataSource.</param>
88+
/// <returns>True indicating success, False indicating failure.</returns>
89+
public bool AddDataSource(string dataSourceName, DataSource dataSource)
90+
{
91+
return _dataSourceNameToDataSource.TryAdd(dataSourceName, dataSource);
92+
}
93+
94+
/// <summary>
95+
/// Updates the DataSourceNameToDataSource dictionary with the new datasource.
96+
/// </summary>
97+
/// <param name="dataSourceName">Name of datasource</param>
98+
/// <param name="dataSource">Updated datasource value.</param>
99+
/// <exception cref="DataApiBuilderException">Not found exception if key is not found.</exception>
100+
public void UpdateDataSourceNameToDataSource(string dataSourceName, DataSource dataSource)
101+
{
102+
CheckDataSourceNamePresent(dataSourceName);
103+
_dataSourceNameToDataSource[dataSourceName] = dataSource;
104+
}
105+
106+
/// <summary>
107+
/// Removes the datasource from the DataSourceNameToDataSource dictionary.
108+
/// </summary>
109+
/// <param name="dataSourceName">DataSourceName.</param>
110+
/// <returns>True indicating success, False indicating failure.</returns>
111+
/// <exception cref="DataApiBuilderException">Not found exception if key is not found.</exception>
112+
public bool RemoveDataSource(string dataSourceName)
113+
{
114+
CheckDataSourceNamePresent(dataSourceName);
115+
return _dataSourceNameToDataSource.Remove(dataSourceName);
116+
}
117+
118+
/// <summary>
119+
/// Gets datasourceName from EntityNameToDatasourceName dictionary.
120+
/// </summary>
121+
/// <param name="entityName">entityName</param>
122+
/// <returns>DataSourceName</returns>
123+
public string GetDataSourceNameFromEntityName(string entityName)
124+
{
125+
CheckEntityNamePresent(entityName);
126+
return _entityNameToDataSourceName[entityName];
127+
}
128+
129+
/// <summary>
130+
/// Gets datasource using entityName.
131+
/// </summary>
132+
/// <param name="entityName">entityName.</param>
133+
/// <returns>DataSource using EntityName.</returns>
134+
public DataSource GetDataSourceFromEntityName(string entityName)
135+
{
136+
CheckEntityNamePresent(entityName);
137+
return _dataSourceNameToDataSource[_entityNameToDataSourceName[entityName]];
138+
}
139+
140+
/// <summary>
141+
/// Add entity to the EntityNameToDataSourceName dictionary.
142+
/// </summary>
143+
/// <param name="entityName">EntityName</param>
144+
/// <param name="dataSourceName">DatasourceName.</param>
145+
/// <returns>True indicating success, False indicating failure.</returns>
146+
/// <exception cref="DataApiBuilderException">Not found exception if key is not found.</exception>
147+
public bool AddEntity(string entityName, string dataSourceName)
148+
{
149+
CheckDataSourceNamePresent(dataSourceName);
150+
return _entityNameToDataSourceName.TryAdd(entityName, dataSourceName);
151+
}
152+
153+
/// <summary>
154+
/// Updates the EntityNameToDataSourceName dictionary with the new Entity to datasource mapping.
155+
/// </summary>
156+
/// <param name="entityName">EntityName.</param>
157+
/// <param name="dataSourceName">dataSourceName.</param>
158+
/// <exception cref="DataApiBuilderException"></exception>
159+
public void UpdateEntityNameToDataSourceName(string entityName, string dataSourceName)
160+
{
161+
CheckDataSourceNamePresent(dataSourceName);
162+
CheckEntityNamePresent(entityName);
163+
_entityNameToDataSourceName[entityName] = dataSourceName;
164+
}
165+
166+
/// <summary>
167+
/// Removes the entity from the EntityNameToDataSourceName dictionary.
168+
/// </summary>
169+
/// <param name="entityName">Name of Entity</param>
170+
/// <exception cref="DataApiBuilderException">Not found exception if key is not found.</exception>
171+
public bool RemoveEntity(string entityName)
172+
{
173+
CheckEntityNamePresent(entityName);
174+
return _entityNameToDataSourceName.Remove(entityName);
175+
}
176+
177+
/// <summary>
178+
/// Get the default datasource name.
179+
/// </summary>
180+
/// <returns>default datasourceName.</returns>
181+
#pragma warning disable CA1024 // Use properties where appropriate. Reason: Do not want datasource serialized and want to keep it private to restrict set;
182+
public string GetDefaultDataSourceName()
183+
#pragma warning restore CA1024 // Use properties where appropriate
184+
{
185+
return _defaultDataSourceName;
186+
}
187+
15188
/// <summary>
16189
/// Serializes the RuntimeConfig object to JSON for writing to file.
17190
/// </summary>
18191
/// <returns></returns>
19-
public string ToJson()
192+
public string ToJson(JsonSerializerOptions? jsonSerializerOptions = null)
193+
{
194+
// get default serializer options if none provided.
195+
jsonSerializerOptions = jsonSerializerOptions ?? RuntimeConfigLoader.GetSerializationOptions();
196+
return JsonSerializer.Serialize(this, jsonSerializerOptions);
197+
}
198+
199+
private void CheckDataSourceNamePresent(string dataSourceName)
200+
{
201+
if (!_dataSourceNameToDataSource.ContainsKey(dataSourceName))
202+
{
203+
throw new DataApiBuilderException($"{nameof(dataSourceName)}:{dataSourceName} could not be found within the config", HttpStatusCode.BadRequest, DataApiBuilderException.SubStatusCodes.DataSourceNotFound);
204+
}
205+
}
206+
207+
private void CheckEntityNamePresent(string entityName)
20208
{
21-
return JsonSerializer.Serialize(this, RuntimeConfigLoader.GetSerializationOptions());
209+
if (!_entityNameToDataSourceName.ContainsKey(entityName))
210+
{
211+
throw new DataApiBuilderException($"{nameof(entityName)}:{entityName} could not be found within the config", HttpStatusCode.BadRequest, DataApiBuilderException.SubStatusCodes.EntityNotFound);
212+
}
22213
}
23214
}

‎src/Config/RuntimeConfigLoader.cs‎

Lines changed: 44 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -42,18 +42,24 @@ public RuntimeConfigLoader(string? connectionString = null)
4242
public abstract string GetPublishedDraftSchemaLink();
4343

4444
/// <summary>
45-
/// Parses a JSON string into a <c>RuntimeConfig</c> object
45+
/// Parses a JSON string into a <c>RuntimeConfig</c> object for single database scenario.
4646
/// </summary>
4747
/// <param name="json">JSON that represents the config file.</param>
4848
/// <param name="config">The parsed config, or null if it parsed unsuccessfully.</param>
4949
/// <returns>True if the config was parsed, otherwise false.</returns>
50+
/// <param name="logger">logger to log messages</param>
51+
/// <param name="connectionString">connectionString to add to config if specified</param>
5052
/// <param name="replaceEnvVar">Whether to replace environment variable with its
5153
/// value or not while deserializing. By default, no replacement happens.</param>
54+
/// <param name="dataSourceName"> datasource name for which to add connection string</param>
55+
/// <param name="datasourceNameToConnectionString"> dictionary of datasource name to connection string</param>
5256
public static bool TryParseConfig(string json,
5357
[NotNullWhen(true)] out RuntimeConfig? config,
5458
ILogger? logger = null,
5559
string? connectionString = null,
56-
bool replaceEnvVar = false)
60+
bool replaceEnvVar = false,
61+
string dataSourceName = "",
62+
Dictionary<string, string>? datasourceNameToConnectionString = null)
5763
{
5864
JsonSerializerOptions options = GetSerializationOptions(replaceEnvVar);
5965

@@ -66,22 +72,51 @@ public static bool TryParseConfig(string json,
6672
return false;
6773
}
6874

75+
// retreive current connection string from config
6976
string updatedConnectionString = config.DataSource.ConnectionString;
7077

78+
// set dataSourceName to default if not provided
79+
if (string.IsNullOrEmpty(dataSourceName))
80+
{
81+
dataSourceName = config.GetDefaultDataSourceName();
82+
}
83+
7184
if (!string.IsNullOrEmpty(connectionString))
7285
{
86+
// update connection string if provided.
7387
updatedConnectionString = connectionString;
7488
}
7589

76-
// Add Application Name for telemetry for MsSQL
77-
// Do this only when environment variables have been replaced since
78-
// otherwise parsing the connection string may result in an exception
79-
if (config.DataSource.DatabaseType is DatabaseType.MSSQL && replaceEnvVar)
90+
if (datasourceNameToConnectionString is null)
8091
{
81-
updatedConnectionString = GetConnectionStringWithApplicationName(updatedConnectionString);
92+
datasourceNameToConnectionString = new Dictionary<string, string>();
8293
}
8394

84-
config = config with { DataSource = config.DataSource with { ConnectionString = updatedConnectionString } };
95+
// add to dictionary if datasourceName is present (will either be the default or the one provided)
96+
datasourceNameToConnectionString.TryAdd(dataSourceName, updatedConnectionString);
97+
98+
// iterate over dictionary and update runtime config with connection strings.
99+
foreach ((string dataSourceKey, string connectionValue) in datasourceNameToConnectionString)
100+
{
101+
string updatedConnection = connectionValue;
102+
103+
DataSource ds = config.GetDataSourceFromDataSourceName(dataSourceKey);
104+
105+
// Add Application Name for telemetry for MsSQL
106+
if (ds.DatabaseType is DatabaseType.MSSQL && replaceEnvVar)
107+
{
108+
updatedConnection = GetConnectionStringWithApplicationName(connectionValue);
109+
}
110+
111+
ds = ds with { ConnectionString = updatedConnection };
112+
config.UpdateDataSourceNameToDataSource(dataSourceName, ds);
113+
114+
if (string.Equals(dataSourceKey, config.GetDefaultDataSourceName(), StringComparison.OrdinalIgnoreCase))
115+
{
116+
config = config with { DataSource = ds };
117+
}
118+
119+
}
85120
}
86121
catch (JsonException ex)
87122
{
@@ -113,6 +148,7 @@ public static bool TryParseConfig(string json,
113148
/// By default, no replacement happens.</param>
114149
public static JsonSerializerOptions GetSerializationOptions(bool replaceEnvVar = false)
115150
{
151+
116152
JsonSerializerOptions options = new()
117153
{
118154
PropertyNameCaseInsensitive = false,

0 commit comments

Comments
 (0)