Skip to content

Commit

Permalink
feat: create db bootstrap API
Browse files Browse the repository at this point in the history
  • Loading branch information
@Alxandr
Alxandr committed Oct 4, 2024
1 parent e0a4def commit 9648724
Show file tree
Hide file tree
Showing 15 changed files with 831 additions and 68 deletions.
128 changes: 60 additions & 68 deletions src/Directory.Packages.props
View file
Original file line number Diff line number Diff line change
@@ -1,70 +1,62 @@
<Project>

<PropertyGroup>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>

<ItemGroup>
<PackageVersion Include="Altinn.Authorization.ServiceDefaults.Npgsql.Yuniql" Version="2.2.2" />
<PackageVersion Include="Altinn.Authorization.ServiceDefaults" Version="2.2.2" />

<!-- Microsoft Extensions-->
<PackageVersion Include="Microsoft.Extensions.Caching.Abstractions" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.AzureKeyVault" Version="3.1.24" />
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Logging.ApplicationInsights" Version="2.22.0" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Telemetry.Abstractions" Version="8.7.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />

<!---ASP.NET -->
<PackageVersion Include="Microsoft.AspNetCore.Hosting" Version="2.2.7" />
<PackageVersion Include="Microsoft.AspNetCore.Hosting.Abstractions" Version="2.2.0" />
<PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="8.0.7" />
<PackageVersion Include="Microsoft.FeatureManagement.AspNetCore" Version="3.5.0" />
<PackageVersion Include="Swashbuckle.AspNetCore" Version="6.4.0" />

<!-- StyleCop-->
<PackageVersion Include="StyleCop.Analyzers" Version=" 1.2.0-beta.556" />

<!-- Azure -->
<PackageVersion Include="Microsoft.Extensions.Azure" Version=" 1.7.4" />
<PackageVersion Include="Microsoft.Azure.AppConfiguration.AspNetCore"
Version=" 8.0.0-preview.3" />
<PackageVersion Include="Azure.Identity" Version=" 1.12.0" />

<!-- Postgres -->
<PackageVersion Include="Npgsql" Version=" 8.0.3" />
<PackageVersion Include="Yuniql.PostgreSql" Version=" 1.3.15" />
<PackageVersion Include="Yuniql.Core" Version=" 1.3.15" />
<PackageVersion Include="Yuniql.AspNetCore" Version=" 1.2.25" />
<PackageVersion Include="Npgsql.OpenTelemetry" Version=" 8.0.3" />

<!-- Open Telemetry -->
<PackageVersion Include="OpenTelemetry" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Exporter.Console" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version=" 1.9.0" />
<PackageVersion Include="Azure.Monitor.OpenTelemetry.AspNetCore" Version=" 1.3.0-beta.1" />

<!-- Mass Transit -->
<PackageVersion Include="MassTransit.Azure.ServiceBus.Core" Version=" 8.2.6-develop.1998" />
<PackageVersion Include="MassTransit.Extensions.DependencyInjection" Version=" 7.3.1" />

<!-- Test -->
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version=" 17.10.0" />
<PackageVersion Include="Moq" Version=" 4.20.70" />
<PackageVersion Include="xunit" Version=" 2.9.0" />
<PackageVersion Include="xunit.runner.visualstudio" Version=" 2.8.2" />
<PackageVersion Include="coverlet.collector" Version=" 6.0.2" />

<PackageVersion
Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="8.0.8" />
<PackageVersion
Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="8.0.8" />

</ItemGroup>
<PropertyGroup>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>
<PackageVersion Include="Altinn.Authorization.ServiceDefaults.Npgsql.Yuniql" Version="2.2.2" />
<PackageVersion Include="Altinn.Authorization.ServiceDefaults" Version="2.2.2" />
<PackageVersion Include="Azure.ResourceManager.KeyVault" Version="1.3.0" />
<PackageVersion Include="Azure.ResourceManager.PostgreSql" Version="1.1.3" />
<PackageVersion Include="Azure.Security.KeyVault.Secrets" Version="4.6.0" />
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.8" />
<PackageVersion Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="8.0.8" />
<!-- Microsoft Extensions-->
<PackageVersion Include="Microsoft.Extensions.Caching.Abstractions" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.AzureKeyVault" Version="3.1.24" />
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Logging.ApplicationInsights" Version="2.22.0" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Telemetry.Abstractions" Version="8.7.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
<!---ASP.NET -->
<PackageVersion Include="Microsoft.AspNetCore.Hosting" Version="2.2.7" />
<PackageVersion Include="Microsoft.AspNetCore.Hosting.Abstractions" Version="2.2.0" />
<PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="8.0.7" />
<PackageVersion Include="Microsoft.FeatureManagement.AspNetCore" Version="3.5.0" />
<PackageVersion Include="Nerdbank.Streams" Version="2.11.79" />
<PackageVersion Include="Spectre.Console" Version="0.49.1" />
<PackageVersion Include="Swashbuckle.AspNetCore" Version="6.4.0" />
<!-- StyleCop-->
<PackageVersion Include="StyleCop.Analyzers" Version=" 1.2.0-beta.556" />
<!-- Azure -->
<PackageVersion Include="Microsoft.Extensions.Azure" Version=" 1.7.4" />
<PackageVersion Include="Microsoft.Azure.AppConfiguration.AspNetCore" Version=" 8.0.0-preview.3" />
<PackageVersion Include="Azure.Identity" Version="1.12.1" />
<!-- Postgres -->
<PackageVersion Include="Npgsql" Version="8.0.4" />
<PackageVersion Include="Yuniql.PostgreSql" Version=" 1.3.15" />
<PackageVersion Include="Yuniql.Core" Version=" 1.3.15" />
<PackageVersion Include="Yuniql.AspNetCore" Version=" 1.2.25" />
<PackageVersion Include="Npgsql.OpenTelemetry" Version=" 8.0.3" />
<!-- Open Telemetry -->
<PackageVersion Include="OpenTelemetry" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Exporter.Console" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version=" 1.9.0" />
<PackageVersion Include="Azure.Monitor.OpenTelemetry.AspNetCore" Version=" 1.3.0-beta.1" />
<!-- Mass Transit -->
<PackageVersion Include="MassTransit.Azure.ServiceBus.Core" Version=" 8.2.6-develop.1998" />
<PackageVersion Include="MassTransit.Extensions.DependencyInjection" Version=" 7.3.1" />
<!-- Test -->
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version=" 17.10.0" />
<PackageVersion Include="Moq" Version=" 4.20.70" />
<PackageVersion Include="xunit" Version=" 2.9.0" />
<PackageVersion Include="xunit.runner.visualstudio" Version=" 2.8.2" />
<PackageVersion Include="coverlet.collector" Version=" 6.0.2" />
<PackageVersion Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="8.0.8" />
<PackageVersion Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="8.0.8" />
</ItemGroup>
</Project>
21 changes: 21 additions & 0 deletions ...zation.DeployApi/src/Altinn.Authorization.DeployApi/Altinn.Authorization.DeployApi.csproj
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk.Web">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Azure.Identity" />
<PackageReference Include="Azure.ResourceManager.KeyVault" />
<PackageReference Include="Azure.ResourceManager.PostgreSql" />
<PackageReference Include="Azure.Security.KeyVault.Secrets" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" />
<PackageReference Include="Nerdbank.Streams" />
<PackageReference Include="Npgsql" />
<PackageReference Include="Spectre.Console" />
</ItemGroup>

</Project>
138 changes: 138 additions & 0 deletions ...ployApi/src/Altinn.Authorization.DeployApi/BootstrapDatabase/BootstrapDatabasePipeline.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
using Altinn.Authorization.DeployApi.Pipelines;
using Azure.Core;
using Azure.ResourceManager;
using Azure.ResourceManager.KeyVault;
using Azure.ResourceManager.PostgreSql.FlexibleServers;
using Azure.Security.KeyVault.Secrets;
using Npgsql;
using System.Text.Json.Serialization;

namespace Altinn.Authorization.DeployApi.BootstrapDatabase;

internal sealed class BootstrapDatabasePipeline
: Pipeline
{
[JsonPropertyName("resources")]
public required ResourcesConfig Resources { get; init; }

[JsonPropertyName("databaseName")]
public required string DatabaseName { get; init; }

[JsonPropertyName("schemas")]
public required IReadOnlyDictionary<string, SchemaBootstrapModel> Schemas { get; init; }

protected internal override async Task ExecuteAsync(PipelineContext context, CancellationToken cancellationToken)
{
var cred = context.GetRequiredService<TokenCredential>();
var client = new ArmClient(cred, defaultSubscriptionId: Resources.SubscriptionId);

var subscription = await context.RunTask(
"Get subscription info",
(_, ct) => client.GetDefaultSubscriptionAsync(ct),
cancellationToken);

var resourceGroup = await context.RunTask(
"Get resource group info",
(_, ct) => subscription.GetResourceGroupAsync(Resources.ResourceGroupName, ct),
cancellationToken);

var keyVault = await context.RunTask(
"Get key vault info",
(_, ct) => resourceGroup.Value.GetKeyVaultAsync(Resources.KeyVaultName, ct),
cancellationToken);

var server = await context.RunTask(
"Get server info",
(_, ct) => resourceGroup.Value.GetPostgreSqlFlexibleServerAsync(Resources.ServerName, ct),
cancellationToken);

var token = await context.RunTask(
"Get db auth token",
(_, ct) => cred.GetTokenAsync(new(["https://ossrdbms-aad.database.windows.net/.default"]), ct).AsTask(),
cancellationToken);

var secretClient = new SecretClient(keyVault.Value.Data.Properties.VaultUri, cred);

var serverUrl = server.Value.Data.FullyQualifiedDomainName;
var connStringBuilder = new NpgsqlConnectionStringBuilder()
{
Host = serverUrl,
Database = "postgres",
Username = Resources.User,
Password = token.Token,
Port = 5432,
SslMode = SslMode.Require,
Pooling = false,
};

var serverConnString = connStringBuilder.ToString();
await using var serverConn = new NpgsqlConnection(serverConnString);
await context.RunTask(
"Connecting to database server",
(_, ct) => serverConn.OpenAsync(ct),
cancellationToken);

var migratorUser = await context.RunTask(new CreateDatabaseRoleTask(secretClient, serverConn, $"{DatabaseName}_migrator", Resources.User), cancellationToken);
var appUser = await context.RunTask(new CreateDatabaseRoleTask(secretClient, serverConn, $"{DatabaseName}_app", Resources.User), cancellationToken);
await context.RunTask(new CreateDatabaseTask(serverConn, DatabaseName), cancellationToken);
await context.RunTask(new GrantDatabasePrivilegesTask(serverConn, DatabaseName, migratorUser.RoleName, "CREATE, CONNECT"), cancellationToken);
await context.RunTask(new GrantDatabasePrivilegesTask(serverConn, DatabaseName, appUser.RoleName, "CONNECT"), cancellationToken);

connStringBuilder.Database = DatabaseName;
var dbConnString = connStringBuilder.ToString();
await using var dbConn = new NpgsqlConnection(dbConnString);
await context.RunTask(
$"Connecting to database '[cyan]{DatabaseName}[/]'",
(_, ct) => dbConn.OpenAsync(ct),
cancellationToken);

foreach (var (schemaName, schemaCfg) in Schemas)
{
await context.RunTask(new CreateDatabaseSchemaTask(dbConn, migratorUser.RoleName, schemaName, schemaCfg), cancellationToken);
}

connStringBuilder = new NpgsqlConnectionStringBuilder()
{
Host = serverUrl,
Database = DatabaseName,
Username = Resources.User,
Password = token.Token,
Port = 5432,
SslMode = SslMode.Require,
};

var connectionStrings = new Dictionary<string, string>();

connStringBuilder.Username = migratorUser.RoleName;
connStringBuilder.Password = migratorUser.Password;
connectionStrings[$"db-{DatabaseName}-migrator"] = connStringBuilder.ToString();

connStringBuilder.Username = appUser.RoleName;
connStringBuilder.Password = appUser.Password;
connectionStrings[$"db-{DatabaseName}-app"] = connStringBuilder.ToString();

await context.RunTask(new SaveConnectionStringsTask(secretClient, connectionStrings), cancellationToken);
}

public record ResourcesConfig
{
[JsonPropertyName("subscriptionId")]
public required string SubscriptionId { get; init; }

[JsonPropertyName("resourceGroup")]
public required string ResourceGroupName { get; init; }

[JsonPropertyName("serverName")]
public required string ServerName { get; init; }

[JsonPropertyName("user")]
public required string User { get; init; }

[JsonPropertyName("keyVaultName")]
public required string KeyVaultName { get; init; }
}

public record SchemaBootstrapModel
{
}
}
99 changes: 99 additions & 0 deletions ....DeployApi/src/Altinn.Authorization.DeployApi/BootstrapDatabase/CreateDatabaseRoleTask.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
using System.Security.Cryptography;
using Altinn.Authorization.DeployApi.Tasks;
using Azure;
using Azure.Security.KeyVault.Secrets;
using Npgsql;
using Spectre.Console;

namespace Altinn.Authorization.DeployApi.BootstrapDatabase;

internal sealed class CreateDatabaseRoleTask
: StepTask<CreateDatabaseRoleTask.Result>
{
private readonly SecretClient _secrets;
private readonly NpgsqlConnection _conn;
private readonly string _roleName;
private readonly string _adminRole;

public CreateDatabaseRoleTask(SecretClient secrets, NpgsqlConnection conn, string roleName, string adminRole)
{
_secrets = secrets;
_conn = conn;
_roleName = roleName;
_adminRole = adminRole;
}

public override string Name => $"Creating role '[cyan]{_roleName}[/]'";

public override async Task<Result> ExecuteAsync(ProgressTask task, CancellationToken cancellationToken)
{
var pw = await GetOrCreatePassword(task, cancellationToken);
var secretName = $"-db-{_roleName.Replace('_', '-')}-pw";
var secret = await _secrets.GetSecretAsync(secretName, cancellationToken: cancellationToken);

await using var cmd = _conn.CreateCommand();
cmd.CommandText =
/*strpsql*/$"""
CREATE ROLE "{_roleName}"
WITH LOGIN
PASSWORD '{pw.Password}'
""";

try
{
await cmd.ExecuteNonQueryAsync(cancellationToken);
}
catch (PostgresException ex) when (ex.SqlState == "42710")
{
if (pw.Updated)
{
cmd.CommandText =
/*strpsql*/$"""
ALTER ROLE "{_roleName}"
WITH LOGIN
PASSWORD '{pw.Password}'
""";

await cmd.ExecuteNonQueryAsync(cancellationToken);
}
}

cmd.CommandText =
/*strpsql*/$"""
GRANT "{_roleName}" TO "{_adminRole}"
""";

await cmd.ExecuteNonQueryAsync(cancellationToken);

return new Result(_roleName, pw.Password);
}

private async Task<PasswordResult> GetOrCreatePassword(ProgressTask task, CancellationToken cancellationToken)
{
var secretName = $"-db-{_roleName.Replace('_', '-')}-pw";
Response<KeyVaultSecret> secret;
bool updated = false;
try
{
secret = await _secrets.GetSecretAsync(secretName, cancellationToken: cancellationToken);
}
catch (RequestFailedException ex) when (ex.ErrorCode == "SecretNotFound")
{
var pw = GenerateRandomPw();
secret = await _secrets.SetSecretAsync(secretName, pw, cancellationToken: cancellationToken);
updated = true;
}

return new PasswordResult(secret.Value.Value, updated);
}

private static string GenerateRandomPw()
{
const string VALID_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*()_+";
return RandomNumberGenerator.GetString(VALID_CHARS.AsSpan(), 64);
}

private record PasswordResult(string Password, bool Updated);

internal record Result(string RoleName, string Password);
}
Loading

0 comments on commit 9648724

Please sign in to comment.