Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: create db bootstrap API #11

Merged
merged 4 commits into from
Oct 7, 2024
Merged

feat: create db bootstrap API #11

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9648724
feat: create db bootstrap API
Alxandr Oct 4, 2024
689c3d8
style: sort packages
Alxandr Oct 4, 2024
0d03c37
chore: fix pr issues
Alxandr Oct 7, 2024
c7bbcb2
chore: add sln
Alxandr Oct 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 53 additions & 69 deletions src/Directory.Packages.props
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,70 +1,54 @@
<Project>

<PropertyGroup>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>

<ItemGroup>
<PackageVersion Include="Altinn.Authorization.ServiceDefaults.Npgsql.Yuniql" Version="2.2.2" />
<PackageVersion Include="Altinn.Authorization.ServiceDefaults" Version="2.2.2" />

<!-- Microsoft Extensions-->
<PackageVersion Include="Microsoft.Extensions.Caching.Abstractions" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.AzureKeyVault" Version="3.1.24" />
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Logging.ApplicationInsights" Version="2.22.0" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Telemetry.Abstractions" Version="8.7.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />

<!---ASP.NET -->
<PackageVersion Include="Microsoft.AspNetCore.Hosting" Version="2.2.7" />
<PackageVersion Include="Microsoft.AspNetCore.Hosting.Abstractions" Version="2.2.0" />
<PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="8.0.7" />
<PackageVersion Include="Microsoft.FeatureManagement.AspNetCore" Version="3.5.0" />
<PackageVersion Include="Swashbuckle.AspNetCore" Version="6.4.0" />

<!-- StyleCop-->
<PackageVersion Include="StyleCop.Analyzers" Version=" 1.2.0-beta.556" />

<!-- Azure -->
<PackageVersion Include="Microsoft.Extensions.Azure" Version=" 1.7.4" />
<PackageVersion Include="Microsoft.Azure.AppConfiguration.AspNetCore"
Version=" 8.0.0-preview.3" />
<PackageVersion Include="Azure.Identity" Version=" 1.12.0" />

<!-- Postgres -->
<PackageVersion Include="Npgsql" Version=" 8.0.3" />
<PackageVersion Include="Yuniql.PostgreSql" Version=" 1.3.15" />
<PackageVersion Include="Yuniql.Core" Version=" 1.3.15" />
<PackageVersion Include="Yuniql.AspNetCore" Version=" 1.2.25" />
<PackageVersion Include="Npgsql.OpenTelemetry" Version=" 8.0.3" />

<!-- Open Telemetry -->
<PackageVersion Include="OpenTelemetry" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Exporter.Console" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version=" 1.9.0" />
<PackageVersion Include="Azure.Monitor.OpenTelemetry.AspNetCore" Version=" 1.3.0-beta.1" />

<!-- Mass Transit -->
<PackageVersion Include="MassTransit.Azure.ServiceBus.Core" Version=" 8.2.6-develop.1998" />
<PackageVersion Include="MassTransit.Extensions.DependencyInjection" Version=" 7.3.1" />

<!-- Test -->
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version=" 17.10.0" />
<PackageVersion Include="Moq" Version=" 4.20.70" />
<PackageVersion Include="xunit" Version=" 2.9.0" />
<PackageVersion Include="xunit.runner.visualstudio" Version=" 2.8.2" />
<PackageVersion Include="coverlet.collector" Version=" 6.0.2" />

<PackageVersion
Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="8.0.8" />
<PackageVersion
Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="8.0.8" />

</ItemGroup>
</Project>
<PropertyGroup>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>
<PackageVersion Include="Altinn.Authorization.ServiceDefaults.Npgsql.Yuniql" Version="2.2.2" />
<PackageVersion Include="Altinn.Authorization.ServiceDefaults" Version="2.2.2" />
<PackageVersion Include="Azure.Identity" Version="1.12.1" />
<PackageVersion Include="Azure.Monitor.OpenTelemetry.AspNetCore" Version=" 1.3.0-beta.1" />
<PackageVersion Include="Azure.ResourceManager.KeyVault" Version="1.3.0" />
<PackageVersion Include="Azure.ResourceManager.PostgreSql" Version="1.1.3" />
<PackageVersion Include="Azure.Security.KeyVault.Secrets" Version="4.6.0" />
<PackageVersion Include="coverlet.collector" Version=" 6.0.2" />
<PackageVersion Include="MassTransit.Azure.ServiceBus.Core" Version=" 8.2.6-develop.1998" />
<PackageVersion Include="MassTransit.Extensions.DependencyInjection" Version=" 7.3.1" />
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.8" />
<PackageVersion Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="8.0.8" />
<PackageVersion Include="Microsoft.AspNetCore.Hosting.Abstractions" Version="2.2.0" />
<PackageVersion Include="Microsoft.AspNetCore.Hosting" Version="2.2.7" />
<PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="8.0.7" />
<PackageVersion Include="Microsoft.Azure.AppConfiguration.AspNetCore" Version=" 8.0.0-preview.3" />
<PackageVersion Include="Microsoft.Extensions.Azure" Version=" 1.7.4" />
<PackageVersion Include="Microsoft.Extensions.Caching.Abstractions" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.AzureKeyVault" Version="3.1.24" />
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="8.0.8" />
<PackageVersion Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="8.0.8" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Logging.ApplicationInsights" Version="2.22.0" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageVersion Include="Microsoft.Extensions.Telemetry.Abstractions" Version="8.7.0" />
<PackageVersion Include="Microsoft.FeatureManagement.AspNetCore" Version="3.5.0" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version=" 17.10.0" />
<PackageVersion Include="Moq" Version=" 4.20.70" />
<PackageVersion Include="Nerdbank.Streams" Version="2.11.79" />
<PackageVersion Include="Npgsql.OpenTelemetry" Version=" 8.0.3" />
<PackageVersion Include="Npgsql" Version="8.0.4" />
<PackageVersion Include="OpenTelemetry.Exporter.Console" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version=" 1.9.0" />
<PackageVersion Include="OpenTelemetry" Version=" 1.9.0" />
<PackageVersion Include="Spectre.Console" Version="0.49.1" />
<PackageVersion Include="StyleCop.Analyzers" Version=" 1.2.0-beta.556" />
<PackageVersion Include="Swashbuckle.AspNetCore" Version="6.4.0" />
<PackageVersion Include="xunit.runner.visualstudio" Version=" 2.8.2" />
<PackageVersion Include="xunit" Version=" 2.9.0" />
<PackageVersion Include="Yuniql.AspNetCore" Version=" 1.2.25" />
<PackageVersion Include="Yuniql.Core" Version=" 1.3.15" />
<PackageVersion Include="Yuniql.PostgreSql" Version=" 1.3.15" />
</ItemGroup>
</Project>
21 changes: 21 additions & 0 deletions ...zation.DeployApi/src/Altinn.Authorization.DeployApi/Altinn.Authorization.DeployApi.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk.Web">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Azure.Identity" />
<PackageReference Include="Azure.ResourceManager.KeyVault" />
<PackageReference Include="Azure.ResourceManager.PostgreSql" />
<PackageReference Include="Azure.Security.KeyVault.Secrets" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" />
<PackageReference Include="Nerdbank.Streams" />
<PackageReference Include="Npgsql" />
<PackageReference Include="Spectre.Console" />
</ItemGroup>

</Project>
138 changes: 138 additions & 0 deletions ...ployApi/src/Altinn.Authorization.DeployApi/BootstrapDatabase/BootstrapDatabasePipeline.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
using Altinn.Authorization.DeployApi.Pipelines;
using Azure.Core;
using Azure.ResourceManager;
using Azure.ResourceManager.KeyVault;
using Azure.ResourceManager.PostgreSql.FlexibleServers;
using Azure.Security.KeyVault.Secrets;
using Npgsql;
using System.Text.Json.Serialization;

namespace Altinn.Authorization.DeployApi.BootstrapDatabase;

internal sealed class BootstrapDatabasePipeline
: Pipeline
{
[JsonPropertyName("resources")]
public required ResourcesConfig Resources { get; init; }

[JsonPropertyName("databaseName")]
public required string DatabaseName { get; init; }

[JsonPropertyName("schemas")]
public required IReadOnlyDictionary<string, SchemaBootstrapModel> Schemas { get; init; }

protected internal override async Task ExecuteAsync(PipelineContext context, CancellationToken cancellationToken)
{
var cred = context.GetRequiredService<TokenCredential>();
var client = new ArmClient(cred, defaultSubscriptionId: Resources.SubscriptionId);

var subscription = await context.RunTask(
"Get subscription info",
(_, ct) => client.GetDefaultSubscriptionAsync(ct),
cancellationToken);

var resourceGroup = await context.RunTask(
"Get resource group info",
(_, ct) => subscription.GetResourceGroupAsync(Resources.ResourceGroupName, ct),
cancellationToken);

var keyVault = await context.RunTask(
"Get key vault info",
(_, ct) => resourceGroup.Value.GetKeyVaultAsync(Resources.KeyVaultName, ct),
cancellationToken);

var server = await context.RunTask(
"Get server info",
(_, ct) => resourceGroup.Value.GetPostgreSqlFlexibleServerAsync(Resources.ServerName, ct),
cancellationToken);

var token = await context.RunTask(
"Get db auth token",
(_, ct) => cred.GetTokenAsync(new(["https://ossrdbms-aad.database.windows.net/.default"]), ct).AsTask(),
cancellationToken);

var secretClient = new SecretClient(keyVault.Value.Data.Properties.VaultUri, cred);

var serverUrl = server.Value.Data.FullyQualifiedDomainName;
var connStringBuilder = new NpgsqlConnectionStringBuilder()
{
Host = serverUrl,
Database = "postgres",
Username = Resources.User,
Password = token.Token,
Port = 5432,
SslMode = SslMode.Require,
Pooling = false,
};

var serverConnString = connStringBuilder.ToString();
await using var serverConn = new NpgsqlConnection(serverConnString);
await context.RunTask(
"Connecting to database server",
(_, ct) => serverConn.OpenAsync(ct),
cancellationToken);

var migratorUser = await context.RunTask(new CreateDatabaseRoleTask(secretClient, serverConn, $"{DatabaseName}_migrator", Resources.User), cancellationToken);
andreasisnes marked this conversation as resolved.
Show resolved Hide resolved
var appUser = await context.RunTask(new CreateDatabaseRoleTask(secretClient, serverConn, $"{DatabaseName}_app", Resources.User), cancellationToken);
await context.RunTask(new CreateDatabaseTask(serverConn, DatabaseName), cancellationToken);
await context.RunTask(new GrantDatabasePrivilegesTask(serverConn, DatabaseName, migratorUser.RoleName, "CREATE, CONNECT"), cancellationToken);
await context.RunTask(new GrantDatabasePrivilegesTask(serverConn, DatabaseName, appUser.RoleName, "CONNECT"), cancellationToken);

connStringBuilder.Database = DatabaseName;
var dbConnString = connStringBuilder.ToString();
await using var dbConn = new NpgsqlConnection(dbConnString);
await context.RunTask(
$"Connecting to database '[cyan]{DatabaseName}[/]'",
(_, ct) => dbConn.OpenAsync(ct),
cancellationToken);

foreach (var (schemaName, schemaCfg) in Schemas)
{
await context.RunTask(new CreateDatabaseSchemaTask(dbConn, migratorUser.RoleName, schemaName, schemaCfg), cancellationToken);
}

connStringBuilder = new NpgsqlConnectionStringBuilder()
{
Host = serverUrl,
Database = DatabaseName,
Username = Resources.User,
Password = token.Token,
Port = 5432,
SslMode = SslMode.Require,
};

var connectionStrings = new Dictionary<string, string>();

connStringBuilder.Username = migratorUser.RoleName;
connStringBuilder.Password = migratorUser.Password;
connectionStrings[$"db-{DatabaseName}-migrator"] = connStringBuilder.ToString();

connStringBuilder.Username = appUser.RoleName;
connStringBuilder.Password = appUser.Password;
connectionStrings[$"db-{DatabaseName}-app"] = connStringBuilder.ToString();

await context.RunTask(new SaveConnectionStringsTask(secretClient, connectionStrings), cancellationToken);
}

public record ResourcesConfig
{
[JsonPropertyName("subscriptionId")]
public required string SubscriptionId { get; init; }

[JsonPropertyName("resourceGroup")]
public required string ResourceGroupName { get; init; }

[JsonPropertyName("serverName")]
public required string ServerName { get; init; }

[JsonPropertyName("user")]
public required string User { get; init; }

[JsonPropertyName("keyVaultName")]
public required string KeyVaultName { get; init; }
}

public record SchemaBootstrapModel
{
}
}
99 changes: 99 additions & 0 deletions ....DeployApi/src/Altinn.Authorization.DeployApi/BootstrapDatabase/CreateDatabaseRoleTask.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
using System.Security.Cryptography;
using Altinn.Authorization.DeployApi.Tasks;
using Azure;
using Azure.Security.KeyVault.Secrets;
using Npgsql;
using Spectre.Console;

namespace Altinn.Authorization.DeployApi.BootstrapDatabase;

internal sealed class CreateDatabaseRoleTask
: StepTask<CreateDatabaseRoleTask.Result>
{
private readonly SecretClient _secrets;
private readonly NpgsqlConnection _conn;
private readonly string _roleName;
private readonly string _adminRole;

public CreateDatabaseRoleTask(SecretClient secrets, NpgsqlConnection conn, string roleName, string adminRole)
{
_secrets = secrets;
_conn = conn;
_roleName = roleName;
_adminRole = adminRole;
}

public override string Name => $"Creating role '[cyan]{_roleName}[/]'";

public override async Task<Result> ExecuteAsync(ProgressTask task, CancellationToken cancellationToken)
{
var pw = await GetOrCreatePassword(task, cancellationToken);
var secretName = $"-db-{_roleName.Replace('_', '-')}-pw";
var secret = await _secrets.GetSecretAsync(secretName, cancellationToken: cancellationToken);

await using var cmd = _conn.CreateCommand();
cmd.CommandText =
/*strpsql*/$"""
CREATE ROLE "{_roleName}"
WITH LOGIN
PASSWORD '{pw.Password}'
""";

try
{
await cmd.ExecuteNonQueryAsync(cancellationToken);
}
catch (PostgresException ex) when (ex.SqlState == "42710")
andreasisnes marked this conversation as resolved.
Show resolved Hide resolved
{
if (pw.Updated)
{
cmd.CommandText =
/*strpsql*/$"""
ALTER ROLE "{_roleName}"
WITH LOGIN
PASSWORD '{pw.Password}'
""";

await cmd.ExecuteNonQueryAsync(cancellationToken);
}
}

cmd.CommandText =
/*strpsql*/$"""
GRANT "{_roleName}" TO "{_adminRole}"
""";

await cmd.ExecuteNonQueryAsync(cancellationToken);

return new Result(_roleName, pw.Password);
}

private async Task<PasswordResult> GetOrCreatePassword(ProgressTask task, CancellationToken cancellationToken)
{
var secretName = $"-db-{_roleName.Replace('_', '-')}-pw";
Response<KeyVaultSecret> secret;
bool updated = false;
try
{
secret = await _secrets.GetSecretAsync(secretName, cancellationToken: cancellationToken);
}
catch (RequestFailedException ex) when (ex.ErrorCode == "SecretNotFound")
{
var pw = GenerateRandomPw();
secret = await _secrets.SetSecretAsync(secretName, pw, cancellationToken: cancellationToken);
updated = true;
}

return new PasswordResult(secret.Value.Value, updated);
}

private static string GenerateRandomPw()
{
const string VALID_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*()_+";
return RandomNumberGenerator.GetString(VALID_CHARS.AsSpan(), 64);
}

private record PasswordResult(string Password, bool Updated);

internal record Result(string RoleName, string Password);
}
Loading
Loading