From c78a376d62a987f8db56c33dc0008900460b4231 Mon Sep 17 00:00:00 2001 From: Adam Grabski Date: Thu, 22 Aug 2024 21:58:03 +0200 Subject: [PATCH 1/2] Added the ability to generate compose files with enviroment variables, as parameter values --- docs/Writerside/topics/Generate-Command.md | 64 +++++++++++-------- .../Actions/Secrets/PopulateInputsAction.cs | 1 - .../PopulateInputsWithEnvVariablesAction.cs | 28 ++++++++ src/Aspirate.Commands/Commands/BaseCommand.cs | 7 +- .../Commands/BaseCommandOptions.cs | 1 + .../Commands/Generate/GenerateCommand.cs | 45 ++++++------- .../Generate/GenerateCommandHandler.cs | 33 +++++++--- .../Commands/Generate/GenerateOptions.cs | 3 + .../UseEnvVariablesAsParameterValuesOption.cs | 16 +++++ .../ServiceCollectionExtensions.cs | 3 +- .../Json/JsonExpressionProcessor.cs | 9 ++- .../Commands/Contracts/IGenerateOptions.cs | 1 + .../Models/Aspirate/AspirateState.cs | 11 ++-- 13 files changed, 152 insertions(+), 70 deletions(-) create mode 100644 src/Aspirate.Commands/Actions/Secrets/PopulateInputsWithEnvVariablesAction.cs create mode 100644 src/Aspirate.Commands/Options/UseEnvVariablesAsParameterValuesOption.cs diff --git a/docs/Writerside/topics/Generate-Command.md b/docs/Writerside/topics/Generate-Command.md index 005769a5..04b55d0a 100644 --- a/docs/Writerside/topics/Generate-Command.md +++ b/docs/Writerside/topics/Generate-Command.md @@ -22,6 +22,13 @@ aspirate generate --output-format compose ``` Your docker-compose file will be at the path `%output-dir%/docker-compose.yaml` directory by default. +It will have all parameter references replaced by their values. + +To generate a compose file that replaces parameter references with enviroment variables, that you can then pass during deployment, use the `--use-env-variables-as-parameter-values` flag: + +```bash +aspirate generate --output-format compose --use-env-variables-as-parameter-values +``` When using the `--output-format compose` flag, you can also build certain dockerfiles using the compose file. This will skip the build and push in Aspirate. @@ -49,31 +56,32 @@ a Helm chart is what's classed as an "Ejected Deployment" and is not managed by ## Cli Options (Optional) -| Option | Alias | Environmental Variable Counterpart | Description | -|-------------------------------|-------|--------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| --project-path | -p | `ASPIRATE_PROJECT_PATH` | The path to the aspire project. | -| --aspire-manifest | -m | `ASPIRATE_ASPIRE_MANIFEST_PATH` | The aspire manifest file to use | -| --output-path | -o | `ASPIRATE_OUTPUT_PATH` | The path to the output directory. Defaults to `%output-dir%` | -| --skip-build | | `ASPIRATE_SKIP_BUILD` | Skips build and Push of containers. | -| --disable-state | | `ASPIRATE_DISABLE_STATE` | Disable aspirate state management. | -| --namespace | | `ASPIRATE_NAMESPACE` | Generates a Kubernetes Namespace resource, and applies the namespace to all generated resources. Will be used at deployment time. | -| --skip-final | -sf | `ASPIRATE_SKIP_FINAL_KUSTOMIZE_GENERATION` | Skips The final generation of the kustomize manifest, which is the parent top level file | -| --container-image-tag | -ct | `ASPIRATE_CONTAINER_IMAGE_TAG` | The Container Image Tag to use as the fall-back value for all containers. | -| --container-registry | -cr | `ASPIRATE_CONTAINER_REGISTRY` | The Container Registry to use as the fall-back value for all containers. | -| --container-repository-prefix | | `ASPIRATE_CONTAINER_REPOSITORY_PREFIX` | The Container Repository Prefix to use as the fall-back value for all containers. | -| --container-builder | | `ASPIRATE_CONTAINER_BUILDER` | The Container Builder: can be `docker` or `podman`. The default is `docker`. | -| --image-pull-policy | | `ASPIRATE_IMAGE_PULL_POLICY` | The image pull policy to use for all containers in generated manifests. Can be `Always`, `Never` or `IfNotPresent`. For your local docker desktop cluster - use `IfNotPresent` | -| --disable-secrets | | `ASPIRATE_DISABLE_SECRETS` | Disables secrets management features. | -| --output-format | | `ASPIRATE_OUTPUT_FORMAT` | Sets the output manifest format. Defaults to `kustomize`. Can be `kustomize`, `helm` or `compose`. | -| --runtime-identifier | | `ASPIRATE_RUNTIME_IDENTIFIER` | Sets the runtime identifier for project builds. Defaults to `linux-x64`. | -| --secret-password | | `ASPIRATE_SECRET_PASSWORD` | If using secrets, or you have a secret file - Specify the password to decrypt them | -| --non-interactive | | `ASPIRATE_NON_INTERACTIVE` | Disables interactive mode for the command | -| --private-registry | | `ASPIRATE_PRIVATE_REGISTRY` | Enables usage of a private registry - which will produce image pull secret. | -| --private-registry-url | | `ASPIRATE_PRIVATE_REGISTRY_URL` | The url for the private registry | -| --private-registry-username | | `ASPIRATE_PRIVATE_REGISTRY_USERNAME` | The username for the private registry. This is required if passing `--private-registry`. | -| --private-registry-password | | `ASPIRATE_PRIVATE_REGISTRY_PASSWORD` | The password for the private registry. This is required if passing `--private-registry`. | -| --private-registry-email | | `ASPIRATE_PRIVATE_REGISTRY_EMAIL` | The email for the private registry. This is purely optional and will default to `aspirate@aspirate.com`. | -| --include-dashboard | | `ASPIRATE_INCLUDE_DASHBOARD` | Boolean flag to specify if the Aspire dashboard should also be included in deployments. | -| --compose-build | | | Can be included one or more times to set certain dockerfile resource building to be handled by the compose file. This will skip build and push in aspirate. | -| --launch-profile | -lp | `ASPIRATE_LAUNCH_PROFILE` | The launch profile to use when building the Aspire Manifest. | -| --replace-secrets | | `ASPIRATE_REPLACE_SECRETS` | The secret state will be completely reinitialised, prompting for a new password. All input values and secrets will be re generated / prompted, and stored in the state. | \ No newline at end of file +| Option | Alias | Environmental Variable Counterpart | Description | +|---------------------------------------|-------|--------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| --project-path | -p | `ASPIRATE_PROJECT_PATH` | The path to the aspire project. | +| --aspire-manifest | -m | `ASPIRATE_ASPIRE_MANIFEST_PATH` | The aspire manifest file to use | +| --output-path | -o | `ASPIRATE_OUTPUT_PATH` | The path to the output directory. Defaults to `%output-dir%` | +| --skip-build | | `ASPIRATE_SKIP_BUILD` | Skips build and Push of containers. | +| --disable-state | | `ASPIRATE_DISABLE_STATE` | Disable aspirate state management. | +| --namespace | | `ASPIRATE_NAMESPACE` | Generates a Kubernetes Namespace resource, and applies the namespace to all generated resources. Will be used at deployment time. | +| --skip-final | -sf | `ASPIRATE_SKIP_FINAL_KUSTOMIZE_GENERATION` | Skips The final generation of the kustomize manifest, which is the parent top level file | +| --container-image-tag | -ct | `ASPIRATE_CONTAINER_IMAGE_TAG` | The Container Image Tag to use as the fall-back value for all containers. | +| --container-registry | -cr | `ASPIRATE_CONTAINER_REGISTRY` | The Container Registry to use as the fall-back value for all containers. | +| --container-repository-prefix | | `ASPIRATE_CONTAINER_REPOSITORY_PREFIX` | The Container Repository Prefix to use as the fall-back value for all containers. | +| --container-builder | | `ASPIRATE_CONTAINER_BUILDER` | The Container Builder: can be `docker` or `podman`. The default is `docker`. | +| --image-pull-policy | | `ASPIRATE_IMAGE_PULL_POLICY` | The image pull policy to use for all containers in generated manifests. Can be `Always`, `Never` or `IfNotPresent`. For your local docker desktop cluster - use `IfNotPresent` | +| --disable-secrets | | `ASPIRATE_DISABLE_SECRETS` | Disables secrets management features. | +| --output-format | | `ASPIRATE_OUTPUT_FORMAT` | Sets the output manifest format. Defaults to `kustomize`. Can be `kustomize`, `helm` or `compose`. | +| --runtime-identifier | | `ASPIRATE_RUNTIME_IDENTIFIER` | Sets the runtime identifier for project builds. Defaults to `linux-x64`. | +| --secret-password | | `ASPIRATE_SECRET_PASSWORD` | If using secrets, or you have a secret file - Specify the password to decrypt them | +| --non-interactive | | `ASPIRATE_NON_INTERACTIVE` | Disables interactive mode for the command | +| --private-registry | | `ASPIRATE_PRIVATE_REGISTRY` | Enables usage of a private registry - which will produce image pull secret. | +| --private-registry-url | | `ASPIRATE_PRIVATE_REGISTRY_URL` | The url for the private registry | +| --private-registry-username | | `ASPIRATE_PRIVATE_REGISTRY_USERNAME` | The username for the private registry. This is required if passing `--private-registry`. | +| --private-registry-password | | `ASPIRATE_PRIVATE_REGISTRY_PASSWORD` | The password for the private registry. This is required if passing `--private-registry`. | +| --private-registry-email | | `ASPIRATE_PRIVATE_REGISTRY_EMAIL` | The email for the private registry. This is purely optional and will default to `aspirate@aspirate.com`. | +| --include-dashboard | | `ASPIRATE_INCLUDE_DASHBOARD` | Boolean flag to specify if the Aspire dashboard should also be included in deployments. | +| --compose-build | | | Can be included one or more times to set certain dockerfile resource building to be handled by the compose file. This will skip build and push in aspirate. | +| --launch-profile | -lp | `ASPIRATE_LAUNCH_PROFILE` | The launch profile to use when building the Aspire Manifest. | +| --replace-secrets | | `ASPIRATE_REPLACE_SECRETS` | The secret state will be completely reinitialised, prompting for a new password. All input values and secrets will be re generated / prompted, and stored in the state. | +|--use-env-variables-as-parameter-values| | `ASPIRATE_USE_ENV_VARIABLES` | Replace parameter references with enviroment variable names | diff --git a/src/Aspirate.Commands/Actions/Secrets/PopulateInputsAction.cs b/src/Aspirate.Commands/Actions/Secrets/PopulateInputsAction.cs index a9e34013..8bb2dcc5 100644 --- a/src/Aspirate.Commands/Actions/Secrets/PopulateInputsAction.cs +++ b/src/Aspirate.Commands/Actions/Secrets/PopulateInputsAction.cs @@ -1,5 +1,4 @@ namespace Aspirate.Commands.Actions.Secrets; - public sealed class PopulateInputsAction( IPasswordGenerator passwordGenerator, IServiceProvider serviceProvider, diff --git a/src/Aspirate.Commands/Actions/Secrets/PopulateInputsWithEnvVariablesAction.cs b/src/Aspirate.Commands/Actions/Secrets/PopulateInputsWithEnvVariablesAction.cs new file mode 100644 index 00000000..c5667c7e --- /dev/null +++ b/src/Aspirate.Commands/Actions/Secrets/PopulateInputsWithEnvVariablesAction.cs @@ -0,0 +1,28 @@ +namespace Aspirate.Commands.Actions.Secrets; + +public sealed class PopulateInputsWithEnvVariablesAction(IServiceProvider serviceProvider) : BaseAction(serviceProvider) +{ + public override Task ExecuteAsync() + { + Logger.WriteRuler("[purple]Generating enviroment variable placeholders for inputs[/]"); + + var parameterResources = CurrentState.LoadedAspireManifestResources.Where(x => x.Value is ParameterResource).ToArray(); + + if (parameterResources.Length == 0) + { + return Task.FromResult(true); + } + + foreach (var parameter in parameterResources) + { + if (parameter.Value is ParameterResource resource) + { + resource.Value = $"${{{resource.Name.Replace('-', '_').ToUpperInvariant()}}}"; + } + } + + Logger.MarkupLine($"[green]({EmojiLiterals.CheckMark}) Done: [/] Input values have all been assigned enviroment variables."); + + return Task.FromResult(true); + } +} diff --git a/src/Aspirate.Commands/Commands/BaseCommand.cs b/src/Aspirate.Commands/Commands/BaseCommand.cs index aa3646b2..a74be9dd 100644 --- a/src/Aspirate.Commands/Commands/BaseCommand.cs +++ b/src/Aspirate.Commands/Commands/BaseCommand.cs @@ -40,9 +40,10 @@ private async Task ConstructCommand(TOptions options, IServiceCollection se await stateService.RestoreState(stateOptions); handler.CurrentState.PopulateStateFromOptions(options); - - LoadSecrets(options, secretService, handler); - + if (options.RequiresSecrets) + { + LoadSecrets(options, secretService, handler); + } var exitCode = await handler.HandleAsync(options); await stateService.SaveState(stateOptions); diff --git a/src/Aspirate.Commands/Commands/BaseCommandOptions.cs b/src/Aspirate.Commands/Commands/BaseCommandOptions.cs index 6d7c50b0..e01ed98a 100644 --- a/src/Aspirate.Commands/Commands/BaseCommandOptions.cs +++ b/src/Aspirate.Commands/Commands/BaseCommandOptions.cs @@ -8,4 +8,5 @@ public abstract class BaseCommandOptions : ICommandOptions public bool? DisableState { get; set; } public string? SecretPassword { get; set; } public string? LaunchProfile { get; set; } + public virtual bool RequiresSecrets => true; } diff --git a/src/Aspirate.Commands/Commands/Generate/GenerateCommand.cs b/src/Aspirate.Commands/Commands/Generate/GenerateCommand.cs index dcaa0826..f4c9adb4 100644 --- a/src/Aspirate.Commands/Commands/Generate/GenerateCommand.cs +++ b/src/Aspirate.Commands/Commands/Generate/GenerateCommand.cs @@ -6,27 +6,28 @@ public sealed class GenerateCommand : BaseCommand HandleAsync(GenerateOptions options) return outputFormat.Name switch { nameof(OutputFormat.Kustomize) => GenerateKustomizeManifests(), - nameof(OutputFormat.DockerCompose) => GenerateDockerComposeManifests(), + nameof(OutputFormat.DockerCompose) => GenerateDockerComposeManifests(options.UseEnvVariablesAsParameterValues ?? false), nameof(OutputFormat.Helm) => GenerateHelmManifests(), _ => throw new ArgumentOutOfRangeException(nameof(options.OutputFormat), $"The output format '{options.OutputFormat}' is not supported."), }; } - private ActionExecutor BaseGenerateActionSequence() => - ActionExecutor + private ActionExecutor BaseGenerateActionSequence(bool useEnvVariablesAsParameterValues = false) + { + var result = ActionExecutor .QueueAction(nameof(LoadConfigurationAction)) .QueueAction(nameof(GenerateAspireManifestAction)) .QueueAction(nameof(LoadAspireManifestAction)) - .QueueAction(nameof(IncludeAspireDashboardAction)) - .QueueAction(nameof(PopulateInputsAction)) + .QueueAction(nameof(IncludeAspireDashboardAction)); + if (!useEnvVariablesAsParameterValues) + { + result.QueueAction(nameof(PopulateInputsAction)); + } + else + { + result.QueueAction(nameof(PopulateInputsWithEnvVariablesAction)); + } + result .QueueAction(nameof(SubstituteValuesAspireManifestAction)) .QueueAction(nameof(ApplyDaprAnnotationsAction)) .QueueAction(nameof(PopulateContainerDetailsForProjectsAction)) .QueueAction(nameof(BuildAndPushContainersFromProjectsAction)) - .QueueAction(nameof(BuildAndPushContainersFromDockerfilesAction)) - .QueueAction(nameof(SaveSecretsAction)); + .QueueAction(nameof(BuildAndPushContainersFromDockerfilesAction)); + if (!useEnvVariablesAsParameterValues) + { + result.QueueAction(nameof(SaveSecretsAction)); + } + + return result; + } private ActionExecutor BaseKubernetesActionSequence() => BaseGenerateActionSequence() .QueueAction(nameof(AskImagePullPolicyAction)); - private Task GenerateDockerComposeManifests() => - BaseGenerateActionSequence() + private Task GenerateDockerComposeManifests(bool useEnvVariablesAsParameterValues) => + BaseGenerateActionSequence(useEnvVariablesAsParameterValues) .QueueAction(nameof(GenerateDockerComposeManifestAction)) .ExecuteCommandsAsync(); diff --git a/src/Aspirate.Commands/Commands/Generate/GenerateOptions.cs b/src/Aspirate.Commands/Commands/Generate/GenerateOptions.cs index d49c326e..0c653099 100644 --- a/src/Aspirate.Commands/Commands/Generate/GenerateOptions.cs +++ b/src/Aspirate.Commands/Commands/Generate/GenerateOptions.cs @@ -30,4 +30,7 @@ public sealed class GenerateOptions : BaseCommandOptions, public bool? WithPrivateRegistry { get; set; } public bool? IncludeDashboard { get; set; } public bool? ReplaceSecrets { get; set; } + public bool? UseEnvVariablesAsParameterValues { get; set; } + + public override bool RequiresSecrets => (!UseEnvVariablesAsParameterValues) ?? true; } diff --git a/src/Aspirate.Commands/Options/UseEnvVariablesAsParameterValuesOption.cs b/src/Aspirate.Commands/Options/UseEnvVariablesAsParameterValuesOption.cs new file mode 100644 index 00000000..092ce310 --- /dev/null +++ b/src/Aspirate.Commands/Options/UseEnvVariablesAsParameterValuesOption.cs @@ -0,0 +1,16 @@ +namespace Aspirate.Commands.Options; + +public sealed class UseEnvVariablesAsParameterValuesOption : BaseOption +{ + private static readonly string[] _aliases = ["--use-env-variables-as-parameter-values"]; + + private UseEnvVariablesAsParameterValuesOption() : base(_aliases, "ASPIRATE_USE_ENV_VARIABLES", false) + { + Name = nameof(IGenerateOptions.UseEnvVariablesAsParameterValues); + Description = "Replace parameter references with enviroment variable names"; + Arity = ArgumentArity.ZeroOrOne; + IsRequired = false; + } + + public static UseEnvVariablesAsParameterValuesOption Instance { get; } = new(); +} diff --git a/src/Aspirate.Commands/ServiceCollectionExtensions.cs b/src/Aspirate.Commands/ServiceCollectionExtensions.cs index ee7415a3..378e34a3 100644 --- a/src/Aspirate.Commands/ServiceCollectionExtensions.cs +++ b/src/Aspirate.Commands/ServiceCollectionExtensions.cs @@ -1,4 +1,4 @@ -namespace Aspirate.Commands; +namespace Aspirate.Commands; /// /// Extension methods for IServiceCollection to register services for AspirateState and AspirateActions. @@ -39,6 +39,7 @@ public static IServiceCollection AddAspirateActions(this IServiceCollection serv .RegisterAction() .RegisterAction() .RegisterAction() + .RegisterAction() .RegisterAction() .RegisterAction() .RegisterAction() diff --git a/src/Aspirate.Processors/Transformation/Json/JsonExpressionProcessor.cs b/src/Aspirate.Processors/Transformation/Json/JsonExpressionProcessor.cs index 175a39b5..4073f34b 100644 --- a/src/Aspirate.Processors/Transformation/Json/JsonExpressionProcessor.cs +++ b/src/Aspirate.Processors/Transformation/Json/JsonExpressionProcessor.cs @@ -26,7 +26,7 @@ public void ResolveJsonExpressions(JsonNode? jsonNode, JsonNode rootNode) } while (_unresolvedExpressionPointers.Count > 0); } - [GeneratedRegex(@"\{([\w\.-]+)\}")] + [GeneratedRegex(@"(\$|)\{([\w\.-]+)\}")] private static partial Regex PlaceholderPatternRegex(); public static IJsonExpressionProcessor CreateDefaultExpressionProcessor() => @@ -103,7 +103,12 @@ private void ReplaceWithResolvedExpression(JsonNode rootNode, JsonNode jsonValue for (var i = 0; i < matches.Count; i++) { var match = matches[i]; - var jsonPath = match.Groups[1].Value; + if (!string.IsNullOrEmpty(match.Groups[1].Value)) + { + continue; + } + + var jsonPath = match.Groups[2].Value; var pathParts = jsonPath.Split('.'); if (pathParts.Length == 1) { diff --git a/src/Aspirate.Shared/Interfaces/Commands/Contracts/IGenerateOptions.cs b/src/Aspirate.Shared/Interfaces/Commands/Contracts/IGenerateOptions.cs index 1736f483..18e2d565 100644 --- a/src/Aspirate.Shared/Interfaces/Commands/Contracts/IGenerateOptions.cs +++ b/src/Aspirate.Shared/Interfaces/Commands/Contracts/IGenerateOptions.cs @@ -8,4 +8,5 @@ public interface IGenerateOptions bool? SkipFinalKustomizeGeneration { get; set; } string? ImagePullPolicy { get; set; } string? OutputFormat { get; set; } + bool? UseEnvVariablesAsParameterValues { get; set; } } diff --git a/src/Aspirate.Shared/Models/Aspirate/AspirateState.cs b/src/Aspirate.Shared/Models/Aspirate/AspirateState.cs index 91eb3873..b5bf3824 100644 --- a/src/Aspirate.Shared/Models/Aspirate/AspirateState.cs +++ b/src/Aspirate.Shared/Models/Aspirate/AspirateState.cs @@ -135,13 +135,13 @@ public class AspirateState : public List AspireComponentsToProcess { get; set; } = []; [JsonIgnore] - public Dictionary LoadedAspireManifestResources { get; set; } = new(); + public Dictionary LoadedAspireManifestResources { get; set; } = []; [JsonIgnore] public string? PrivateRegistryPassword { get; set; } [JsonIgnore] - public Dictionary FinalResources { get; } = new(); + public Dictionary FinalResources { get; } = []; [JsonIgnore] public bool StateWasLoadedFromPrevious { get; set; } @@ -179,6 +179,9 @@ public class AspirateState : [JsonIgnore] public string? SecretPassword { get; set; } + [JsonIgnore] + public bool? UseEnvVariablesAsParameterValues { get; set; } + public void AppendToFinalResources(string key, Resource resource) => FinalResources.Add(key, resource); @@ -186,10 +189,10 @@ public bool IsNotDeployable(Resource resource) { if (OutputFormat.Equals("compose", StringComparison.OrdinalIgnoreCase)) { - return (resource is ParameterResource or ValueResource); + return resource is ParameterResource or ValueResource; } - return (resource is DaprResource or ParameterResource or ValueResource); + return resource is DaprResource or ParameterResource or ValueResource; } [JsonIgnore] From 5cb6ef8f5969d6233462ac2595354c7078f58166 Mon Sep 17 00:00:00 2001 From: Adam Grabski Date: Thu, 22 Aug 2024 21:59:18 +0200 Subject: [PATCH 2/2] missing period --- docs/Writerside/topics/Generate-Command.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Writerside/topics/Generate-Command.md b/docs/Writerside/topics/Generate-Command.md index 04b55d0a..de3ac722 100644 --- a/docs/Writerside/topics/Generate-Command.md +++ b/docs/Writerside/topics/Generate-Command.md @@ -84,4 +84,4 @@ a Helm chart is what's classed as an "Ejected Deployment" and is not managed by | --compose-build | | | Can be included one or more times to set certain dockerfile resource building to be handled by the compose file. This will skip build and push in aspirate. | | --launch-profile | -lp | `ASPIRATE_LAUNCH_PROFILE` | The launch profile to use when building the Aspire Manifest. | | --replace-secrets | | `ASPIRATE_REPLACE_SECRETS` | The secret state will be completely reinitialised, prompting for a new password. All input values and secrets will be re generated / prompted, and stored in the state. | -|--use-env-variables-as-parameter-values| | `ASPIRATE_USE_ENV_VARIABLES` | Replace parameter references with enviroment variable names | +|--use-env-variables-as-parameter-values| | `ASPIRATE_USE_ENV_VARIABLES` | Replace parameter references with enviroment variable names. |