feat: Add configuration adapter and enhance sample project with new input files and settings

Author: Stephane Royer

src/Paillave.Etl.FromConfigurationConnectors/ConfigurationAdapter.cs

@@ -0,0 +1,86 @@
+using System;
+using System.Linq;
+using System.Text.Json.Nodes;
+using Microsoft.Extensions.Configuration;
+using Paillave.Etl.Core;
+
+namespace Paillave.Etl.FromConfigurationConnectors;
+
+public class ConfigurationAdapter(IConfiguration configuration, ConfigurationFileValueConnectorParser parser, Func<string, string> resolveSensitiveValues)
+{
+    private const string code = "DEVITEMSOURCE";
+    public IFileValueProvider GetFileValueProvider(string sectionName)
+        => CreateFileValueConnectors(sectionName, ConnectorType.Provider).GetProvider(code);
+    public IFileValueProcessor GetFileValueProcessor(string sectionName)
+        => CreateFileValueConnectors(sectionName, ConnectorType.Processor).GetProcessor(code);
+
+    private IFileValueConnectors CreateFileValueConnectors(string sectionName, ConnectorType connectorType)
+    {
+        IConfigurationSection configurationSection = configuration.GetSection(sectionName);
+        var section = configurationSection.ToJsonNode();
+        var definition = BuildConnectorConfiguration(section, code, connectorType) ?? throw new InvalidOperationException("No configuration found for a connector");
+        var fileValueConnectors = parser.GetConnectors(definition, resolveSensitiveValues);
+        return fileValueConnectors;
+    }
+    private enum ConnectorType
+    {
+        Provider,
+        Processor
+    }
+    private string? BuildConnectorConfiguration(JsonNode? configuration, string code, ConnectorType connectorType)
+    {
+        if (configuration is null)
+            return null;
+
+        var obj = new JsonObject()
+        {
+            ["src"] = new JsonObject()
+            {
+                ["Type"] = configuration?["Type"]?.DeepClone(),
+                ["Connection"] = configuration?["Connection"]?.DeepClone()
+            }
+        };
+        switch (connectorType)
+        {
+            case ConnectorType.Provider:
+                obj["src"]!["Providers"] = new JsonObject()
+                {
+                    [code] = configuration?["Provider"]?.DeepClone() ?? new JsonObject()
+                };
+                break;
+            case ConnectorType.Processor:
+                obj["src"]!["Processors"] = new JsonObject()
+                {
+                    [code] = configuration?["Processor"]?.DeepClone() ?? new JsonObject()
+                };
+                break;
+        }
+        return obj.ToJsonString();
+    }
+}
+public static class ConfigurationJsonExtensions
+{
+    public static JsonNode? ToJsonNode(this IConfigurationSection section)
+    {
+        if (!section.Exists()) return null;
+
+        var children = section.GetChildren().ToList();
+        if (children.Count == 0)
+        {
+            return section.Value is null ? JsonValue.Create((string?)null) : JsonValue.Create(section.Value);
+        }
+
+        if (children.All(c => int.TryParse(c.Key, out _)))
+        {
+            var arr = new JsonArray();
+            foreach (var child in children.OrderBy(c => int.Parse(c.Key)))
+                arr.Add(child.ToJsonNode());
+            return arr;
+        }
+
+        var obj = new JsonObject();
+        foreach (var child in children)
+            obj[child.Key] = child.ToJsonNode();
+        return obj;
+    }
+}

src/Paillave.Etl.FromConfigurationConnectors/Paillave.Etl.FromConfigurationConnectors.csproj

@@ -11,7 +11,8 @@
     <ProjectReference Include="..\Paillave.Etl\Paillave.Etl.csproj" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" allowedVersions="13.0.1"/>
-    <PackageReference Include="NJsonSchema" Version="11.1.0" allowedVersions="10.6.6"/>
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.5" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" allowedVersions="13.0.1" />
+    <PackageReference Include="NJsonSchema" Version="11.1.0" allowedVersions="10.6.6" />
   </ItemGroup>
 </Project>

src/Paillave.Etl/Core/FileValueProviderBase.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using System.IO;
 using System.Text.Json.Nodes;
 using System.Threading;
 
@@ -14,15 +16,25 @@ namespace Paillave.Etl.Core;
 
     public virtual string TypeName => this.GetType().Name;
 
-    public void Provide(object input, Action<IFileValue, FileReference> pushFileValue, CancellationToken cancellationToken)
+    public void Provide(object? input, Action<IFileValue, FileReference> pushFileValue, CancellationToken cancellationToken)
         => Provide(input, pushFileValue, connectionParameters, providerParameters, cancellationToken);
-    public void Provide(object input, Action<IFileValue> pushFileValue, CancellationToken cancellationToken)
+    public void Provide(object? input, Action<IFileValue> pushFileValue, CancellationToken cancellationToken)
         => Provide(input, (fileValue, FileReference) => pushFileValue(fileValue), cancellationToken);
-    protected abstract void Provide(object input, Action<IFileValue, FileReference> pushFileValue, TConnectionParameters connectionParameters, TProviderParameters providerParameters, CancellationToken cancellationToken);
+    protected abstract void Provide(object? input, Action<IFileValue, FileReference> pushFileValue, TConnectionParameters connectionParameters, TProviderParameters providerParameters, CancellationToken cancellationToken);
     public void Test() => Test(connectionParameters, providerParameters);
     protected abstract void Test(TConnectionParameters connectionParameters, TProviderParameters providerParameters);
     public void PushValues(object input, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
         => Provide(input, push, cancellationToken);
 
     public abstract IFileValue Provide(JsonNode? fileSpecific);
+
+    public IAsyncEnumerable<FileReference> Provide(object? input = null, CancellationToken cancellationToken = default)
+    {
+        throw new NotImplementedException();
+    }
+
+    public Stream Open(JsonNode fileSpecific)
+    {
+        throw new NotImplementedException();
+    }
 }

src/Paillave.Etl/Core/IFileValueProvider.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using System.IO;
 using System.Text.Json.Nodes;
 using System.Threading;
 
@@ -9,7 +11,9 @@ public interface IFileValueProvider
     string Code { get; }
     ProcessImpact PerformanceImpact { get; }
     ProcessImpact MemoryFootPrint { get; }
-    void Provide(object input, Action<IFileValue, FileReference> pushFileValue, CancellationToken cancellationToken);
+    void Provide(object? input, Action<IFileValue, FileReference> pushFileValue, CancellationToken cancellationToken);
     IFileValue Provide(JsonNode fileSpecific);
+    // IAsyncEnumerable<FileReference> Provide(object? input = null, CancellationToken cancellationToken = default);
+    // Stream Open(JsonNode fileSpecific);
     void Test();
 }

src/Tutorials/Paillave.Etl.Samples/InputFiles2/Input01.Portfolios.csv

@@ -0,0 +1,4 @@
+SicavCode,SicavName,SicavType,PortfolioCode,PortfolioName
+S1,Sicav1,UCITS,P1,Portfolio1
+S2,Sicav2,AIFM,P2,Portfolio2
+S1,Sicav1,UCITS,P3,Portfolio3

src/Tutorials/Paillave.Etl.Samples/InputFiles2/Input01.Positions.csv

@@ -0,0 +1,12 @@
+PortfolioCode,SecurityCode,Isin,SecurityName,SecurityClass,Issuer,Date,Value
+P1,S1,ISIN1,Security1,A,,20201020,10
+P1,S2,ISIN2,Security2,B,,20201020,11
+P1,S3,,Security3,,Issuer1,20201020,12
+P2,S4,ISIN4,Security4,C,,20201020,20
+P2,S2,ISIN2,Security2,B,,20201020,21
+P2,S3,,Security3,,Issuer1,20201020,22
+P2,S2,ISIN2,Security2,B,,20201020,31
+P2,S3,,Security3,,Issuer1,20201020,32
+P3,S1,ISIN1,Security1,A,,20201020,42
+P3,S2,ISIN2,Security2,B,,20201020,31
+P3,S3,,Security3,,Issuer1,20201020,32

src/Tutorials/Paillave.Etl.Samples/InputFiles2/Input02.Positions.csv

@@ -0,0 +1,12 @@
+PortfolioCode,SecurityCode,Isin,SecurityName,SecurityClass,Issuer,Date,Value
+P1,S1,ISIN1,Security1,A,,20201021,11
+P1,S2,ISIN2,Security2,B,,20201021,12
+P1,S3,,Security3,,Issuer1,20201021,13
+P2,S5,ISIN5,Security5,D,,20201021,21
+P2,S2,ISIN2,Security2,B,,20201021,22
+P2,S3,,Security3,,Issuer1,20201021,33
+P2,S3,,Security3,,Issuer1,20201021,33
+P2,S2,ISIN2,Security2,B,,20201021,32
+P2,S3,,Security3,,Issuer1,20201021,4
+P3,S2,ISIN2,Security2,B,,20201021,32
+P3,S3,,Security3,,Issuer1,20201021,4

src/Tutorials/Paillave.Etl.Samples/InputFiles2/PortfoliosWeight.csv

@@ -0,0 +1,7 @@
+Code,Date,Weight
+P1,"Tue 20 Oct, 2020",33
+P1,"Wed 21 Oct, 2020",36
+P2,"Tue 20 Oct, 2020",126
+P2,"Wed 21 Oct, 2020",145
+P3,"Tue 20 Oct, 2020",105
+P3,"Wed 21 Oct, 2020",36

src/Tutorials/Paillave.Etl.Samples/Paillave.Etl.Samples.csproj

@@ -26,6 +26,10 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.KeyPerFile" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="9.0.0" />
   </ItemGroup>
 
   <PropertyGroup>

src/Tutorials/Paillave.Etl.Samples/Program2.cs

@@ -10,6 +10,10 @@
 using Paillave.Etl.Core;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
+using Paillave.Etl.FromConfigurationConnectors;
+using Paillave.Etl.Mail;
+using Paillave.Etl.GraphApi;
 
 namespace Paillave.Etl.Samples;
 
@@ -35,6 +39,30 @@ static async Task Main(string[] args)
     //         /// <returns></returns>
     static async Task SimplyImportAsync(string[] args)
     {
+
+
+        var configuration = ApplicationConfigurationBuilder.GetConfiguration(args);
+
+
+        var configAdapter = new ConfigurationAdapter(
+            configuration,
+            new ConfigurationFileValueConnectorParser(
+                new MailProviderProcessorAdapter(),
+                new FileSystemProviderProcessorAdapter(),
+                new GraphApiProviderProcessorAdapter()),
+            s => s);
+
+        var provider = configAdapter.GetFileValueProvider("MyProvider");
+        var processor = configAdapter.GetFileValueProcessor("MyProcessor");
+        provider.Provide(new(), (fv, fr) =>
+        {
+            Console.WriteLine(fv.Name);
+            processor.Process(fv, i => { }, default);
+        }, default);
+
+        processor.Process(new InMemoryFileValue(new MemoryStream(), "TestFile.txt"), i => { }, default);
+
+
         var services = new ServiceCollection()
             .AddPooledDbContextFactory<DataAccess.TestDbContext>(options => options.UseSqlServer(@"Server=localhost,1433;Initial Catalog=TestEtl;Persist Security Info=False;User ID=SA;Password=<YourStrong@Passw0rd>;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=true;Connection Timeout=300;", options => options
                 .CommandTimeout(2000)
@@ -168,4 +196,22 @@ static async Task SimplyImportAsync(string[] args)
     //             var res = await processRunner.ExecuteAsync(args, executionOptions);
     //         }
     //     }
+}
+
+public static class ApplicationConfigurationBuilder
+{
+    public static IConfigurationRoot GetConfiguration(string[]? args = null) => new ConfigurationBuilder()
+        .SetBasePath(Directory.GetCurrentDirectory())
+        .AddJsonFile("appsettings.json", true, true)
+        .AddJsonFile($"appsettings.{Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT")}.json", true, true)
+        .AddKeyPerFile("appsettings", true, true)
+        .AddEnvironmentVariables()
+        .AddEnvironmentVariables("SAMPLEETLNET_")
+        .AddUserSecrets(typeof(Program).Assembly)
+        .AddCommandLine(args ?? [], new Dictionary<string, string>
+        {
+            ["-cnx"] = "ConnectionStrings:SqlServer",
+            ["--connectionString"] = "ConnectionStrings:SqlServer"
+        })
+        .Build();
 }