Merge pull request #501 from serilog-mssql/dev

Release 6.5.0

.github/workflows/pr-validation.yml

@@ -4,6 +4,10 @@ on:
   pull_request:
     branches: [ dev, main ]
 
+  # Run every biweekly to discover failures due to environment changes
+  schedule:
+    - cron: '0 0 1,15 * *'
+
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 

CHANGES.md

@@ -1,3 +1,7 @@
+# 6.5.0
+* Implemented #488: Support OpenTelemetry TraceId and SpanId as provided by Serilog core
+* Include README in NuGet package
+
 # 6.4.0
 * Implemented #436: Truncate additional columns (thanks to @nhart12)
 * Fixed issue #489: allow varchar on standard columns (thanks to @nhart12)

CONTRIBUTING.md

@@ -0,0 +1,3 @@
+# Contributing
+
+Please refer to the [Serilog Contributing Guidelines](https://github.com/serilog/serilog/blob/dev/CONTRIBUTING.md) which also apply to the MSSqlServer Sink.

Directory.Packages.props

@@ -23,9 +23,9 @@
     <PackageVersion Include="Moq" Version="4.18.2" />
     <PackageVersion Include="xunit" Version="2.4.2" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.4.5" />
-    <PackageVersion Include="Serilog" Version="2.12.0" />
+    <PackageVersion Include="Serilog" Version="3.1.1" />
     <PackageVersion Include="Serilog.Extensions.Hosting" Version="5.0.1" />
     <PackageVersion Include="Serilog.Settings.Configuration" Version="3.4.0" />
     <PackageVersion Include="Serilog.Sinks.PeriodicBatching" Version="3.1.0" />
   </ItemGroup>
-</Project>
+</Project>

README.md

@@ -406,18 +406,22 @@ By default (and consistent with the SQL DDL to create a table shown earlier) the
  - `StandardColumn.Exception`
  - `StandardColumn.Properties`
 
-There is one additional Standard Column which is not included by default (for backwards-compatibility reasons):
+There are the following additional standard columns which are not included by default (for backwards-compatibility reasons):
 
 - `StandardColumn.LogEvent`
+- `StandardColumn.TraceId`
+- `StandardColumn.SpanId`
 
 You can change this list as long as the underlying table definition is consistent:
 
 ```csharp
 // we don't need XML data
 columnOptions.Store.Remove(StandardColumn.Properties);
 
-// we do want JSON data
+// we do want JSON data and OpenTelemetry
 columnOptions.Store.Add(StandardColumn.LogEvent);
+columnOptions.Store.Add(StandardColumn.TraceId);
+columnOptions.Store.Add(StandardColumn.SpanId);
 ```
 
 In addition to any special properties described below, each Standard Column also has the usual column properties like `ColumnName` as described in the topic [SqlColumn Objects](#sqlcolumn-objects).
@@ -454,7 +458,7 @@ In case `DataLength` is set to a specific value different from -1, any message l
 
 This column stores the log event message with the property placeholders. It defaults to `nvarchar(max)`. The `DataType` property can only be set to character-storage types.
 
-In case `DataLength` is set to a specific value different from -1, any template text longer than that length will be effectively truncated to that size. Any truncating ignores all differences between the tokens in the template meaning that a template might get cut off in the middle of a property token. Example: `DataLength` is set to 20 and the message template is "a long {NumberOfCharacters} template text" (without the quotes), the final template stored in the database will be: "a long {NumberOfC..." (again without quotes).
+If `DataLength` is set to a value different to -1 longer text will be truncated. See [Message column](#message) for details.
 
 ### Level
 
@@ -484,7 +488,7 @@ When the `ConvertToUtc` property is set to `true`, the time stamp is adjusted to
 
 When an exception is logged as part of the log event, the exception message is stored here automatically. The `DataType` must be `nvarchar`.
 
-Similar to the columns `Message` and `MessageTemplate`, setting `DataLength` of `Exception` to a specific value different from -1 will effectively truncate any exception message to the stated length in `DataLength`.
+Similar to the columns `Message` and `MessageTemplate`, setting `DataLength` to a specific value different from -1 will effectively truncate any exception message to the stated length in `DataLength`. See [Message column](#message ) for details.
 
 ### Properties
 
@@ -506,13 +510,22 @@ If `OmitElementIfEmpty` is set then if a property is empty, it will not be seria
 
 This column stores log event property values as JSON. Typically you will use either this column or the XML-based `Properties` column, but not both. This column's `DataType` must always be `nvarchar`.
 
-The `ExcludeAddtionalProperties` and `ExcludeStandardColumns` properties are described in the [Custom Property Columns](#custom-property-columns) topic.
+By default this column is not used unless it is added to the `ColumnOptions.Store` property as documented [above](#standard-columns).
 
 The content of this column is rendered as JSON by default or with a custom ITextFormatter passed by the caller as parameter `logEventFormatter`. Details can be found in [Sink Configuration](#sink-configuration).
 
+### TraceId and SpanId
+
+These two columns store the OpenTelemetry `TraceId` and `SpanId` log event properties which are documented [here](https://github.com/serilog/serilog/issues/1923). The `DataType` of these columns must be `nvarchar` or `varchar`.
+
+By default these columns are not used unless they are added to the `ColumnOptions.Store` property as documented [above](#standard-columns).
+
 ## Custom Property Columns
 
-By default, any log event properties you include in your log statements will be saved to the XML `Properties` column or the JSON `LogEvent` column. But they can also be stored in their own individual columns via the `AdditionalColumns` collection. This adds overhead to write operations but is very useful for frequently-queried properties. Only `ColumnName` is required; the default configuration is `varchar(max)`. If you specify a DataLength on a column of character data types (NVarChar, VarChar, Char, NChar) the string will be automatically truncated to the datalength to fit in the column.
+By default, any log event properties you include in your log statements will be saved to the XML `Properties` column or the JSON `LogEvent` column. But they can also be stored in their own individual columns via the `AdditionalColumns` collection. This adds overhead to write operations but is very useful for frequently-queried properties. Only `ColumnName` is required; the default configuration is `varchar(max)`.
+
+If you specify a DataLength other than -1 on a column of character data types (NVarChar, VarChar, Char, NChar) longer text will be truncated to the specified length. See [Message column](#message ) for details.
+
 
 ```csharp
 var columnOptions = new ColumnOptions
@@ -601,7 +614,7 @@ As the name suggests, `columnOptionSection` is an entire configuration section i
     "disableTriggers": true,
     "clusteredColumnstoreIndex": false,
     "primaryKeyColumnName": "Id",
-    "addStandardColumns": [ "LogEvent" ],
+    "addStandardColumns": [ "LogEvent", "TraceId", "SpanId" ],
     "removeStandardColumns": [ "MessageTemplate", "Properties" ],
     "additionalColumns": [
         { "ColumnName": "EventType", "DataType": "int", "AllowNull": false },
@@ -665,6 +678,8 @@ Keys and values are case-sensitive. Case must match **_exactly_** as shown below
     <!-- ColumnOptions parameters -->
     <AddStandardColumns>
         <add Name="LogEvent"/>
+        <add Name="TraceId"/>
+        <add Name="SpanId"/>
     </AddStandardColumns>
     <RemoveStandardColumns>
         <remove Name="Properties"/>

sample/WorkerServiceDemo/appsettings.json

@@ -21,7 +21,7 @@
           },
           "restrictedToMinimumLevel": "Information",
           "columnOptionsSection": {
-            "addStandardColumns": [ "LogEvent" ],
+            "addStandardColumns": [ "LogEvent", "TraceId", "SpanId" ],
             "removeStandardColumns": [ "MessageTemplate", "Properties" ],
             "timeStamp": {
               "columnName": "Timestamp",
@@ -54,10 +54,10 @@
           "sinkOptionsSection": {
             "tableName": "LogEventsAudit",
             "autoCreateSqlDatabase": true,
-            "autoCreateSqlTable": true,
+            "autoCreateSqlTable": true
           },
           "columnOptionsSection": {
-            "addStandardColumns": [ "LogEvent" ],
+            "addStandardColumns": [ "LogEvent", "TraceId", "SpanId" ],
             "removeStandardColumns": [ "MessageTemplate", "Properties" ],
             "timeStamp": {
               "columnName": "Timestamp",

src/Serilog.Sinks.MSSqlServer/Configuration/Implementations/Microsoft.Extensions.Configuration/MicrosoftExtensionsColumnOptionsProvider.cs

@@ -147,6 +147,18 @@ private static void ReadStandardColumns(IConfigurationSection config, ColumnOpti
             section = config.GetSection("messageTemplate");
             if (section != null)
                 SetCommonColumnOptions(section, columnOptions.MessageTemplate);
+
+            section = config.GetSection("traceId");
+            if (section != null)
+            {
+                SetCommonColumnOptions(section, columnOptions.TraceId);
+            }
+
+            section = config.GetSection("spanId");
+            if (section != null)
+            {
+                SetCommonColumnOptions(section, columnOptions.SpanId);
+            }
         }
 
         private static void ReadMiscColumnOptions(IConfigurationSection config, ColumnOptions columnOptions)

src/Serilog.Sinks.MSSqlServer/Configuration/Implementations/System.Configuration/MSSqlServerConfigurationSection.cs

@@ -106,6 +106,18 @@ public StandardColumnConfigLevel Level
             get => (StandardColumnConfigLevel)base[nameof(Level)];
         }
 
+        [ConfigurationProperty(nameof(TraceId))]
+        public StandardColumnConfigTraceId TraceId
+        {
+            get => (StandardColumnConfigTraceId)base[nameof(TraceId)];
+        }
+
+        [ConfigurationProperty(nameof(SpanId))]
+        public StandardColumnConfigSpanId SpanId
+        {
+            get => (StandardColumnConfigSpanId)base[nameof(SpanId)];
+        }
+
         [ConfigurationProperty(nameof(LogEvent))]
         public StandardColumnConfigLogEvent LogEvent
         {

src/Serilog.Sinks.MSSqlServer/Configuration/Implementations/System.Configuration/StandardColumnConfigSpanId.cs

@@ -0,0 +1,24 @@
+using System.Configuration;
+
+// Disable XML comment warnings for internal config classes which are required to have public members
+#pragma warning disable 1591
+
+namespace Serilog.Sinks.MSSqlServer
+{
+    public class StandardColumnConfigSpanId : ColumnConfig
+    {
+        public StandardColumnConfigSpanId() : base()
+        { }
+
+        // override to set IsRequired = false
+        [ConfigurationProperty("ColumnName", IsRequired = false, IsKey = true)]
+        public override string ColumnName
+        {
+            get => base.ColumnName;
+            set => base.ColumnName = value;
+        }
+    }
+}
+
+#pragma warning restore 1591
+

src/Serilog.Sinks.MSSqlServer/Configuration/Implementations/System.Configuration/StandardColumnConfigTraceId.cs

@@ -0,0 +1,24 @@
+using System.Configuration;
+
+// Disable XML comment warnings for internal config classes which are required to have public members
+#pragma warning disable 1591
+
+namespace Serilog.Sinks.MSSqlServer
+{
+    public class StandardColumnConfigTraceId : ColumnConfig
+    {
+        public StandardColumnConfigTraceId() : base()
+        { }
+
+        // override to set IsRequired = false
+        [ConfigurationProperty("ColumnName", IsRequired = false, IsKey = true)]
+        public override string ColumnName
+        {
+            get => base.ColumnName;
+            set => base.ColumnName = value;
+        }
+    }
+}
+
+#pragma warning restore 1591
+

src/Serilog.Sinks.MSSqlServer/Configuration/Implementations/System.Configuration/SystemConfigurationColumnOptionsProvider.cs

@@ -105,6 +105,8 @@ private static void ReadStandardColumns(MSSqlServerConfigurationSection config,
             SetCommonColumnOptions(config.Id, columnOptions.Id);
             SetCommonColumnOptions(config.Level, columnOptions.Level);
             SetCommonColumnOptions(config.LogEvent, columnOptions.LogEvent);
+            SetCommonColumnOptions(config.TraceId, columnOptions.TraceId);
+            SetCommonColumnOptions(config.SpanId, columnOptions.SpanId);
             SetCommonColumnOptions(config.Message, columnOptions.Message);
             SetCommonColumnOptions(config.MessageTemplate, columnOptions.MessageTemplate);
             SetCommonColumnOptions(config.PropertiesColumn, columnOptions.Properties);

src/Serilog.Sinks.MSSqlServer/Serilog.Sinks.MSSqlServer.csproj

@@ -1,8 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <Description>A Serilog sink that writes events to Microsoft SQL Server</Description>
-    <VersionPrefix>6.4.0</VersionPrefix>
+    <Description>A Serilog sink that writes events to Microsoft SQL Server and Azure SQL</Description>
+    <VersionPrefix>6.5.0</VersionPrefix>
     <Authors>Michiel van Oudheusden;Christian Kadluba;Serilog Contributors</Authors>
     <TargetFrameworks>netstandard2.0;net462;net472;net6.0</TargetFrameworks>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
@@ -14,6 +14,7 @@
     <PackageId>Serilog.Sinks.MSSqlServer</PackageId>
     <PackageTags>serilog;sinks;mssqlserver</PackageTags>
     <PackageIcon>serilog-sink-nuget.png</PackageIcon>
+    <PackageReadmeFile>README.md</PackageReadmeFile>
     <PackageProjectUrl>https://github.com/serilog-mssql/serilog-sinks-mssqlserver</PackageProjectUrl>
     <PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
     <RepositoryUrl>https://github.com/serilog-mssql/serilog-sinks-mssqlserver</RepositoryUrl>
@@ -67,6 +68,7 @@
 
   <ItemGroup>
     <None Include="Images\serilog-sink-nuget.png" Pack="true" PackagePath="" />
+    <None Include="../../README.md" Pack="true" PackagePath="" />
   </ItemGroup>
 
   <ItemGroup>

src/Serilog.Sinks.MSSqlServer/Sinks/MSSqlServer/ColumnOptions/ColumnOptions.cs

@@ -20,6 +20,8 @@ public ColumnOptions()
             // Apply any defaults in the individual Standard Column constructors.
             Id = new IdColumnOptions();
             Level = new LevelColumnOptions();
+            TraceId = new TraceIdColumnOptions();
+            SpanId = new SpanIdColumnOptions();
             Properties = new PropertiesColumnOptions();
             Message = new MessageColumnOptions();
             MessageTemplate = new MessageTemplateColumnOptions();
@@ -113,6 +115,16 @@ public ICollection<StandardColumn> Store
         /// </summary>
         public LevelColumnOptions Level { get; private set; }
 
+        /// <summary>
+        /// Options for the TraceId column.
+        /// </summary>
+        public TraceIdColumnOptions TraceId { get; private set; }
+
+        /// <summary>
+        /// Options for the SpanId column.
+        /// </summary>
+        public SpanIdColumnOptions SpanId { get; private set; }
+
         /// <summary>
         /// Options for the Properties column.
         /// </summary>
@@ -152,6 +164,8 @@ internal SqlColumn GetStandardColumnOptions(StandardColumn standardColumn)
             {
                 case StandardColumn.Id: return Id;
                 case StandardColumn.Level: return Level;
+                case StandardColumn.TraceId: return TraceId;
+                case StandardColumn.SpanId: return SpanId;
                 case StandardColumn.TimeStamp: return TimeStamp;
                 case StandardColumn.LogEvent: return LogEvent;
                 case StandardColumn.Message: return Message;

src/Serilog.Sinks.MSSqlServer/Sinks/MSSqlServer/ColumnOptions/SpanIdColumnOptions.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Data;
+
+namespace Serilog.Sinks.MSSqlServer
+{
+    public partial class ColumnOptions // Standard Column options are inner classes for backwards-compatibility.
+    {
+        /// <summary>
+        /// Options for the SpanId column.
+        /// </summary>
+        public class SpanIdColumnOptions : SqlColumn
+        {
+            /// <summary>
+            /// Constructor.
+            /// </summary>
+            public SpanIdColumnOptions() : base()
+            {
+                StandardColumnIdentifier = StandardColumn.SpanId;
+                DataType = SqlDbType.NVarChar;
+            }
+
+            /// <summary>
+            /// The SpanId column defaults to NVarChar and must be of a character-storage data type.
+            /// </summary>
+            public new SqlDbType DataType
+            {
+                get => base.DataType;
+                set
+                {
+                    if (!SqlDataTypes.VariableCharacterColumnTypes.Contains(value))
+                        throw new ArgumentException("The Standard Column \"SpanId\" must be NVarChar or VarChar.");
+                    base.DataType = value;
+                }
+            }
+        }
+    }
+}

src/Serilog.Sinks.MSSqlServer/Sinks/MSSqlServer/ColumnOptions/TraceIdColumnOptions.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Data;
+
+namespace Serilog.Sinks.MSSqlServer
+{
+    public partial class ColumnOptions // Standard Column options are inner classes for backwards-compatibility.
+    {
+        /// <summary>
+        /// Options for the TraceId column.
+        /// </summary>
+        public class TraceIdColumnOptions : SqlColumn
+        {
+            /// <summary>
+            /// Constructor.
+            /// </summary>
+            public TraceIdColumnOptions() : base()
+            {
+                StandardColumnIdentifier = StandardColumn.TraceId;
+                DataType = SqlDbType.NVarChar;
+            }
+
+            /// <summary>
+            /// The TraceId column defaults to NVarChar and must be of a character-storage data type.
+            /// </summary>
+            public new SqlDbType DataType
+            {
+                get => base.DataType;
+                set
+                {
+                    if (!SqlDataTypes.VariableCharacterColumnTypes.Contains(value))
+                        throw new ArgumentException("The Standard Column \"TraceId\" must be NVarChar or VarChar.");
+                    base.DataType = value;
+                }
+            }
+        }
+    }
+}

src/Serilog.Sinks.MSSqlServer/Sinks/MSSqlServer/Output/StandardColumnDataGenerator.cs

@@ -50,6 +50,10 @@ public KeyValuePair<string, object> GetStandardColumnNameAndValue(StandardColumn
                     return new KeyValuePair<string, object>(_columnOptions.MessageTemplate.ColumnName, logEvent.MessageTemplate.Text.TruncateOutput(_columnOptions.MessageTemplate.DataLength));
                 case StandardColumn.Level:
                     return new KeyValuePair<string, object>(_columnOptions.Level.ColumnName, _columnOptions.Level.StoreAsEnum ? (object)logEvent.Level : logEvent.Level.ToString());
+                case StandardColumn.TraceId:
+                    return new KeyValuePair<string, object>(_columnOptions.TraceId.ColumnName, logEvent.TraceId);
+                case StandardColumn.SpanId:
+                    return new KeyValuePair<string, object>(_columnOptions.SpanId.ColumnName, logEvent.SpanId);
                 case StandardColumn.TimeStamp:
                     return GetTimeStampStandardColumnNameAndValue(logEvent);
                 case StandardColumn.Exception: