Normalize line endings before reading csv

ClubDoorman/SpamHamClassifier.cs

@@ -91,7 +91,10 @@ private async Task Train()
             var stopWords = (await File.ReadAllTextAsync("data/exclude-tokens.txt")).Split(',').Select(x => x.Trim()).ToArray();
 
             List<MessageData> dataset;
-            using (var reader = new StreamReader(SpamHamDataset))
+            // Right now dataset is around 160KB, but if/when it will be huge we would need to normalize line endings without loading everything into memory
+            var csvContent = await File.ReadAllTextAsync(SpamHamDataset);
+            var normalizedCsvContent = NormalizeLineEndings(csvContent);
+            using (var reader = new StringReader(normalizedCsvContent))
             using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
             {
                 dataset = csv.GetRecords<MessageData>().ToList();
@@ -129,4 +132,6 @@ private async Task Train()
             _logger.LogError(e, "Exception during training");
         }
     }
+
+    private static string NormalizeLineEndings(string input) => input.Replace("\r\n", "\n").Replace("\r", "\n");
 }