Prevent duplicate Gallic culture when converting to TFE (#1513) #patch

#1467

ImperatorToCK3.UnitTests/CK3/Characters/CharacterCollectionTests.cs

@@ -41,6 +41,8 @@ public class CharacterCollectionTests {
 	private static readonly CultureCollection cultures;
 
 	static CharacterCollectionTests() {
+		var colorFactory = new ColorFactory();
+
 		var states = new StateCollection();
 		var countries = new CountryCollection();
 		ImperatorToCK3.Imperator.Provinces.ProvinceCollection irProvinces = new();
@@ -54,9 +56,9 @@ static CharacterCollectionTests() {
 		AreaCollection areas = new();
 		areas.LoadAreas(irModFS, irProvinces);
 		irRegionMapper = new ImperatorRegionMapper(areas);
-		irRegionMapper.LoadRegions(irModFS, new ColorFactory());
+		irRegionMapper.LoadRegions(irModFS, colorFactory);
 
-		cultures = new CultureCollection(new PillarCollection());
+		cultures = new CultureCollection(colorFactory, new PillarCollection());
 	}
 
 	[Fact]

ImperatorToCK3.UnitTests/CK3/Religions/ReligionCollectionTests.cs

@@ -175,7 +175,7 @@ public void OptionalFaithIsNotLoadedIfInvalidatedByExistingFaiths() {
 		religions.LoadReligions(ck3ModFS, colorFactory);
 		Assert.Contains(religions.Faiths, f => f.Id == "religion_a_faith");
 
-		religions.LoadOptionalFaiths("TestFiles/configurables/optional_faiths.txt", colorFactory);
+		religions.LoadConverterFaiths("TestFiles/configurables/optional_faiths.txt", colorFactory);
 		// Optional berber_pagan is invalidated by religion_a_faith, so it should not be loaded.
 		Assert.DoesNotContain(religions.Faiths, r => r.Id == "berber_pagan");
 	}
@@ -186,7 +186,7 @@ public void OptionalFaithCanBeLoaded() {
 		var colorFactory = new ColorFactory();
 		Assert.DoesNotContain(religions.Faiths, r => r.Id == "religion_a_faith");
 
-		religions.LoadOptionalFaiths("TestFiles/configurables/optional_faiths.txt", colorFactory);
+		religions.LoadConverterFaiths("TestFiles/configurables/optional_faiths.txt", colorFactory);
 		Assert.Contains(religions.Faiths, r => r.Id == "berber_pagan");
 	}
 }

ImperatorToCK3.UnitTests/CK3/Titles/LandedTitlesTests.cs

@@ -57,7 +57,7 @@ static LandedTitlesTests() {
 
 	public LandedTitlesTests() {
 		PillarCollection pillars = new();
-		cultures = new CultureCollection(pillars);
+		cultures = new CultureCollection(new ColorFactory(), pillars);
 	}
 
 	[Fact]

ImperatorToCK3.UnitTests/CK3/Titles/RulerTermTests.cs

@@ -56,7 +56,7 @@ public void ImperatorRulerTermIsCorrectlyConverted() {
 			govMapper,
 			new LocDB("english"),
 			new ReligionMapper(ck3Religions, irRegionMapper, ck3RegionMapper),
-			new CultureMapper(irRegionMapper, ck3RegionMapper, new CultureCollection(new PillarCollection())),
+			new CultureMapper(irRegionMapper, ck3RegionMapper, new CultureCollection(new ColorFactory(), new PillarCollection())),
 			new NicknameMapper("TestFiles/configurables/nickname_map.txt"),
 			new ProvinceMapper(),
 			new Configuration()

ImperatorToCK3.UnitTests/CK3/Titles/TitleTests.cs

@@ -56,7 +56,7 @@ private class TitleBuilder {
 		private DefiniteFormMapper definiteFormMapper = new("TestFiles/configurables/definite_form_names.txt");
 
 		private readonly ReligionMapper religionMapper;
-		private readonly CultureMapper cultureMapper = new(irRegionMapper, new CK3RegionMapper(), new CultureCollection(new PillarCollection()));
+		private readonly CultureMapper cultureMapper = new(irRegionMapper, new CK3RegionMapper(), new CultureCollection(new ColorFactory(), new PillarCollection()));
 		private readonly NicknameMapper nicknameMapper = new("TestFiles/configurables/nickname_map.txt");
 		private readonly Date ck3BookmarkDate = new(867, 1, 1);
 		private readonly CharacterCollection characters = new();

ImperatorToCK3.UnitTests/Mappers/TagTitle/TagTitleMapperTests.cs

@@ -39,7 +39,7 @@ public class TagTitleMapperTests {
 
 	static TagTitleMapperTests() {
 		var pillars = new PillarCollection();
-		cultures = new CultureCollection(pillars);
+		cultures = new CultureCollection(ColorFactory, pillars);
 	}
 
 	public TagTitleMapperTests() {

ImperatorToCK3.UnitTests/Outputter/CoatOfArmsOutputterTests.cs

@@ -60,7 +60,7 @@ public void CoaIsOutputtedForCountryWithFlagSet() {
 			new SuccessionLawMapper(),
 			new DefiniteFormMapper(),
 			new ReligionMapper(ck3Religions, irRegionMapper, ck3RegionMapper),
-			new CultureMapper(irRegionMapper, ck3RegionMapper, new CultureCollection(new PillarCollection())),
+			new CultureMapper(irRegionMapper, ck3RegionMapper, new CultureCollection(new ColorFactory(), new PillarCollection())),
 			new NicknameMapper(),
 			new CharacterCollection(),
 			new Date(400, 1, 1),
@@ -102,7 +102,7 @@ public void CoaIsNotOutputtedForCountryWithoutFlagSet() {
 			new SuccessionLawMapper(),
 			new DefiniteFormMapper(),
 			new ReligionMapper(ck3Religions, irRegionMapper, ck3RegionMapper),
-			new CultureMapper(irRegionMapper, ck3RegionMapper, new CultureCollection(new PillarCollection())),
+			new CultureMapper(irRegionMapper, ck3RegionMapper, new CultureCollection(new ColorFactory(), new PillarCollection())),
 			new NicknameMapper(),
 			new CharacterCollection(),
 			new Date(400, 1, 1),

ImperatorToCK3.UnitTests/Outputter/DynastiesOutputterTests.cs

@@ -29,7 +29,7 @@ public void DynastiesAreOutputted() {
 		AreaCollection areas = new();
 		ImperatorRegionMapper irRegionMapper = new(areas);
 		irRegionMapper.LoadRegions(irModFS, new ColorFactory());
-		CultureMapper cultureMapper = new(irRegionMapper, new CK3RegionMapper(), new CultureCollection(new PillarCollection()));
+		CultureMapper cultureMapper = new(irRegionMapper, new CK3RegionMapper(), new CultureCollection(new ColorFactory(), new PillarCollection()));
 
 		var characters = new CharacterCollection();
 		var dynasties = new DynastyCollection();

ImperatorToCK3.UnitTests/TestHelpers/TestCK3CultureCollection.cs

@@ -1,11 +1,11 @@
 using commonItems;
-using commonItems.Collections;
 using commonItems.Colors;
 using ImperatorToCK3.CK3.Cultures;
+using System.Linq;
 
 namespace ImperatorToCK3.UnitTests.TestHelpers; 
 
-public class TestCK3CultureCollection() : CultureCollection(TestCulturalPillars) {
+public class TestCK3CultureCollection() : CultureCollection(new ColorFactory(), TestCulturalPillars) {
 	private static readonly PillarCollection TestCulturalPillars = new();
 
 	static TestCK3CultureCollection() {
@@ -15,13 +15,11 @@ static TestCK3CultureCollection() {
 	public void GenerateTestCulture(string id) {
 		const string nameListId = "name_list_test";
 		var nameList = new NameList(nameListId, new BufferedReader());
-		var culture = new Culture(
-			id,
-			new BufferedReader($"heritage=test_heritage name_list={nameListId}"),
-			TestCulturalPillars,
-			new IdObjectCollection<string, NameList> {nameList},
-			new ColorFactory()
-		);
+		var cultureData = new CultureData {
+			Heritage = TestCulturalPillars.Heritages.First(p => p.Id == "test_heritage"),
+			NameLists = {nameList}
+		};
+		var culture = new Culture(id, cultureData);
 		Add(culture);
 	}
 }

ImperatorToCK3/CK3/Cultures/Culture.cs

@@ -1,57 +1,63 @@
 using commonItems;
 using commonItems.Collections;
 using commonItems.Colors;
-using ImperatorToCK3.Exceptions;
-using System;
+using commonItems.Serialization;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
 
 namespace ImperatorToCK3.CK3.Cultures; 
 
-public sealed class Culture : IIdentifiable<string> {
+public sealed class Culture : IIdentifiable<string>, IPDXSerializable {
 	public string Id { get; }
-	public Color Color { get; private set; } = new(0, 0, 0);
-	public Pillar Heritage { get; private set; }
-	private SortedSet<string> traditionIds = new();
+	public Color Color { get; }
+	public OrderedSet<string> ParentCultureIds { get; set; } = new();
+	public Pillar Heritage { get; }
+	private readonly OrderedSet<string> traditionIds;
 	public IReadOnlyCollection<string> TraditionIds => traditionIds;
-	public OrderedSet<NameList> NameLists { get; }
+	private readonly OrderedSet<NameList> nameLists;
+	public IReadOnlyCollection<NameList> NameLists => nameLists;
+	private readonly List<KeyValuePair<string, StringOfItem>> attributes;
+	public IReadOnlyCollection<KeyValuePair<string, StringOfItem>> Attributes => attributes;
 
-	public Culture(string id, BufferedReader cultureReader, PillarCollection pillars, IdObjectCollection<string, NameList> nameLists, ColorFactory colorFactory) {
+	public Culture(string id, CultureData cultureData) {
 		Id = id;
 
-		NameLists = new OrderedSet<NameList>();
-		var parser = new Parser();
-		parser.RegisterKeyword("color", reader => {
-			try {
-				Color = colorFactory.GetColor(reader);
-			} catch (Exception e) {
-				Logger.Warn($"Culture {id} has invalid color! {e.Message}");
-			}
-		});
-		parser.RegisterKeyword("heritage", reader => {
-			var heritageId = reader.GetString();
-			Heritage = pillars.Heritages.First(p => p.Id == heritageId);
-		});
-		parser.RegisterKeyword("traditions", reader => {
-			traditionIds = new SortedSet<string>(reader.GetStrings());
-		});
-		parser.RegisterKeyword("name_list", reader => {
-			var nameListId = reader.GetString();
-			if (nameLists.TryGetValue(nameListId, out var nameList)) {
-				NameLists.Add(nameList);
-			} else {
-				Logger.Warn($"Culture {id} has unrecognized name list: {nameListId}");
-			}
-		});
-		parser.IgnoreUnregisteredItems();
-		parser.ParseStream(cultureReader);
-
-		if (Heritage is null) {
-			throw new ConverterException($"Culture {id} has no heritage defined!");
+		Color = cultureData.Color!;
+		ParentCultureIds = cultureData.ParentCultureIds;
+		Heritage = cultureData.Heritage!;
+		traditionIds = cultureData.TraditionIds;
+		nameLists = cultureData.NameLists;
+		attributes = cultureData.Attributes;
+	}
+
+	public string Serialize(string indent, bool withBraces) {
+		var contentIndent = indent;
+		if (withBraces) {
+			contentIndent += '\t';
 		}
-		if (NameLists.Count == 0) {
-			throw new ConverterException($"Culture {id} has no name list defined!");
+
+		var sb = new StringBuilder();
+		if (withBraces) {
+			sb.AppendLine("{");
+		}
+
+		sb.Append(contentIndent).AppendLine($"color={Color.OutputRgb()}");
+		if (ParentCultureIds.Any()) {
+			sb.Append(contentIndent).AppendLine($"parents={PDXSerializer.Serialize(ParentCultureIds)}");
 		}
+		sb.Append(contentIndent).AppendLine($"heritage={Heritage.Id}");
+		sb.Append(contentIndent).AppendLine($"traditions={PDXSerializer.Serialize(TraditionIds)}");
+		foreach (var nameList in NameLists) {
+			sb.Append(contentIndent).AppendLine($"name_list={nameList.Id}");
+		}
+		sb.AppendLine(PDXSerializer.Serialize(Attributes, indent: contentIndent, withBraces: false));
+
+		if (withBraces) {
+			sb.Append(indent).Append('}');
+		}
+
+		return sb.ToString();
 	}
 
 	public IEnumerable<string> MaleNames => NameLists.SelectMany(l => l.MaleNames);

ImperatorToCK3/CK3/Cultures/CultureCollection.cs

@@ -2,21 +2,106 @@
 using commonItems.Collections;
 using commonItems.Colors;
 using commonItems.Mods;
+using System;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace ImperatorToCK3.CK3.Cultures; 
 
 public class CultureCollection : IdObjectCollection<string, Culture> {
-	public CultureCollection(PillarCollection pillarCollection) {
+	public CultureCollection(ColorFactory colorFactory, PillarCollection pillarCollection) {
 		this.pillarCollection = pillarCollection;
+		InitCultureDataParser(colorFactory);
+	}
+
+	private void InitCultureDataParser(ColorFactory colorFactory) {
+		cultureDataParser.RegisterKeyword("INVALIDATED_BY", reader => {
+			cultureData.InvalidatingCultureIds = reader.GetStrings();
+		});
+		cultureDataParser.RegisterKeyword("color", reader => {
+			try {
+				cultureData.Color = colorFactory.GetColor(reader);
+			} catch (Exception e) {
+				Logger.Warn($"Found invalid color when parsing culture! {e.Message}");
+			}
+		});
+		cultureDataParser.RegisterKeyword("parents", reader => {
+			cultureData.ParentCultureIds = reader.GetStrings().ToOrderedSet();
+		});
+		cultureDataParser.RegisterKeyword("heritage", reader => {
+			var heritageId = reader.GetString();
+			cultureData.Heritage = pillarCollection.Heritages.First(p => p.Id == heritageId);
+		});
+		cultureDataParser.RegisterKeyword("traditions", reader => {
+			cultureData.TraditionIds = reader.GetStrings().ToOrderedSet();
+		});
+		cultureDataParser.RegisterKeyword("name_list", reader => {
+			var nameListId = reader.GetString();
+			if (nameListCollection.TryGetValue(nameListId, out var nameList)) {
+				cultureData.NameLists.Add(nameList);
+			} else {
+				Logger.Warn($"Found unrecognized name list when parsing culture: {nameListId}");
+			}
+		});
+		cultureDataParser.RegisterRegex(CommonRegexes.String, (reader, keyword) => {
+			cultureData.Attributes.Add(new KeyValuePair<string, StringOfItem>(keyword, reader.GetStringOfItem()));
+		});
+		cultureDataParser.IgnoreAndLogUnregisteredItems();
 	}
 
-	public void LoadCultures(ModFilesystem ck3ModFS, ColorFactory colorFactory) {
+	public void LoadCultures(ModFilesystem ck3ModFS) {
 		var parser = new Parser();
-		parser.RegisterRegex(CommonRegexes.String, (reader, cultureId) => {
-			AddOrReplace(new Culture(cultureId, reader, pillarCollection, nameListCollection, colorFactory));
-		});
+		parser.RegisterRegex(CommonRegexes.String, (reader, cultureId) => LoadCulture(cultureId, reader));
 		parser.IgnoreAndLogUnregisteredItems();
 		parser.ParseGameFolder("common/culture/cultures", ck3ModFS, "txt", true, logFilePaths: true);
+
+		ReplaceInvalidatedParents();
+	}
+
+	public void LoadConverterCultures(string converterCulturesPath) {
+		var parser = new Parser();
+		parser.RegisterRegex(CommonRegexes.String, (reader, cultureId) => LoadCulture(cultureId, reader));
+		parser.IgnoreAndLogUnregisteredItems();
+		parser.ParseFile(converterCulturesPath);
+
+		ReplaceInvalidatedParents();
+	}
+
+	private void LoadCulture(string cultureId, BufferedReader cultureReader) {
+		cultureDataParser.ParseStream(cultureReader);
+
+		if (cultureData.InvalidatingCultureIds.Any()) {
+			foreach (var existingCulture in this) {
+				if (!cultureData.InvalidatingCultureIds.Contains(existingCulture.Id)) {
+					continue;
+				}
+				Logger.Debug($"Culture {cultureId} is invalidated by existing {existingCulture.Id}.");
+				cultureReplacements[cultureId] = existingCulture.Id;
+				return;
+			}
+			Logger.Debug($"Loading optional culture {cultureId}...");
+		}
+		if (cultureData.Heritage is null) {
+			Logger.Warn($"Culture {cultureId} has no heritage defined! Skipping.");
+			return;
+		}
+		if (cultureData.NameLists.Count == 0) {
+			Logger.Warn($"Culture {cultureId} has no name list defined! Skipping.");
+			return;
+		}
+		AddOrReplace(new Culture(cultureId, cultureData));
+
+		// Reset culture data for the next culture.
+		cultureData = new CultureData();
+	}
+
+	private void ReplaceInvalidatedParents() {
+		// Replace invalidated cultures in parent culture lists.
+		foreach (var culture in this) {
+			culture.ParentCultureIds = culture.ParentCultureIds
+				.Select(id => cultureReplacements.TryGetValue(id, out var replacementId) ? replacementId : id)
+				.ToOrderedSet();
+		}
 	}
 
 	public void LoadNameLists(ModFilesystem ck3ModFS) {
@@ -25,9 +110,14 @@ public void LoadNameLists(ModFilesystem ck3ModFS) {
 			nameListCollection.AddOrReplace(new NameList(nameListId, reader));
 		});
 		parser.IgnoreAndLogUnregisteredItems();
-		parser.ParseGameFolder("common/culture/name_lists", ck3ModFS, "txt", true, logFilePaths: true);
+		parser.ParseGameFolder("common/culture/name_lists", ck3ModFS, "txt", recursive: true, logFilePaths: true);
 	}
+
+	private readonly IDictionary<string, string> cultureReplacements = new Dictionary<string, string>(); // replaced culture -> replacing culture
 
 	private readonly PillarCollection pillarCollection;
 	private readonly IdObjectCollection<string, NameList> nameListCollection = new();
+
+	private CultureData cultureData = new();
+	private readonly Parser cultureDataParser = new();
 }

ImperatorToCK3/CK3/Cultures/CultureData.cs

@@ -0,0 +1,17 @@
+using commonItems;
+using commonItems.Collections;
+using commonItems.Colors;
+using System.Collections.Generic;
+
+namespace ImperatorToCK3.CK3.Cultures;
+
+public record CultureData {
+	public IEnumerable<string> InvalidatingCultureIds { get; set; } = new List<string>();
+	public Color? Color { get; set; }
+	public OrderedSet<string> ParentCultureIds { get; set; } = new();
+	public Pillar? Heritage { get; set; }
+	public OrderedSet<string> TraditionIds { get; set; } = new();
+	public OrderedSet<NameList> NameLists { get; } = new();
+
+	public List<KeyValuePair<string, StringOfItem>> Attributes { get; } = new();
+}