Release 1.30 (#229)

* Improve deduplication of publications based on DOI (#225)

* Fix error in publication deduplication. Handle case where publication DOI is null. (#226)

* TTV model update for dim_publication.publication_status_code (#227)

* Upgrade Keycloak to version 23.0.4 (#228)

aspnetcore/src/api.Tests/Services_Tests/DuplicateHandlerServiceTest.cs

@@ -1,15 +1,15 @@
-using Xunit;
-using System.Collections.Generic;
+using Xunit;
+using System.Collections.Generic;
 using api.Services;
 using api.Models;
 using api.Models.Common;
 using api.Models.ProfileEditor;
 using api.Models.ProfileEditor.Items;
 using api.Models.Ttv;
 
-namespace api.Tests
-{
-    [Collection("Duplicate handler service tests.")]
+namespace api.Tests
+{
+    [Collection("Duplicate handler service tests.")]
     public class DuplicateHandlerServiceTests_HasSameDoiButIsDifferentPublication
     {
         [Fact(DisplayName = "Virta and ORCID publication have the same DOI, they are considered as the same publications: Virta publication type code is not A3, A4, B2, B3, D2, D3 or E1")]
@@ -302,11 +302,11 @@ public void addPublicationToProfileEditorData_020()
                 DimFieldDisplaySettings_FieldIdentifier = Constants.FieldIdentifiers.ACTIVITY_PUBLICATION
             };
 
-            // Create ProfileDataRaw for ORCID publication 1. The same DOI and name as in Virta publication.
+            // Create ProfileDataRaw for ORCID publication 1. The same DOI (in uppercase letters) and name as in Virta publication.
             ProfileDataFromSql profileDataOrcid1 = new()
             {
                 DimProfileOnlyPublication_PublicationId = "publicationId456",
-                DimProfileOnlyPublication_Doi = "doi123",
+                DimProfileOnlyPublication_Doi = "DOI123",
                 DimProfileOnlyPublication_PublicationName = "name123",
                 DimFieldDisplaySettings_FieldIdentifier = Constants.FieldIdentifiers.ACTIVITY_PUBLICATION_PROFILE_ONLY
             };
@@ -352,6 +352,63 @@ public void addPublicationToProfileEditorData_020()
             Assert.Equal(2, publications3.Count);
         }
 
+        [Fact(DisplayName = "AddPublicationToProfileEditorData_HandleDoiNull")]
+        public void addPublicationToProfileEditorData_021()
+        {
+            DuplicateHandlerService duplicateHandlerService = new();
+
+            // Datasources
+            ProfileEditorSource profileEditorSourceVirta = new()
+            {
+                Id = 1,
+                RegisteredDataSource = "Virta",
+                Organization = new Organization() { }
+            };
+            ProfileEditorSource profileEditorSourceOrcid = new()
+            {
+                Id = 2,
+                RegisteredDataSource = "ORCID",
+                Organization = new Organization() { }
+            };
+
+            // Create ProfileDataRaw for Virta publication 1
+            ProfileDataFromSql profileDataVirta1 = new()
+            {
+                DimPublication_PublicationId = "publicationId123",
+                DimPublication_Doi = null, // DOI missing
+                DimPublication_PublicationName = "name123",
+                DimPublication_PublicationTypeCode = "A4",
+                DimFieldDisplaySettings_FieldIdentifier = Constants.FieldIdentifiers.ACTIVITY_PUBLICATION
+            };
+
+            // Create ProfileDataRaw for ORCID publication 1. The same DOI (in uppercase letters) and name as in Virta publication.
+            ProfileDataFromSql profileDataOrcid1 = new()
+            {
+                DimProfileOnlyPublication_PublicationId = "publicationId456",
+                DimProfileOnlyPublication_Doi = "doi123",
+                DimProfileOnlyPublication_PublicationName = "name123",
+                DimFieldDisplaySettings_FieldIdentifier = Constants.FieldIdentifiers.ACTIVITY_PUBLICATION_PROFILE_ONLY
+            };
+
+            // Create empty list of publications
+            List<ProfileEditorPublication> publications = new();
+
+            // Add Virta publication
+            List<ProfileEditorPublication> publications1 = duplicateHandlerService.AddPublicationToProfileEditorData(
+                dataSource: profileEditorSourceVirta,
+                profileData: profileDataVirta1,
+                publications: publications
+            );
+            // Add ORCID publication
+            List<ProfileEditorPublication> publications2 = duplicateHandlerService.AddPublicationToProfileEditorData(
+                dataSource: profileEditorSourceOrcid,
+                profileData: profileDataOrcid1,
+                publications: publications1
+            );
+
+            // Check that publication list contains two publications
+            Assert.Equal(2, publications2.Count);
+        }
 
         [Fact(DisplayName = "TestPublicationYearHandling")]
         public void testPublicationYearHandling()
@@ -367,5 +424,5 @@ public void testPublicationYearHandling()
             // Publication year null
             Assert.Null(duplicateHandlerService.HandlePublicationYear(null));
         }
-    }
+    }
 }

aspnetcore/src/api.Tests/Services_Tests/UserProfileServiceTest.cs

@@ -1,4 +1,4 @@
-using Xunit;
+using Xunit;
 using api.Services;
 using api.Models.Common;
 using api.Models.Ttv;
@@ -7,9 +7,9 @@
 using System;
 using api.Models.ProfileEditor;
 
-namespace api.Tests
-{
-    [Collection("User profile service tests")]
+namespace api.Tests
+{
+    [Collection("User profile service tests")]
     public class UserProfileServiceTests
     {
         [Fact(DisplayName = "Get FieldIdentifiers")]
@@ -500,5 +500,43 @@ public void ResearchActivityDeduplication()
                 "Research activities are not duplicates, Sv name differs"
             );
         }
-    }
+
+        [Fact(DisplayName = "Get profile editor source")]
+        public void GetProfileEditorSource()
+        {
+            // Arrange
+            LanguageService languageService = new LanguageService();
+            UserProfileService userProfileService = new(languageService: languageService);
+            ProfileDataFromSql p = new ()
+            {
+                DimRegisteredDataSource_Id = 1234,
+                DimRegisteredDataSource_Name = "TestRegisteredDataSourceName",
+                DimRegisteredDataSource_DimOrganization_NameFi = "TestOrganizationNameFi",
+                DimRegisteredDataSource_DimOrganization_NameEn = "TestOrganizationNameEn",
+                DimRegisteredDataSource_DimOrganization_NameSv = "TestOrganizationNameSv",
+                DimRegisteredDataSource_DimOrganization_DimSector_SectorId = "TestSectorId"
+            };
+            ProfileEditorSource expectedProfileEditorSource = new ()
+            {
+                Id = 1234,
+                RegisteredDataSource = "TestRegisteredDataSourceName",
+                Organization = new Organization()
+                {
+                    NameFi = "TestOrganizationNameFi",
+                    NameEn = "TestOrganizationNameEn",
+                    NameSv = "TestOrganizationNameSv",
+                    SectorId = "TestSectorId"
+                }
+            };
+            // Act
+            ProfileEditorSource actualProfileEditorSource = userProfileService.GetProfileEditorSource(p);
+            // Assert
+            Assert.Equal(expectedProfileEditorSource.Id, actualProfileEditorSource.Id);
+            Assert.Equal(expectedProfileEditorSource.RegisteredDataSource, actualProfileEditorSource.RegisteredDataSource);
+            Assert.Equal(expectedProfileEditorSource.Organization.NameFi, actualProfileEditorSource.Organization.NameFi);
+            Assert.Equal(expectedProfileEditorSource.Organization.NameEn, actualProfileEditorSource.Organization.NameEn);
+            Assert.Equal(expectedProfileEditorSource.Organization.NameSv, actualProfileEditorSource.Organization.NameSv);
+            Assert.Equal(expectedProfileEditorSource.Organization.SectorId, actualProfileEditorSource.Organization.SectorId);
+        }
+    }
 }

aspnetcore/src/api/Models/Ttv/DimPublication.cs

@@ -11,7 +11,7 @@ public partial class DimPublication
 
     public string PublicationId { get; set; }
 
-    public string PublicationStatusCode { get; set; }
+    public int PublicationStatusCode { get; set; }
 
     public string PublicationOrgId { get; set; }
 
@@ -147,6 +147,8 @@ public partial class DimPublication
 
     public virtual DimReferencedatum PublicationCountryCodeNavigation { get; set; }
 
+    public virtual DimReferencedatum PublicationStatusCodeNavigation { get; set; }
+
     public virtual DimReferencedatum PublicationTypeCode2Navigation { get; set; }
 
     public virtual DimReferencedatum PublicationTypeCodeNavigation { get; set; }

aspnetcore/src/api/Models/Ttv/DimReferencedatum.cs

@@ -73,6 +73,8 @@ public partial class DimReferencedatum
 
     public virtual ICollection<DimPublication> DimPublicationPublicationCountryCodeNavigations { get; set; } = new List<DimPublication>();
 
+    public virtual ICollection<DimPublication> DimPublicationPublicationStatusCodeNavigations { get; set; } = new List<DimPublication>();
+
     public virtual ICollection<DimPublication> DimPublicationPublicationTypeCode2Navigations { get; set; } = new List<DimPublication>();
 
     public virtual ICollection<DimPublication> DimPublicationPublicationTypeCodeNavigations { get; set; } = new List<DimPublication>();

aspnetcore/src/api/Models/Ttv/TtvContext.cs

@@ -2499,9 +2499,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 .IsRequired()
                 .HasMaxLength(255)
                 .HasColumnName("publication_org_id");
-            entity.Property(e => e.PublicationStatusCode)
-                .HasMaxLength(255)
-                .HasColumnName("publication_status_code");
+            entity.Property(e => e.PublicationStatusCode).HasColumnName("publication_status_code");
             entity.Property(e => e.PublicationTypeCode).HasColumnName("publication_type_code");
             entity.Property(e => e.PublicationTypeCode2).HasColumnName("publication_type_code2");
             entity.Property(e => e.PublicationYear).HasColumnName("publication_year");
@@ -2567,6 +2565,11 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 .OnDelete(DeleteBehavior.ClientSetNull)
                 .HasConstraintName("publication_country_code");
 
+            entity.HasOne(d => d.PublicationStatusCodeNavigation).WithMany(p => p.DimPublicationPublicationStatusCodeNavigations)
+                .HasForeignKey(d => d.PublicationStatusCode)
+                .OnDelete(DeleteBehavior.ClientSetNull)
+                .HasConstraintName("publication_status_code");
+
             entity.HasOne(d => d.PublicationTypeCodeNavigation).WithMany(p => p.DimPublicationPublicationTypeCodeNavigations)
                 .HasForeignKey(d => d.PublicationTypeCode)
                 .OnDelete(DeleteBehavior.ClientSetNull)

aspnetcore/src/api/Services/DuplicateHandlerService.cs

@@ -1,5 +1,5 @@
 
-using System.Collections.Generic;
+using System.Collections.Generic;
 using api.Models.Common;
 using api.Models.ProfileEditor;
 using api.Models.ProfileEditor.Items;
@@ -85,8 +85,8 @@ public List<ProfileEditorPublication> AddPublicationToProfileEditorData(ProfileE
                 // Check duplicate DOI.
                 if (
                     IsOrcidPublication(profileData) &&
-                    profileData.DimProfileOnlyPublication_Doi != "" &&
-                    profileData.DimProfileOnlyPublication_Doi == publication.Doi &&
+                    !string.IsNullOrWhiteSpace(profileData.DimProfileOnlyPublication_Doi) && !string.IsNullOrWhiteSpace(publication.Doi) &&
+                    profileData.DimProfileOnlyPublication_Doi.ToLower() == publication.Doi.ToLower() &&
                     !HasSameDoiButIsDifferentPublication(profileData.DimProfileOnlyPublication_PublicationName, publication)
                 )
                 {

aspnetcore/src/api/Services/IDuplicateHandlerService.cs

@@ -10,5 +10,6 @@ public interface IDuplicateHandlerService
         List<ProfileEditorPublication> AddPublicationToProfileEditorData(ProfileEditorSource dataSource, ProfileDataFromSql profileData, List<ProfileEditorPublication> publications);
         bool HasSameDoiButIsDifferentPublication(string orcidPublicationName, ProfileEditorPublication publication);
         bool IsOrcidPublication(ProfileDataFromSql profileData);
+        int? HandlePublicationYear(int? dimDateYear);
     }
 }

aspnetcore/src/api/Services/UserProfileService.cs

@@ -70,6 +70,11 @@ public UserProfileService(IDataSourceHelperService dataSourceHelperService)
             _dataSourceHelperService = dataSourceHelperService;
         }
 
+        public UserProfileService(ILanguageService languageService)
+        {
+            _languageService = languageService;
+        }
+
         /*
          * Get FieldIdentifiers.
          */
@@ -1041,11 +1046,36 @@ await AddTtvDataToUserProfile(
             await _ttvContext.SaveChangesAsync();
         }
 
+        /*
+         * Create object indincating data source. Used for every profile data item.
+         */
+        public ProfileEditorSource GetProfileEditorSource(ProfileDataFromSql p)
+        {
+            // Organization name translation
+            NameTranslation nameTranslationSourceOrganization = _languageService.GetNameTranslation(
+                nameFi: p.DimRegisteredDataSource_DimOrganization_NameFi,
+                nameEn: p.DimRegisteredDataSource_DimOrganization_NameEn,
+                nameSv: p.DimRegisteredDataSource_DimOrganization_NameSv
+            );
+
+            ProfileEditorSource profileEditorSource = new()
+            {
+                Id = p.DimRegisteredDataSource_Id,
+                RegisteredDataSource = p.DimRegisteredDataSource_Name,
+                Organization = new Organization()
+                {
+                    NameFi = nameTranslationSourceOrganization.NameFi,
+                    NameEn = nameTranslationSourceOrganization.NameEn,
+                    NameSv = nameTranslationSourceOrganization.NameSv,
+                    SectorId = p.DimRegisteredDataSource_DimOrganization_DimSector_SectorId
+                }
+            };
+            return profileEditorSource;
+        }
 
 
         /*
-         *  Get profile data. New version using data structure,
-         *  where each item contains a list of data sources.
+         *  Get profile data.
          */
         public async Task<ProfileEditorDataResponse> GetProfileDataAsync(int userprofileId, LogUserIdentification logUserIdentification, bool forElasticsearch = false)
         {
@@ -1065,28 +1095,13 @@ public async Task<ProfileEditorDataResponse> GetProfileDataAsync(int userprofile
             // Helper list, which is used in deduplication of research activities
             List<ProfileDataFromSql> profileOnlyResearchActivityRowsToDeduplicate = new();
 
+            // Helper lists, which are used in DOI based deduplication of ORCID publications
+            List<ProfileDataFromSql> profileOnlyPublicationsToDeduplicate = new();
+
             foreach (ProfileDataFromSql p in profileDataList)
             {
-                // Organization name translation
-                NameTranslation nameTranslationSourceOrganization = _languageService.GetNameTranslation(
-                    nameFi: p.DimRegisteredDataSource_DimOrganization_NameFi,
-                    nameEn: p.DimRegisteredDataSource_DimOrganization_NameEn,
-                    nameSv: p.DimRegisteredDataSource_DimOrganization_NameSv
-                );
-
                 // Source object containing registered data source and organization name.
-                ProfileEditorSource profileEditorSource = new()
-                {
-                    Id = p.DimRegisteredDataSource_Id,
-                    RegisteredDataSource = p.DimRegisteredDataSource_Name,
-                    Organization = new Organization()
-                    {
-                        NameFi = nameTranslationSourceOrganization.NameFi,
-                        NameEn = nameTranslationSourceOrganization.NameEn,
-                        NameSv = nameTranslationSourceOrganization.NameSv,
-                        SectorId = p.DimRegisteredDataSource_DimOrganization_DimSector_SectorId
-                    }
-                };
+                ProfileEditorSource profileEditorSource = GetProfileEditorSource(p);
 
                 // Add data source into list of unique data sources.
                 if (!uniqueDataSourceIds.Contains(profileEditorSource.Id))
@@ -1467,7 +1482,7 @@ public async Task<ProfileEditorDataResponse> GetProfileDataAsync(int userprofile
                         );
                         break;
 
-                    // Publication
+                    // Publication (DimPublication)
                     case Constants.FieldIdentifiers.ACTIVITY_PUBLICATION:
                         profileDataResponse.activity.publications =
                             _duplicateHandlerService.AddPublicationToProfileEditorData(
@@ -1477,14 +1492,10 @@ public async Task<ProfileEditorDataResponse> GetProfileDataAsync(int userprofile
                             );
                         break;
 
-                    // Publication (ORCID)
+                    // Publication (DimProfileOnlyPublication)
+                    // Collect items into a helper list. They will be deduplicated later.
                     case Constants.FieldIdentifiers.ACTIVITY_PUBLICATION_PROFILE_ONLY:
-                        profileDataResponse.activity.publications =
-                            _duplicateHandlerService.AddPublicationToProfileEditorData(
-                                dataSource: profileEditorSource,
-                                profileData: p,
-                                publications: profileDataResponse.activity.publications
-                            );
+                        profileOnlyPublicationsToDeduplicate.Add(p);
                         break;
 
                     // Research activity
@@ -1931,6 +1942,19 @@ public async Task<ProfileEditorDataResponse> GetProfileDataAsync(int userprofile
                 }
             }
 
+            /*
+             * ORCID publication deduplication
+             */
+            foreach (ProfileDataFromSql p in profileOnlyPublicationsToDeduplicate)
+            {
+                ProfileEditorSource profileEditorSource = GetProfileEditorSource(p);
+                profileDataResponse.activity.publications = _duplicateHandlerService.AddPublicationToProfileEditorData(
+                    dataSource: profileEditorSource,
+                    profileData: p,
+                    publications: profileDataResponse.activity.publications
+                );
+            }
+
             /*
              * Research activity deduplication
              *