Skip to content

Commit

Permalink
[PERFORMANCE] ChangeLogService and OperationProcessor performance upg…
Browse files Browse the repository at this point in the history 
…rade (#256)
  • Loading branch information
@lxatstariongroup
lxatstariongroup authored Mar 28, 2022
1 parent 1507638 commit 83e63ee
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 37 deletions.
1 change: 1 addition & 0 deletions CDP4WebServices.API/Modules/10-25/EngineeringModelApi.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,7 @@ protected override Response PostResponseData(dynamic routeParams)
{
transaction?.Dispose();
connection?.Dispose();
sw.Stop();
}
}

Expand Down
16 changes: 16 additions & 0 deletions CDP4WebServices.API/Services/ChangeLog/ChangeLogService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,13 @@ public bool TryAppendModelChangeLogData(NpgsqlTransaction transaction, string pa
{
var sw = Stopwatch.StartNew();
Logger.Info("Starting to append changelog data");

var isCachedDtoReadEnabled = this.TransactionManager.IsCachedDtoReadEnabled(transaction);

if (!isCachedDtoReadEnabled)
{
this.TransactionManager.SetCachedDtoReadEnabled(true);
}

var isFullAccessEnabled = this.TransactionManager.IsFullAccessEnabled();
var result = false;
Expand Down Expand Up @@ -306,6 +313,8 @@ public bool TryAppendModelChangeLogData(NpgsqlTransaction transaction, string pa
operationData.Update.Add(modelLogEntryClasslessDTO);
}

// New things that need to be read that are not yet in cache at this moment in time
this.TransactionManager.SetCachedDtoReadEnabled(false);
this.OperationProcessor.Process(operationData, transaction, partition);

result = true;
Expand All @@ -322,8 +331,15 @@ public bool TryAppendModelChangeLogData(NpgsqlTransaction transaction, string pa
{
this.TransactionManager.SetFullAccessState(false);
}

if (!isCachedDtoReadEnabled && this.TransactionManager.IsCachedDtoReadEnabled(transaction))
{
this.TransactionManager.SetCachedDtoReadEnabled(false);
}
}

sw.Stop();

Logger.Info($"Finished appending to changelog data in {sw.ElapsedMilliseconds} [ms]");

return result;
Expand Down
117 changes: 80 additions & 37 deletions CDP4WebServices.API/Services/Operations/OperationProcessor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ namespace CDP4WebServices.API.Services.Operations

using CDP4Orm.Dao;
using CDP4Orm.Dao.Resolve;

using CDP4WebServices.API.Services.Authorization;
using CDP4WebServices.API.Services.Operations.SideEffects;

Expand All @@ -44,6 +43,7 @@ namespace CDP4WebServices.API.Services.Operations
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Security;
Expand Down Expand Up @@ -818,6 +818,33 @@ private void DeletePersistedItem(NpgsqlTransaction transaction, string partition
service.DeleteConcept(transaction, partition, thing);
}

/// <summary>
/// Try and get persisted items from the data store.
/// </summary>
/// <param name="transaction">
/// The current transaction to the database.
/// </param>
/// <param name="partition">
/// The database partition (schema) where the requested resource will be stored.
/// </param>
/// <param name="service">
/// The service instance for the requested type.
/// </param>
/// <param name="iids">
/// The id of the item to retrieve.
/// </param>
/// <returns>
/// The retrieved <see cref="IEnumerable{T}"/> or type <see cref="Thing"/> instances
/// </returns>
/// <param name="securityContext">
/// The security Context used for permission checking.
/// </param>
private IEnumerable<Thing> GetPersistedItems(NpgsqlTransaction transaction, string partition, IPersistService service, IEnumerable<Guid> iids, ISecurityContext securityContext)
{
return service.GetShallow(
transaction, partition, iids, securityContext);
}

/// <summary>
/// Try and get persisted item from the data store.
/// </summary>
Expand Down Expand Up @@ -960,55 +987,71 @@ private void ApplyCreateOperations(CdpPostOperation operation, NpgsqlTransaction
{
// re-order create
this.ReorderCreateOrder(operation);
foreach (var createInfo in operation.Create.Select(x => x.GetInfoPlaceholder()))

if (operation.Create.Any())
{
var service = this.ServiceProvider.MapToPersitableService(createInfo.TypeName);
var operationInfoPlaceholders = operation.Create.Select(x => x.GetInfoPlaceholder());
var securityContext = new RequestSecurityContext { ContainerReadAllowed = true };

securityContext.Credentials = this.RequestUtils.Context.AuthenticatedCredentials;

var resolvedInfo = this.operationThingCache[createInfo];

// check that item doen not exist:
var persistedItem = this.GetPersistedItem(transaction, resolvedInfo.Partition, service, createInfo.Iid, securityContext);
var typeGroups = operationInfoPlaceholders.GroupBy(x => x.TypeName).Select(x => new {TypeName = x.Key, Things = x.ToList()});

if (persistedItem != null)
foreach (var typeGroup in typeGroups)
{
throw new InvalidOperationException(
string.Format("Item '{0}' with Iid: '{1}' already exists", createInfo.TypeName, createInfo.Iid));
}
var partitions = this.operationThingCache.Where(x => typeGroup.Things.Contains(x.Key)).Select(x => x.Value.Partition).Distinct();
foreach (var partition in partitions)
{
var service = this.ServiceProvider.MapToPersitableService(typeGroup.TypeName);
var typeGroupIids = typeGroup.Things.Select(x => x.Iid);

var persistedItems = this.GetPersistedItems(transaction, partition, service, typeGroupIids, securityContext);

// get the (cached) containment information for this create request
var resolvedContainerInfo = this.GetContainerInfo(resolvedInfo.Thing);
foreach (var createInfo in typeGroup.Things)
{
var resolvedInfo = this.operationThingCache[createInfo];

// keep a copy of the orginal thing to pass to the after create hook
var originalThing = resolvedInfo.Thing.DeepClone<Thing>();
var persistedItem = persistedItems.FirstOrDefault(x => x.Iid.Equals(createInfo.Iid));

if (this.operationOriginalThingCache.All(x => x.Iid != originalThing.Iid))
{
this.operationOriginalThingCache.Add(originalThing);
}
if (persistedItem != null)
{
throw new InvalidOperationException(
string.Format("Item '{0}' with Iid: '{1}' already exists", createInfo.TypeName, createInfo.Iid));
}

// call before create hook
if (!this.OperationSideEffectProcessor.BeforeCreate(resolvedInfo.Thing, resolvedContainerInfo.Thing, transaction, resolvedInfo.Partition, securityContext))
{
Logger.Warn("Skipping create operation of thing {0} with id {1} as a consequence of the side-effect.", createInfo.TypeName, createInfo.Iid);
continue;
}
// get the (cached) containment information for this create request
var resolvedContainerInfo = this.GetContainerInfo(resolvedInfo.Thing);

if (resolvedInfo.ContainerInfo.ContainmentSequence != -1)
{
service.CreateConcept(transaction, resolvedInfo.Partition, resolvedInfo.Thing, resolvedContainerInfo.Thing, resolvedInfo.ContainerInfo.ContainmentSequence);
}
else
{
service.CreateConcept(transaction, resolvedInfo.Partition, resolvedInfo.Thing, resolvedContainerInfo.Thing);
}
// keep a copy of the orginal thing to pass to the after create hook
var originalThing = resolvedInfo.Thing.DeepClone<Thing>();

var createdItem = this.GetPersistedItem(transaction, resolvedInfo.Partition, service, createInfo.Iid, securityContext);
if (this.operationOriginalThingCache.All(x => x.Iid != originalThing.Iid))
{
this.operationOriginalThingCache.Add(originalThing);
}

// call before create hook
if (resolvedInfo.Thing is ParameterValueSet || !this.OperationSideEffectProcessor.BeforeCreate(resolvedInfo.Thing, resolvedContainerInfo.Thing, transaction, resolvedInfo.Partition, securityContext))
{
Logger.Warn("Skipping create operation of thing {0} with id {1} as a consequence of the side-effect.", createInfo.TypeName, createInfo.Iid);
continue;
}

if (resolvedInfo.ContainerInfo.ContainmentSequence != -1)
{
service.CreateConcept(transaction, resolvedInfo.Partition, resolvedInfo.Thing, resolvedContainerInfo.Thing, resolvedInfo.ContainerInfo.ContainmentSequence);
}
else
{
service.CreateConcept(transaction, resolvedInfo.Partition, resolvedInfo.Thing, resolvedContainerInfo.Thing);
}

var createdItem = this.GetPersistedItem(transaction, resolvedInfo.Partition, service, createInfo.Iid, securityContext);

// call after create hook
this.OperationSideEffectProcessor.AfterCreate(createdItem, resolvedContainerInfo.Thing, originalThing, transaction, resolvedInfo.Partition, securityContext);
// call after create hook
this.OperationSideEffectProcessor.AfterCreate(createdItem, resolvedContainerInfo.Thing, originalThing, transaction, resolvedInfo.Partition, securityContext);
}
}
}
}
}

Expand Down

0 comments on commit 83e63ee

Please sign in to comment.