Skip to content

Commit

Permalink
Improve UI responsiveness when doing node search (#15773)
Browse files Browse the repository at this point in the history 
Co-authored-by: chubakueno <[email protected]>
  • Loading branch information
@pinzart90 @chubakueno
pinzart90 and chubakueno authored Feb 3, 2025
1 parent 0f8b928 commit 6fbe022
Show file tree
Hide file tree
Showing 9 changed files with 186 additions and 90 deletions.
37 changes: 15 additions & 22 deletions src/DynamoCore/Models/DynamoModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1592,7 +1592,6 @@ private void InitializeIncludedNodes()

var cnbNode = new CodeBlockNodeSearchElement(cbnData, LibraryServices);
SearchModel?.Add(cnbNode);
LuceneUtility.AddNodeTypeToSearchIndex(cnbNode, iDoc);

var symbolSearchElement = new NodeModelSearchElement(symbolData)
{
Expand All @@ -1611,10 +1610,8 @@ private void InitializeIncludedNodes()
};

SearchModel?.Add(symbolSearchElement);
LuceneUtility.AddNodeTypeToSearchIndex(symbolSearchElement, iDoc);

SearchModel?.Add(outputSearchElement);
LuceneUtility.AddNodeTypeToSearchIndex(outputSearchElement, iDoc);
LuceneUtility.AddNodeTypeToSearchIndexBulk([cnbNode, symbolSearchElement, outputSearchElement], iDoc);

}

Expand Down Expand Up @@ -1755,6 +1752,7 @@ internal void LoadNodeLibrary(Assembly assem, bool suppressZeroTouchLibraryLoad
private void LoadNodeModels(List<TypeLoadData> nodes, bool isPackageMember)
{
var iDoc = LuceneUtility.InitializeIndexDocumentForNodes();
List<NodeSearchElement> nodeSearchElements = [];
foreach (var type in nodes)
{
// Protect ourselves from exceptions thrown by malformed third party nodes.
Expand All @@ -1769,14 +1767,15 @@ private void LoadNodeModels(List<TypeLoadData> nodes, bool isPackageMember)
// TODO: get search element some other way
if (ele != null)
{
LuceneUtility.AddNodeTypeToSearchIndex(ele, iDoc);
nodeSearchElements.Add(ele);
}
}
catch (Exception e)
{
Logger.Log(e);
}
}
LuceneUtility.AddNodeTypeToSearchIndexBulk(nodeSearchElements, iDoc);
}

private void InitializePreferences()
Expand Down Expand Up @@ -3478,26 +3477,20 @@ internal void HideUnhideNamespace(bool hide, string library, string namespc)
internal void AddZeroTouchNodesToSearch(IEnumerable<FunctionGroup> functionGroups)
{
var iDoc = LuceneUtility.InitializeIndexDocumentForNodes();
List<NodeSearchElement> nodes = new();
foreach (var funcGroup in functionGroups)
AddZeroTouchNodeToSearch(funcGroup, iDoc);
}

private void AddZeroTouchNodeToSearch(FunctionGroup funcGroup, Document iDoc)
{
foreach (var functionDescriptor in funcGroup.Functions)
{
AddZeroTouchNodeToSearch(functionDescriptor, iDoc);
}
}

private void AddZeroTouchNodeToSearch(FunctionDescriptor functionDescriptor, Document iDoc)
{
if (functionDescriptor.IsVisibleInLibrary)
{
var ele = new ZeroTouchSearchElement(functionDescriptor);
SearchModel?.Add(ele);
LuceneUtility.AddNodeTypeToSearchIndex(ele, iDoc);
foreach (var functionDescriptor in funcGroup.Functions)
{
if (functionDescriptor.IsVisibleInLibrary)
{
var ele = new ZeroTouchSearchElement(functionDescriptor);
SearchModel?.Add(ele);
nodes.Add(ele);
}
}
}
LuceneUtility.AddNodeTypeToSearchIndexBulk(nodes, iDoc);
}

/// <summary>
Expand Down
41 changes: 32 additions & 9 deletions src/DynamoCore/Search/NodeSearchModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Xml;
using Dynamo.Configuration;
using Dynamo.Graph.Nodes;
Expand All @@ -10,7 +11,6 @@
using Dynamo.Utilities;
using DynamoUtilities;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.QueryParsers.Classic;
using Lucene.Net.Search;

Expand Down Expand Up @@ -231,29 +231,42 @@ internal string ProcessNodeCategory(string category, ref SearchElementGroup grou
return category.Substring(0, index);
}

internal IEnumerable<NodeSearchElement> Search(string search, LuceneSearchUtility luceneSearchUtility)
/// <summary>
/// Search for nodes by using a search key.
/// </summary>
/// <param name="search"></param>
/// <param name="luceneSearchUtility"></param>
/// <param name="ctk">Cancellation token to short circuit the search.</param>
/// <returns></returns>
/// <exception cref="Exception"></exception>
internal IEnumerable<NodeSearchElement> Search(string search, LuceneSearchUtility luceneSearchUtility, CancellationToken ctk = default)
{

ctk.ThrowIfCancellationRequested();

if (luceneSearchUtility != null)
{
//The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected
luceneSearchUtility.dirReader = luceneSearchUtility.writer != null ? luceneSearchUtility.writer.GetReader(applyAllDeletes: true) : DirectoryReader.Open(luceneSearchUtility.indexDir);
luceneSearchUtility.Searcher = new IndexSearcher(luceneSearchUtility.dirReader);
if (luceneSearchUtility.Searcher == null)
{
throw new Exception("Invalid IndexSearcher found");
}

string searchTerm = search.Trim();
var candidates = new List<NodeSearchElement>();

var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, LuceneConfig.NodeIndexFields, luceneSearchUtility.Analyzer)
{
AllowLeadingWildcard = true,
DefaultOperator = LuceneConfig.DefaultOperator,
FuzzyMinSim = LuceneConfig.MinimumSimilarity
};

Query query = parser.Parse(luceneSearchUtility.CreateSearchQuery(LuceneConfig.NodeIndexFields, searchTerm));
Query query = parser.Parse(luceneSearchUtility.CreateSearchQuery(LuceneConfig.NodeIndexFields, searchTerm, false, ctk));
TopDocs topDocs = luceneSearchUtility.Searcher.Search(query, n: LuceneConfig.DefaultResultsCount);

for (int i = 0; i < topDocs.ScoreDocs.Length; i++)
{
ctk.ThrowIfCancellationRequested();

// read back a Lucene doc from results
Document resultDoc = luceneSearchUtility.Searcher.Doc(topDocs.ScoreDocs[i].Doc);

Expand All @@ -268,7 +281,7 @@ internal IEnumerable<NodeSearchElement> Search(string search, LuceneSearchUtilit
}
else
{
var foundNode = FindModelForNodeNameAndCategory(name, cat, parameters);
var foundNode = FindModelForNodeNameAndCategory(name, cat, parameters, ctk);
if (foundNode != null)
{
candidates.Add(foundNode);
Expand All @@ -280,8 +293,18 @@ internal IEnumerable<NodeSearchElement> Search(string search, LuceneSearchUtilit
return null;
}

internal NodeSearchElement FindModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters)
/// <summary>
/// Finds the node model that corresponds to the input nodeName, nodeCategory and parameters.
/// </summary>
/// <param name="nodeName"></param>
/// <param name="nodeCategory"></param>
/// <param name="parameters"></param>
/// <param name="ctk">Cancellation token to short circuit the operation.</param>
/// <returns></returns>
internal NodeSearchElement FindModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters, CancellationToken ctk = default)
{
ctk.ThrowIfCancellationRequested();

var result = Entries.Where(e => {
if (e.Name.Replace(" ", string.Empty).Equals(nodeName) && e.FullCategoryName.Equals(nodeCategory))
{
Expand Down
51 changes: 39 additions & 12 deletions src/DynamoCore/Utilities/LuceneSearchUtility.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using Dynamo.Configuration;
using Dynamo.Models;
using Dynamo.Search.SearchElements;
Expand All @@ -23,7 +24,6 @@
using Lucene.Net.Search;
using Lucene.Net.Store;
using Lucene.Net.Util;
using Newtonsoft.Json;

namespace Dynamo.Utilities
{
Expand All @@ -50,7 +50,7 @@ internal class LuceneSearchUtility
internal Lucene.Net.Store.Directory indexDir;

/// <summary>
/// Lucene Index write
/// Lucene Index write. Make sure to call InitializeIndexSearcher after every index change.
/// </summary>
internal IndexWriter writer;

Expand Down Expand Up @@ -190,6 +190,16 @@ internal void CreateLuceneIndexWriter(OpenMode mode = OpenMode.CREATE)
}
}

/// <summary>
/// InitializeIndexSearcher initializes the dirReader and Searcher of this class.
/// This method should be called after every index change.
/// </summary>
private void InitializeIndexSearcher()
{
dirReader = writer != null ? writer.GetReader(applyAllDeletes: true) : DirectoryReader.Open(indexDir);
Searcher = new(dirReader);
}

/// <summary>
/// Initialize Lucene index document object for reuse
/// </summary>
Expand Down Expand Up @@ -252,11 +262,8 @@ internal void UpdateIndexedNodesInfo(List<NodeSearchElement> nodeList)
if(nodeList.Any())
{
writer.DeleteAll();
foreach(var node in nodeList)
{
var iDoc = InitializeIndexDocumentForNodes();
AddNodeTypeToSearchIndex(node, iDoc);
}
var iDoc = InitializeIndexDocumentForNodes();
AddNodeTypeToSearchIndexBulk(nodeList, iDoc);
}
}

Expand Down Expand Up @@ -321,8 +328,9 @@ internal void SetDocumentFieldValue(Document doc, string field, string value, bo
/// <param name="fields">All fields to be searched in.</param>
/// <param name="SearchTerm">Search key to be searched for.</param>
/// <param name="IsPackageContext">Set this to true if the search context is packages instead of nodes.</param>
/// <param name="ctk">Cancellation token to short circuit the search.</param>
/// <returns></returns>
internal string CreateSearchQuery(string[] fields, string SearchTerm, bool IsPackageContext = false)
internal string CreateSearchQuery(string[] fields, string SearchTerm, bool IsPackageContext = false, CancellationToken ctk = default)
{
//By Default the search will be normal
SearchType searchType = SearchType.Normal;
Expand Down Expand Up @@ -353,6 +361,7 @@ internal string CreateSearchQuery(string[] fields, string SearchTerm, bool IsPac
foreach (string f in fields)
{
Occur occurQuery = Occur.SHOULD;
ctk.ThrowIfCancellationRequested();

searchTerm = QueryParser.Escape(SearchTerm);
if (searchType == SearchType.ByDotCategory)
Expand Down Expand Up @@ -383,7 +392,7 @@ internal string CreateSearchQuery(string[] fields, string SearchTerm, bool IsPac
continue;

//Adds the FuzzyQuery and 4 WildcardQueries (3 of them contain regular expressions), with the normal weights
AddQueries(searchTerm, f, searchType, booleanQuery, occurQuery, fuzzyLogicMaxEdits);
AddQueries(searchTerm, f, searchType, booleanQuery, occurQuery, fuzzyLogicMaxEdits, ctk);

if (searchType == SearchType.ByEmptySpace)
{
Expand All @@ -396,7 +405,7 @@ internal string CreateSearchQuery(string[] fields, string SearchTerm, bool IsPac
if (string.IsNullOrEmpty(s)) continue;

//Adds the FuzzyQuery and 4 WildcardQueries (3 of them contain regular expressions), with the weights for Queries with RegularExpressions
AddQueries(s, f, searchType, booleanQuery, occurQuery, LuceneConfig.FuzzySearchMinEdits, true);
AddQueries(s, f, searchType, booleanQuery, occurQuery, LuceneConfig.FuzzySearchMinEdits, ctk, true);
}
}
}
Expand All @@ -412,9 +421,12 @@ internal string CreateSearchQuery(string[] fields, string SearchTerm, bool IsPac
/// <param name="booleanQuery">The Boolean query in which the Wildcard queries will be added</param>
/// <param name="occurQuery">Occur type can be Should or Must</param>
/// <param name="fuzzyLogicMaxEdits">Max edit lenght for Fuzzy queries</param>
/// <param name="ctk">Cancellation token to short circuit the operation.</param>
/// <param name="termSplit">Indicates if the SearchTerm has been split by empty space or not</param>
private void AddQueries(string searchTerm, string field, SearchType searchType, BooleanQuery booleanQuery, Occur occurQuery, int fuzzyLogicMaxEdits, bool termSplit = false)
private void AddQueries(string searchTerm, string field, SearchType searchType, BooleanQuery booleanQuery, Occur occurQuery, int fuzzyLogicMaxEdits, CancellationToken ctk = default, bool termSplit = false)
{
ctk.ThrowIfCancellationRequested();

string querySearchTerm = searchTerm.Replace(" ", string.Empty);

FuzzyQuery fuzzyQuery;
Expand Down Expand Up @@ -582,14 +594,16 @@ internal void CommitWriterChanges()
{
//Commit the info indexed if index writer exists
writer?.Commit();

InitializeIndexSearcher();
}

/// <summary>
/// Add node information to existing Lucene index
/// </summary>
/// <param name="node">node info that will be indexed</param>
/// <param name="doc">Lucene document in which the node info will be indexed</param>
internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
internal void AddNodeTypeToSearchIndex_uninitialized(NodeSearchElement node, Document doc)
{
if (addedFields == null) return;
// During DynamoModel initialization, the index writer should still be valid here
Expand Down Expand Up @@ -628,6 +642,19 @@ internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)

writer?.AddDocument(doc);
}
internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
{
AddNodeTypeToSearchIndex_uninitialized(node,doc);
InitializeIndexSearcher();
}
internal void AddNodeTypeToSearchIndexBulk(List<NodeSearchElement> nodes, Document doc)
{
foreach(var node in nodes)
{
AddNodeTypeToSearchIndex_uninitialized(node, doc);
}
InitializeIndexSearcher();
}
}

/// <summary>
Expand Down
8 changes: 3 additions & 5 deletions src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -733,11 +733,9 @@ internal void SearchAutoCompleteCandidates(string input)
LuceneSearch.LuceneUtilityNodeAutocomplete = new LuceneSearchUtility(dynamoViewModel.Model, LuceneSearchUtility.DefaultStartConfig);

//Memory indexing process for Node Autocomplete (indexing just the nodes returned by the NodeAutocomplete service so we limit the scope of the query search)
foreach (var node in searchElementsCache.Select(x => x.Model))
{
var doc = LuceneUtility.InitializeIndexDocumentForNodes();
LuceneUtility.AddNodeTypeToSearchIndex(node, doc);
}
var doc = LuceneUtility.InitializeIndexDocumentForNodes();
List<NodeSearchElement> nodeSearchElements = [.. searchElementsCache.Select(x => x.Model)];
LuceneUtility.AddNodeTypeToSearchIndexBulk(nodeSearchElements, doc);

//Write the Lucene documents to memory
LuceneUtility.CommitWriterChanges();
Expand Down
Loading

0 comments on commit 6fbe022

Please sign in to comment.