Improve querying by ids

- GetByIds now always return a Replica<HashSet<T>> - Previously some GetByIds were implemented as IAsyncEnumerable - GetBulk methods are added that return the IAsyncEnumerable The GetBulk methods are only added for endpoints that do not support ids=all (at least for now) - v2/items - v2/recipes - v2/commerce/prices - v2/commerce/listings GetBulk will take a list of ids and partition them into chunks of 200, then fetch those chunks in parallel. You can optionally configure the chunk size and degree of parallelism. As a user, you do not really perceive that this chunking is happening in the background. It is an effective substitute for ids=all. Perhaps the only downside is you cannot access the response headers of individial chunks.
sliekens · Sep 10, 2023 · ee27f65 · ee27f65
1 parent d31a94f
commit ee27f65
Show file tree

Hide file tree

Showing 8 changed files with 178 additions and 106 deletions.
diff --git a/GW2SDK.Tests/Features/Commerce/CommerceQueryTest.cs b/GW2SDK.Tests/Features/Commerce/CommerceQueryTest.cs
@@ -74,25 +74,26 @@ public async Task Item_prices_can_be_filtered_by_id()
             35984
         };
 
-        var actual = await sut.Commerce.GetItemPricesByIds(ids).ToListAsync();
+        var actual = await sut.Commerce.GetItemPricesByIds(ids);
 
         Assert.Collection(
             ids,
-            first => Assert.Contains(actual, found => found.Id == first),
-            second => Assert.Contains(actual, found => found.Id == second),
-            third => Assert.Contains(actual, found => found.Id == third)
+            first => Assert.Contains(actual.Value, found => found.Id == first),
+            second => Assert.Contains(actual.Value, found => found.Id == second),
+            third => Assert.Contains(actual.Value, found => found.Id == third)
         );
     }
 
-    [Fact(
-        Skip =
-            "This test is best used interactively, otherwise it will hit rate limits in this as well as other tests."
-    )]
+    [Fact]
     public async Task Item_prices_can_be_enumerated()
     {
         var sut = Composer.Resolve<Gw2Client>();
 
-        await foreach (var actual in sut.Commerce.GetItemPrices())
+        // You wouldn't want to use Take() in production code
+        //   but enumerating all entries is too expensive for a test
+        // This code will actually try to fetch more than 600 entries
+        //  but the extra requests will be cancelled when this test completes
+        await foreach (var actual in sut.Commerce.GetItemPricesBulk(degreeOfParalllelism: 3).Take(600))
         {
             Assert.True(actual.Id > 0);
             if (actual.TotalSupply == 0)
@@ -190,25 +191,26 @@ public async Task Order_books_can_be_filtered_by_id()
             35984
         };
 
-        var actual = await sut.Commerce.GetOrderBooksByIds(ids).ToListAsync();
+        var actual = await sut.Commerce.GetOrderBooksByIds(ids);
 
         Assert.Collection(
             ids,
-            first => Assert.Contains(actual, found => found.Id == first),
-            second => Assert.Contains(actual, found => found.Id == second),
-            third => Assert.Contains(actual, found => found.Id == third)
+            first => Assert.Contains(actual.Value, found => found.Id == first),
+            second => Assert.Contains(actual.Value, found => found.Id == second),
+            third => Assert.Contains(actual.Value, found => found.Id == third)
         );
     }
 
-    [Fact(
-        Skip =
-            "This test is best used interactively, otherwise it will hit rate limits in this as well as other tests."
-    )]
+    [Fact]
     public async Task Order_books_can_be_enumerated()
     {
         var sut = Composer.Resolve<Gw2Client>();
 
-        await foreach (var actual in sut.Commerce.GetOrderBooks())
+        // You wouldn't want to use Take() in production code
+        //   but enumerating all entries is too expensive for a test
+        // This code will actually try to fetch more than 600 entries
+        //  but the extra requests will be cancelled when this test completes
+        await foreach (var actual in sut.Commerce.GetOrderBooksBulk(degreeOfParalllelism: 3).Take(600))
         {
             Assert.True(actual.Id > 0);
             if (actual.TotalSupply == 0)

diff --git a/GW2SDK.Tests/Features/Crafting/Recipes.cs b/GW2SDK.Tests/Features/Crafting/Recipes.cs
@@ -7,26 +7,22 @@ namespace GuildWars2.Tests.Features.Crafting;
 
 public class Recipes
 {
-    [Fact(
-        Skip =
-            "This test is best used interactively, otherwise it will hit rate limits in this as well as other tests."
-    )]
+    [Fact]
     public async Task Can_be_enumerated()
     {
         var sut = Composer.Resolve<Gw2Client>();
 
-        var actual = await sut.Crafting.GetRecipes().ToListAsync();
-
-        Assert.All(
-            actual,
-            entry =>
-            {
-                entry.Has_id();
-                entry.Has_output_item_id();
-                entry.Has_item_count();
-                entry.Has_min_rating_between_0_and_500();
-                entry.Has_time_to_craft();
-            }
-        );
+        // You wouldn't want to use Take() in production code
+        //   but enumerating all entries is too expensive for a test
+        // This code will actually try to fetch more than 600 entries
+        //  but the extra requests will be cancelled when this test completes
+        await foreach (var actual in sut.Crafting.GetRecipesBulk(degreeOfParalllelism: 3).Take(600))
+        {
+            actual.Has_id();
+            actual.Has_output_item_id();
+            actual.Has_item_count();
+            actual.Has_min_rating_between_0_and_500();
+            actual.Has_time_to_craft();
+        }
     }
 }
diff --git a/GW2SDK.Tests/Features/Crafting/RecipesByFilter.cs b/GW2SDK.Tests/Features/Crafting/RecipesByFilter.cs
@@ -20,10 +20,10 @@ public async Task Can_be_filtered_by_id()
             3
         };
 
-        var actual = await sut.Crafting.GetRecipesByIds(ids).ToListAsync();
+        var actual = await sut.Crafting.GetRecipesByIds(ids);
 
         Assert.All(
-            actual,
+            actual.Value,
             entry =>
             {
                 entry.Has_id();

diff --git a/GW2SDK.Tests/Features/Items/ItemsQueryTest.cs b/GW2SDK.Tests/Features/Items/ItemsQueryTest.cs
@@ -42,13 +42,13 @@ public async Task Items_can_be_filtered_by_id()
             56
         };
 
-        var actual = await sut.Items.GetItemsByIds(ids).ToListAsync();
+        var actual = await sut.Items.GetItemsByIds(ids);
 
         Assert.Collection(
             ids,
-            first => Assert.Contains(actual, found => found.Id == first),
-            second => Assert.Contains(actual, found => found.Id == second),
-            third => Assert.Contains(actual, found => found.Id == third)
+            first => Assert.Contains(actual.Value, found => found.Id == first),
+            second => Assert.Contains(actual.Value, found => found.Id == second),
+            third => Assert.Contains(actual.Value, found => found.Id == third)
         );
     }
 
@@ -63,15 +63,16 @@ public async Task Items_can_be_filtered_by_page()
         Assert.Equal(3, actual.PageContext.PageSize);
     }
 
-    [Fact(
-        Skip =
-            "This test is best used interactively, otherwise it will hit rate limits in this as well as other tests."
-    )]
+    [Fact]
     public async Task Items_can_be_enumerated()
     {
         var sut = Composer.Resolve<Gw2Client>();
 
-        await foreach (var actual in sut.Items.GetItems())
+        // You wouldn't want to use Take() in production code
+        //   but enumerating all entries is too expensive for a test
+        // This code will actually try to fetch more than 600 entries
+        //  but the extra requests will be cancelled when this test completes
+        await foreach (var actual in sut.Items.GetItemsBulk(degreeOfParalllelism: 3).Take(600))
         {
             actual.Validate();
         }

diff --git a/GW2SDK/Features/Commerce/CommerceQuery.cs b/GW2SDK/Features/Commerce/CommerceQuery.cs
@@ -67,42 +67,60 @@ public Task<Replica<ItemPrice>> GetItemPriceById(
         return request.SendAsync(http, cancellationToken);
     }
 
-    public IAsyncEnumerable<ItemPrice> GetItemPricesByIds(
+    public Task<Replica<HashSet<ItemPrice>>> GetItemPricesByIds(
         IReadOnlyCollection<int> itemIds,
         MissingMemberBehavior missingMemberBehavior = default,
+        CancellationToken cancellationToken = default
+    )
+    {
+        ItemPricesByIdsRequest request = new(itemIds)
+        {
+            MissingMemberBehavior = missingMemberBehavior
+        };
+        return request.SendAsync(http, cancellationToken);
+    }
+
+    public IAsyncEnumerable<ItemPrice> GetItemPricesBulk(
+        IReadOnlyCollection<int> itemIds,
+        MissingMemberBehavior missingMemberBehavior = default,
+        int degreeOfParalllelism = BulkQuery.DefaultDegreeOfParalllelism,
+        int chunkSize = BulkQuery.DefaultChunkSize,
         IProgress<ResultContext>? progress = default,
         CancellationToken cancellationToken = default
     )
     {
         var producer = BulkQuery.Create<int, ItemPrice>(
             async (chunk, ct) =>
             {
-                ItemPricesByIdsRequest request = new(chunk)
-                {
-                    MissingMemberBehavior = missingMemberBehavior
-                };
-                var response = await request.SendAsync(http, ct).ConfigureAwait(false);
+                var response = await GetItemPricesByIds(chunk, missingMemberBehavior, ct)
+                    .ConfigureAwait(false);
                 return response.Value;
             }
         );
 
         return producer.QueryAsync(
             itemIds,
+            degreeOfParalllelism,
+            chunkSize,
             progress: progress,
             cancellationToken: cancellationToken
         );
     }
 
-    public async IAsyncEnumerable<ItemPrice> GetItemPrices(
+    public async IAsyncEnumerable<ItemPrice> GetItemPricesBulk(
         MissingMemberBehavior missingMemberBehavior = default,
+        int degreeOfParalllelism = BulkQuery.DefaultDegreeOfParalllelism,
+        int chunkSize = BulkQuery.DefaultChunkSize,
         IProgress<ResultContext>? progress = default,
         [EnumeratorCancellation] CancellationToken cancellationToken = default
     )
     {
         var index = await GetItemPricesIndex(cancellationToken).ConfigureAwait(false);
-        var producer = GetItemPricesByIds(
+        var producer = GetItemPricesBulk(
             index.Value,
             missingMemberBehavior,
+            degreeOfParalllelism,
+            chunkSize,
             progress,
             cancellationToken
         );
@@ -138,42 +156,60 @@ public Task<Replica<OrderBook>> GetOrderBookById(
         return request.SendAsync(http, cancellationToken);
     }
 
-    public IAsyncEnumerable<OrderBook> GetOrderBooksByIds(
+    public Task<Replica<HashSet<OrderBook>>> GetOrderBooksByIds(
+        IReadOnlyCollection<int> itemIds,
+        MissingMemberBehavior missingMemberBehavior = default,
+        CancellationToken cancellationToken = default
+    )
+    {
+        OrderBooksByIdsRequest request = new(itemIds)
+        {
+            MissingMemberBehavior = missingMemberBehavior
+        };
+        return request.SendAsync(http, cancellationToken);
+    }
+
+    public IAsyncEnumerable<OrderBook> GetOrderBooksBulk(
         IReadOnlyCollection<int> itemIds,
         MissingMemberBehavior missingMemberBehavior = default,
+        int degreeOfParalllelism = BulkQuery.DefaultDegreeOfParalllelism,
+        int chunkSize = BulkQuery.DefaultChunkSize,
         IProgress<ResultContext>? progress = default,
         CancellationToken cancellationToken = default
     )
     {
         var producer = BulkQuery.Create<int, OrderBook>(
             async (chunk, ct) =>
             {
-                OrderBooksByIdsRequest request = new(chunk)
-                {
-                    MissingMemberBehavior = missingMemberBehavior
-                };
-                var response = await request.SendAsync(http, ct).ConfigureAwait(false);
+                var response = await GetOrderBooksByIds(chunk, missingMemberBehavior, ct)
+                    .ConfigureAwait(false);
                 return response.Value;
             }
         );
 
         return producer.QueryAsync(
             itemIds,
+            degreeOfParalllelism,
+            chunkSize,
             progress: progress,
             cancellationToken: cancellationToken
         );
     }
 
-    public async IAsyncEnumerable<OrderBook> GetOrderBooks(
+    public async IAsyncEnumerable<OrderBook> GetOrderBooksBulk(
         MissingMemberBehavior missingMemberBehavior = default,
+        int degreeOfParalllelism = BulkQuery.DefaultDegreeOfParalllelism,
+        int chunkSize = BulkQuery.DefaultChunkSize,
         IProgress<ResultContext>? progress = default,
         [EnumeratorCancellation] CancellationToken cancellationToken = default
     )
     {
         var index = await GetOrderBooksIndex(cancellationToken).ConfigureAwait(false);
-        var producer = GetOrderBooksByIds(
+        var producer = GetOrderBooksBulk(
             index.Value,
             missingMemberBehavior,
+            degreeOfParalllelism,
+            chunkSize,
             progress,
             cancellationToken
         );

diff --git a/GW2SDK/Features/Crafting/Models/Disciplines/CraftingQuery.cs b/GW2SDK/Features/Crafting/Models/Disciplines/CraftingQuery.cs
@@ -124,30 +124,17 @@ public Task<Replica<Recipe>> GetRecipeById(
         return request.SendAsync(http, cancellationToken);
     }
 
-    public IAsyncEnumerable<Recipe> GetRecipesByIds(
+    public Task<Replica<HashSet<Recipe>>> GetRecipesByIds(
         IReadOnlyCollection<int> recipeIds,
         MissingMemberBehavior missingMemberBehavior = default,
-        IProgress<ResultContext>? progress = default,
         CancellationToken cancellationToken = default
     )
     {
-        var producer = BulkQuery.Create<int, Recipe>(
-            async (chunk, ct) =>
-            {
-                RecipesByIdsRequest request = new(chunk)
-                {
-                    MissingMemberBehavior = missingMemberBehavior
-                };
-                var response = await request.SendAsync(http, ct).ConfigureAwait(false);
-                return response.Value;
-            }
-        );
-
-        return producer.QueryAsync(
-            recipeIds,
-            progress: progress,
-            cancellationToken: cancellationToken
-        );
+        RecipesByIdsRequest request = new(recipeIds)
+        {
+            MissingMemberBehavior = missingMemberBehavior
+        };
+        return request.SendAsync(http, cancellationToken);
     }
 
     public Task<Replica<HashSet<Recipe>>> GetRecipesByPage(
@@ -165,16 +152,47 @@ public Task<Replica<HashSet<Recipe>>> GetRecipesByPage(
         return request.SendAsync(http, cancellationToken);
     }
 
-    public async IAsyncEnumerable<Recipe> GetRecipes(
+    public IAsyncEnumerable<Recipe> GetRecipesBulk(
+        IReadOnlyCollection<int> recipeIds,
+        MissingMemberBehavior missingMemberBehavior = default,
+        int degreeOfParalllelism = BulkQuery.DefaultDegreeOfParalllelism,
+        int chunkSize = BulkQuery.DefaultChunkSize,
+        IProgress<ResultContext>? progress = default,
+        CancellationToken cancellationToken = default
+    )
+    {
+        var producer = BulkQuery.Create<int, Recipe>(
+            async (chunk, ct) =>
+            {
+                var response = await GetRecipesByIds(recipeIds, missingMemberBehavior, ct)
+                    .ConfigureAwait(false);
+                return response.Value;
+            }
+        );
+
+        return producer.QueryAsync(
+            recipeIds,
+            degreeOfParalllelism,
+            chunkSize,
+            progress: progress,
+            cancellationToken: cancellationToken
+        );
+    }
+
+    public async IAsyncEnumerable<Recipe> GetRecipesBulk(
         MissingMemberBehavior missingMemberBehavior = default,
+        int degreeOfParalllelism = BulkQuery.DefaultDegreeOfParalllelism,
+        int chunkSize = BulkQuery.DefaultChunkSize,
         IProgress<ResultContext>? progress = default,
         [EnumeratorCancellation] CancellationToken cancellationToken = default
     )
     {
         var index = await GetRecipesIndex(cancellationToken).ConfigureAwait(false);
-        var producer = GetRecipesByIds(
+        var producer = GetRecipesBulk(
             index.Value,
             missingMemberBehavior,
+            degreeOfParalllelism,
+            chunkSize,
             progress,
             cancellationToken
         );