Feat Throttled List

src/IpfsPinSync.js

@@ -99,14 +99,22 @@ export default class IpfsPinSync {
 
       let pinsExistToCheck = true;
       let pageCount = 0,
-        pageLimit = 250;
+        pageLimit = 1000;
       let pinList = [];
 
+      const listLimiter = new Bottleneck({
+        reservoir: 25,
+        reservoirRefreshInterval: 60 * 1000,
+        reservoirRefreshAmount: 25,
+        maxConcurrent: 1
+      });
+      const throttledListPins = listLimiter.wrap(axios);
+
       while (pinsExistToCheck === true) {
         //Request Page of Pins from Provider
         listConfig.params.pageLimit = pageLimit;
         listConfig.params.pageOffset = pageCount * pageLimit;
-        const listRequest = await axios(listConfig);
+        const listRequest = await throttledListPins(listConfig);
         for (let pin of listRequest.data.rows) {
           pinList.push({
             pin: {
@@ -129,43 +137,54 @@ export default class IpfsPinSync {
     }
 
     async #list (client) {
-        let pinsExistToCheck = true
-        let earliestPinInList = null
-        let pinList = [];
-
-        while (pinsExistToCheck === true) {
-            // Get 1000 Successful Pins
-            let pinsGetOptions = {
-                limit: 500,
-                status: new Set([Status.Pinned, Status.Pinning, Status.Queued]) // requires a set, and not an array
-            }
-            if (earliestPinInList != null) {
-                pinsGetOptions.before = earliestPinInList
-            }
-
-            const {count, results} = await client.pinsGet(pinsGetOptions)
-
-            console.log(count, results)
-            pinList = pinList.concat(Array.from(results));
-
-            earliestPinInList = this.#getOldestPinCreateDate(results)
+      let pinsExistToCheck = true
+      let earliestPinInList = null
+      let pinList = [];
 
-            console.log(`Results Length: ${results.size}`)
-            if (results.size !== 1000) {
-                pinsExistToCheck = false;
-            }
-        }
+      const listLimiter = new Bottleneck({
+        reservoir: 25,
+        reservoirRefreshInterval: 10 * 1000,
+        reservoirRefreshAmount: 25,
+        maxConcurrent: 1
+      });
+      const throttledGetPins = listLimiter.wrap((options) => {
+        return client.pinsGet(options)
+      });
+
+      const pinsToBatch = 500;
+      while (pinsExistToCheck === true) {
+          // Get 500 Successful Pins
+          let pinsGetOptions = {
+              limit: pinsToBatch,
+              status: new Set([Status.Pinned, Status.Pinning, Status.Queued]) // requires a set, and not an array
+          }
+          if (earliestPinInList != null) {
+              pinsGetOptions.before = earliestPinInList
+          }
+
+          const {count, results} = await throttledGetPins(pinsGetOptions)
+
+          console.log(count, results)
+          pinList = pinList.concat(Array.from(results));
+
+          earliestPinInList = this.#getOldestPinCreateDate(results)
+
+          console.log(`Results Length: ${results.size}`)
+          if (results.size !== pinsToBatch) {
+              pinsExistToCheck = false;
+          }
+      }
 
-        return pinList;
+      return pinList;
     }
 
     async sync (progressCallback) {
         const syncLimiter = new Bottleneck({
             reservoir: 25,
             reservoirRefreshInterval: 1000,
             reservoirRefreshAmount: 25,
-            maxConcurrent: 10
-        })
+            maxConcurrent: 4
+        });
         const throttledAddPin = syncLimiter.wrap(this.#addPin);
 
         const sourcePins = await this.listSource();