feat(weavedrive): making multi-url more robust. Use customFetch in all calls to fetch

extensions/weavedrive/src/index.js

@@ -16,29 +16,41 @@ module.exports = function weaveDrive(mod, FS) {
       FS.streams[fd].node.cache = new Uint8Array(0)
     },
 
+    joinUrl ({ url, path }) {
+      if (!path) return url
+      if (path.startsWith('/')) return this.joinUrl({ url, path: path.slice(1) })
+
+      url = new URL(url)
+      url.pathname += path
+      return url.toString()
+    },
+
     async customFetch(path, options) {
-      let urlList = null
-      if (mod.ARWEAVE.includes(',')) {
-        urlList = mod.ARWEAVE.split(',').map(url => url.trim())
-      }
-      if (urlList && urlList.length > 0) {
-        /**
-         * Try a list of gateways instead of a single one
-         */
-        for (const url of urlList) {
-          const response = await fetch(`${url}${path}`, options)
-          if (response.ok) {
-            return response
-          }
-        }
-        /**
-         * None succeeded so fall back to mod.ARWEAVE so that
-         * if this fails we return a proper error response
-         */
-        return await fetch(`${mod.ARWEAVE}${path}`, options)
-      } else {
-        return await fetch(`${mod.ARWEAVE}${path}`, options)
-      }
+      /**
+       * mod.ARWEAVE may be a comma-delimited list of urls.
+       * So we parse it into an array that we sequentially consume
+       * using fetch, and return the first successful response.
+       * 
+       * The first url is considered "primary". So if all urls fail
+       * to produce a successful response, then we return the primary's
+       * error response
+       */
+      const urlList = mod.ARWEAVE.includes(',')
+        ? mod.ARWEAVE.split(',').map(url => url.trim())
+        : [mod.ARWEAVE]
+
+      let p
+      for (const url of urlList) {
+        const res = fetch(this.joinUrl({ url, path }), options)
+        if (await res.then((r) => r.ok).catch(() => false)) return res
+        if (!p) p = res
+      }
+
+      /**
+       * None succeeded so fallback to the primary and accept
+       * whatever it returned
+       */
+      return p
     },
 
     async create(id) {
@@ -482,9 +494,9 @@ module.exports = function weaveDrive(mod, FS) {
             owners: ${attestors},
             block: {min: 0, max: ${blockHeight}},
             tags: [
-              { name: "Data-Protocol", values: ["ao"] },
               { name: "Type", values: ["Attestation"] },
               { name: "Message", values: ["${ID}"]}
+              { name: "Data-Protocol", values: ["ao"] },
             ]
           ) 
           {
@@ -510,9 +522,9 @@ module.exports = function weaveDrive(mod, FS) {
               owners: ${attestors},
               block: {min: 0, max: ${blockHeight}},
               tags: [
-                { name: "Data-Protocol", values: ["WeaveDrive"] },
                 { name: "Type", values: ["Available"]},
                 { name: "ID", values: ["${ID}"]}
+                { name: "Data-Protocol", values: ["WeaveDrive"] },
               ]
             ) 
             {
@@ -588,42 +600,13 @@ module.exports = function weaveDrive(mod, FS) {
     },
 
     async gqlQuery(query, variables) {
-      let urlList = null
-      const headers = new Headers({})
-      headers.append("content-type", "application/json")
-      if (mod.ARWEAVE.includes(',')) {
-        urlList = mod.ARWEAVE.split(',').map(url => url.trim())
-      }
-      if (urlList && urlList.length > 0) {
-        /**
-         * Try a list of gateways instead of a single one
-         */
-        for (const url of urlList) {
-          const response = await fetch(`${url}/graphql`, {
-            method: 'POST',
-            body: JSON.stringify({ query, variables }),
-            headers
-          })
-          if (response.ok) {
-            return response
-          }
-        }
-        /**
-         * None succeeded so fall back to mod.ARWEAVE so that
-         * if this fails we return a proper error response
-         */
-        return await fetch(`${mod.ARWEAVE}/graphql`, {
-          method: 'POST',
-          body: JSON.stringify({ query, variables }),
-          headers
-        })
-      } else {
-        return await fetch(`${mod.ARWEAVE}/graphql`, {
-          method: 'POST',
-          body: JSON.stringify({ query, variables }),
-          headers
-        })
+      const options = {
+        method: 'POST',
+        body: JSON.stringify({ query, variables }),
+        headers: { 'Content-Type': 'application/json'}
       }
+
+      return this.customFetch('graphql', options)
     }
   }
 }

extensions/weavedrive/test/index.test.js

@@ -297,43 +297,45 @@ describe('Individual Mode', () => {
   })
 })
 
-// test weavedrive feature of acceptint multiple gateways
-test('read block, multi url', async () => {
-  const handle = await AoLoader(wasm, {
-    ...options,
-    ARWEAVE: 'https://arweave.net,https://g8way.io'
+// test weavedrive feature of accepting multiple gateways
+describe('multi-url', () => {
+  const urls =  'https://arweave.net/does-not-exist,https://g8way.io'
+  test('read block', async () => {
+    const handle = await AoLoader(wasm, {
+      ...options,
+      ARWEAVE: urls
+    })
+    const result = await handle(memory, {
+      ...Msg,
+      Data: `
+        return #require('WeaveDrive').getBlock('1439784').txs
+      `
+    }, { Process, Module })
+    memory = result.Memory
+    assert.equal(result.Output.data, '20')
   })
-  const result = await handle(memory, {
-    ...Msg,
-    Data: `
-    return #require('WeaveDrive').getBlock('1439784').txs
-`
-  }, { Process, Module })
-  memory = result.Memory
-})
-
-
-test('read tx, multi url', async () => {
-  const handle = await AoLoader(wasm, {
-    ...options,
-    ARWEAVE: 'https://arweave.net,https://g8way.io'
+
+
+  test('read tx', async () => {
+    const handle = await AoLoader(wasm, {
+      ...options,
+      ARWEAVE: urls
+    })
+    const result = await handle(memory, {
+      ...Msg,
+      Data: `
+        local results = {}
+        local drive = require('WeaveDrive')
+        local txs = drive.getBlock('1439784').txs
+        for i=1,#txs do
+          local tx = drive.getTx(txs[i])
+        end
+        return results
+      `
+    }, { Process, Module })
+    memory = result.Memory
+    assert.ok(true)
   })
-  const result = await handle(memory, {
-    ...Msg,
-    Data: `
-local results = {}
-local drive = require('WeaveDrive')
-local txs = drive 
-  .getBlock('1439784').txs
-for i=1,#txs do
-  local tx = drive.getTx(txs[i])
-end
-
-return results
-    `
-  }, { Process, Module })
-  memory = result.Memory
-  assert.ok(true)
 })
 
 test('read data item tx', async () => {
@@ -479,4 +481,27 @@ test('boot loader set to tx id', async function () {
   }, { Process: ProcessBootLoaderTx, Module })
 
   assert.equal(result.Output.data, '<TICKER>')
-})
+})
+
+describe('joinUrl', () => {
+  const wd = weaveDrive()
+  const joinUrl = wd.joinUrl.bind(wd)
+
+  test('should return the url', () => {
+    assert.equal(joinUrl({ url: 'https://arweave.net/graphql' }), 'https://arweave.net/graphql')
+    assert.equal(joinUrl({ url: 'https://arweave.net/graphql?foo=bar' }), 'https://arweave.net/graphql?foo=bar')
+    assert.equal(joinUrl({ url: 'https://arweave.net/graphql', path: undefined }), 'https://arweave.net/graphql')
+  })
+
+  test('should append the path', () => {
+    assert.equal(joinUrl({ url: 'https://arweave.net', path: 'graphql' }), 'https://arweave.net/graphql')
+    assert.equal(joinUrl({ url: 'https://arweave.net', path: '/graphql' }), 'https://arweave.net/graphql')
+    assert.equal(joinUrl({ url: 'https://arweave.net/', path: 'graphql' }), 'https://arweave.net/graphql')
+    assert.equal(joinUrl({ url: 'https://arweave.net/', path: '/graphql' }), 'https://arweave.net/graphql')
+
+    assert.equal(joinUrl({ url: 'https://arweave.net?foo=bar', path: 'graphql' }), 'https://arweave.net/graphql?foo=bar')
+    assert.equal(joinUrl({ url: 'https://arweave.net?foo=bar', path: '/graphql' }), 'https://arweave.net/graphql?foo=bar')
+    assert.equal(joinUrl({ url: 'https://arweave.net/?foo=bar', path: 'graphql' }), 'https://arweave.net/graphql?foo=bar')
+    assert.equal(joinUrl({ url: 'https://arweave.net/?foo=bar', path: '/graphql' }), 'https://arweave.net/graphql?foo=bar')
+  })
+})