Merge pull request #343 from permaweb/VinceJuliano/boot-loader-342

Vince juliano/boot loader 342

README.md

@@ -71,6 +71,7 @@ aos [name]
 | `--cu-url [url]`                              | The cu-url flag allows you to specify a custom Computer Unit to connect to.                                                                                                                                                        | 0-1      |
 | `--mu-url [url]`                                | The mu-url flag allows you to specify a custom Memory Unit to connect to.                                                                                                                                                          | 0-1      |
 | `--sqlite` | 0-1 | Use sqlite aos module when spawning new process |
+| `--data [filepath]` | 0-1 | Set file contents as the data field of your Process. Will run when the process boots up if the On-Boot tag is set to Data, if On-Boot is set to Data make sure the file contains valid lua code |
 
 ### Commands
 
@@ -83,6 +84,20 @@ When running the console, you can type `dot` commands to instruct the console to
 | `.load-blueprint [name]` | This command will grab a lua file from the blueprints directory and load it into your process.                                               |
 | `.exit`                  | This command will exit you console, but you can also do `Ctrl-C` or `Ctrl-D`                                                                 |
 
+### Boot Loading
+
+When booting up a new aos you can specify a script as the Data field of your process and use the On-Boot tag to run at startup. If On-Boot is set to Data, the script can be read from a local file
+
+```sh
+aos boottest --tag-name="On-Boot" --tag-value="Data" --data=../bootup.lua
+```
+
+Or On-Boot can be a tx id, which must also contain a valid lua script.
+
+```sh
+aos boottest --tag-name="On-Boot" --tag-value="arweavetxid"
+```
+
 ## License
 
 The ao and aos codebases are offered under the BSL 1.1 license for the duration

extensions/weavedrive/client/main.lua

@@ -33,4 +33,16 @@ function drive.getTx(txId)
   return contents
 end
 
+function drive.getData(txId)
+  local file = io.open('/data/' .. txId)
+  if not file then
+    return nil, "File not found!"
+  end
+  local contents = file:read(
+    file:seek('end')
+  )
+  file:close()
+  return contents
+end
+
 return drive

extensions/weavedrive/package.json

@@ -4,7 +4,7 @@
   "main": "./dist/index.cjs",
   "license": "MIT",
   "devDependencies": {
-    "@permaweb/ao-loader": "^0.0.33",
+    "@permaweb/ao-loader": "^0.0.35",
     "@permaweb/weavedrive": "^0.0.2",
     "esbuild": "^0.21.1"
   }

extensions/weavedrive/src/index.js

@@ -16,6 +16,31 @@ module.exports = function weaveDrive(mod, FS) {
       FS.streams[fd].node.cache = new Uint8Array(0)
     },
 
+    async customFetch(path, options) {
+      let urlList = null
+      if(mod.ARWEAVE.includes(',')) {
+        urlList = mod.ARWEAVE.split(',').map(url => url.trim())
+      }
+      if(urlList && urlList.length > 0) {
+        /**
+         * Try a list of gateways instead of a single one
+         */
+        for (const url of urlList) {
+          const response = await fetch(`${url}${path}`, options)
+          if (response.ok) {
+            return response
+          }
+        }
+        /**
+         * None succeeded so fall back to mod.ARWEAVE so that
+         * if this fails we return a proper error response
+         */
+        return await fetch(`${mod.ARWEAVE}${path}`, options)
+      } else {
+        return await fetch(`${mod.ARWEAVE}${path}`, options)
+      }
+    },
+
     async create(id) {
       var properties = { isDevice: false, contents: null }
 
@@ -25,10 +50,17 @@ module.exports = function weaveDrive(mod, FS) {
       }
 
       // Create the file in the emscripten FS
-      // TODO: might make sense to create the `data` folder here if does not exist
+
+      // This check/mkdir was added for AOP 6 Boot loader because create is
+      // called first because were only loading Data, we needed to create
+      // the directory. See: https://github.com/permaweb/aos/issues/342
+      if(!FS.analyzePath('/data/').exists){
+        FS.mkdir('/data/');
+      }
+
       var node = FS.createFile('/', 'data/' + id, properties, true, false);
       // Set initial parameters
-      var bytesLength = await fetch(`${mod.ARWEAVE}/${id}`, { method: 'HEAD' }).then(res => res.headers.get('Content-Length'))
+      var bytesLength = await this.customFetch(`/${id}`, { method: 'HEAD' }).then(res => res.headers.get('Content-Length'))
       node.total_size = Number(bytesLength)
       node.cache = new Uint8Array(0)
       node.position = 0;
@@ -43,15 +75,16 @@ module.exports = function weaveDrive(mod, FS) {
       return stream;
     },
     async createBlockHeader(id) {
+      const customFetch = this.customFetch
       // todo: add a bunch of retries
       async function retry(x) {
         return new Promise(r => {
           setTimeout(function () {
-            r(fetch(`${mod.ARWEAVE}/block/height/${id}`))
+            r(customFetch(`/block/height/${id}`))
           }, x * 10000)
         })
       }
-      var result = await fetch(`${mod.ARWEAVE}/block/height/${id}`)
+      var result = await this.customFetch(`/block/height/${id}`)
         .then(res => !res.ok ? retry(1) : res)
         .then(res => !res.ok ? retry(2) : res)
         .then(res => !res.ok ? retry(3) : res)
@@ -66,6 +99,7 @@ module.exports = function weaveDrive(mod, FS) {
       return stream;
     },
     async createTxHeader(id) {
+      const customFetch = this.customFetch
       async function toAddress(owner) {
         return Arweave.utils.bufferTob64Url(
           await Arweave.crypto.hash(Arweave.utils.b64UrlToBuffer(owner))
@@ -74,12 +108,12 @@ module.exports = function weaveDrive(mod, FS) {
       async function retry(x) {
         return new Promise(r => {
           setTimeout(function () {
-            r(fetch(`${mod.ARWEAVE}/tx/${id}`))
+            r(customFetch(`/tx/${id}`))
           }, x * 10000)
         })
       }
       // todo: add a bunch of retries
-      var result = await fetch(`${mod.ARWEAVE}/tx/${id}`)
+      var result = await this.customFetch(`/tx/${id}`)
         .then(res => !res.ok ? retry(1) : res)
         .then(res => !res.ok ? retry(2) : res)
         .then(res => !res.ok ? retry(3) : res)
@@ -197,7 +231,7 @@ module.exports = function weaveDrive(mod, FS) {
       //console.log("WeaveDrive: fd: ", fd, " Read length: ", to_read, " Reading ahead:", to - to_read - stream.position)
 
       // Fetch with streaming
-      const response = await fetch(`${mod.ARWEAVE}/${stream.node.name}`, {
+      const response = await this.customFetch(`/${stream.node.name}`, {
         method: "GET",
         redirect: "follow",
         headers: { "Range": `bytes=${stream.position}-${to}` }
@@ -313,6 +347,11 @@ module.exports = function weaveDrive(mod, FS) {
         return true
       }
 
+      // Check if we are attempting to load the On-Boot id, if so allow it
+      // this was added for AOP 6 Boot loader See: https://github.com/permaweb/aos/issues/342
+      const bootTag = mod.Process.Tags.find((t) => t.name === 'On-Boot')?.value;
+      if (bootTag && (bootTag === ID)) return true;
+
       // Check that this module or process set the WeaveDrive tag on spawn
       const blockHeight = mod.blockHeight
       const moduleExtensions = this.getTagValues("Extension", mod.module.tags)

extensions/weavedrive/test/index.test.js

@@ -4,6 +4,7 @@ const { test } = require('node:test')
 const assert = require('assert')
 const weaveDrive = require('../src/index.js')
 const wasm = fs.readFileSync('./aosqlite.wasm')
+const bootLoaderWasm = fs.readFileSync('./bootloader.wasm')
 
 let memory = null
 
@@ -146,3 +147,112 @@ return require('json').encode({ A = #results, B = #results2 })
 
   assert.equal(res.A, res.B)
 })
+
+// test weavedrive feature of acceptint multiple gateways
+test('read block, multi url', async () => {
+  const handle = await AoLoader(wasm, { 
+    ...options, 
+    ARWEAVE: 'https://arweave.net,https://g8way.io'
+  })
+  const result = await handle(memory, {
+    ...Msg,
+    Data: `
+    return #require('WeaveDrive').getBlock('1439784').txs
+`
+  }, { Process, Module })
+  memory = result.Memory
+})
+
+
+test('read tx, multi url', async () => {
+  const handle = await AoLoader(wasm, { 
+    ...options, 
+    ARWEAVE: 'https://arweave.net,https://g8way.io'
+  })
+  const result = await handle(memory, {
+    ...Msg,
+    Data: `
+local results = {}
+local drive = require('WeaveDrive')
+local txs = drive 
+  .getBlock('1439784').txs
+for i=1,#txs do
+  local tx = drive.getTx(txs[i])
+end
+
+return results
+    `
+  }, { Process, Module })
+  memory = result.Memory
+  console.log(result)
+  assert.ok(true)
+})
+
+
+let memoryBootLoader = null
+
+ /*
+  * The Process is also the first message when aop 6 boot loader
+  * is enabled in the network
+  */
+const ProcessBootLoaderData = {
+  Id: 'PROCESS',
+  Owner: 'PROCESS',
+  Target: 'PROCESS',
+  Tags: [
+    { name: 'Data-Protocol', value: 'ao' },
+    { name: 'Variant', value: 'ao.TN.1' },
+    { name: 'Type', value: 'Process' },
+    { name: 'Extension', value: 'WeaveDrive' },
+    { name: 'On-Boot', value: 'Data' },
+    { name: 'Module', value: 'MODULE' },
+    { name: 'Authority', value: 'PROCESS' }
+  ],
+  Data: `
+Test = 1
+    `,
+  From: 'PROCESS',
+  Module: 'MODULE',
+  "Block-Height": 1000,
+  Timestamp: Date.now()
+}
+
+const optionsBootLoaderData = { ...options, Process: ProcessBootLoaderData, mode: null}
+
+test('boot loader set to Data', async function () {
+  const handle = await AoLoader(bootLoaderWasm, optionsBootLoaderData)
+  await handle(memoryBootLoader, {
+    ...ProcessBootLoaderData
+  }, { Process: ProcessBootLoaderData, Module })
+})
+
+const ProcessBootLoaderTx = {
+  Id: 'PROCESS',
+  Owner: 'PROCESS',
+  Target: 'PROCESS',
+  Tags: [
+    { name: 'Data-Protocol', value: 'ao' },
+    { name: 'Variant', value: 'ao.TN.1' },
+    { name: 'Type', value: 'Process' },
+    { name: 'Extension', value: 'WeaveDrive' },
+    { name: 'On-Boot', value: 'Fmtgzy1Chs-5ZuUwHpQjQrQ7H7v1fjsP0Bi8jVaDIKA' },
+    { name: 'Module', value: 'MODULE' },
+    { name: 'Authority', value: 'PROCESS' }
+  ],
+  Data: `
+Test = 1
+    `,
+  From: 'PROCESS',
+  Module: 'MODULE',
+  "Block-Height": 1000,
+  Timestamp: Date.now()
+}
+
+const optionsBootLoaderTx = { ...options, Process: ProcessBootLoaderTx, mode: null}
+
+test('boot loader set to tx id', async function () {
+  const handle = await AoLoader(bootLoaderWasm, optionsBootLoaderTx)
+  await handle(memoryBootLoader, {
+    ...ProcessBootLoaderTx
+  }, { Process: ProcessBootLoaderTx, Module })
+})

process/boot.lua

@@ -0,0 +1,34 @@
+-- This is for aop6 Boot Loader
+-- See: https://github.com/permaweb/aos/issues/342
+-- For the Process as the first Message, if On-Boot
+-- has the value 'data' then evaluate the data
+-- if it is a tx id, then download and evaluate the tx
+
+local drive = { _version = "0.0.1" }
+
+function drive.getData(txId)
+  local file = io.open('/data/' .. txId)
+  if not file then
+    return nil, "File not found!"
+  end
+  local contents = file:read(
+    file:seek('end')
+  )
+  file:close()
+  return contents
+end
+
+return function (ao)
+  local eval = require(".eval")(ao)
+  return function (msg)
+    if msg.Tags['On-Boot'] == nil then
+      return
+    end
+    if msg.Tags['On-Boot'] == 'Data' then
+      eval(msg)
+    else
+      local loadedVal = drive.getData(msg.Tags['On-Boot'])
+      eval({ Data = loadedVal })
+    end
+  end 
+end

process/process.lua

@@ -299,7 +299,18 @@ function process.handle(msg, _)
     end,
     require('.eval')(ao)
   )
+
+  -- Added for aop6 boot loader
+  -- See: https://github.com/permaweb/aos/issues/342
+  Handlers.once("_boot",
+    function (msg)
+      return msg.Tags.Type == "Process" and Owner == msg.From
+    end,
+    require('.boot')(ao)
+  )
+
   Handlers.append("_default", function () return true end, require('.default')(insertInbox))
+
   -- call evaluate from handlers passing env
   msg.reply =
     function(replyMsg)
@@ -376,6 +387,25 @@ function process.handle(msg, _)
         test = Dump(HANDLER_PRINT_LOGS)
       }
     })
+    HANDLER_PRINT_LOGS = {} -- clear logs
+    return response
+  elseif msg.Tags.Type == "Process" and Owner == msg.From then 
+    local response = nil
+
+    -- detect if there was any output from the boot loader call
+    for _, value in pairs(HANDLER_PRINT_LOGS) do
+      if value ~= "" then
+        -- there was output from the Boot Loader eval so we want to print it
+        response = ao.result({ Output = { data = table.concat(HANDLER_PRINT_LOGS, "\n"), prompt = Prompt(), print = true } })
+        break
+      end
+    end
+
+    if response == nil then 
+      -- there was no output from the Boot Loader eval, so we shouldn't print it
+      response = ao.result({ Output = { data = "", prompt = Prompt() } })
+    end
+
     HANDLER_PRINT_LOGS = {} -- clear logs
     return response
   else