various fixes for Docker

.dockerignore

@@ -1 +1,3 @@
 .env
+node_modules
+data-*

.gitignore

@@ -4,3 +4,4 @@ tmp
 *.env
 .DS_Store
 .vscode
+data-*

Dockerfile

@@ -16,7 +16,7 @@ FROM node:lts-alpine3.19
 RUN apk add gcompat
 
 # Set the working directory in the container
-WORKDIR /usr/src/app
+WORKDIR /app
 
 # Copy package.json and package-lock.json (if available) to the working directory
 COPY package*.json ./

bin/auth-server-example.js

@@ -14,9 +14,11 @@ const appName = 'Auth-Server-Example'
 const authPrivateKey = await ecdsa.importKeyJwk(json.parse(env.ensureConf('auth-private-key')))
 const port = 5173
 
+console.log('Auth Server Config', { port })
+
 const app = uws.App({})
 
-app.put('/ydoc/:room', async (res, req) => {
+app.put('/ydoc/:room', async function updated(res, req) {
   let aborted = false
   res.onAborted(() => {
     aborted = true
@@ -36,64 +38,88 @@ app.put('/ydoc/:room', async (res, req) => {
         return
       }
       const ydocUpdate = new Uint8Array(ydocUpdateData)
+      Y.logUpdateV2(ydocUpdate)
       const ydoc = new Y.Doc()
       Y.applyUpdateV2(ydoc, ydocUpdate)
-      console.log(`Ydoc in room "${room}" updated. New codemirror content: "${ydoc.getText('codemirror')}"`)
+      console.log('/ydoc', { room, content: dumpDoc(ydoc) })
       res.endWithoutBody()
     }
   })
 })
 
 // This example server always grants read-write permission to all requests.
 // Modify it to your own needs or implement the same API in your own backend!
-app.get('/auth/token', async (res, _req) => {
+app.get('/auth/token', async function authn(res, _req) {
+  const yuserid = 'user1'
   let aborted = false
   res.onAborted(() => {
     aborted = true
   })
   const token = await jwt.encodeJwt(authPrivateKey, {
     iss: appName,
     exp: time.getUnixTime() + 1000 * 60 * 60, // access expires in an hour
-    yuserid: 'user1'
+    yuserid
   })
   if (aborted) return
-  res.cork(() => {
-    res.end(token)
-  })
+  res.cork(() => res.end(token))
+  console.log('/auth/token', { yuserid }, '=>', token)
 })
 
-app.get('/auth/perm/:room/:userid', async (res, req) => {
+app.get('/auth/perm/:room/:userid', async function authz(res, req) {
   const yroom = req.getParameter(0)
   const yuserid = req.getParameter(1)
-  res.end(json.stringify({
-    yroom,
-    yaccess: 'rw',
-    yuserid
-  }))
+  const response = json.stringify({ yroom, yaccess: 'rw', yuserid })
+  console.log('/auth/perm', { yroom, yuserid }, '=>', response)
+  res.end(response)
 })
 
 /**
  * Resolves when the server started.
  */
-export const authServerStarted = promise.create((resolve, reject) => {
+export const whenStarted = promise.create((resolve, reject) => {
   const server = app.listen(port, (token) => {
     if (token) {
-      logging.print(logging.GREEN, `[${appName}] Listening to port ${port}`)
+      console.log(`Listening on port ${port}`)
       resolve()
     } else {
-      const err = error.create(`[${appName}] Failed to lisen to port ${port}`)
+      const err = error.create(`[${appName}] Failed to listen on port ${port}`)
       reject(err)
       throw err
     }
   })
 
   // Gracefully shut down the server when running in Docker
-  process.on("SIGTERM", shutDown)
-  process.on("SIGINT", shutDown)
+  process.on('SIGTERM', shutDown)
+  process.on('SIGINT', shutDown)
 
   function shutDown() {
-    console.log("Received SIGTERM/SIGINT - shutting down")
+    console.log('Received SIGTERM/SIGINT - shutting down')
     server.close()
     process.exit(0)
   }
 })
+
+/**
+ * Hacky function to dump latest version of a general Y.Doc to a plain object.
+ * Ideally you would assume a specific schema and use the approprate typed getter.
+ *
+ * @param {Y.Doc} ydoc
+ */
+function dumpDoc(ydoc) {
+  /**
+   * @type {Object<string, any>}
+   */
+  const doc = {}
+  ydoc.share.forEach((type, key) => {
+    // Hack to "get" types in order for yDoc.toJSON() to show the data.
+    let value =
+      type._start !== null //
+        ? ydoc.getText(key).toJSON()
+        : type._map.size !== 0
+        ? ydoc.getMap(key).toJSON()
+        : 'unknown'
+
+    doc[key] = value
+  })
+  return doc
+}

bin/server.js

@@ -10,6 +10,8 @@ const postgresUrl = env.getConf('postgres')
 const s3Endpoint = env.getConf('s3-endpoint')
 const checkPermCallbackUrl = env.ensureConf('AUTH_PERM_CALLBACK')
 
+console.log('Server Config ', { port, redisPrefix, postgresUrl, s3Endpoint, checkPermCallbackUrl })
+
 let store
 if (s3Endpoint) {
   console.log('using s3 store')
@@ -30,4 +32,13 @@ if (s3Endpoint) {
   store = createMemoryStorage()
 }
 
-server.createYWebsocketServer({ port, store, checkPermCallbackUrl, redisPrefix })
+const instance = await server.createYWebsocketServer({ port, store, checkPermCallbackUrl, redisPrefix })
+
+// Gracefully shut down the server when running in Docker
+process.on('SIGTERM', shutDown)
+process.on('SIGINT', shutDown)
+function shutDown() {
+  console.log('Received SIGTERM/SIGINT - shutting down')
+  instance.destroy()
+  process.exit(0)
+}

bin/worker.js

@@ -7,6 +7,8 @@ const redisPrefix = env.getConf('redis-prefix') || 'y'
 const postgresUrl = env.getConf('postgres')
 const s3Endpoint = env.getConf('s3-endpoint')
 
+console.log('Worker Config', { redisPrefix, postgresUrl, s3Endpoint })
+
 let store
 if (s3Endpoint) {
   console.log('using s3 store')
@@ -27,4 +29,13 @@ if (s3Endpoint) {
   store = createMemoryStorage()
 }
 
-api.createWorker(store, redisPrefix)
+const wk = await api.createWorker(store, redisPrefix)
+
+// Gracefully shut down the server when running in Docker
+process.on('SIGTERM', shutDown)
+process.on('SIGINT', shutDown)
+function shutDown() {
+  console.log('Received SIGTERM/SIGINT - shutting down')
+  wk.client.destroy()
+  process.exit(0)
+}

compose.yaml

@@ -1,25 +1,64 @@
+x-vars:
+  shared: &shared
+    env_file: .env # only needed for AUTH_*_KEY env vars
+    # overrides to connect services inside of Docker overlay network
+    environment: &shared-env
+      REDIS: redis://redis:6379
+      POSTGRES: postgres://pg:5432
+      S3_ENDPOINT: s3
+      AUTH_PERM_CALLBACK: http://auth:5173/auth/perm
+      YDOC_UPDATE_CALLBACK: http://backup:5173/ydoc
+  core: &core
+    <<: *shared
+    build: .
+    depends_on: [s3, redis, pg]
+
 services:
   server:
-    build: .
-    ports:
-      - "3002:3002"
+    <<: *core
+    ports: ['3002:3002']
     command: node ./bin/server.js --no-colors
-    env_file: .env
-    depends_on:
-      - minio
-      - redis
+
   worker:
-    build: .
+    <<: *core
     command: node ./bin/worker.js --no-colors
-    env_file: .env
-    depends_on:
-      - minio
-      - redis
+
+  # microservices - also included in demo1 server
+  auth: # this is usually provided by the demo app
+    <<: *core
+    command: node ./bin/auth-server-example --no-colors
+
+  backup:
+    <<: *core
+    command: node ./bin/auth-server-example --no-colors
+
+  # demos - start these from inside demos/*/compose.yml
+  # demo1:
+  #   <<: *shared
+  #   build: demos/auth-express
+  #   ports: ["5173:5173"]
+  #   # profiles: [demo1]
+  #   depends_on: [server]
+
+  # demo2:
+  #   <<: *shared
+  #   build: demos/blocksuite
+  #   ports: ["5174:5173"]
+  #   # profiles: [demo2]
+  #   depends_on: [server]
+
+  # Backing services - WARNING: secure if you want to use in production
   redis:
     image: redis:alpine
-  minio:
-    image: quay.io/minio/minio
-    ports:
-      - 9000:9000
-      - 9001:9001
-    command: server /data --console-address :9001
+    volumes: ['./data-redis:/data']
+    command: redis-server --dir /data
+  s3:
+    image: minio/minio
+    volumes: ['./data-minio:/data']
+    command: server /data --quiet
+  pg:
+    image: postgres:alpine
+    volumes: ['./data-postgres:/data']
+    environment:
+      PGDATA: /data
+      POSTGRES_HOST_AUTH_METHOD: trust

demos/auth-express/Dockerfile

@@ -16,7 +16,7 @@ FROM node:lts-alpine3.19
 RUN apk add gcompat
 
 # Set the working directory in the container
-WORKDIR /usr/src/app
+WORKDIR /app
 
 # Copy package.json and package-lock.json (if available) to the working directory
 COPY package*.json ./
@@ -26,6 +26,7 @@ RUN npm install
 
 # Bundle your app source inside the Docker image
 COPY . .
+RUN npm run dist
 
 # Make port 3002 available to the world outside this container,
 # assuming your app runs on port 3002

demos/auth-express/demo.js

@@ -25,12 +25,17 @@ export const usercolors = [
 
 export const userColor = usercolors[random.uint32() % usercolors.length]
 
+const syncUrl = `ws://localhost:3002`
+const authUrl = `http://${location.host}/auth/token`
 const room = 'y-redis-demo-app'
-const authTokenRequest = await fetch(`http://${location.host}/auth/token`)
+
+console.log("Demo Config", { syncUrl, authUrl, room })
+
+const authTokenRequest = await fetch(authUrl)
 const authToken = await authTokenRequest.text()
 
 const ydoc = new Y.Doc()
-const provider = new WebsocketProvider('ws://localhost:3002', room, ydoc, { params: { yauth: authToken } })
+const provider = new WebsocketProvider(syncUrl, room, ydoc, { params: { yauth: authToken } })
 const ytext = ydoc.getText('codemirror')
 
 provider.awareness.setLocalStateField('user', {