chore: upgrade to turbo v2 and reduce ci quality gate runtime (#4324)

* chore: upgrade to turbo v2 and reduce ci quality gate runtime * chore: upgrade to yarn v4 and deprecate turborepo * chore: reconfigure modules and fix tests * chore: switch to github-hosted runner * fix: dependency resolving * chore: clean redundant step * chore: headless test * chore: remove headed param * fix: resolve dependency version * fix: ubuntu no-sandbox test --------- Co-authored-by: Hien To <[email protected]> Co-authored-by: Louis <[email protected]>
janhq · Dec 29, 2024 · bd0e525 · bd0e525
1 parent da4336c
commit bd0e525
Show file tree

Hide file tree

Showing 48 changed files with 30,330 additions and 299 deletions.
diff --git a/.github/workflows/jan-electron-linter-and-test.yml b/.github/workflows/jan-electron-linter-and-test.yml
@@ -66,7 +66,7 @@ jobs:
 
   test-on-macos:
     if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
-    runs-on: [self-hosted, macOS, macos-desktop]
+    runs-on: macos-latest
     steps:
       - name: Getting the repo
         uses: actions/checkout@v3
@@ -100,18 +100,13 @@ jobs:
 
       - name: Linter and test
         run: |
-          npm config set registry ${{ secrets.NPM_PROXY }} --global
-          yarn config set registry ${{ secrets.NPM_PROXY }} --global
           make test
         env:
           CSC_IDENTITY_AUTO_DISCOVERY: 'false'
-          # TURBO_API: '${{ secrets.TURBO_API }}'
-          # TURBO_TEAM: 'macos'
-          # TURBO_TOKEN: '${{ secrets.TURBO_TOKEN }}'
 
   test-on-macos-pr-target:
     if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
-    runs-on: [self-hosted, macOS, macos-desktop]
+    runs-on: macos-latest
     steps:
       - name: Getting the repo
         uses: actions/checkout@v3
@@ -131,8 +126,6 @@ jobs:
 
       - name: Linter and test
         run: |
-          npm config set registry https://registry.npmjs.org --global
-          yarn config set registry https://registry.npmjs.org --global
           make test
         env:
           CSC_IDENTITY_AUTO_DISCOVERY: 'false'
@@ -182,13 +175,8 @@ jobs:
       - name: Linter and test
         shell: powershell
         run: |
-          npm config set registry ${{ secrets.NPM_PROXY }} --global
-          yarn config set registry ${{ secrets.NPM_PROXY }} --global
           make test
-        # env:
-        #   TURBO_API: '${{ secrets.TURBO_API }}'
-        #   TURBO_TEAM: 'windows'
-        #   TURBO_TOKEN: '${{ secrets.TURBO_TOKEN }}'
+
   test-on-windows-pr:
     if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)
     runs-on: windows-desktop-default-windows-security
@@ -230,17 +218,11 @@ jobs:
       - name: Linter and test
         shell: powershell
         run: |
-          npm config set registry ${{ secrets.NPM_PROXY }} --global
-          yarn config set registry ${{ secrets.NPM_PROXY }} --global
           make test
-        # env:
-        #   TURBO_API: '${{ secrets.TURBO_API }}'
-        #   TURBO_TEAM: 'windows'
-        #   TURBO_TOKEN: '${{ secrets.TURBO_TOKEN }}'
 
   test-on-windows-pr-target:
     if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
-    runs-on: windows-desktop-default-windows-security
+    runs-on: windows-latest
     steps:
       - name: Getting the repo
         uses: actions/checkout@v3
@@ -268,12 +250,10 @@ jobs:
       - name: Linter and test
         shell: powershell
         run: |
-          npm config set registry https://registry.npmjs.org --global
-          yarn config set registry https://registry.npmjs.org --global
           make test
 
   test-on-ubuntu:
-    runs-on: [self-hosted, Linux, ubuntu-desktop]
+    runs-on: ubuntu-latest
     if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     steps:
       - name: Getting the repo
@@ -311,13 +291,7 @@ jobs:
         run: |
           export DISPLAY=$(w -h | awk 'NR==1 {print $2}')
           echo -e "Display ID: $DISPLAY"
-          npm config set registry ${{ secrets.NPM_PROXY }} --global
-          yarn config set registry ${{ secrets.NPM_PROXY }} --global
           make test
-        # env:
-        #   TURBO_API: '${{ secrets.TURBO_API }}'
-        #   TURBO_TEAM: 'linux'
-        #   TURBO_TOKEN: '${{ secrets.TURBO_TOKEN }}'
 
       - uses: actions/upload-artifact@v4
         if: always()
@@ -327,7 +301,7 @@ jobs:
           retention-days: 2
 
   coverage-check:
-    runs-on: [self-hosted, Linux, ubuntu-desktop]
+    runs-on: ubuntu-latest
     needs: base_branch_cov
     if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     steps:
@@ -341,6 +315,9 @@ jobs:
         with:
           node-version: 20
 
+      - name: Install yarn
+        run: npm install -g yarn
+
       - name: 'Cleanup cache'
         continue-on-error: true
         run: |
@@ -356,15 +333,9 @@ jobs:
         run: |
           export DISPLAY=$(w -h | awk 'NR==1 {print $2}')
           echo -e "Display ID: $DISPLAY"
-          npm config set registry ${{ secrets.NPM_PROXY }} --global
-          yarn config set registry ${{ secrets.NPM_PROXY }} --global
           make lint
           yarn build:test
           yarn test:coverage
-        # env:
-        #   TURBO_API: '${{ secrets.TURBO_API }}'
-        #   TURBO_TEAM: 'linux'
-        #   TURBO_TOKEN: '${{ secrets.TURBO_TOKEN }}'
 
       - name: Generate Code Coverage report
         id: code-coverage
@@ -377,7 +348,7 @@ jobs:
           show-annotations: 'warning'
 
   test-on-ubuntu-pr-target:
-    runs-on: [self-hosted, Linux, ubuntu-desktop]
+    runs-on: ubuntu-latest
     if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Getting the repo
@@ -400,6 +371,4 @@ jobs:
         run: |
           export DISPLAY=$(w -h | awk 'NR==1 {print $2}')
           echo -e "Display ID: $DISPLAY"
-          npm config set registry https://registry.npmjs.org --global
-          yarn config set registry https://registry.npmjs.org --global
           make test
diff --git a/.github/workflows/publish-npm-core.yml b/.github/workflows/publish-npm-core.yml
@@ -1,4 +1,4 @@
-name: Publish plugin models Package to npmjs
+name: Publish core Package to npmjs
 on:
   push:
     tags: ["v[0-9]+.[0-9]+.[0-9]+-core"]
@@ -45,7 +45,7 @@ jobs:
           node-version: "20.x"
           registry-url: "https://registry.npmjs.org"
 
-      - run: cd core && yarn install && yarn build
+      - run: cd core && corepack enable && corepack prepare [email protected] --activate && yarn --version && yarn install && yarn build
 
       - run: cd core && yarn publish --access public
         if: github.event_name == 'push'

diff --git a/.github/workflows/publish-npm-joi.yml b/.github/workflows/publish-npm-joi.yml
@@ -1,4 +1,4 @@
-name: Publish plugin models Package to npmjs
+name: Publish joi Package to npmjs
 on:
   push:
     tags: ["v[0-9]+.[0-9]+.[0-9]+-joi"]
@@ -45,7 +45,7 @@ jobs:
           node-version: "20.x"
           registry-url: "https://registry.npmjs.org"
 
-      - run: cd joi && yarn install && yarn build
+      - run: cd joi && corepack enable && corepack prepare [email protected] --activate && yarn --version && yarn install && yarn build
 
       - run: cd joi && yarn publish --access public
         if: github.event_name == 'push'

diff --git a/.gitignore b/.gitignore
@@ -8,7 +8,6 @@ error.log
 node_modules
 *.tgz
 !charts/server/charts/*.tgz
-yarn.lock
 dist
 build
 .DS_Store
@@ -48,3 +47,6 @@ coverage
 test_results.html
 *.tsbuildinfo
 electron/shared/**
+
+# docs
+docs/yarn.lock
diff --git a/.yarnrc.yml b/.yarnrc.yml
@@ -0,0 +1,3 @@
+nmHoistingLimits: workspaces
+nodeLinker: node-modules
+checksumBehavior: update
diff --git a/Makefile b/Makefile
@@ -10,20 +10,23 @@ REPORT_PORTAL_DESCRIPTION ?= "Jan App report"
 all:
 	@echo "Specify a target to run"
 
+# Config yarn version
+
+config-yarn:
+	corepack enable
+	corepack prepare [email protected] --activate
+	yarn --version
+	yarn config set -H enableImmutableInstalls false
+
 # Builds the UI kit
-build-joi:
-ifeq ($(OS),Windows_NT)
-	cd joi && yarn config set network-timeout 300000 && yarn install && yarn build
-else
+build-joi: config-yarn
 	cd joi && yarn install && yarn build
-endif
 
 # Installs yarn dependencies and builds core and extensions
 install-and-build: build-joi
 ifeq ($(OS),Windows_NT)
-	yarn config set network-timeout 300000
+	echo "skip"
 endif
-	yarn global add [email protected]
 	yarn build:core
 	yarn build:server
 	yarn install
@@ -117,9 +120,8 @@ build: check-file-counts
 
 clean:
 ifeq ($(OS),Windows_NT)
-	-powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist, build, out, .turbo -Recurse -Directory | Remove-Item -Recurse -Force"
-	-powershell -Command "Get-ChildItem -Path . -Include package-lock.json -Recurse -File | Remove-Item -Recurse -Force"
-	-powershell -Command "Get-ChildItem -Path . -Include yarn.lock -Recurse -File | Remove-Item -Recurse -Force"
+	-powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist, build, out, .turbo, .yarn -Recurse -Directory | Remove-Item -Recurse -Force"
+	-powershell -Command "Get-ChildItem -Path . -Include package-lock.json, tsconfig.tsbuildinfo -Recurse -File | Remove-Item -Recurse -Force"
 	-powershell -Command "Remove-Item -Recurse -Force ./pre-install/*.tgz"
 	-powershell -Command "Remove-Item -Recurse -Force ./extensions/*/*.tgz"
 	-powershell -Command "Remove-Item -Recurse -Force ./electron/pre-install/*.tgz"
@@ -131,8 +133,8 @@ else ifeq ($(shell uname -s),Linux)
 	find . -name "build" -type d -exec rm -rf '{}' +
 	find . -name "out" -type d -exec rm -rf '{}' +
 	find . -name ".turbo" -type d -exec rm -rf '{}' +
+	find . -name ".yarn" -type d -exec rm -rf '{}' +
 	find . -name "packake-lock.json" -type f -exec rm -rf '{}' +
-	find . -name "yarn.lock" -type f -exec rm -rf '{}' +
 	find . -name "package-lock.json" -type f -exec rm -rf '{}' +
 	rm -rf ./pre-install/*.tgz
 	rm -rf ./extensions/*/*.tgz
@@ -146,8 +148,8 @@ else
 	find . -name "build" -type d -exec rm -rf '{}' +
 	find . -name "out" -type d -exec rm -rf '{}' +
 	find . -name ".turbo" -type d -exec rm -rf '{}' +
+	find . -name ".yarn" -type d -exec rm -rf '{}' +
 	find . -name "package-lock.json" -type f -exec rm -rf '{}' +
-	find . -name "yarn.lock" -type f -exec rm -rf '{}' +
 	rm -rf ./pre-install/*.tgz
 	rm -rf ./extensions/*/*.tgz
 	rm -rf ./electron/pre-install/*.tgz

diff --git a/core/package.json b/core/package.json
@@ -8,8 +8,9 @@
   ],
   "homepage": "https://jan.ai",
   "license": "AGPL-3.0",
+  "browser": "dist/core.es5.js",
   "main": "dist/core.es5.js",
-  "module": "dist/core.cjs.js",
+  "module": "dist/index.cjs.js",
   "typings": "dist/types/index.d.ts",
   "files": [
     "dist",
@@ -36,18 +37,24 @@
     "lint": "tslint --project tsconfig.json -t codeFrame 'src/**/*.ts' 'test/**/*.ts'",
     "test": "jest",
     "prebuild": "rimraf dist",
-    "build": "tsc --module commonjs && rollup -c rollup.config.ts",
-    "start": "rollup -c rollup.config.ts -w"
+    "build": "rollup -c rollup.config.ts"
   },
   "devDependencies": {
+    "@npmcli/arborist": "^9.0.0",
     "@rollup/plugin-replace": "^5.0.5",
-    "@types/jest": "^29.5.12",
-    "@types/node": "^20.11.4",
+    "@types/jest": "^29.5.14",
+    "@types/node": "^22.10.0",
+    "@types/pacote": "^11.1.8",
+    "@types/request": "^2.48.12",
+    "electron": "33.2.1",
     "eslint": "8.57.0",
     "eslint-plugin-jest": "^27.9.0",
     "jest": "^29.7.0",
     "jest-junit": "^16.0.0",
     "jest-runner": "^29.7.0",
+    "pacote": "^21.0.0",
+    "request": "^2.88.2",
+    "request-progress": "^3.0.0",
     "rimraf": "^3.0.2",
     "rollup": "^2.38.5",
     "rollup-plugin-commonjs": "^9.1.8",
@@ -62,5 +69,6 @@
   "dependencies": {
     "rxjs": "^7.8.1",
     "ulidx": "^2.3.0"
-  }
+  },
+  "packageManager": "[email protected]"
 }
diff --git a/core/rollup.config.ts b/core/rollup.config.ts
@@ -23,9 +23,12 @@ export default [
       // Allow json resolution
       json(),
       // Compile TypeScript files
-      typescript({ useTsconfigDeclarationDir: true }),
+      typescript({
+        useTsconfigDeclarationDir: true,
+        exclude: ['**/*.test.ts', 'src/node/**'],
+      }),
       // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
-      commonjs(),
+      // commonjs(),
       // Allow node_modules resolution, so you can use 'external' to control
       // which external modules to include in the bundle
       // https://github.com/rollup/rollup-plugin-node-resolve#usage
@@ -44,7 +47,9 @@ export default [
   },
   {
     input: `src/node/index.ts`,
-    output: [{ file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true }],
+    output: [
+      { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true },
+    ],
     // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
     external: [
       'fs/promises',
@@ -70,7 +75,10 @@ export default [
       // Allow json resolution
       json(),
       // Compile TypeScript files
-      typescript({ useTsconfigDeclarationDir: true }),
+      typescript({
+        useTsconfigDeclarationDir: true,
+        exclude: ['**/*.test.ts', 'src/browser/**'],
+      }),
       // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
       commonjs(),
       // Allow node_modules resolution, so you can use 'external' to control

diff --git a/core/src/node/api/processors/app.test.ts b/core/src/node/api/processors/app.test.ts
@@ -4,16 +4,6 @@ jest.mock('../../helper', () => ({
 }))
 import { App } from './app'
 
-it('should call stopServer', () => {
-  const app = new App()
-  const stopServerMock = jest.fn().mockResolvedValue('Server stopped')
-  jest.mock('@janhq/server', () => ({
-    stopServer: stopServerMock,
-  }))
-  app.stopServer()
-  expect(stopServerMock).toHaveBeenCalled()
-})
-
 it('should correctly retrieve basename', () => {
   const app = new App()
   const result = app.baseName('/path/to/file.txt')
@@ -23,15 +13,17 @@ it('should correctly retrieve basename', () => {
 it('should correctly identify subdirectories', () => {
   const app = new App()
   const basePath = process.platform === 'win32' ? 'C:\\path\\to' : '/path/to'
-  const subPath = process.platform === 'win32' ? 'C:\\path\\to\\subdir' : '/path/to/subdir'
+  const subPath =
+    process.platform === 'win32' ? 'C:\\path\\to\\subdir' : '/path/to/subdir'
   const result = app.isSubdirectory(basePath, subPath)
   expect(result).toBe(true)
 })
 
 it('should correctly join multiple paths', () => {
   const app = new App()
   const result = app.joinPath(['path', 'to', 'file'])
-  const expectedPath = process.platform === 'win32' ? 'path\\to\\file' : 'path/to/file'
+  const expectedPath =
+    process.platform === 'win32' ? 'path\\to\\file' : 'path/to/file'
   expect(result).toBe(expectedPath)
 })
 
@@ -52,5 +44,7 @@ it('should retrieve the directory name from a file path (Unix/Windows)', async (
 it('should retrieve the directory name when using file protocol', async () => {
   const app = new App()
   const path = 'file:/models/file.txt'
-  expect(await app.dirName(path)).toBe(process.platform === 'win32' ? 'app\\models' : 'app/models')
+  expect(await app.dirName(path)).toBe(
+    process.platform === 'win32' ? 'app\\models' : 'app/models'
+  )
 })