From 090041c0df83a5e8a260a341ab8518cd1ef7b1e6 Mon Sep 17 00:00:00 2001 From: Alex Wegener Date: Wed, 30 Oct 2024 17:27:47 +0100 Subject: [PATCH] local llm integration --- .cursorrules | 59 ++++++++ .env.example | 2 + .eslintrc.js | 65 ++++----- .vscode/launch.json | 22 +++ README.md | 79 +++++++++++ package-lock.json | 153 +++++++++++++++++---- package.json | 6 +- src/app/api/docker/route.ts | 20 +-- src/app/api/llm/ollama/health/route.ts | 21 +++ src/app/api/llm/ollama/list/route.ts | 21 +++ src/app/api/llm/ollama/metrics/route.ts | 26 ++++ src/app/api/llm/ollama/pull/route.ts | 173 ++++++++++++++++++++++++ src/app/api/llm/ollama/status/route.ts | 104 ++++++++++++++ src/app/api/llm/route.ts | 31 +++-- src/components/Settings.tsx | 80 +++++------ src/components/chat/ChatComponent.tsx | 56 +++++--- src/components/chat/ModelSelector.tsx | 96 ++++++++++--- src/components/llm/LocalModels.tsx | 77 +++++++++++ src/components/llm/ModelManager.tsx | 114 ++++++++++++++++ src/components/ui/alert.tsx | 59 ++++++++ src/components/ui/sheet.tsx | 140 +++++++++++++++++++ src/hooks/useDockerHandlers.ts | 6 +- src/hooks/useModelManager.ts | 133 ++++++++++++++++++ src/lib/context/manager.ts | 10 ++ src/lib/context/types.ts | 14 ++ src/lib/docker/tools.ts | 11 ++ src/lib/docker/types.ts | 56 ++++++++ src/lib/functions/registry.ts | 9 ++ src/lib/llm/ollama-client.ts | 123 +++++++++++++++++ src/lib/llm/ollama.ts | 165 ++++++++++++++++++++++ src/lib/llm/provider.ts | 14 ++ src/lib/llm/tokens.ts | 46 +++++++ src/lib/llm/types.ts | 124 +++++++++++++++-- src/lib/utils.ts | 6 +- src/lib/utils/api.ts | 13 ++ src/lib/utils/langchain.ts | 14 ++ src/middleware.ts | 39 ++++-- src/services/dockerService.ts | 27 +--- src/services/llm.service.ts | 86 ++++++++++-- tsconfig.json | 1 + 40 files changed, 2082 insertions(+), 219 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 src/app/api/llm/ollama/health/route.ts create mode 100644 src/app/api/llm/ollama/list/route.ts create mode 100644 src/app/api/llm/ollama/metrics/route.ts create mode 100644 src/app/api/llm/ollama/pull/route.ts create mode 100644 src/app/api/llm/ollama/status/route.ts create mode 100644 src/components/llm/LocalModels.tsx create mode 100644 src/components/llm/ModelManager.tsx create mode 100644 src/components/ui/alert.tsx create mode 100644 src/components/ui/sheet.tsx create mode 100644 src/hooks/useModelManager.ts create mode 100644 src/lib/context/manager.ts create mode 100644 src/lib/context/types.ts create mode 100644 src/lib/docker/tools.ts create mode 100644 src/lib/docker/types.ts create mode 100644 src/lib/functions/registry.ts create mode 100644 src/lib/llm/ollama-client.ts create mode 100644 src/lib/llm/ollama.ts create mode 100644 src/lib/llm/tokens.ts create mode 100644 src/lib/utils/api.ts create mode 100644 src/lib/utils/langchain.ts diff --git a/.cursorrules b/.cursorrules index 6cf0188..bda2546 100644 --- a/.cursorrules +++ b/.cursorrules @@ -96,6 +96,7 @@ - Always use TypeScript for better type safety and maintainability - Maintain clean code principles and follow best practices - Use strict and explicit typing; never use the 'any' keyword +- Follow DRY principle wherever possible // Project structure @@ -246,3 +247,61 @@ Ensure all required dependencies are properly installed and typed: - Support function execution tracking - Add message history support - Handle message reconstruction + +rules: + +- name: "TypeScript Strict Mode" + pattern: "*.ts,*.tsx" + rules: + - "No 'any' types allowed" + - "Use explicit return types" + - "Enable strict null checks" + - "Use interface over type where possible" + +- name: "Code Organization" + pattern: "src/**/*" + rules: + - "Use feature-based folder structure" + - "Keep components pure and focused" + - "Separate business logic from UI" + - "Use hooks for shared logic" + +- name: "API Design" + pattern: "src/app/api/**/*" + rules: + - "Use proper HTTP methods" + - "Validate inputs" + - "Handle errors gracefully" + - "Stream responses when appropriate" + +- name: "State Management" + pattern: "src/**/*" + rules: + - "Use React Query for server state" + - "Use local state for UI" + - "Avoid prop drilling" + - "Keep state close to where it's used" + +- name: "Testing" + pattern: "**/*.test.ts,**/*.test.tsx" + rules: + - "Write unit tests for utils" + - "Write integration tests for API" + - "Test error cases" + - "Mock external dependencies" + +- name: "Documentation" + pattern: "**/*" + rules: + - "Document public APIs" + - "Add JSDoc for complex functions" + - "Keep README up to date" + - "Document environment setup" + +- name: "Code Quality" + pattern: "*.ts,*.tsx" + rules: + - "No while(true) loops - use explicit conditions" + - "No constant conditions in loops or conditionals" + - "Use proper loop termination conditions" + - "Avoid infinite loops" diff --git a/.env.example b/.env.example index a9e4cad..733471c 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,8 @@ NEXT_PUBLIC_API_KEY=your_public_api_key_here +NEXT_PUBLIC_OLLAMA_URL=http://localhost:11434 + # Secret API keys for server-side use API_KEY=your_secret_api_key_here diff --git a/.eslintrc.js b/.eslintrc.js index c2c7b1d..63e2647 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,9 +1,9 @@ module.exports = { root: true, - parser: "@typescript-eslint/parser", + parser: '@typescript-eslint/parser', parserOptions: { ecmaVersion: 2021, - sourceType: "module", + sourceType: 'module', ecmaFeatures: { jsx: true, }, @@ -14,48 +14,43 @@ module.exports = { es6: true, }, extends: [ - "eslint:recommended", - "plugin:@typescript-eslint/recommended", - "plugin:react/recommended", - "plugin:react-hooks/recommended", - "plugin:jsx-a11y/recommended", - "plugin:import/errors", - "plugin:import/warnings", - "plugin:import/typescript", - "plugin:prettier/recommended", - "next/core-web-vitals", + 'eslint:recommended', + 'plugin:@typescript-eslint/recommended', + 'plugin:react/recommended', + 'plugin:react-hooks/recommended', + 'plugin:jsx-a11y/recommended', + 'plugin:import/errors', + 'plugin:import/warnings', + 'plugin:import/typescript', + 'plugin:prettier/recommended', + 'next/core-web-vitals', ], - plugins: ["@typescript-eslint", "react", "jsx-a11y", "import", "prettier"], + plugins: ['@typescript-eslint', 'react', 'jsx-a11y', 'import', 'prettier'], rules: { - "no-console": ["warn", { allow: ["warn", "error"] }], - "@typescript-eslint/explicit-module-boundary-types": "off", - "@typescript-eslint/no-unused-vars": ["error", { argsIgnorePattern: "^_" }], - "@typescript-eslint/no-explicit-any": "error", - "react/prop-types": "off", - "react/react-in-jsx-scope": "off", - "jsx-a11y/anchor-is-valid": "off", - "import/order": [ - "error", + // TypeScript specific rules + '@typescript-eslint/no-unused-vars': [ + 'warn', { - groups: [ - "builtin", - "external", - "internal", - "parent", - "sibling", - "index", - ], - "newlines-between": "always", - alphabetize: { order: "asc", caseInsensitive: true }, + argsIgnorePattern: '^_', + destructuredArrayIgnorePattern: '^_', + varsIgnorePattern: '^_', }, ], - "prettier/prettier": "error", + 'no-console': ['warn', { allow: ['warn', 'error'] }], + '@typescript-eslint/explicit-module-boundary-types': 'off', + '@typescript-eslint/no-explicit-any': 'error', + 'react/prop-types': 'off', + 'react/react-in-jsx-scope': 'off', + 'jsx-a11y/anchor-is-valid': 'off', + // Import rules + 'import/order': 'off', + 'prettier/prettier': 'error', }, settings: { react: { - version: "detect", + version: 'detect', }, - "import/resolver": { + 'import/resolver': { typescript: {}, }, }, diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..2ed12bf --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Next.js: debug server-side", + "type": "node", + "request": "launch", + "runtimeArgs": ["--inspect"], + "program": "${workspaceFolder}/node_modules/.bin/next", + "args": ["dev"], + "skipFiles": ["/**"] + // "outFiles": ["${workspaceFolder}/.next/**/*.js"] + }, + { + "name": "Next.js: debug client-side", + "type": "chrome", + "request": "launch", + "url": "http://localhost:3000", + "webRoot": "${workspaceFolder}" + } + ] +} diff --git a/README.md b/README.md index 7c4a78a..216e399 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ A Next.js application that uses a large language model to control a computer thr > - ✅ Model selection > - ✅ Model tracking > - ✅ Message history +> - ✅ Local model support +> - ✅ Model download tracking > - 🔳 Context management > - 🔳 Function calling > - ⬜ Streaming support @@ -103,6 +105,7 @@ A Next.js application that uses a large language model to control a computer thr - Node.js (LTS version) - Docker - Python 3.11.6 (for certain features) +- Ollama (for local models) - See [Ollama Setup](#ollama-setup) section ## Installation @@ -159,6 +162,82 @@ The application includes a custom Docker environment with: - Firefox ESR with pre-configured extensions - Various utility applications +## Ollama Setup + +### Installation + +#### macOS + +```bash +# Using Homebrew +brew install ollama + +# Start Ollama service +ollama serve +``` + +#### Linux + +```bash +# Install Ollama +curl -fsSL https://ollama.com/install.sh | sh + +# Start Ollama service +systemctl start ollama +``` + +#### Windows + +1. Install WSL2 if not already installed: + +```bash +wsl --install +``` + +2. Install Ollama in WSL2: + +```bash +curl -fsSL https://ollama.com/install.sh | sh +``` + +3. Start Ollama service in WSL2: + +```bash +ollama serve +``` + +### Configuration + +Add the following to your `.env` file: + +```env +# Ollama Configuration +NEXT_PUBLIC_OLLAMA_URL=http://localhost:11434 +``` + +### Troubleshooting + +1. Check if Ollama is running: + +```bash +curl http://localhost:11434/api/health +``` + +2. If not running, start the service: + +```bash +# macOS/Linux +ollama serve + +# Windows (in WSL2) +wsl -d Ubuntu -u root ollama serve +``` + +3. Common issues: + - Port 11434 is already in use + - Insufficient disk space + - GPU drivers not properly installed (for GPU acceleration) + ## Contributing 1. Ensure you follow the project's coding standards: diff --git a/package-lock.json b/package-lock.json index 1edbbf4..1022af6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,11 +29,13 @@ "dockerode": "^3.3.5", "electron": "^24.3.0", "langchain": "^0.1.21", + "lodash": "^4.17.21", "lucide-react": "^0.453.0", "next": "^15.0.1", "node-pty": "^0.10.1", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-error-boundary": "^4.1.2", "ssh2": "^1.11.0", "tailwind-merge": "^2.5.4", "tailwindcss-animate": "^1.0.7", @@ -45,6 +47,7 @@ "@types/dockerode": "^3.3.31", "@types/electron": "^1.6.10", "@types/jest": "^29.5.11", + "@types/lodash": "^4.17.13", "@types/node": "^20.2.5", "@types/react": "^18.2.7", "@types/react-dom": "^18.2.4", @@ -58,6 +61,7 @@ "eslint-config-prettier": "^8.8.0", "eslint-plugin-import": "^2.27.5", "eslint-plugin-jsx-a11y": "^6.7.1", + "eslint-plugin-prettier": "^5.2.1", "eslint-plugin-react": "^7.32.2", "eslint-plugin-react-hooks": "^4.6.0", "jest": "^29.7.0", @@ -118,9 +122,9 @@ } }, "node_modules/@anthropic-ai/sdk/node_modules/@types/node": { - "version": "18.19.59", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.59.tgz", - "integrity": "sha512-vizm2EqwV/7Zay+A6J3tGl9Lhr7CjZe2HmWS988sefiEmsyP9CeXEleho6i4hJk/8UtZAo0bWN4QPZZr83RxvQ==", + "version": "18.19.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.60.tgz", + "integrity": "sha512-cYRj7igVqgxhlHFdBHHpU2SNw3+dN2x0VTZJtLYk6y/ieuGN4XiBgtDjYVktM/yk2y/8pKMileNc6IoEzEJnUw==", "license": "MIT", "dependencies": { "undici-types": "~5.26.4" @@ -2847,6 +2851,19 @@ "node": ">=14" } }, + "node_modules/@pkgr/core": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", + "integrity": "sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, "node_modules/@radix-ui/number": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.0.tgz", @@ -4015,6 +4032,13 @@ "@types/node": "*" } }, + "node_modules/@types/lodash": { + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.13.tgz", + "integrity": "sha512-lfx+dftrEZcdBPczf9d0Qv0x+j/rfNCMuC6OcfXmO8gkfeNAY88PgKUbvG56whcN23gc27yenwF6oJZXGFpYxg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/minimatch": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", @@ -4031,9 +4055,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "20.17.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.1.tgz", - "integrity": "sha512-j2VlPv1NnwPJbaCNv69FO/1z4lId0QmGvpT41YxitRtWlg96g/j8qcv2RKsLKe2F6OJgyXhupN1Xo17b2m139Q==", + "version": "20.17.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.2.tgz", + "integrity": "sha512-OOHK4sjXqkL7yQ7VEEHcf6+0jSvKjWqwnaCtY7AKD/VLEvRHMsxxu7eI8ErnjxHS8VwmekD4PeVCpu4qZEZSxg==", "license": "MIT", "dependencies": { "undici-types": "~6.19.2" @@ -4122,9 +4146,9 @@ } }, "node_modules/@types/ssh2/node_modules/@types/node": { - "version": "18.19.59", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.59.tgz", - "integrity": "sha512-vizm2EqwV/7Zay+A6J3tGl9Lhr7CjZe2HmWS988sefiEmsyP9CeXEleho6i4hJk/8UtZAo0bWN4QPZZr83RxvQ==", + "version": "18.19.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.60.tgz", + "integrity": "sha512-cYRj7igVqgxhlHFdBHHpU2SNw3+dN2x0VTZJtLYk6y/ieuGN4XiBgtDjYVktM/yk2y/8pKMileNc6IoEzEJnUw==", "dev": true, "license": "MIT", "dependencies": { @@ -5632,9 +5656,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001673", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001673.tgz", - "integrity": "sha512-WTrjUCSMp3LYX0nE12ECkV0a+e6LC85E0Auz75555/qr78Oc8YWhEPNfDd6SHdtlCMSzqtuXY0uyEMNRcsKpKw==", + "version": "1.0.30001674", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001674.tgz", + "integrity": "sha512-jOsKlZVRnzfhLojb+Ykb+gyUSp9Xb57So+fAiFlLzzTKpqg8xxSav0e40c8/4F/v9N8QSvrRRaLeVzQbLqomYw==", "funding": [ { "type": "opencollective", @@ -7238,16 +7262,16 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.47", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.47.tgz", - "integrity": "sha512-zS5Yer0MOYw4rtK2iq43cJagHZ8sXN0jDHDKzB+86gSBSAI4v07S97mcq+Gs2vclAxSh1j7vOAHxSVgduiiuVQ==", + "version": "1.5.49", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.49.tgz", + "integrity": "sha512-ZXfs1Of8fDb6z7WEYZjXpgIRF6MEu8JdeGA0A40aZq6OQbS+eJpnnV49epZRna2DU/YsEjSQuGtQPPtvt6J65A==", "dev": true, "license": "ISC" }, "node_modules/electron/node_modules/@types/node": { - "version": "18.19.59", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.59.tgz", - "integrity": "sha512-vizm2EqwV/7Zay+A6J3tGl9Lhr7CjZe2HmWS988sefiEmsyP9CeXEleho6i4hJk/8UtZAo0bWN4QPZZr83RxvQ==", + "version": "18.19.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.60.tgz", + "integrity": "sha512-cYRj7igVqgxhlHFdBHHpU2SNw3+dN2x0VTZJtLYk6y/ieuGN4XiBgtDjYVktM/yk2y/8pKMileNc6IoEzEJnUw==", "license": "MIT", "dependencies": { "undici-types": "~5.26.4" @@ -7796,6 +7820,37 @@ "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8 || ^9" } }, + "node_modules/eslint-plugin-prettier": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.2.1.tgz", + "integrity": "sha512-gH3iR3g4JfF+yYPaJYkN7jEl9QbweL/YfkoRlNnuIEHEz1vHVlCmWOS+eGGiRuzHQXdJFCOTxRgvju9b8VUmrw==", + "dev": true, + "license": "MIT", + "dependencies": { + "prettier-linter-helpers": "^1.0.0", + "synckit": "^0.9.1" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint-plugin-prettier" + }, + "peerDependencies": { + "@types/eslint": ">=8.0.0", + "eslint": ">=8.0.0", + "eslint-config-prettier": "*", + "prettier": ">=3.0.0" + }, + "peerDependenciesMeta": { + "@types/eslint": { + "optional": true + }, + "eslint-config-prettier": { + "optional": true + } + } + }, "node_modules/eslint-plugin-react": { "version": "7.37.2", "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.37.2.tgz", @@ -8170,6 +8225,13 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-diff": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", + "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/fast-glob": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", @@ -11278,9 +11340,9 @@ } }, "node_modules/langchain/node_modules/@types/node": { - "version": "18.19.59", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.59.tgz", - "integrity": "sha512-vizm2EqwV/7Zay+A6J3tGl9Lhr7CjZe2HmWS988sefiEmsyP9CeXEleho6i4hJk/8UtZAo0bWN4QPZZr83RxvQ==", + "version": "18.19.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.60.tgz", + "integrity": "sha512-cYRj7igVqgxhlHFdBHHpU2SNw3+dN2x0VTZJtLYk6y/ieuGN4XiBgtDjYVktM/yk2y/8pKMileNc6IoEzEJnUw==", "license": "MIT", "dependencies": { "undici-types": "~5.26.4" @@ -11428,7 +11490,6 @@ "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "devOptional": true, "license": "MIT" }, "node_modules/lodash.memoize": { @@ -12212,9 +12273,9 @@ } }, "node_modules/openai/node_modules/@types/node": { - "version": "18.19.59", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.59.tgz", - "integrity": "sha512-vizm2EqwV/7Zay+A6J3tGl9Lhr7CjZe2HmWS988sefiEmsyP9CeXEleho6i4hJk/8UtZAo0bWN4QPZZr83RxvQ==", + "version": "18.19.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.60.tgz", + "integrity": "sha512-cYRj7igVqgxhlHFdBHHpU2SNw3+dN2x0VTZJtLYk6y/ieuGN4XiBgtDjYVktM/yk2y/8pKMileNc6IoEzEJnUw==", "license": "MIT", "dependencies": { "undici-types": "~5.26.4" @@ -12773,6 +12834,19 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/prettier-linter-helpers": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", + "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-diff": "^1.1.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", @@ -12924,6 +12998,18 @@ "react": "^18.3.1" } }, + "node_modules/react-error-boundary": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/react-error-boundary/-/react-error-boundary-4.1.2.tgz", + "integrity": "sha512-GQDxZ5Jd+Aq/qUxbCm1UtzmL/s++V7zKgE8yMktJiCQXCCFZnMZh9ng+6/Ne6PjNSXH0L9CjeOEREfRnq6Duag==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.12.5" + }, + "peerDependencies": { + "react": ">=16.13.1" + } + }, "node_modules/react-is": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", @@ -14140,6 +14226,23 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/synckit": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.2.tgz", + "integrity": "sha512-vrozgXDQwYO72vHjUb/HnFbQx1exDjoKzqx23aXEg2a9VIg2TSFZ8FmeZpTjUCFMYw7mpX4BE2SFu8wI7asYsw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@pkgr/core": "^0.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, "node_modules/tailwind-merge": { "version": "2.5.4", "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.5.4.tgz", diff --git a/package.json b/package.json index 31cdbc3..6f597a4 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "author": "Alex Wegener, lx-0", "license": "ISC", "scripts": { - "dev": "next dev", + "dev": "NODE_OPTIONS='--inspect' next dev", "build": "next build", "start": "next start", "lint": "eslint . --ext .ts,.tsx", @@ -36,11 +36,13 @@ "dockerode": "^3.3.5", "electron": "^24.3.0", "langchain": "^0.1.21", + "lodash": "^4.17.21", "lucide-react": "^0.453.0", "next": "^15.0.1", "node-pty": "^0.10.1", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-error-boundary": "^4.1.2", "ssh2": "^1.11.0", "tailwind-merge": "^2.5.4", "tailwindcss-animate": "^1.0.7", @@ -52,6 +54,7 @@ "@types/dockerode": "^3.3.31", "@types/electron": "^1.6.10", "@types/jest": "^29.5.11", + "@types/lodash": "^4.17.13", "@types/node": "^20.2.5", "@types/react": "^18.2.7", "@types/react-dom": "^18.2.4", @@ -65,6 +68,7 @@ "eslint-config-prettier": "^8.8.0", "eslint-plugin-import": "^2.27.5", "eslint-plugin-jsx-a11y": "^6.7.1", + "eslint-plugin-prettier": "^5.2.1", "eslint-plugin-react": "^7.32.2", "eslint-plugin-react-hooks": "^4.6.0", "jest": "^29.7.0", diff --git a/src/app/api/docker/route.ts b/src/app/api/docker/route.ts index 7682a36..95303e7 100644 --- a/src/app/api/docker/route.ts +++ b/src/app/api/docker/route.ts @@ -258,21 +258,28 @@ export async function GET(request: Request) { const buildId = searchParams.get('buildId'); const statusId = searchParams.get('statusId'); const containerId = searchParams.get('containerId'); + const apiKey = searchParams.get('apiKey'); + + // Check API key + if (!apiKey || apiKey !== process.env.API_KEY) { + return new Response(JSON.stringify({ error: 'Unauthorized' }), { + status: 401, + headers: { 'Content-Type': 'application/json' }, + }); + } if (buildId) { - // Existing build progress stream logic + // Existing build progress stream logic with headers return new Response( new ReadableStream({ start(controller) { const listener = (event: any) => { - // console.log('Emitting build event:', event); controller.enqueue(`data: ${JSON.stringify(event)}\n\n`); }; buildEmitter.on('progress', listener); request.signal.addEventListener('abort', () => { - console.log('Build stream client disconnected'); buildEmitter.removeListener('progress', listener); }); }, @@ -286,27 +293,22 @@ export async function GET(request: Request) { } ); } else if (statusId && containerId) { - // New container status stream + // Container status stream with headers return new Response( new ReadableStream({ start(controller) { const listener = (event: any) => { - // console.log('Emitting status event:', event); controller.enqueue(`data: ${JSON.stringify(event)}\n\n`); }; statusEmitter.on('status', listener); - - // Initial status check emitContainerStatus(containerId); - // Set up interval for status updates const intervalId = setInterval(() => { emitContainerStatus(containerId); }, 5000); request.signal.addEventListener('abort', () => { - console.log('Status stream client disconnected'); statusEmitter.removeListener('status', listener); clearInterval(intervalId); }); diff --git a/src/app/api/llm/ollama/health/route.ts b/src/app/api/llm/ollama/health/route.ts new file mode 100644 index 0000000..5164933 --- /dev/null +++ b/src/app/api/llm/ollama/health/route.ts @@ -0,0 +1,21 @@ +import { OllamaService } from '@/lib/llm/ollama'; +import { NextRequest, NextResponse } from 'next/server'; + +export async function GET(req: NextRequest): Promise { + const searchParams = req.nextUrl.searchParams; + const apiKey = searchParams.get('apiKey'); + + // Validate API key + if (!apiKey || apiKey !== process.env.API_KEY) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + } + + try { + const ollama = OllamaService.getInstance(); + const isHealthy = await ollama.checkHealth(); + return NextResponse.json({ healthy: isHealthy }); + } catch (error) { + console.error('Health check failed:', error); + return NextResponse.json({ healthy: false }); + } +} diff --git a/src/app/api/llm/ollama/list/route.ts b/src/app/api/llm/ollama/list/route.ts new file mode 100644 index 0000000..7d6d078 --- /dev/null +++ b/src/app/api/llm/ollama/list/route.ts @@ -0,0 +1,21 @@ +import { OllamaService } from '@/lib/llm/ollama'; +import { NextRequest, NextResponse } from 'next/server'; + +export async function GET(req: NextRequest): Promise { + const searchParams = req.nextUrl.searchParams; + const apiKey = searchParams.get('apiKey'); + + // Validate API key + if (!apiKey || apiKey !== process.env.API_KEY) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + } + + try { + const ollama = OllamaService.getInstance(); + const models = await ollama.listModels(); + return NextResponse.json({ models }); + } catch (error) { + console.error('Failed to list models:', error); + return NextResponse.json({ models: [] }); + } +} diff --git a/src/app/api/llm/ollama/metrics/route.ts b/src/app/api/llm/ollama/metrics/route.ts new file mode 100644 index 0000000..0224dfc --- /dev/null +++ b/src/app/api/llm/ollama/metrics/route.ts @@ -0,0 +1,26 @@ +import { OllamaService } from '@/lib/llm/ollama'; +import { ModelResourceMetrics } from '@/lib/llm/types'; +import { NextApiRequest, NextApiResponse } from 'next'; + +type MetricsResponse = ModelResourceMetrics | { message: string }; + +export async function GET(req: NextApiRequest, res: NextApiResponse) { + if (req.method !== 'GET') { + return res.status(405).json({ message: 'Method not allowed' }); + } + + const { modelId } = req.query; + + if (typeof modelId !== 'string') { + return res.status(400).json({ message: 'Model ID is required' }); + } + + try { + const ollama = OllamaService.getInstance(); + const metrics = await ollama.getModelMetrics(modelId); + res.status(200).json(metrics); + } catch (error) { + console.error('Failed to fetch model metrics:', error); + res.status(500).json({ message: 'Failed to fetch model metrics' }); + } +} diff --git a/src/app/api/llm/ollama/pull/route.ts b/src/app/api/llm/ollama/pull/route.ts new file mode 100644 index 0000000..8d4ebc1 --- /dev/null +++ b/src/app/api/llm/ollama/pull/route.ts @@ -0,0 +1,173 @@ +import { OllamaService } from '@/lib/llm/ollama'; +import { OllamaModelStatus } from '@/lib/llm/types'; +import { throttle } from 'lodash'; +import { NextRequest, NextResponse } from 'next/server'; + +export const config = { + runtime: 'edge', +}; + +interface DebugMessage { + type: 'debug'; + message: string; + data?: { + status?: string; + progress?: number; + error?: string; + [key: string]: unknown; + }; +} + +export async function GET(req: NextRequest): Promise { + const searchParams = req.nextUrl.searchParams; + const modelId = searchParams.get('modelId'); + const apiKey = searchParams.get('apiKey'); + + console.log('Pull request received for model:', modelId); + + // Check API key + if (!apiKey || apiKey !== process.env.API_KEY) { + console.error('Unauthorized pull request'); + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + } + + if (!modelId) { + console.error('No model ID provided'); + return NextResponse.json({ error: 'Model ID is required' }, { status: 400 }); + } + + const encoder = new TextEncoder(); + + // Create ReadableStream for SSE + const stream = new ReadableStream({ + start: async (controller) => { + // Helper function to send debug info + const sendDebug = async (message: string, data?: DebugMessage['data']) => { + const debugMessage: DebugMessage = { + type: 'debug', + message, + data, + }; + console.log(`[Pull Debug] ${message}`, data); + try { + controller.enqueue(encoder.encode(`data: ${JSON.stringify(debugMessage)}\n\n`)); + } catch (error) { + console.error('Error sending debug message:', error); + } + }; + + // Helper function to send status update + const sendStatus = (status: OllamaModelStatus) => { + try { + controller.enqueue(encoder.encode(`data: ${JSON.stringify(status)}\n\n`)); + } catch (error) { + console.error('Error sending status:', error); + throw error; + } + }; + + // Create throttled versions of both functions + const throttledSendStatus = throttle(sendStatus, 1000); // Update once per second + const throttledSendDebug = throttle(sendDebug, 2000); // Debug messages every 2 seconds + + let keepAliveInterval: ReturnType | undefined; + + try { + const ollama = OllamaService.getInstance(); + await sendDebug('Starting pull process'); // Initial debug not throttled + + // Create initial status + const initialStatus: OllamaModelStatus = { + name: modelId, + status: 'downloading', + progress: 0, + lastUpdated: new Date(), + }; + sendStatus(initialStatus); // Initial status not throttled + + // Start pull process with progress callback + await ollama.pullModel(modelId, (status: OllamaModelStatus) => { + try { + // Use throttled debug for progress updates + if (status.status === 'downloading') { + throttledSendDebug('Received status update', { + status: status.status, + progress: status.progress, + error: status.error, + }); + throttledSendStatus(status); + } else { + // But send terminal states (ready/error) immediately with debug + sendDebug('Received status update', { + status: status.status, + progress: status.progress, + error: status.error, + }); + sendStatus(status); + } + + // If we're done or have an error, clean up + if (status.status === 'ready' || status.status === 'error') { + if (keepAliveInterval) { + clearInterval(keepAliveInterval); + } + controller.close(); + } + } catch (error) { + console.error('Error writing to stream:', error); + sendDebug('Error writing status', { error: String(error) }); + if (keepAliveInterval) { + clearInterval(keepAliveInterval); + } + controller.close(); + } + }); + + // Keep-alive interval + keepAliveInterval = setInterval(() => { + try { + controller.enqueue(encoder.encode(': ping\n\n')); + } catch (error) { + console.error('Error sending keepalive:', error); + if (keepAliveInterval) { + clearInterval(keepAliveInterval); + } + controller.close(); + } + }, 15000); + + // Handle client disconnect + req.signal.addEventListener('abort', () => { + console.log('Client disconnected, cleaning up'); + if (keepAliveInterval) { + clearInterval(keepAliveInterval); + } + controller.close(); + }); + } catch (error) { + console.error('Stream error:', error); + // Send error status + const errorStatus: OllamaModelStatus = { + name: modelId, + status: 'error', + error: error instanceof Error ? error.message : 'Unknown error', + lastUpdated: new Date(), + }; + sendStatus(errorStatus); + if (keepAliveInterval) { + clearInterval(keepAliveInterval); + } + controller.close(); + } + }, + }); + + // Return streaming response + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }, + }); +} diff --git a/src/app/api/llm/ollama/status/route.ts b/src/app/api/llm/ollama/status/route.ts new file mode 100644 index 0000000..abe61b4 --- /dev/null +++ b/src/app/api/llm/ollama/status/route.ts @@ -0,0 +1,104 @@ +import { OllamaService } from '@/lib/llm/ollama'; +import { OllamaModelStatus } from '@/lib/llm/types'; +import { NextRequest, NextResponse } from 'next/server'; + +export async function GET(req: NextRequest): Promise { + console.log('Status request received', req.nextUrl.searchParams); + + // Get query parameters + const searchParams = req.nextUrl.searchParams; + const modelId = searchParams.get('modelId'); + const apiKey = searchParams.get('apiKey'); + + // Validate inputs + if (apiKey !== process.env.NEXT_PUBLIC_API_KEY) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + } + + if (!modelId) { + return NextResponse.json({ error: 'Model ID is required' }, { status: 400 }); + } + + // Create ReadableStream for SSE + const stream = new ReadableStream({ + start: async (controller) => { + try { + // Get Ollama service instance + const ollama = OllamaService.getInstance(); + + // Get initial status by checking if model is in list + const models = await ollama.listModels(); + const modelInfo = models.find((m) => m.name === modelId); + + const initialStatus: OllamaModelStatus = { + name: modelId, + status: modelInfo ? 'ready' : 'checking', + progress: 0, + lastUpdated: new Date(), + }; + + // Debug log + console.log('Sending initial status:', initialStatus); + + // Send initial status + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(initialStatus)}\n\n`)); + + // Set up periodic status checks + const statusInterval = setInterval(async () => { + try { + // Check model status by listing models + const currentModels = await ollama.listModels(); + const currentModel = currentModels.find((m) => m.name === modelId); + + // Get metrics if model exists + const metrics = currentModel ? await ollama.getModelMetrics(modelId) : undefined; + + const status: OllamaModelStatus = { + name: modelId, + status: currentModel ? 'ready' : 'checking', + lastUpdated: new Date(), + metrics, + }; + + // Send status update + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(status)}\n\n`)); + } catch (error) { + console.error('Error checking status:', error); + } + }, 5000); // Check every 5 seconds + + // Keep-alive interval + const keepAlive = setInterval(() => { + try { + controller.enqueue(new TextEncoder().encode(': ping\n\n')); + } catch (error) { + console.error('Error sending keepalive:', error); + clearInterval(keepAlive); + clearInterval(statusInterval); + controller.close(); + } + }, 15000); + + // Handle client disconnect + req.signal.addEventListener('abort', () => { + console.log('Client disconnected, cleaning up'); + clearInterval(keepAlive); + clearInterval(statusInterval); + controller.close(); + }); + } catch (error) { + console.error('Stream error:', error); + controller.error(error); + } + }, + }); + + // Return streaming response + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }, + }); +} diff --git a/src/app/api/llm/route.ts b/src/app/api/llm/route.ts index 4e126c9..b1ce2e3 100644 --- a/src/app/api/llm/route.ts +++ b/src/app/api/llm/route.ts @@ -1,26 +1,37 @@ import { LLMService } from '@/services/llm.service'; -import { AIMessage, HumanMessage, SystemMessage } from '@langchain/core/messages'; +import { + AIMessage, + AIMessageFields, + BaseMessageFields, + HumanMessage, + SystemMessage, +} from '@langchain/core/messages'; import { NextRequest, NextResponse } from 'next/server'; -export async function POST(req: NextRequest) { +export async function POST(req: NextRequest): Promise { try { const { message, modelId, options } = await req.json(); + // Validate required fields if (!modelId) { return NextResponse.json({ error: 'Model ID is required' }, { status: 400 }); } - // Reconstruct Langchain message instances + if (!message) { + return NextResponse.json({ error: 'Message is required' }, { status: 400 }); + } + + // Reconstruct Langchain message instances if history exists const history = options?.history - ?.map((msg: any) => { + ?.map((msg: { type: string; id: string[]; kwargs: Record }) => { if (msg.type === 'constructor') { switch (msg.id[2]) { case 'HumanMessage': - return new HumanMessage(msg.kwargs); + return new HumanMessage(msg.kwargs as BaseMessageFields); case 'AIMessage': - return new AIMessage(msg.kwargs); + return new AIMessage(msg.kwargs as AIMessageFields); case 'SystemMessage': - return new SystemMessage(msg.kwargs); + return new SystemMessage(msg.kwargs as BaseMessageFields); default: return null; } @@ -29,6 +40,7 @@ export async function POST(req: NextRequest) { }) .filter(Boolean); + // Send message through LLM service const llmService = LLMService.getInstance(); const response = await llmService.sendMessage(message, modelId, { ...options, @@ -39,7 +51,10 @@ export async function POST(req: NextRequest) { } catch (error) { console.error('LLM API error:', error); return NextResponse.json( - { error: error instanceof Error ? error.message : 'Failed to process LLM request' }, + { + error: error instanceof Error ? error.message : 'Failed to process LLM request', + timestamp: new Date().toISOString(), + }, { status: 500 } ); } diff --git a/src/components/Settings.tsx b/src/components/Settings.tsx index fa3d6e6..880323b 100644 --- a/src/components/Settings.tsx +++ b/src/components/Settings.tsx @@ -1,53 +1,47 @@ -import { Button } from '@/components/ui/button'; -import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; -import { cn } from '@/lib/utils'; -import { ArrowLeftFromLine } from 'lucide-react'; -import React, { useEffect, useRef } from 'react'; +import { LocalModels } from '@/components/llm/LocalModels'; +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from '@/components/ui/sheet'; +import { ErrorBoundary } from 'react-error-boundary'; interface SettingsProps { isOpen: boolean; onClose: () => void; } -const Settings: React.FC = ({ isOpen, onClose }) => { - const settingsRef = useRef(null); +function ErrorFallback({ error }: { error: Error }) { + return ( +
+

Error loading models

+

{error.message}

+
+ ); +} - useEffect(() => { - const handleClickOutside = (event: MouseEvent) => { - if (settingsRef.current && !settingsRef.current.contains(event.target as Node) && isOpen) { - onClose(); - } - }; +export default function Settings({ isOpen, onClose }: SettingsProps) { + return ( + + + + Settings + Configure your local models and application settings. + - document.addEventListener('mousedown', handleClickOutside); - return () => { - document.removeEventListener('mousedown', handleClickOutside); - }; - }, [isOpen, onClose]); +
+ {/* Existing settings sections */} - return ( - - - Settings - - - {/* Add your settings options here */} -

Settings content goes here

- -
- -
- + {/* Add Local Models section with error boundary */} +
+ + + +
+
+ + ); -}; - -export default Settings; +} diff --git a/src/components/chat/ChatComponent.tsx b/src/components/chat/ChatComponent.tsx index fba2df4..69699d6 100644 --- a/src/components/chat/ChatComponent.tsx +++ b/src/components/chat/ChatComponent.tsx @@ -3,7 +3,6 @@ import ChatMessage from '@/components/chat/ChatMessage'; import { DockerMenu } from '@/components/chat/DockerMenu'; import { Button } from '@/components/ui/button'; import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; -import { ScrollArea } from '@/components/ui/scroll-area'; import { useChatMessages } from '@/hooks/useChatMessages'; import { useDockerHandlers } from '@/hooks/useDockerHandlers'; import { AVAILABLE_MODELS, convertToLangchainMessage } from '@/lib/llm/types'; @@ -11,7 +10,7 @@ import { cn } from '@/lib/utils'; import { LLMApiService } from '@/services/llm-api.service'; import { AIMessage, HumanMessage, SystemMessage } from '@langchain/core/messages'; import { Settings as SettingsIcon } from 'lucide-react'; -import { useCallback, useEffect, useRef, useState } from 'react'; +import { useEffect, useRef, useState } from 'react'; import ChatCopyButton from './ChatCopyButton'; interface ChatComponentProps { @@ -107,16 +106,30 @@ const ChatComponent: React.FC = ({ } }; - const scrollToBottom = useCallback(() => { - if (scrollAreaRef.current) { - const scrollElement = scrollAreaRef.current; - scrollElement.scrollTop = scrollElement.scrollHeight; - } - }, []); + // effects on messages change + const chatMessagesEndRef = useRef(null); + const isThrottling = useRef(false); + // Effect on messages change useEffect(() => { - scrollToBottom(); - }, [chatMessages, scrollToBottom]); + // automatically scroll to bottom of chat + const scrollToBottom = () => { + chatMessagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); + }; + + if (!isThrottling.current) { + // Execute the function immediately + scrollToBottom(); + + // Set throttling flag to true + isThrottling.current = true; + + // Set timeout to reset throttling after the desired period + setTimeout(() => { + isThrottling.current = false; + }, 100); // Adjust the throttle duration as needed + } + }, [chatMessages]); return ( @@ -144,18 +157,17 @@ const ChatComponent: React.FC = ({
- - {chatMessages.map((message, index) => ( - - ))} - + {chatMessages.map((message, index) => ( + + ))} +
void; } -export const ModelSelector: React.FC = ({ - models, - selectedModel, - onModelSelect, -}) => { - const selectedModelInfo = models.find((model) => model.id === selectedModel); +export const ModelSelector: React.FC = ({ selectedModel, onModelSelect }) => { + const [installedLocalModels, setInstalledLocalModels] = useState([]); + const [loading, setLoading] = useState(true); + + // Fetch installed local models + useEffect(() => { + const fetchLocalModels = async () => { + try { + const client = OllamaClient.getInstance(); + const models = await client.listModels(); + setInstalledLocalModels(models.map((m) => m.id)); + } catch (error) { + console.error('Failed to fetch local models:', error); + } finally { + setLoading(false); + } + }; + + fetchLocalModels(); + }, []); + + const selectedModelInfo = AVAILABLE_MODELS.find((model) => model.id === selectedModel); + + // Filter models based on installation status + const modelsByProvider = AVAILABLE_MODELS.reduce( + (acc, model) => { + // Include non-local models and installed local models + if (model.provider !== 'local' || installedLocalModels.includes(model.id)) { + const provider = model.provider; + if (!acc[provider]) acc[provider] = []; + acc[provider].push(model); + } + return acc; + }, + {} as Record + ); + + // Get local models that need to be downloaded + const downloadableModels = AVAILABLE_MODELS.filter( + (model) => model.provider === 'local' && !installedLocalModels.includes(model.id) + ); return ( ); diff --git a/src/components/llm/LocalModels.tsx b/src/components/llm/LocalModels.tsx new file mode 100644 index 0000000..dae4950 --- /dev/null +++ b/src/components/llm/LocalModels.tsx @@ -0,0 +1,77 @@ +import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert'; +import { OllamaClient } from '@/lib/llm/ollama-client'; +import { AVAILABLE_MODELS, OllamaModelInfo } from '@/lib/llm/types'; +import { AlertCircle } from 'lucide-react'; +import { useEffect, useState } from 'react'; +import { ModelManager } from './ModelManager'; + +export function LocalModels() { + const [installedModels, setInstalledModels] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + // Get the list of available local models from our predefined list + const availableLocalModels = AVAILABLE_MODELS.filter((model) => model.provider === 'local'); + + useEffect(() => { + const fetchModels = async () => { + try { + setLoading(true); + setError(null); + const client = OllamaClient.getInstance(); + + // First check if Ollama is running + const isHealthy = await client.checkHealth(); + if (!isHealthy) { + throw new Error('Ollama service is not running. Please start Ollama and try again.'); + } + + const models = await client.listModels(); + setInstalledModels(models); + } catch (err) { + console.error('Failed to fetch models:', err); + setError(err instanceof Error ? err.message : 'Failed to fetch models'); + } finally { + setLoading(false); + } + }; + + fetchModels(); + }, []); + + if (loading) { + return
Loading local models...
; + } + + if (error) { + return ( + + + Error + {error} + + ); + } + + return ( +
+

Local Models

+
+ {availableLocalModels.map((model) => { + const isInstalled = installedModels.some((m) => m.name === model.id); + return ( + { + console.log('Model status changed:', model.id, status); + }} + /> + ); + })} +
+
+ ); +} diff --git a/src/components/llm/ModelManager.tsx b/src/components/llm/ModelManager.tsx new file mode 100644 index 0000000..ae3950a --- /dev/null +++ b/src/components/llm/ModelManager.tsx @@ -0,0 +1,114 @@ +import { Button } from '@/components/ui/button'; +import { Card } from '@/components/ui/card'; +import { useModelManager } from '@/hooks/useModelManager'; +import { LLMModel, OllamaModelStatus } from '@/lib/llm/types'; +import { CheckCircle, ChevronDown, ChevronRight, Clock, Loader2, XCircle } from 'lucide-react'; +import { useCallback, useEffect, useState } from 'react'; + +interface ModelStatusDisplayProps { + modelId: string; + status: OllamaModelStatus; + debug?: string[]; +} + +const ModelStatusDisplay: React.FC = ({ modelId, status, debug }) => { + const [showDebug, setShowDebug] = useState(false); + + return ( +
+
+ {status.status === 'downloading' && status.progress ? ( +
+ + Downloading: {Math.round(status.progress)}% +
+ ) : status.status === 'ready' ? ( +
+ + Ready +
+ ) : status.status === 'error' ? ( +
+ + Error + {status.error && ({status.error})} +
+ ) : ( +
+ + Not Downloaded +
+ )} +
+ {debug && debug.length > 0 && ( +
+ + {showDebug && ( +
+ {debug.map((msg, i) => ( +
+ {msg} +
+ ))} +
+ )} +
+ )} +
+ ); +}; + +interface ModelManagerProps { + modelId: string; + model: LLMModel; + isInstalled: boolean; + onStatusChange: (status: OllamaModelStatus) => void; +} + +export function ModelManager({ modelId, model, isInstalled, onStatusChange }: ModelManagerProps) { + const [debug, setDebug] = useState([]); + const { status, handleDownload } = useModelManager(modelId); + + useEffect(() => { + onStatusChange(status); + }, [status, onStatusChange]); + + const addDebug = useCallback((message: string) => { + setDebug((prev) => [...prev, `${new Date().toISOString()} - ${message}`].slice(-50)); + }, []); + + useEffect(() => { + addDebug(`Status changed: ${status.status} (${status.progress || 0}%)`); + }, [status, addDebug]); + + return ( + +
+
+
+ {model.name} + {model.description} +
+ +
+ {!isInstalled && status.status !== 'ready' && ( + + )} +
+ + ); +} diff --git a/src/components/ui/alert.tsx b/src/components/ui/alert.tsx new file mode 100644 index 0000000..5afd41d --- /dev/null +++ b/src/components/ui/alert.tsx @@ -0,0 +1,59 @@ +import * as React from "react" +import { cva, type VariantProps } from "class-variance-authority" + +import { cn } from "@/lib/utils" + +const alertVariants = cva( + "relative w-full rounded-lg border px-4 py-3 text-sm [&>svg+div]:translate-y-[-3px] [&>svg]:absolute [&>svg]:left-4 [&>svg]:top-4 [&>svg]:text-foreground [&>svg~*]:pl-7", + { + variants: { + variant: { + default: "bg-background text-foreground", + destructive: + "border-destructive/50 text-destructive dark:border-destructive [&>svg]:text-destructive", + }, + }, + defaultVariants: { + variant: "default", + }, + } +) + +const Alert = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes & VariantProps +>(({ className, variant, ...props }, ref) => ( +
+)) +Alert.displayName = "Alert" + +const AlertTitle = React.forwardRef< + HTMLParagraphElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +AlertTitle.displayName = "AlertTitle" + +const AlertDescription = React.forwardRef< + HTMLParagraphElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +AlertDescription.displayName = "AlertDescription" + +export { Alert, AlertTitle, AlertDescription } diff --git a/src/components/ui/sheet.tsx b/src/components/ui/sheet.tsx new file mode 100644 index 0000000..417e7e1 --- /dev/null +++ b/src/components/ui/sheet.tsx @@ -0,0 +1,140 @@ +"use client" + +import * as React from "react" +import * as SheetPrimitive from "@radix-ui/react-dialog" +import { Cross2Icon } from "@radix-ui/react-icons" +import { cva, type VariantProps } from "class-variance-authority" + +import { cn } from "@/lib/utils" + +const Sheet = SheetPrimitive.Root + +const SheetTrigger = SheetPrimitive.Trigger + +const SheetClose = SheetPrimitive.Close + +const SheetPortal = SheetPrimitive.Portal + +const SheetOverlay = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +SheetOverlay.displayName = SheetPrimitive.Overlay.displayName + +const sheetVariants = cva( + "fixed z-50 gap-4 bg-background p-6 shadow-lg transition ease-in-out data-[state=closed]:duration-300 data-[state=open]:duration-500 data-[state=open]:animate-in data-[state=closed]:animate-out", + { + variants: { + side: { + top: "inset-x-0 top-0 border-b data-[state=closed]:slide-out-to-top data-[state=open]:slide-in-from-top", + bottom: + "inset-x-0 bottom-0 border-t data-[state=closed]:slide-out-to-bottom data-[state=open]:slide-in-from-bottom", + left: "inset-y-0 left-0 h-full w-3/4 border-r data-[state=closed]:slide-out-to-left data-[state=open]:slide-in-from-left sm:max-w-sm", + right: + "inset-y-0 right-0 h-full w-3/4 border-l data-[state=closed]:slide-out-to-right data-[state=open]:slide-in-from-right sm:max-w-sm", + }, + }, + defaultVariants: { + side: "right", + }, + } +) + +interface SheetContentProps + extends React.ComponentPropsWithoutRef, + VariantProps {} + +const SheetContent = React.forwardRef< + React.ElementRef, + SheetContentProps +>(({ side = "right", className, children, ...props }, ref) => ( + + + + + + Close + + {children} + + +)) +SheetContent.displayName = SheetPrimitive.Content.displayName + +const SheetHeader = ({ + className, + ...props +}: React.HTMLAttributes) => ( +
+) +SheetHeader.displayName = "SheetHeader" + +const SheetFooter = ({ + className, + ...props +}: React.HTMLAttributes) => ( +
+) +SheetFooter.displayName = "SheetFooter" + +const SheetTitle = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +SheetTitle.displayName = SheetPrimitive.Title.displayName + +const SheetDescription = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +SheetDescription.displayName = SheetPrimitive.Description.displayName + +export { + Sheet, + SheetPortal, + SheetOverlay, + SheetTrigger, + SheetClose, + SheetContent, + SheetHeader, + SheetFooter, + SheetTitle, + SheetDescription, +} diff --git a/src/hooks/useDockerHandlers.ts b/src/hooks/useDockerHandlers.ts index d01e2b3..025a1c2 100644 --- a/src/hooks/useDockerHandlers.ts +++ b/src/hooks/useDockerHandlers.ts @@ -56,7 +56,8 @@ export function useDockerHandlers({ const followBuildProgress = useCallback( (messageId: string, subprocessId: string): Promise => { return new Promise((resolve) => { - const eventSource = new EventSource(`/api/docker?buildId=${Date.now()}`); + const apiKey = process.env.NEXT_PUBLIC_API_KEY || ''; + const eventSource = new EventSource(`/api/docker?buildId=${Date.now()}&apiKey=${apiKey}`); let buildLog = ''; eventSource.onmessage = (event) => { @@ -299,8 +300,9 @@ export function useDockerHandlers({ const setupStatusStream = () => { if (state.containerId) { + const apiKey = process.env.NEXT_PUBLIC_API_KEY || ''; eventSource = new EventSource( - `/api/docker?statusId=${Date.now()}&containerId=${state.containerId}` + `/api/docker?statusId=${Date.now()}&containerId=${state.containerId}&apiKey=${apiKey}` ); eventSource.onmessage = (event) => { diff --git a/src/hooks/useModelManager.ts b/src/hooks/useModelManager.ts new file mode 100644 index 0000000..596ff51 --- /dev/null +++ b/src/hooks/useModelManager.ts @@ -0,0 +1,133 @@ +import { OllamaClient } from '@/lib/llm/ollama-client'; +import { OllamaModelStatus } from '@/lib/llm/types'; +import { useCallback, useEffect, useState } from 'react'; + +export function useModelManager(modelId: string) { + const [status, setStatus] = useState({ + name: modelId, + status: 'checking', + lastUpdated: new Date(), + }); + + // Check initial status and set up event stream + useEffect(() => { + const client = OllamaClient.getInstance(); + + // Get initial status + const checkHealth = async () => { + try { + const isHealthy = await client.checkHealth(); + if (!isHealthy) { + setStatus((prev) => ({ + ...prev, + status: 'error', + error: 'Ollama service is not healthy', + lastUpdated: new Date(), + })); + return; + } + + // Get initial model status + const currentStatus = client.getModelStatus(modelId); + if (currentStatus) { + setStatus(currentStatus); + } + } catch (error) { + console.error('Failed to check health:', error); + setStatus((prev) => ({ + ...prev, + status: 'error', + error: 'Failed to check health', + lastUpdated: new Date(), + })); + } + }; + + checkHealth(); + + // Set up event stream for status updates + const eventSource = client.setupModelStatusStream(modelId); + + // Handle incoming status updates + eventSource.onmessage = (event: MessageEvent) => { + try { + const newStatus = JSON.parse(event.data) as OllamaModelStatus; + setStatus(newStatus); + } catch (error) { + console.error('Failed to parse status update:', error); + } + }; + + eventSource.onerror = () => { + console.error('Status EventSource error'); + setStatus((prev) => ({ + ...prev, + status: 'error', + error: 'Connection error', + lastUpdated: new Date(), + })); + }; + + return () => { + eventSource.close(); + }; + }, [modelId]); + + const handleDownload = useCallback(async () => { + try { + setStatus((prev) => ({ + ...prev, + status: 'downloading', + progress: 0, + lastUpdated: new Date(), + })); + + const client = OllamaClient.getInstance(); + const eventSource = client.setupPullStream(modelId); + + // Handle download progress + eventSource.onmessage = (event: MessageEvent) => { + try { + const data = JSON.parse(event.data) as OllamaModelStatus; + setStatus((prev) => ({ + ...prev, + ...data, + lastUpdated: new Date(), + })); + + // Close event source when download is complete + if (data.status === 'ready' || data.status === 'error') { + eventSource.close(); + } + } catch (error) { + console.error('Failed to parse pull status:', error); + } + }; + + // Handle errors + eventSource.onerror = () => { + console.error('Pull event stream error'); + setStatus((prev) => ({ + ...prev, + status: 'error', + error: 'Failed to download model', + lastUpdated: new Date(), + })); + eventSource.close(); + }; + } catch (error) { + console.error('Failed to start model download:', error); + setStatus((prev) => ({ + ...prev, + status: 'error', + error: error instanceof Error ? error.message : 'Failed to start download', + lastUpdated: new Date(), + })); + } + }, [modelId]); + + return { + status, + handleDownload, + }; +} diff --git a/src/lib/context/manager.ts b/src/lib/context/manager.ts new file mode 100644 index 0000000..b057a16 --- /dev/null +++ b/src/lib/context/manager.ts @@ -0,0 +1,10 @@ +import { ContextWindow, Message, Tokenizer } from './types'; + +interface ContextManager { + window: ContextWindow; + tokenizer: Tokenizer; + + addMessage(msg: Message): void; + pruneHistory(): void; + getContextForModel(modelId: string): Message[]; +} diff --git a/src/lib/context/types.ts b/src/lib/context/types.ts new file mode 100644 index 0000000..642599e --- /dev/null +++ b/src/lib/context/types.ts @@ -0,0 +1,14 @@ +import { BaseMessage } from '@langchain/core/messages'; + +export interface ContextWindow { + maxTokens: number; + currentTokens: number; + messages: BaseMessage[]; +} + +export interface Tokenizer { + countTokens(text: string): Promise; + countMessageTokens(messages: BaseMessage[]): Promise; +} + +export type Message = BaseMessage; diff --git a/src/lib/docker/tools.ts b/src/lib/docker/tools.ts new file mode 100644 index 0000000..3c12d91 --- /dev/null +++ b/src/lib/docker/tools.ts @@ -0,0 +1,11 @@ +import { CommandResult, ContainerManager, ImageBuilder, StatusMonitor, UserInput } from './types'; + +interface DockerTooling { + containerManager: ContainerManager; + imageBuilder: ImageBuilder; + statusMonitor: StatusMonitor; + + executeCommand(cmd: string): Promise; + captureScreen(): Promise; + sendInput(input: UserInput): Promise; +} diff --git a/src/lib/docker/types.ts b/src/lib/docker/types.ts new file mode 100644 index 0000000..d9e4417 --- /dev/null +++ b/src/lib/docker/types.ts @@ -0,0 +1,56 @@ +export interface ContainerManager { + start(options: ContainerOptions): Promise; + stop(containerId: string): Promise; + remove(containerId: string): Promise; +} + +export interface ImageBuilder { + build(dockerfile: string, options: BuildOptions): Promise; + pull(image: string): Promise; +} + +export interface StatusMonitor { + getStatus(containerId: string): Promise; + subscribe(containerId: string, callback: StatusCallback): () => void; +} + +export interface CommandResult { + exitCode: number; + stdout: string; + stderr: string; +} + +export interface UserInput { + type: 'keyboard' | 'mouse'; + data: KeyboardEvent | MouseEvent; +} + +export interface ContainerOptions { + image: string; + ports?: Record; + env?: Record; +} + +export interface BuildOptions { + tag: string; + context: string; + args?: Record; +} + +export interface ContainerStatus { + id: string; + state: 'running' | 'stopped' | 'error'; + details?: string; + metrics?: ContainerMetrics; +} + +export interface ContainerMetrics { + cpu: number; + memory: number; + network: { + rx: number; + tx: number; + }; +} + +export type StatusCallback = (status: ContainerStatus) => void; diff --git a/src/lib/functions/registry.ts b/src/lib/functions/registry.ts new file mode 100644 index 0000000..bc77331 --- /dev/null +++ b/src/lib/functions/registry.ts @@ -0,0 +1,9 @@ +import { FunctionDefinition } from '../llm/types'; + +interface FunctionRegistry { + functions: Map; + + register(def: FunctionDefinition): void; + validate(name: string, params: unknown): boolean; + execute(name: string, params: unknown): Promise; +} diff --git a/src/lib/llm/ollama-client.ts b/src/lib/llm/ollama-client.ts new file mode 100644 index 0000000..015d08d --- /dev/null +++ b/src/lib/llm/ollama-client.ts @@ -0,0 +1,123 @@ +import { EventEmitter } from 'events'; +import { + OllamaHealthResponse, + OllamaListResponse, + OllamaMetricsResponse, + OllamaModelInfo, + OllamaModelStatus, +} from './types'; + +export class OllamaClient { + private static instance: OllamaClient; + private modelStatusEmitter: EventEmitter; + private modelStatuses: Map; + private apiKey: string; + + private constructor() { + this.modelStatusEmitter = new EventEmitter(); + this.modelStatuses = new Map(); + this.apiKey = process.env.NEXT_PUBLIC_API_KEY || ''; + } + + public static getInstance(): OllamaClient { + if (!OllamaClient.instance) { + OllamaClient.instance = new OllamaClient(); + } + return OllamaClient.instance; + } + + public async listModels(): Promise { + try { + const response = await fetch(`/api/llm/ollama/list?apiKey=${this.apiKey}`); + if (!response.ok) throw new Error('Failed to fetch models'); + const data = (await response.json()) as OllamaListResponse; + return data.models; + } catch (error) { + console.error('Failed to list models:', error); + return []; + } + } + + public async checkHealth(): Promise { + try { + const response = await fetch(`/api/llm/ollama/health?apiKey=${this.apiKey}`); + if (!response.ok) return false; + const data = (await response.json()) as OllamaHealthResponse; + return data.healthy; + } catch (error) { + console.error('Health check failed:', error); + return false; + } + } + + public async getModelMetrics(modelId: string): Promise { + try { + const response = await fetch( + `/api/llm/ollama/metrics?modelId=${modelId}&apiKey=${this.apiKey}` + ); + if (!response.ok) throw new Error('Failed to fetch metrics'); + return await response.json(); + } catch (error) { + console.error('Failed to fetch model metrics:', error); + return { memoryUsage: 0 }; + } + } + + public setupModelStatusStream(modelId: string): EventSource { + const eventSource = new EventSource( + `/api/llm/ollama/status?modelId=${modelId}&apiKey=${this.apiKey}` + ); + + eventSource.onmessage = (event) => { + try { + const status = JSON.parse(event.data) as OllamaModelStatus; + this.updateModelStatus(modelId, status); + } catch (error) { + console.error('Failed to parse status update:', error); + } + }; + + return eventSource; + } + + public setupPullStream(modelId: string): EventSource { + return new EventSource(`/api/llm/ollama/pull?modelId=${modelId}&apiKey=${this.apiKey}`); + } + + public subscribeToModelStatus( + modelId: string, + callback: (status: OllamaModelStatus) => void + ): () => void { + const handleStatusUpdate = (_modelId: string, status: OllamaModelStatus) => { + if (_modelId === modelId) { + callback(status); + } + }; + + this.modelStatusEmitter.on('statusUpdate', handleStatusUpdate); + return () => { + this.modelStatusEmitter.off('statusUpdate', handleStatusUpdate); + }; + } + + private updateModelStatus(modelId: string, status: Partial): void { + const currentStatus = this.modelStatuses.get(modelId) || { + name: modelId, + status: 'checking', + lastUpdated: new Date(), + }; + + const newStatus: OllamaModelStatus = { + ...currentStatus, + ...status, + lastUpdated: new Date(), + }; + + this.modelStatuses.set(modelId, newStatus); + this.modelStatusEmitter.emit('statusUpdate', modelId, newStatus); + } + + public getModelStatus(modelId: string): OllamaModelStatus | undefined { + return this.modelStatuses.get(modelId); + } +} diff --git a/src/lib/llm/ollama.ts b/src/lib/llm/ollama.ts new file mode 100644 index 0000000..e4dc1b6 --- /dev/null +++ b/src/lib/llm/ollama.ts @@ -0,0 +1,165 @@ +import { getOllamaHeaders } from '@/lib/utils/api'; +import { ModelResourceMetrics, OllamaModelInfo, OllamaModelStatus } from './types'; + +// Define a type for the model +interface Model { + name: string; + digest: string; + size: number; + modified_at: string; + details?: { + parameter_size: number; + quantization_level: string; + }; +} + +export class OllamaService { + private static instance: OllamaService; + private baseUrl: string; + + private constructor() { + this.baseUrl = process.env.NEXT_PUBLIC_OLLAMA_URL || 'http://localhost:11434'; + } + + public static getInstance(): OllamaService { + if (!OllamaService.instance) { + OllamaService.instance = new OllamaService(); + } + return OllamaService.instance; + } + + public async listModels(): Promise { + try { + const response = await fetch(`${this.baseUrl}/api/tags`, { + headers: getOllamaHeaders(), + }); + if (!response.ok) throw new Error('Failed to fetch models'); + + const { models } = await response.json(); + return models.map((model: Model) => ({ + id: model.name, + name: model.name, + provider: 'local' as const, + digest: model.digest, + size: model.size, + modified_at: model.modified_at, + status: 'ready', + contextLength: 4096, + description: model.details + ? `${model.details.parameter_size} parameters, ${model.details.quantization_level} quantization` + : 'Local model', + details: model.details, + })); + } catch (error) { + console.error('Failed to list models:', error); + throw error; + } + } + + public async pullModel( + modelName: string, + onProgress?: (status: OllamaModelStatus) => void + ): Promise { + try { + const response = await fetch(`${this.baseUrl}/api/pull`, { + method: 'POST', + headers: getOllamaHeaders(), + body: JSON.stringify({ name: modelName }), + }); + + if (!response.ok) { + throw new Error('Failed to start model pull'); + } + + const reader = response.body?.getReader(); + if (!reader) { + throw new Error('No response body'); + } + + // Process the stream using async iteration + const processStream = async () => { + let buffer = ''; + let isDone = false; + + try { + while (!isDone) { + const { done, value } = await reader.read(); + isDone = done; + + if (done) break; + + // Append new chunk to buffer and split by newlines + buffer += new TextDecoder().decode(value); + const lines = buffer.split('\n'); + + // Process all complete lines + for (let i = 0; i < lines.length - 1; i++) { + const line = lines[i].trim(); + if (!line) continue; + + try { + const progress = JSON.parse(line); + + // Create status update based on progress + const status: OllamaModelStatus = { + name: modelName, + status: progress.status === 'success' ? 'ready' : 'downloading', + progress: + progress.completed && progress.total + ? (progress.completed / progress.total) * 100 + : undefined, + downloadedSize: progress.completed, + totalSize: progress.total, + error: progress.error, + lastUpdated: new Date(), + }; + + // Call the progress callback if provided + onProgress?.(status); + + // If we got an error or success, we're done + if (progress.status === 'success' || progress.error) { + isDone = true; + break; + } + } catch (error) { + console.warn('Failed to parse progress:', error); + } + } + + // Keep the last incomplete line in the buffer + buffer = lines[lines.length - 1]; + } + } finally { + reader.releaseLock(); + } + }; + + // Start processing the stream + await processStream(); + return response; + } catch (error) { + console.error('Failed to pull model:', error); + throw error; + } + } + + public async checkHealth(): Promise { + try { + const response = await fetch(this.baseUrl, { + headers: getOllamaHeaders(), + }); + return response.ok; + } catch (error) { + console.error('Health check failed:', error); + return false; + } + } + + public async getModelMetrics(modelId: string): Promise { + // Implement actual metrics fetching from Ollama when available + return { + memoryUsage: 0, + }; + } +} diff --git a/src/lib/llm/provider.ts b/src/lib/llm/provider.ts index db2271e..39dbfe3 100644 --- a/src/lib/llm/provider.ts +++ b/src/lib/llm/provider.ts @@ -1,4 +1,5 @@ import { ChatAnthropic } from '@langchain/anthropic'; +import { ChatOllama } from '@langchain/community/chat_models/ollama'; import { BaseChatModel } from '@langchain/core/language_models/chat_models'; import { AIMessage, HumanMessage, SystemMessage } from '@langchain/core/messages'; import { ChatOpenAI } from '@langchain/openai'; @@ -38,6 +39,19 @@ export class LLMProvider { temperature: this.config.temperature, maxTokens: this.config.maxTokens, }); + case 'local': + if (!this.config.ollamaConfig?.baseUrl) { + throw new Error('Ollama base URL is required for local models'); + } + + return new ChatOllama({ + model: this.config.model, + baseUrl: this.config.ollamaConfig.baseUrl, + temperature: this.config.temperature, + numPredict: this.config.maxTokens, + // Additional Ollama-specific settings + ...this.config.ollamaConfig.parameters, + }); default: throw new Error(`Unsupported provider: ${this.config.provider}`); } diff --git a/src/lib/llm/tokens.ts b/src/lib/llm/tokens.ts new file mode 100644 index 0000000..b6e0424 --- /dev/null +++ b/src/lib/llm/tokens.ts @@ -0,0 +1,46 @@ +import { getMessageContent } from '@/lib/utils/langchain'; +import { BaseMessage } from '@langchain/core/messages'; +import { getEncoding, Tiktoken } from 'js-tiktoken'; +import { LLMProvider } from './types'; + +export class TokenCounter { + private static instance: TokenCounter; + private encoders: Map; + + private constructor() { + this.encoders = new Map(); + } + + public static getInstance(): TokenCounter { + if (!TokenCounter.instance) { + TokenCounter.instance = new TokenCounter(); + } + return TokenCounter.instance; + } + + public async countTokens(text: string, provider: LLMProvider): Promise { + if (provider === 'anthropic') { + // Use Claude's token counting approximation + return Math.ceil(text.length / 4); + } + + // OpenAI uses tiktoken + const encoder = await this.getEncoder(provider); + return encoder?.encode(text).length; + } + + public async countMessageTokens(messages: BaseMessage[], provider: LLMProvider): Promise { + let totalTokens = 0; + for (const message of messages) { + totalTokens += (await this.countTokens(getMessageContent(message), provider)) ?? 0; + } + return totalTokens; + } + + private async getEncoder(provider: LLMProvider) { + if (!this.encoders.has(provider)) { + this.encoders.set(provider, await getEncoding('cl100k_base')); + } + return this.encoders.get(provider); + } +} diff --git a/src/lib/llm/types.ts b/src/lib/llm/types.ts index 1f5bdad..c7b8d7f 100644 --- a/src/lib/llm/types.ts +++ b/src/lib/llm/types.ts @@ -4,13 +4,31 @@ import { AIMessage, BaseMessage, HumanMessage, SystemMessage } from '@langchain/ // Core LLM types export type LLMProvider = 'openai' | 'anthropic' | 'local'; +// Add Ollama-specific types +export interface OllamaConfig { + baseUrl: string; + numGpu?: number; + threads?: number; + contextSize?: number; + // Ollama-specific parameters + parameters?: { + numPredict?: number; + topK?: number; + topP?: number; + temperature?: number; + repeatPenalty?: number; + }; +} + +// Update LLMConfig to include Ollama settings export interface LLMConfig { provider: LLMProvider; model: string; - apiKey: string; + apiKey?: string; // Optional now as local models don't need it baseUrl?: string; temperature?: number; maxTokens?: number; + ollamaConfig?: OllamaConfig; // Add Ollama configuration } export interface FunctionDefinition { @@ -33,21 +51,37 @@ export interface LLMResponse { }; } -export interface LLMModel { +// Base model information interface that all providers should implement +export interface BaseModelInfo { id: string; name: string; provider: LLMProvider; - contextWindow: number; + size?: number; // in MB + status: 'not_downloaded' | 'downloading' | 'ready' | 'error'; + modified_at?: string; + contextLength: number; + quantization?: string; + description?: string; +} + +// Ollama-specific model information +export interface OllamaModelInfo extends BaseModelInfo { + provider: 'local'; + digest: string; // Ollama-specific + modified_at: string; // Required for Ollama +} + +// Update the LLMModel interface to extend BaseModelInfo +export interface LLMModel extends BaseModelInfo { maxOutputTokens: number; trainingCutOffDate?: string; - description?: string; } const createModel = ( id: string, name: string, provider: LLMProvider, - contextWindow: number, + contextLength: number, maxOutputTokens: number, description: string, trainingCutOffDate?: string @@ -55,10 +89,11 @@ const createModel = ( id, name, provider, - contextWindow, + contextLength, maxOutputTokens, - trainingCutOffDate, description, + trainingCutOffDate, + status: 'not_downloaded', // Default status }); export const AVAILABLE_MODELS: LLMModel[] = [ @@ -138,6 +173,35 @@ export const AVAILABLE_MODELS: LLMModel[] = [ // 'Faster and cheaper reasoning model particularly good at coding, math, and science', // 'Oct 2023' // ), + + // Local Models (via Ollama) + createModel( + 'nemotron:latest', + 'Nemotron 70B', + 'local', + 32768, + 4096, + "NVIDIA's latest open model. Optimized for enterprise use, strong at reasoning and coding. 70B parameters.", + 'Oct 2024' + ), + createModel( + 'nemotron-mini:latest', + 'Nemotron Mini 4B', + 'local', + 16384, + 4096, + 'Lightweight version of Nemotron. 4B parameters, efficient for everyday tasks.', + 'Oct 2024' + ), + createModel( + 'llama3.2:3b', + 'Llama 3.2 3B', + 'local', + 4096, + 2048, + 'Latest Llama model optimized for efficiency. Good balance of performance and resource usage.', + 'Oct 2024' + ), ]; // Sort models by provider and capability @@ -147,7 +211,7 @@ export const AVAILABLE_MODELS_SORTED = AVAILABLE_MODELS.sort((a, b) => { return a.provider.localeCompare(b.provider); } // Then by context window size (larger first) - return b.contextWindow - a.contextWindow; + return b.contextLength - a.contextLength; }); // Chat Memory Types @@ -206,3 +270,47 @@ export function createMessage(content: string, role: MessageRole): BaseMessage { return new HumanMessage({ content }); } } + +// Update OllamaModelStatus to include more detailed states +export interface OllamaModelStatus { + name: string; + status: 'checking' | 'ready' | 'downloading' | 'error'; + progress?: number; + error?: string; + downloadedSize?: number; + totalSize?: number; + lastUpdated: Date; + metrics?: ModelResourceMetrics; +} + +// Generic model types that can be used across providers +export interface ModelResourceMetrics { + memoryUsage: number; // in MB + gpuMemoryUsage?: number; // in MB + gpuUtilization?: number; // percentage + temperature?: number; // in Celsius +} + +// Add API response types for better type safety +export interface OllamaHealthResponse { + healthy: boolean; +} + +export interface OllamaListResponse { + models: OllamaModelInfo[]; +} + +export interface OllamaMetricsResponse { + memoryUsage: number; + gpuMemoryUsage?: number; + gpuUtilization?: number; + temperature?: number; +} + +export interface OllamaPullResponse { + status: string; + digest?: string; + total?: number; + completed?: number; + error?: string; +} diff --git a/src/lib/utils.ts b/src/lib/utils.ts index bd0c391..2819a83 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,6 +1,6 @@ -import { clsx, type ClassValue } from "clsx" -import { twMerge } from "tailwind-merge" +import { clsx, type ClassValue } from 'clsx'; +import { twMerge } from 'tailwind-merge'; export function cn(...inputs: ClassValue[]) { - return twMerge(clsx(inputs)) + return twMerge(clsx(inputs)); } diff --git a/src/lib/utils/api.ts b/src/lib/utils/api.ts new file mode 100644 index 0000000..0c8dd1a --- /dev/null +++ b/src/lib/utils/api.ts @@ -0,0 +1,13 @@ +export function getApiHeaders(): HeadersInit { + return { + 'Content-Type': 'application/json', + 'x-api-key': process.env.NEXT_PUBLIC_API_KEY || '', + }; +} + +// Add a new function for Ollama-specific headers +export function getOllamaHeaders(): HeadersInit { + return { + 'Content-Type': 'application/json', + }; +} diff --git a/src/lib/utils/langchain.ts b/src/lib/utils/langchain.ts new file mode 100644 index 0000000..49d1710 --- /dev/null +++ b/src/lib/utils/langchain.ts @@ -0,0 +1,14 @@ +import { BaseMessage } from '@langchain/core/messages'; + +/** + * Get the content of a message, handling arrays of content objects. + * Returns '' if the message content type is not text. + * @param message + * @returns + */ +export const getMessageContent = (message: BaseMessage): string => { + if (message.content instanceof Array) { + return message.content.map((content) => (content.type === 'text' ? content.text : '')).join(''); + } + return message.content; +}; diff --git a/src/middleware.ts b/src/middleware.ts index ba31eb3..601fd05 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -2,24 +2,39 @@ import type { NextRequest } from 'next/server'; import { NextResponse } from 'next/server'; export function middleware(request: NextRequest) { - // Allow GET requests to /api/docker for EventSource - if (request.method === 'GET' && request.nextUrl.pathname === '/api/docker') { - return NextResponse.next(); - } + const isEventStream = request.headers.get('accept') === 'text/event-stream'; + const apiKey = request.headers.get('x-api-key') || request.nextUrl.searchParams.get('apiKey'); - // Check for API key in header for other API routes - const apiKey = request.headers.get('x-api-key'); + // Check if this is an API route + if (request.nextUrl.pathname.startsWith('/api/')) { + // For event streams, check apiKey from query params + if (isEventStream) { + if (!apiKey || apiKey !== process.env.API_KEY) { + return new NextResponse(JSON.stringify({ error: 'Unauthorized' }), { + status: 401, + headers: { 'Content-Type': 'application/json' }, + }); + } + return NextResponse.next(); + } - if (!apiKey || apiKey !== process.env.API_KEY) { - return new NextResponse(JSON.stringify({ message: 'Authentication required' }), { - status: 401, - headers: { 'Content-Type': 'application/json' }, - }); + // For regular API requests, check x-api-key header + if (!apiKey || apiKey !== process.env.API_KEY) { + return new NextResponse(JSON.stringify({ error: 'Unauthorized' }), { + status: 401, + headers: { 'Content-Type': 'application/json' }, + }); + } } return NextResponse.next(); } +// Fix: Matcher paths must start with '/' export const config = { - matcher: '/api/:path*', + matcher: [ + '/api/:path*', // Match all API routes + '/_next/static/:path*', // Match Next.js static files + '/favicon.ico', // Match favicon + ], }; diff --git a/src/services/dockerService.ts b/src/services/dockerService.ts index 0200dcf..683c002 100644 --- a/src/services/dockerService.ts +++ b/src/services/dockerService.ts @@ -1,4 +1,4 @@ -const API_KEY = process.env.NEXT_PUBLIC_API_KEY; +import { getApiHeaders } from '@/lib/utils/api'; interface DockerResponse { ok: boolean; @@ -11,10 +11,7 @@ export const dockerService = { async fetchDockerfiles() { const response = await fetch('/api/docker', { method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-api-key': API_KEY || '', - }, + headers: getApiHeaders(), body: JSON.stringify({ action: 'listDockerfiles' }), }); if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`); @@ -24,10 +21,7 @@ export const dockerService = { async buildImage(dockerfile: string): Promise { const response = await fetch('/api/docker', { method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-api-key': API_KEY || '', - }, + headers: getApiHeaders(), body: JSON.stringify({ action: 'buildImage', dockerfile, @@ -39,10 +33,7 @@ export const dockerService = { async startContainer(imageName: string): Promise { const response = await fetch('/api/docker', { method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-api-key': API_KEY || '', - }, + headers: getApiHeaders(), body: JSON.stringify({ action: 'startContainer', imageName, @@ -55,10 +46,7 @@ export const dockerService = { async stopContainer(containerId: string): Promise { const response = await fetch('/api/docker', { method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-api-key': API_KEY || '', - }, + headers: getApiHeaders(), body: JSON.stringify({ action: 'stopContainer', containerId }), }); return response.json(); @@ -67,10 +55,7 @@ export const dockerService = { async deleteContainer(): Promise { const response = await fetch('/api/docker', { method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-api-key': API_KEY || '', - }, + headers: getApiHeaders(), body: JSON.stringify({ action: 'deleteContainer' }), }); return response.json(); diff --git a/src/services/llm.service.ts b/src/services/llm.service.ts index b4ed30a..045f525 100644 --- a/src/services/llm.service.ts +++ b/src/services/llm.service.ts @@ -1,3 +1,4 @@ +import { OllamaService } from '@/lib/llm/ollama'; import { LLMProvider } from '@/lib/llm/provider'; import { FunctionRegistry } from '@/lib/llm/registry'; import { @@ -5,17 +6,26 @@ import { FunctionDefinition, LLMConfig, LLMRequestOptions, + OllamaModelInfo, } from '@/lib/llm/types'; import { AIMessage, HumanMessage, SystemMessage } from '@langchain/core/messages'; +/** + * LLM Service + * + * This service is responsible for managing LLM providers and functions. + * Used by backend only. + */ export class LLMService { private static instance: LLMService; private providers: Map; private registry: FunctionRegistry; + private ollamaService: OllamaService; private constructor() { this.providers = new Map(); this.registry = FunctionRegistry.getInstance(); + this.ollamaService = OllamaService.getInstance(); } public static getInstance(): LLMService { @@ -25,13 +35,66 @@ export class LLMService { return LLMService.instance; } - private getProvider(modelId: string): LLMProvider { - if (!this.providers.has(modelId)) { - const model = AVAILABLE_MODELS.find((m) => m.id === modelId); - if (!model) { - throw new Error(`Model ${modelId} not found`); + private async setupLocalProvider(modelId: string): Promise { + const model = AVAILABLE_MODELS.find((m) => m.id === modelId); + if (!model) { + throw new Error(`Model ${modelId} not found`); + } + + // Check if Ollama is healthy + const isHealthy = await this.ollamaService.checkHealth(); + if (!isHealthy) { + throw new Error('Ollama service is not available'); + } + + // Check if model is available locally + const availableModels = await this.ollamaService.listModels(); + const modelExists = availableModels.some((m) => m.name === model.id); + + // Pull model if not available + if (!modelExists) { + const pulled = await this.ollamaService.pullModel(model.id); + if (!pulled) { + throw new Error(`Failed to pull model ${model.id}`); } + } + const config: LLMConfig = { + provider: 'local', + model: model.id, + temperature: 0.7, + maxTokens: model.maxOutputTokens, + ollamaConfig: { + baseUrl: process.env.OLLAMA_URL || 'http://localhost:11434', + parameters: { + temperature: 0.7, + numPredict: model.maxOutputTokens, + topK: 40, + topP: 0.9, + repeatPenalty: 1.1, + }, + }, + }; + + return new LLMProvider(config); + } + + private async getProvider(modelId: string): Promise { + const provider = this.providers.get(modelId); + if (provider) { + return provider; + } + + const model = AVAILABLE_MODELS.find((m) => m.id === modelId); + if (!model) { + throw new Error(`Model ${modelId} not found`); + } + + if (model.provider === 'local') { + const provider = await this.setupLocalProvider(modelId); + this.providers.set(modelId, provider); + return provider; + } else { const config: LLMConfig = { provider: model.provider, model: model.id, @@ -39,11 +102,10 @@ export class LLMService { temperature: 0.7, maxTokens: model.maxOutputTokens, }; - - this.providers.set(modelId, new LLMProvider(config)); + const provider = new LLMProvider(config); + this.providers.set(modelId, provider); + return provider; } - - return this.providers.get(modelId)!; } private getApiKey(provider: string): string { @@ -57,6 +119,10 @@ export class LLMService { return key; } + public async getLocalModels(): Promise { + return this.ollamaService.listModels(); + } + public async sendMessage(message: string, modelId: string, options?: LLMRequestOptions) { try { const model = AVAILABLE_MODELS.find((m) => m.id === modelId); @@ -70,7 +136,7 @@ export class LLMService { msg instanceof HumanMessage || msg instanceof AIMessage || msg instanceof SystemMessage ); - const provider = this.getProvider(modelId); + const provider = await this.getProvider(modelId); return await provider.generateResponse(message, { ...options, history, diff --git a/tsconfig.json b/tsconfig.json index 5fb0f5c..06df0f4 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,6 +12,7 @@ "moduleResolution": "node", "resolveJsonModule": true, "isolatedModules": true, + "sourceMap": true, "jsx": "preserve", "incremental": true, "plugins": [