From dc649bf523b059b7ed04e75db63e650352dc1437 Mon Sep 17 00:00:00 2001 From: Louis Date: Wed, 27 Nov 2024 00:14:43 +0700 Subject: [PATCH] fix: 4096 - failed to get huggingface models --- core/src/browser/core.test.ts | 13 ------------ core/src/browser/core.ts | 10 --------- core/src/node/api/processors/download.test.ts | 17 +++++---------- core/src/node/api/processors/download.ts | 21 ------------------- core/src/types/api/index.ts | 1 - web/containers/ModelSearch/index.tsx | 2 +- web/utils/huggingface.test.ts | 13 +++++++----- web/utils/huggingface.ts | 16 +++++++------- 8 files changed, 21 insertions(+), 72 deletions(-) diff --git a/core/src/browser/core.test.ts b/core/src/browser/core.test.ts index f38cc0b404..720ea9dcf3 100644 --- a/core/src/browser/core.test.ts +++ b/core/src/browser/core.test.ts @@ -3,7 +3,6 @@ import { joinPath } from './core' import { openFileExplorer } from './core' import { getJanDataFolderPath } from './core' import { abortDownload } from './core' -import { getFileSize } from './core' import { executeOnMain } from './core' describe('test core apis', () => { @@ -66,18 +65,6 @@ describe('test core apis', () => { expect(result).toBe('aborted') }) - it('should get file size', async () => { - const url = 'http://example.com/file' - globalThis.core = { - api: { - getFileSize: jest.fn().mockResolvedValue(1024), - }, - } - const result = await getFileSize(url) - expect(globalThis.core.api.getFileSize).toHaveBeenCalledWith(url) - expect(result).toBe(1024) - }) - it('should execute function on main process', async () => { const extension = 'testExtension' const method = 'testMethod' diff --git a/core/src/browser/core.ts b/core/src/browser/core.ts index b19e0b339f..7058fc172a 100644 --- a/core/src/browser/core.ts +++ b/core/src/browser/core.ts @@ -28,15 +28,6 @@ const downloadFile: (downloadRequest: DownloadRequest, network?: NetworkConfig) network ) => globalThis.core?.api?.downloadFile(downloadRequest, network) -/** - * Get unit in bytes for a remote file. - * - * @param url - The url of the file. - * @returns {Promise} - A promise that resolves with the file size. - */ -const getFileSize: (url: string) => Promise = (url: string) => - globalThis.core.api?.getFileSize(url) - /** * Aborts the download of a specific file. * @param {string} fileName - The name of the file whose download is to be aborted. @@ -167,7 +158,6 @@ export { getUserHomePath, systemInformation, showToast, - getFileSize, dirName, FileStat, } diff --git a/core/src/node/api/processors/download.test.ts b/core/src/node/api/processors/download.test.ts index 21d94165dc..c4b171a7d9 100644 --- a/core/src/node/api/processors/download.test.ts +++ b/core/src/node/api/processors/download.test.ts @@ -23,6 +23,11 @@ jest.mock('fs', () => ({ createWriteStream: jest.fn(), })) +const requestMock = jest.fn((options, callback) => { + callback(new Error('Test error'), null) +}) +jest.mock('request', () => requestMock) + jest.mock('request-progress', () => { return jest.fn().mockImplementation(() => { return { @@ -54,18 +59,6 @@ describe('Downloader', () => { beforeEach(() => { jest.resetAllMocks() }) - it('should handle getFileSize errors correctly', async () => { - const observer = jest.fn() - const url = 'http://example.com/file' - - const downloader = new Downloader(observer) - const requestMock = jest.fn((options, callback) => { - callback(new Error('Test error'), null) - }) - jest.mock('request', () => requestMock) - - await expect(downloader.getFileSize(observer, url)).rejects.toThrow('Test error') - }) it('should pause download correctly', () => { const observer = jest.fn() diff --git a/core/src/node/api/processors/download.ts b/core/src/node/api/processors/download.ts index ebeb7c299c..709ad96878 100644 --- a/core/src/node/api/processors/download.ts +++ b/core/src/node/api/processors/download.ts @@ -135,25 +135,4 @@ export class Downloader implements Processor { pauseDownload(_observer: any, fileName: any) { DownloadManager.instance.networkRequests[fileName]?.pause() } - - async getFileSize(_observer: any, url: string): Promise { - return new Promise((resolve, reject) => { - const request = require('request') - request( - { - url, - method: 'HEAD', - }, - function (err: any, response: any) { - if (err) { - console.error('Getting file size failed:', err) - reject(err) - } else { - const size: number = response.headers['content-length'] ?? -1 - resolve(size) - } - } - ) - }) - } } diff --git a/core/src/types/api/index.ts b/core/src/types/api/index.ts index 093314a157..b8bf273a7a 100644 --- a/core/src/types/api/index.ts +++ b/core/src/types/api/index.ts @@ -65,7 +65,6 @@ export enum DownloadRoute { pauseDownload = 'pauseDownload', resumeDownload = 'resumeDownload', getDownloadProgress = 'getDownloadProgress', - getFileSize = 'getFileSize', } export enum DownloadEvent { diff --git a/web/containers/ModelSearch/index.tsx b/web/containers/ModelSearch/index.tsx index 10b6466a64..9f2910fef9 100644 --- a/web/containers/ModelSearch/index.tsx +++ b/web/containers/ModelSearch/index.tsx @@ -46,7 +46,7 @@ const ModelSearch = ({ onSearchLocal }: Props) => { errMessage = err.message } toaster({ - title: 'Failed to get Hugging Face models', + title: 'Oops, you may be rate limited, give it a bit more time', description: errMessage, type: 'error', }) diff --git a/web/utils/huggingface.test.ts b/web/utils/huggingface.test.ts index db7dbf3e11..26975f6909 100644 --- a/web/utils/huggingface.test.ts +++ b/web/utils/huggingface.test.ts @@ -3,11 +3,8 @@ import { toHuggingFaceUrl, InvalidHostError, } from './huggingface' -import { getFileSize } from '@janhq/core' -// Mock the getFileSize function jest.mock('@janhq/core', () => ({ - getFileSize: jest.fn(), AllQuantizations: ['q4_0', 'q4_1', 'q5_0', 'q5_1', 'q8_0'], })) @@ -38,9 +35,15 @@ describe('huggingface utils', () => { } ;(global.fetch as jest.Mock).mockResolvedValue({ - json: jest.fn().mockResolvedValue(mockResponse), + json: jest + .fn() + .mockResolvedValueOnce(mockResponse) + .mockResolvedValueOnce([{ + path: 'model-q4_0.gguf', size: 1000000, + },{ + path: 'model-q4_0.gguf', size: 2000 + }]), }) - ;(getFileSize as jest.Mock).mockResolvedValue(1000000) const result = await fetchHuggingFaceRepoData('user/repo') diff --git a/web/utils/huggingface.ts b/web/utils/huggingface.ts index 3e71f3a0f8..1550f183a1 100644 --- a/web/utils/huggingface.ts +++ b/web/utils/huggingface.ts @@ -1,4 +1,4 @@ -import { AllQuantizations, getFileSize, HuggingFaceRepoData } from '@janhq/core' +import { AllQuantizations, HuggingFaceRepoData } from '@janhq/core' /** * Fetches data from a Hugging Face repository. @@ -39,21 +39,19 @@ export const fetchHuggingFaceRepoData = async ( ) } - const promises: Promise[] = [] - // fetching file sizes const url = new URL(sanitizedUrl) const paths = url.pathname.split('/').filter((e) => e.trim().length > 0) + const repoTree: { path: string; size: number }[] = await fetch( + `https://huggingface.co/api/models/${paths[2]}/${paths[3]}/tree/main` + ).then((res) => res.json()) + for (const sibling of data.siblings) { const downloadUrl = `https://huggingface.co/${paths[2]}/${paths[3]}/resolve/main/${sibling.rfilename}` sibling.downloadUrl = downloadUrl - promises.push(getFileSize(downloadUrl)) - } - - const result = await Promise.all(promises) - for (let i = 0; i < data.siblings.length; i++) { - data.siblings[i].fileSize = result[i] + sibling.fileSize = + repoTree.find((file) => file.path === sibling.rfilename)?.size ?? 0 } AllQuantizations.forEach((quantization) => {