From 8c632b09841fbbfeda0e4312774a7c0428b24e47 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Mon, 26 Aug 2024 19:54:30 +0700 Subject: [PATCH 1/5] fix: import model by folder on win --- core/src/node/api/processors/fsExt.ts | 50 +++++++++++++++++++++++++-- web/hooks/useImportModel.ts | 41 +++------------------- 2 files changed, 53 insertions(+), 38 deletions(-) diff --git a/core/src/node/api/processors/fsExt.ts b/core/src/node/api/processors/fsExt.ts index 155732cfce..36a6549e5e 100644 --- a/core/src/node/api/processors/fsExt.ts +++ b/core/src/node/api/processors/fsExt.ts @@ -1,9 +1,10 @@ -import { join } from 'path' -import fs from 'fs' +import { basename, join } from 'path' +import fs, { readdirSync } from 'fs' import { appResourcePath, normalizeFilePath, validatePath } from '../../helper/path' import { getJanDataFolderPath, getJanDataFolderPath as getPath } from '../../helper' import { Processor } from './Processor' import { FileStat } from '../../../types' +import { joinPath } from '../../../browser' export class FSExt implements Processor { observer?: Function @@ -79,4 +80,49 @@ export class FSExt implements Processor { }) }) } + + async getGgufFiles(paths: string[]) { + const sanitizedFilePaths: { + path: string + name: string + size: number + }[] = [] + for (const filePath of paths) { + const fileStats = this.fileStat(filePath, true) + if (!fileStats) continue + if (!fileStats.isDirectory) { + const fileName = await basename(filePath) + sanitizedFilePaths.push({ + path: filePath, + name: fileName, + size: fileStats.size, + }) + } else { + // allowing only one level of directory + const files = await readdirSync(filePath) + + for (const file of files) { + const fullPath = await joinPath([filePath, file]) + const fileStats = await this.fileStat(fullPath, true) + if (!fileStats || fileStats.isDirectory) continue + + sanitizedFilePaths.push({ + path: fullPath, + name: file, + size: fileStats.size, + }) + } + } + } + const unsupportedFiles = sanitizedFilePaths.filter( + (file) => !file.path.endsWith('.gguf') + ) + const supportedFiles = sanitizedFilePaths.filter((file) => + file.path.endsWith('.gguf') + ) + return { + unsupportedFiles, + supportedFiles, + } + } } diff --git a/web/hooks/useImportModel.ts b/web/hooks/useImportModel.ts index 170f03b5ea..3458bb1d30 100644 --- a/web/hooks/useImportModel.ts +++ b/web/hooks/useImportModel.ts @@ -66,44 +66,13 @@ const useImportModel = () => { const sanitizeFilePaths = useCallback( async (filePaths: string[]) => { if (!filePaths || filePaths.length === 0) return - - const sanitizedFilePaths: FilePathWithSize[] = [] - for (const filePath of filePaths) { - const fileStats = await fs.fileStat(filePath, true) - if (!fileStats) continue - - if (!fileStats.isDirectory) { - const fileName = await baseName(filePath) - sanitizedFilePaths.push({ - path: filePath, - name: fileName, - size: fileStats.size, - }) - } else { - // allowing only one level of directory - const files = await fs.readdirSync(filePath) - - for (const file of files) { - const fullPath = await joinPath([filePath, file]) - const fileStats = await fs.fileStat(fullPath, true) - if (!fileStats || fileStats.isDirectory) continue - - sanitizedFilePaths.push({ - path: fullPath, - name: file, - size: fileStats.size, - }) - } - } + const { unsupportedFiles, supportedFiles } = fs.getGgufFiles( + filePaths + ) as { + unsupportedFiles: FilePathWithSize[] + supportedFiles: FilePathWithSize[] } - const unsupportedFiles = sanitizedFilePaths.filter( - (file) => !file.path.endsWith('.gguf') - ) - const supportedFiles = sanitizedFilePaths.filter((file) => - file.path.endsWith('.gguf') - ) - const importingModels: ImportingModel[] = supportedFiles.map( ({ path, name, size }: FilePathWithSize) => ({ importId: uuidv4(), From 119215ad05f1f413454727f16af7111e9fdc2c81 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Mon, 26 Aug 2024 20:18:05 +0700 Subject: [PATCH 2/5] expose getGgufFiles method --- core/src/types/api/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/types/api/index.ts b/core/src/types/api/index.ts index e50dce6de8..e1d1b28da9 100644 --- a/core/src/types/api/index.ts +++ b/core/src/types/api/index.ts @@ -105,6 +105,7 @@ export enum FileManagerRoute { getUserHomePath = 'getUserHomePath', fileStat = 'fileStat', writeBlob = 'writeBlob', + getGgufFiles = 'getGgufFiles', } export type ApiFunction = (...args: any[]) => any From 2293993aa766cb47fc87a523822a98603c76a0ea Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Tue, 27 Aug 2024 04:25:46 +0700 Subject: [PATCH 3/5] fix the missing api --- core/src/browser/fs.ts | 10 +++++++++ core/src/node/api/processors/fsExt.ts | 32 +++++++++++++++++++-------- web/hooks/useImportModel.ts | 4 ++-- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/core/src/browser/fs.ts b/core/src/browser/fs.ts index cca9bb1d3f..9240b38766 100644 --- a/core/src/browser/fs.ts +++ b/core/src/browser/fs.ts @@ -58,6 +58,15 @@ const appendFileSync = (...args: any[]) => globalThis.core.api?.appendFileSync(. const copyFile: (src: string, dest: string) => Promise = (src, dest) => globalThis.core.api?.copyFile(src, dest) +/** + * Gets the list of gguf files in a directory + * + * @param path - The paths to the file. + * @returns {Promise<{any}>} - A promise that resolves with the list of gguf and non-gguf files + */ +const getGgufFiles: (paths: string[]) => Promise = ( + paths) => globalThis.core.api?.getGgufFiles(paths) + /** * Gets the file's stats. * @@ -84,4 +93,5 @@ export const fs = { copyFile, fileStat, writeBlob, + getGgufFiles, } diff --git a/core/src/node/api/processors/fsExt.ts b/core/src/node/api/processors/fsExt.ts index 7256ad6b9c..a1b06f7f30 100644 --- a/core/src/node/api/processors/fsExt.ts +++ b/core/src/node/api/processors/fsExt.ts @@ -4,7 +4,6 @@ import { appResourcePath, normalizeFilePath, validatePath } from '../../helper/p import { defaultAppConfig, getJanDataFolderPath, getJanDataFolderPath as getPath } from '../../helper' import { Processor } from './Processor' import { FileStat } from '../../../types' -import { joinPath } from '../../../browser' export class FSExt implements Processor { observer?: Function @@ -89,23 +88,38 @@ export class FSExt implements Processor { size: number }[] = [] for (const filePath of paths) { - const fileStats = this.fileStat(filePath, true) + const normalizedPath = normalizeFilePath(filePath) + console.log(normalizedPath) + + const isExist = fs.existsSync(normalizedPath) + console.log(11) + if (!isExist) continue + console.log(12) + const fileStats = fs.statSync(normalizedPath) + console.log(13) if (!fileStats) continue - if (!fileStats.isDirectory) { - const fileName = await basename(filePath) + if (!fileStats.isDirectory()) { + console.log(4) + const fileName = await basename(normalizedPath) sanitizedFilePaths.push({ - path: filePath, + path: normalizedPath, name: fileName, size: fileStats.size, }) + console.log(1) } else { + console.log(2) // allowing only one level of directory - const files = await readdirSync(filePath) + const files = await readdirSync(normalizedPath) + console.log(3) for (const file of files) { - const fullPath = await joinPath([filePath, file]) - const fileStats = await this.fileStat(fullPath, true) - if (!fileStats || fileStats.isDirectory) continue + console.log(4, file) + const fullPath = await join(normalizedPath, file) + console.log(fullPath) + const fileStats = await fs.statSync(fullPath) + console.log(6) + if (!fileStats || fileStats.isDirectory()) continue sanitizedFilePaths.push({ path: fullPath, diff --git a/web/hooks/useImportModel.ts b/web/hooks/useImportModel.ts index 3458bb1d30..b23f5a6fb4 100644 --- a/web/hooks/useImportModel.ts +++ b/web/hooks/useImportModel.ts @@ -66,9 +66,9 @@ const useImportModel = () => { const sanitizeFilePaths = useCallback( async (filePaths: string[]) => { if (!filePaths || filePaths.length === 0) return - const { unsupportedFiles, supportedFiles } = fs.getGgufFiles( + const { unsupportedFiles, supportedFiles } = (await fs.getGgufFiles( filePaths - ) as { + )) as unknown as { unsupportedFiles: FilePathWithSize[] supportedFiles: FilePathWithSize[] } From 4a9da1bbe832ab5e3facb148dd451aac0d3d309a Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Tue, 27 Aug 2024 10:06:14 +0700 Subject: [PATCH 4/5] clean log --- core/src/node/api/processors/fsExt.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/core/src/node/api/processors/fsExt.ts b/core/src/node/api/processors/fsExt.ts index a1b06f7f30..ffd51ce223 100644 --- a/core/src/node/api/processors/fsExt.ts +++ b/core/src/node/api/processors/fsExt.ts @@ -92,33 +92,23 @@ export class FSExt implements Processor { console.log(normalizedPath) const isExist = fs.existsSync(normalizedPath) - console.log(11) if (!isExist) continue - console.log(12) const fileStats = fs.statSync(normalizedPath) - console.log(13) if (!fileStats) continue if (!fileStats.isDirectory()) { - console.log(4) const fileName = await basename(normalizedPath) sanitizedFilePaths.push({ path: normalizedPath, name: fileName, size: fileStats.size, }) - console.log(1) } else { - console.log(2) // allowing only one level of directory const files = await readdirSync(normalizedPath) - console.log(3) for (const file of files) { - console.log(4, file) const fullPath = await join(normalizedPath, file) - console.log(fullPath) const fileStats = await fs.statSync(fullPath) - console.log(6) if (!fileStats || fileStats.isDirectory()) continue sanitizedFilePaths.push({ From 9c9f6d0976d25aad1a60dc316d4df4f224938241 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Tue, 27 Aug 2024 10:08:50 +0700 Subject: [PATCH 5/5] clean log --- core/src/node/api/processors/fsExt.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/node/api/processors/fsExt.ts b/core/src/node/api/processors/fsExt.ts index ffd51ce223..4d113e1ee2 100644 --- a/core/src/node/api/processors/fsExt.ts +++ b/core/src/node/api/processors/fsExt.ts @@ -89,7 +89,6 @@ export class FSExt implements Processor { }[] = [] for (const filePath of paths) { const normalizedPath = normalizeFilePath(filePath) - console.log(normalizedPath) const isExist = fs.existsSync(normalizedPath) if (!isExist) continue