Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: import model by folder on win #3463

Merged
merged 6 commits into from
Aug 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions core/src/browser/fs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,15 @@ const appendFileSync = (...args: any[]) => globalThis.core.api?.appendFileSync(.
const copyFile: (src: string, dest: string) => Promise<void> = (src, dest) =>
globalThis.core.api?.copyFile(src, dest)

/**
* Gets the list of gguf files in a directory
*
* @param path - The paths to the file.
* @returns {Promise<{any}>} - A promise that resolves with the list of gguf and non-gguf files
*/
const getGgufFiles: (paths: string[]) => Promise<any> = (
paths) => globalThis.core.api?.getGgufFiles(paths)

/**
* Gets the file's stats.
*
Expand All @@ -84,4 +93,5 @@ export const fs = {
copyFile,
fileStat,
writeBlob,
getGgufFiles,
}
53 changes: 51 additions & 2 deletions core/src/node/api/processors/fsExt.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { join } from 'path'
import fs from 'fs'
import { basename, join } from 'path'
import fs, { readdirSync } from 'fs'
import { appResourcePath, normalizeFilePath, validatePath } from '../../helper/path'
import { defaultAppConfig, getJanDataFolderPath, getJanDataFolderPath as getPath } from '../../helper'
import { Processor } from './Processor'
Expand Down Expand Up @@ -80,4 +80,53 @@ export class FSExt implements Processor {
})
})
}

async getGgufFiles(paths: string[]) {
const sanitizedFilePaths: {
path: string
name: string
size: number
}[] = []
for (const filePath of paths) {
const normalizedPath = normalizeFilePath(filePath)

const isExist = fs.existsSync(normalizedPath)
if (!isExist) continue
const fileStats = fs.statSync(normalizedPath)
if (!fileStats) continue
if (!fileStats.isDirectory()) {
const fileName = await basename(normalizedPath)
sanitizedFilePaths.push({
path: normalizedPath,
name: fileName,
size: fileStats.size,
})
} else {
// allowing only one level of directory
const files = await readdirSync(normalizedPath)

for (const file of files) {
const fullPath = await join(normalizedPath, file)
const fileStats = await fs.statSync(fullPath)
if (!fileStats || fileStats.isDirectory()) continue

sanitizedFilePaths.push({
path: fullPath,
name: file,
size: fileStats.size,
})
}
}
}
const unsupportedFiles = sanitizedFilePaths.filter(
(file) => !file.path.endsWith('.gguf')
)
const supportedFiles = sanitizedFilePaths.filter((file) =>
file.path.endsWith('.gguf')
)
return {
unsupportedFiles,
supportedFiles,
}
}
}
1 change: 1 addition & 0 deletions core/src/types/api/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ export enum FileManagerRoute {
getUserHomePath = 'getUserHomePath',
fileStat = 'fileStat',
writeBlob = 'writeBlob',
getGgufFiles = 'getGgufFiles',
}

export type ApiFunction = (...args: any[]) => any
Expand Down
41 changes: 5 additions & 36 deletions web/hooks/useImportModel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
Model,
ModelExtension,
OptionType,
baseName,

Check warning on line 9 in web/hooks/useImportModel.ts

View workflow job for this annotation

GitHub Actions / test-on-macos

'baseName' is defined but never used

Check warning on line 9 in web/hooks/useImportModel.ts

View workflow job for this annotation

GitHub Actions / test-on-ubuntu

'baseName' is defined but never used

Check warning on line 9 in web/hooks/useImportModel.ts

View workflow job for this annotation

GitHub Actions / test-on-windows-pr

'baseName' is defined but never used
fs,
joinPath,

Check warning on line 11 in web/hooks/useImportModel.ts

View workflow job for this annotation

GitHub Actions / test-on-macos

'joinPath' is defined but never used

Check warning on line 11 in web/hooks/useImportModel.ts

View workflow job for this annotation

GitHub Actions / test-on-ubuntu

'joinPath' is defined but never used

Check warning on line 11 in web/hooks/useImportModel.ts

View workflow job for this annotation

GitHub Actions / test-on-windows-pr

'joinPath' is defined but never used
} from '@janhq/core'

import { atom, useSetAtom } from 'jotai'
Expand Down Expand Up @@ -66,44 +66,13 @@
const sanitizeFilePaths = useCallback(
async (filePaths: string[]) => {
if (!filePaths || filePaths.length === 0) return

const sanitizedFilePaths: FilePathWithSize[] = []
for (const filePath of filePaths) {
const fileStats = await fs.fileStat(filePath, true)
if (!fileStats) continue

if (!fileStats.isDirectory) {
const fileName = await baseName(filePath)
sanitizedFilePaths.push({
path: filePath,
name: fileName,
size: fileStats.size,
})
} else {
// allowing only one level of directory
const files = await fs.readdirSync(filePath)

for (const file of files) {
const fullPath = await joinPath([filePath, file])
const fileStats = await fs.fileStat(fullPath, true)
if (!fileStats || fileStats.isDirectory) continue

sanitizedFilePaths.push({
path: fullPath,
name: file,
size: fileStats.size,
})
}
}
const { unsupportedFiles, supportedFiles } = (await fs.getGgufFiles(
filePaths
)) as unknown as {
unsupportedFiles: FilePathWithSize[]
supportedFiles: FilePathWithSize[]
}

const unsupportedFiles = sanitizedFilePaths.filter(
(file) => !file.path.endsWith('.gguf')
)
const supportedFiles = sanitizedFilePaths.filter((file) =>
file.path.endsWith('.gguf')
)

const importingModels: ImportingModel[] = supportedFiles.map(
({ path, name, size }: FilePathWithSize) => ({
importId: uuidv4(),
Expand Down
Loading