From d6bd493d93acdd5227ca0c118321d416c3b6ada2 Mon Sep 17 00:00:00 2001 From: Realmbird <56459277+Realmbird@users.noreply.github.com> Date: Mon, 3 Jun 2024 22:20:43 -0700 Subject: [PATCH] Added NVIDIA API to new jan after jan rework (#2934) * Added NVIDIA API to new jan * Changed paramters * chore: some small text update - remove databrick since it does not work when I tested - correct some texts --------- Co-authored-by: James Nguyen --- .../inference-nvidia-extension/README.md | 79 +++++++++++++++++++ .../inference-nvidia-extension/package.json | 43 ++++++++++ .../resources/models.json | 31 ++++++++ .../resources/settings.json | 24 ++++++ .../inference-nvidia-extension/src/index.ts | 66 ++++++++++++++++ .../inference-nvidia-extension/tsconfig.json | 14 ++++ .../webpack.config.js | 42 ++++++++++ 7 files changed, 299 insertions(+) create mode 100644 extensions/inference-nvidia-extension/README.md create mode 100644 extensions/inference-nvidia-extension/package.json create mode 100644 extensions/inference-nvidia-extension/resources/models.json create mode 100644 extensions/inference-nvidia-extension/resources/settings.json create mode 100644 extensions/inference-nvidia-extension/src/index.ts create mode 100644 extensions/inference-nvidia-extension/tsconfig.json create mode 100644 extensions/inference-nvidia-extension/webpack.config.js diff --git a/extensions/inference-nvidia-extension/README.md b/extensions/inference-nvidia-extension/README.md new file mode 100644 index 0000000000..65a1b2b593 --- /dev/null +++ b/extensions/inference-nvidia-extension/README.md @@ -0,0 +1,79 @@ +# Nvidia Engine Extension + +Created using Jan extension example + +# Create a Jan Extension using Typescript + +Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 + +## Create Your Own Extension + +To create your own extension, you can use this repository as a template! Just follow the below instructions: + +1. Click the Use this template button at the top of the repository +2. Select Create a new repository +3. Select an owner and name for your new repository +4. Click Create repository +5. Clone your new repository + +## Initial Setup + +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. + +> [!NOTE] +> +> You'll need to have a reasonably modern version of +> [Node.js](https://nodejs.org) handy. If you are using a version manager like +> [`nodenv`](https://github.com/nodenv/nodenv) or +> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the +> root of your repository to install the version specified in +> [`package.json`](./package.json). Otherwise, 20.x or later should work! + +1. :hammer_and_wrench: Install the dependencies + + ```bash + npm install + ``` + +1. :building_construction: Package the TypeScript for distribution + + ```bash + npm run bundle + ``` + +1. :white_check_mark: Check your artifact + + There will be a tgz file in your extension directory now + +## Update the Extension Metadata + +The [`package.json`](package.json) file defines metadata about your extension, such as +extension name, main entry, description and version. + +When you copy this repository, update `package.json` with the name, description for your extension. + +## Update the Extension Code + +The [`src/`](./src/) directory is the heart of your extension! This contains the +source code that will be run when your extension functions are invoked. You can replace the +contents of this directory with your own code. + +There are a few things to keep in mind when writing your extension code: + +- Most Jan Extension functions are processed asynchronously. + In `index.ts`, you will see that the extension function will return a `Promise`. + + ```typescript + import { events, MessageEvent, MessageRequest } from '@janhq/core' + + function onStart(): Promise { + return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.inference(data) + ) + } + ``` + + For more information about the Jan Extension Core module, see the + [documentation](https://github.com/janhq/jan/blob/main/core/README.md). + +So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-nvidia-extension/package.json b/extensions/inference-nvidia-extension/package.json new file mode 100644 index 0000000000..8bd7708bc8 --- /dev/null +++ b/extensions/inference-nvidia-extension/package.json @@ -0,0 +1,43 @@ +{ + "name": "@janhq/inference-nvidia-extension", + "productName": "NVIDIA NIM Inference Engine", + "version": "1.0.1", + "description": "This extension enables NVIDIA chat completion API calls", + "main": "dist/index.js", + "module": "dist/module.js", + "engine": "nvidia", + "author": "Jan ", + "license": "AGPL-3.0", + "scripts": { + "build": "tsc -b . && webpack --config webpack.config.js", + "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install" + }, + "exports": { + ".": "./dist/index.js", + "./main": "./dist/module.js" + }, + "devDependencies": { + "cpx": "^1.5.0", + "rimraf": "^3.0.2", + "webpack": "^5.88.2", + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" + }, + "dependencies": { + "@janhq/core": "file:../../core", + "fetch-retry": "^5.0.6", + "path-browserify": "^1.0.1", + "ulidx": "^2.3.0" + }, + "engines": { + "node": ">=18.0.0" + }, + "files": [ + "dist/*", + "package.json", + "README.md" + ], + "bundleDependencies": [ + "fetch-retry" + ] +} diff --git a/extensions/inference-nvidia-extension/resources/models.json b/extensions/inference-nvidia-extension/resources/models.json new file mode 100644 index 0000000000..b97644fc99 --- /dev/null +++ b/extensions/inference-nvidia-extension/resources/models.json @@ -0,0 +1,31 @@ +[ + { + "sources": [ + { + "url": "https://integrate.api.nvidia.com/v1/chat/completions" + } + ], + "id": "mistralai/mistral-7b-instruct-v0.2", + "object": "model", + "name": "Mistral 7B", + "version": "1.1", + "description": "Mistral 7B with NVIDIA", + "format": "api", + "settings": {}, + "parameters": { + "max_tokens": 1024, + "temperature": 0.3, + "top_p": 1, + "stream": false, + "frequency_penalty": 0, + "presence_penalty": 0, + "stop": null, + "seed": null + }, + "metadata": { + "author": "NVIDIA", + "tags": ["General"] + }, + "engine": "nvidia" + } +] diff --git a/extensions/inference-nvidia-extension/resources/settings.json b/extensions/inference-nvidia-extension/resources/settings.json new file mode 100644 index 0000000000..e7647b5621 --- /dev/null +++ b/extensions/inference-nvidia-extension/resources/settings.json @@ -0,0 +1,24 @@ +[ + { + "key": "chat-completions-endpoint", + "title": "Chat Completions Endpoint", + "description": "The endpoint to use for chat completions. See the [NVIDIA API documentation](https://www.nvidia.com/en-us/ai/) for more information.", + "controllerType": "input", + "controllerProps": { + "placeholder": "https://integrate.api.nvidia.com/v1/chat/completions", + "value": "https://integrate.api.nvidia.com/v1/chat/completions" + } + }, + { + "key": "nvidia-api-key", + "title": "API Key", + "description": "The NVIDIA API uses API keys for authentication. Visit your [API Keys](https://org.ngc.nvidia.com/setup/personal-keys) page to retrieve the API key you'll use in your requests..", + "controllerType": "input", + "controllerProps": { + "placeholder": "nvapi-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "value": "", + "type": "password", + "inputActions": ["unobscure", "copy"] + } + } +] diff --git a/extensions/inference-nvidia-extension/src/index.ts b/extensions/inference-nvidia-extension/src/index.ts new file mode 100644 index 0000000000..9af27d90c7 --- /dev/null +++ b/extensions/inference-nvidia-extension/src/index.ts @@ -0,0 +1,66 @@ +/** + * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. + * The class provides methods for initializing and stopping a model, and for making inference requests. + * It also subscribes to events emitted by the @janhq/core package and handles new message requests. + * @version 1.0.0 + * @module inference-mistral-extension/src/index + */ + +import { RemoteOAIEngine } from '@janhq/core' + +declare const SETTINGS: Array +declare const MODELS: Array + +enum Settings { + apiKey = 'nvidia-api-key', + chatCompletionsEndPoint = 'chat-completions-endpoint', +} +/** + * A class that implements the InferenceExtension interface from the @janhq/core package. + * The class provides methods for initializing and stopping a model, and for making inference requests. + * It also subscribes to events emitted by the @janhq/core package and handles new message requests. + */ +export default class JanNVIDIANIMInferenceEngine extends RemoteOAIEngine { + inferenceUrl: string = '' + provider: string = 'nvidia' + + override async onLoad(): Promise { + super.onLoad() + + // Register Settings + this.registerSettings(SETTINGS) + this.registerModels(MODELS) + + this.apiKey = await this.getSetting(Settings.apiKey, '') + this.inferenceUrl = await this.getSetting( + Settings.chatCompletionsEndPoint, + '' + ) + + if (this.inferenceUrl.length === 0) { + SETTINGS.forEach((setting) => { + if (setting.key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = setting.controllerProps.value as string + } + }) + } + } + + onSettingUpdate(key: string, value: T): void { + if (key === Settings.apiKey) { + this.apiKey = value as string + } else if (key === Settings.chatCompletionsEndPoint) { + if (typeof value !== 'string') return + + if (value.trim().length === 0) { + SETTINGS.forEach((setting) => { + if (setting.key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = setting.controllerProps.value as string + } + }) + } else { + this.inferenceUrl = value + } + } + } +} diff --git a/extensions/inference-nvidia-extension/tsconfig.json b/extensions/inference-nvidia-extension/tsconfig.json new file mode 100644 index 0000000000..2477d58ce5 --- /dev/null +++ b/extensions/inference-nvidia-extension/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "es2016", + "module": "ES6", + "moduleResolution": "node", + "outDir": "./dist", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": false, + "skipLibCheck": true, + "rootDir": "./src" + }, + "include": ["./src"] +} diff --git a/extensions/inference-nvidia-extension/webpack.config.js b/extensions/inference-nvidia-extension/webpack.config.js new file mode 100644 index 0000000000..0e35fc227b --- /dev/null +++ b/extensions/inference-nvidia-extension/webpack.config.js @@ -0,0 +1,42 @@ +const path = require('path') +const webpack = require('webpack') +const packageJson = require('./package.json') +const settingJson = require('./resources/settings.json') +const modelsJson = require('./resources/models.json') + +module.exports = { + experiments: { outputModule: true }, + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', + module: { + rules: [ + { + test: /\.tsx?$/, + use: 'ts-loader', + exclude: /node_modules/, + }, + ], + }, + plugins: [ + new webpack.DefinePlugin({ + SETTINGS: JSON.stringify(settingJson), + ENGINE: JSON.stringify(packageJson.engine), + MODELS: JSON.stringify(modelsJson), + }), + ], + output: { + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format + }, + resolve: { + extensions: ['.ts', '.js'], + fallback: { + path: require.resolve('path-browserify'), + }, + }, + optimization: { + minimize: false, + }, + // Add loaders and other configuration as needed for your project +}