v1.9.5

app/ui/package.json

@@ -1,7 +1,7 @@
 {
  "name": "app",
  "private": true,
- "version": "1.9.4",
+ "version": "1.9.5",
  "type": "module",
  "scripts": {
  "dev": "vite",

app/ui/src/@types/bot.ts

@@ -21,6 +21,7 @@ export type BotSettings = {
  semanticSearchSimilarityScore: string;
  inactivityTimeout: number;
  autoResetSession: boolean;
+ autoSyncDataSources: boolean;
  };
  chatModel: {
  label: string;

app/ui/src/components/Bot/Settings/SettingsBody.tsx

@@ -168,6 +168,7 @@ export const SettingsBody: React.FC<BotSettings> = ({
  semanticSearchSimilarityScore: data.semanticSearchSimilarityScore,
  autoResetSession: data.autoResetSession,
  inactivityTimeout: data.inactivityTimeout,
+ autoSyncDataSources: data.autoSyncDataSources,
  }}
  form={form}
  requiredMark={false}
@@ -453,6 +454,14 @@ export const SettingsBody: React.FC<BotSettings> = ({
  placeholder="Enter inactivity timeout"
  />
  </Form.Item>
+
+ <Form.Item
+ name="autoSyncDataSources"
+ label="Auto Sync Data Source(s)"
+ tooltip="This will automatically re-fetch the URL-based data sources at a certain interval."
+ >
+ <Switch />
+ </Form.Item>
  </div>
  </div>
 

app/ui/src/components/Common/BotForm.tsx

@@ -35,7 +35,7 @@ type Props = {
  form: FormInstance<any>;
  showEmbeddingAndModels: boolean;
  newSelectedSource?: any;
- botConfig: BotConfig
+ botConfig: BotConfig;
 };
 function classNames(...classes: string[]) {
  return classes.filter(Boolean).join(" ");
@@ -47,7 +47,7 @@ export const BotForm = ({
  setSelectedSource,
  form,
  showEmbeddingAndModels,
- botConfig
+ botConfig,
 }: Props) => {
  const youtubeMode = Form.useWatch(["options", "youtube_mode"], form);
  const url = Form.useWatch(["content"], form);
@@ -117,7 +117,7 @@ export const BotForm = ({
  }}
  >
  <Upload.Dragger
- accept={`.pdf,.docx,.csv,.txt,.mp3,.mp4`}
+ accept={`.pdf,.docx,.csv,.txt,.mp3,.mp4,.zip`}
  multiple={true}
  maxCount={botConfig?.fileUploadSizeLimit}
  beforeUpload={(file) => {
@@ -130,6 +130,8 @@ export const BotForm = ({
  "audio/mp4",
  "video/mp4",
  "video/mpeg",
+ "application/zip",
+ "application/x-zip-compressed",
  ]
  .map((type) => type.toLowerCase())
  .join(", ");
@@ -161,12 +163,13 @@ export const BotForm = ({
  <InboxIcon className="h-10 w-10 text-gray-400" />
  </p>
  <p className="ant-upload-text">
- Click or drag PDF, Docx, CSV , TXT, MP3, MP4 files to this
+ Click or drag PDF, Docx, CSV , TXT, MP3, MP4, Zip files to
+ this
  </p>
  <p className="ant-upload-hint">
- Support is available for a single or bulk upload of up to 10
+ {`Support is available for a single or bulk upload of up to ${botConfig?.fileUploadSizeLimit}
  files. Please note that file upload is in beta, so if you
- encounter any issues, kindly report them.
+ encounter any issues, kindly report them.`}
  </p>
  </div>
  </Upload.Dragger>

app/ui/src/routes/settings/application.tsx

@@ -29,6 +29,7 @@ export default function SettingsApplicationRoot() {
  dynamicallyFetchOllamaModels: boolean;
  ollamaURL: string;
  fileUploadSizeLimit: number;
+ refetchDatasource: boolean;
  };
  });
 
@@ -173,9 +174,15 @@ export default function SettingsApplicationRoot() {
  ]}
  tooltip="Default is 10"
  >
-
  <InputNumber size="large" style={{ width: "100%" }} />
-
+ </Form.Item>
+ <Form.Item
+ label="Refetch Data Source"
+ name="refetchDatasource"
+ valuePropName="checked"
+ help="This will refetch the data source at a specific interval."
+ >
+ <Switch />
  </Form.Item>
  </div>
  <div className="bg-gray-50 border-x border-b rounded-b-md rounded-x-md px-4 py-3 text-right sm:px-6 dark:bg-[#141414] dark:border-gray-600">

package.json

@@ -1,6 +1,6 @@
 {
  "name": "dialoqbase",
- "version": "1.9.4",
+ "version": "1.9.5",
  "description": "Create chatbots with ease",
  "scripts": {
  "ui:dev": "pnpm run --filter ui dev",

server/package.json

@@ -61,6 +61,7 @@
  "cohere-ai": "^6.2.1",
  "concurrently": "^7.0.0",
  "copyfiles": "^2.4.1",
+ "cron": "^3.1.7",
  "d3-dsv": "2",
  "date-fns": "^3.6.0",
  "discord.js": "^14.11.0",
@@ -92,6 +93,8 @@
  "turndown": "^7.1.3",
  "unique-names-generator": "^4.7.1",
  "wavefile": "^11.0.0",
+ "yauzl": "^3.1.3",
+ "yauzl-promise": "^4.0.0",
  "yt-transcript": "^0.0.2",
  "ytdl-core": "^4.11.5"
  },
@@ -103,6 +106,8 @@
  "@types/pubsub-js": "^1.8.3",
  "@types/tap": "^15.0.5",
  "@types/turndown": "^5.0.4",
+ "@types/yauzl": "^2.10.3",
+ "@types/yauzl-promise": "^4.0.1",
  "c8": "9.0.0",
  "fastify-tsconfig": "2.0.0",
  "prisma": "^5.9.1",

server/prisma/migrations/q_14_3/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "DialoqbaseSettings" ADD COLUMN "refetchDatasource" BOOLEAN NOT NULL DEFAULT false;

server/prisma/migrations/q_14_4/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Bot" ADD COLUMN "autoSyncDataSources" BOOLEAN DEFAULT false;

server/prisma/schema.prisma

@@ -44,6 +44,7 @@ model Bot {
  bot_api_key String?
  bot_model_api_key String?
  options Json? @default("{}") @db.Json
+ autoSyncDataSources Boolean? @default(false)
  BotAppearance BotAppearance[]
  document BotDocument[]
  BotIntegration BotIntegration[]
@@ -106,6 +107,7 @@ model DialoqbaseSettings {
  defaultEmbeddingModel String @default("dialoqbase_eb_text-embedding-ada-002")
  ollamaURL String? @default("http://host.docker.internal:11434")
  usePuppeteerFetch Boolean? @default(false)
+ refetchDatasource Boolean @default(false)
 }
 
 model BotIntegration {

server/src/app.ts

@@ -13,6 +13,9 @@ import swaggerUi from "@fastify/swagger-ui";
 import { pathToFileURL } from "url";
 import { Worker } from "bullmq";
 import { parseRedisUrl } from "./utils/redis";
+import { CronJob } from 'cron';
+import { processDatasourceCron } from "./cron/index";
+
 declare module "fastify" {
  interface Session {
  is_bot_allowed: boolean;
@@ -31,7 +34,11 @@ const app: FastifyPluginAsync<AppOptions> = async (
 
  void fastify.register(FastifySSEPlugin);
 
- void fastify.register(fastifyMultipart);
+ void fastify.register(fastifyMultipart, {
+ limits: {
+ fileSize: 1 * 1024 * 1024 * 1024,
+ }
+ });
 
  void fastify.register(swagger);
 
@@ -99,8 +106,17 @@ const worker = new Worker("vector", workerUrl, {
  useWorkerThreads: workerThreads === "true",
 });
 
+const job = new CronJob(
+ process.env.DB_CRON_TIME || '0 0 0 * * *',
+ processDatasourceCron,
+ null,
+ true,
+ process.env.DB_CRON_TIMEZONE
+);
+
 process.on("SIGINT", async () => {
  await worker.close();
+ job.stop();
  process.exit();
 });
 

server/src/chain/index.ts

@@ -21,6 +21,17 @@ type RetrievalChainInput = {
  question: string;
 };
 
+const updateTemplateVariables = (template: string) => {
+ // replace template {time} with current time
+ template = template.replace("{time}", new Date().toLocaleTimeString());
+ // replace template {date} with current date
+ template = template.replace("{date}", new Date().toLocaleDateString());
+ // replace template {day} with current day
+ template = template.replace("{day}", new Date().toLocaleString('en-us', { weekday: 'long' }));
+
+ return template;
+}
+
 export function groupMessagesByConversation(messages: any[]) {
  // check if messages are in even numbers if not remove the last message
  if (messages.length % 2 !== 0) {
@@ -109,12 +120,17 @@ export const createChain = ({
  retriever,
  response_template,
 }: {
- llm: BaseLanguageModel<any> | BaseChatModel<any> ;
+ llm: BaseLanguageModel<any> | BaseChatModel<any>;
  question_llm: BaseLanguageModel<any> | BaseChatModel<any>;
  retriever: Runnable;
  question_template: string;
  response_template: string;
 }) => {
+
+ question_template = updateTemplateVariables(question_template);
+
+ response_template = updateTemplateVariables(response_template);
+
  const retrieverChain = createRetrieverChain(
  question_llm,
  retriever,

server/src/cron/index.ts

@@ -0,0 +1,70 @@
+import { PrismaClient } from "@prisma/client";
+import { getSettings } from "../utils/common";
+import { queue } from "../queue/q";
+const prisma = new PrismaClient();
+
+async function processDatasourceCron() {
+ try {
+ await prisma.$connect();
+ const setting = await getSettings(prisma);
+
+ if (!setting.refetchDatasource) {
+ return;
+ }
+
+ console.log("[CRON] Processing datasource cron");
+
+
+ const dataSources = await prisma.botSource.findMany({
+ where: {
+ bot: {
+ autoSyncDataSources: true
+ },
+ type: {
+ in: [
+ "website",
+ "crawl",
+ "sitemap",
+ ]
+ }
+ },
+ include: {
+ bot: true
+ }
+ })
+
+ for (const dataSource of dataSources) {
+
+ await prisma.botDocument.deleteMany({
+ where: {
+ botId: dataSource.botId,
+ sourceId: dataSource.id,
+ },
+ });
+ await queue.add(
+ "process",
+ [
+ {
+ ...dataSource,
+ embedding: dataSource.bot.embedding,
+ },
+ ],
+ {
+ jobId: dataSource.id,
+ removeOnComplete: true,
+ removeOnFail: true,
+ }
+ );
+ }
+
+
+ console.log("[CRON] Finished processing datasource cron");
+
+ } catch (error) {
+ console.error(error);
+ } finally {
+ await prisma.$disconnect();
+ }
+}
+
+export { processDatasourceCron }; 

server/src/handlers/api/v1/admin/type.ts

@@ -5,6 +5,7 @@ export type UpdateDialoqbaseSettingsRequest = {
  allowUserToRegister: boolean;
  usePuppeteerFetch: boolean;
  fileUploadSizeLimit: number;
+ refetchDatasource: boolean;
  };
 };
 

server/src/handlers/api/v1/bot/bot/get.handler.ts

@@ -63,7 +63,7 @@ export const getDatasourceByBotId = async (
  where: {
  botId: id,
  type: {
- notIn: ["crawl", "sitemap"],
+ notIn: ["crawl", "sitemap", "zip"],
  },
  },
  orderBy: {
@@ -77,7 +77,7 @@ export const getDatasourceByBotId = async (
  where: {
  botId: id,
  type: {
- notIn: ["crawl", "sitemap"],
+ notIn: ["crawl", "sitemap", "zip"],
  },
  },
  });
@@ -119,10 +119,9 @@ export const getAllBotsHandler = async (
  reply: FastifyReply
 ) => {
  const prisma = request.server.prisma;
-
  const bots = await prisma.bot.findMany({
  where: {
- user_id: request.user?.is_admin ? undefined : request.user?.user_id
+ user_id: request.user?.user_id
  },
  orderBy: {
  createdAt: "desc",

server/src/handlers/api/v1/bot/bot/upload.handler.ts

@@ -124,7 +124,7 @@ export const createBotFileHandler = async (
  const path = `./uploads/${fileName}`;
  await fs.promises.mkdir("./uploads", { recursive: true });
  await pump(
-  file.file,
+ file.file,
  fs.createWriteStream(path) as any
  );
  const type = fileTypeFinder(file.mimetype);

server/src/plugins/bull.ts

@@ -1,7 +1,7 @@
 import fp from "fastify-plugin";
 import { FastifyPluginAsync } from "fastify";
 import { Queue } from "bullmq";
-import { parseRedisUrl } from "../utils/redis";
+import { queue } from "../queue/q";
 declare module "fastify" {
  interface FastifyInstance {
  queue: Queue<any>;
@@ -13,16 +13,6 @@ const bullPlugin: FastifyPluginAsync = fp(async (server, options) => {
  if (!redis_url) {
  throw new Error("Redis url is not defined");
  }
- const { host, port, password } = parseRedisUrl(redis_url);
-
- const queue = new Queue("vector", {
- connection: {
- host,
- port,
- password,
- username: process?.env?.DB_REDIS_USERNAME,
- },
- });
 
  server.decorate("queue", queue);