Metadata:
@@ -66,4 +89,4 @@ function Search() {
);
}
-export default Search;
\ No newline at end of file
+export default Search;
diff --git a/src/core/rag-application.ts b/src/core/rag-application.ts
index 1d91a12e..c56a1db9 100644
--- a/src/core/rag-application.ts
+++ b/src/core/rag-application.ts
@@ -216,6 +216,12 @@ export class RAGApplication {
return await this.vectorDb.similaritySearch(queryEmbedded, this.searchResultCount);
}
+ public async hybridQuery(query: string, vectorWeight = 0.1, fullTextWeight = 0.9) {
+ const cleanQuery = cleanString(query);
+ const queryEmbedded = await RAGEmbedding.getEmbedding().embedQuery(cleanQuery);
+ return await this.vectorDb.hybridSearch(query, queryEmbedded, this.searchResultCount, vectorWeight, fullTextWeight);
+ }
+
public async getContext(query: string) {
const cleanQuery = cleanString(query);
const rawContext = await this.getEmbeddings(cleanQuery);
@@ -224,7 +230,7 @@ export class RAGApplication {
public async getQueryContext(cleanQuery: string, aggregatePipelineName: string) {
- //TODO: Method override. Create a MQL query with user prompts using LLM.
+ //TODO: Method override. Create a MQL query with user prompts using LLM.
// Generate output query with the user prompt and the context.
let mqlQuery = await this.dbLookup.get(aggregatePipelineName).aggregateQuery;
mqlQuery = JSON.stringify(mqlQuery);
@@ -263,7 +269,7 @@ export class RAGApplication {
// If there is not context lookup provided
sources = [];
}
-
+
return {
sources,
result: await this.model.query(this.queryTemplate, userQuery, context, conversationId),
@@ -274,6 +280,10 @@ export class RAGApplication {
await this.vectorDb.createVectorIndex(RAGEmbedding.getEmbedding().getDimensions());
}
+ public async createTextIndex() {
+ await this.vectorDb.createTextIndex();
+ }
+
public async docsCount() : Promise
{
return await this.vectorDb.docsCount();
}
@@ -281,4 +291,4 @@ export class RAGApplication {
public getDb(key: string): Map {
return this.dbLookup.get(key);
}
-}
\ No newline at end of file
+}
diff --git a/src/interfaces/base-db.ts b/src/interfaces/base-db.ts
index ced439c4..2c96e9da 100644
--- a/src/interfaces/base-db.ts
+++ b/src/interfaces/base-db.ts
@@ -4,8 +4,10 @@ export interface BaseDb {
init({}: { dimensions: number }): Promise;
insertChunks(chunks: InsertChunkData[]): Promise;
similaritySearch(query: number[], k: number): Promise;
+ hybridSearch(textQuery: string, query: number[], k: number, vectorWeight, fullTextWeight): Promise
getVectorCount(): Promise;
createVectorIndex(numDimensions: number): Promise;
+ createTextIndex(): Promise;
docsCount(): Promise;
deleteKeys(uniqueLoaderId: string): Promise;
diff --git a/src/vectorDb/mongo-db-atlas.ts b/src/vectorDb/mongo-db-atlas.ts
index 070a7abf..b651fcb8 100644
--- a/src/vectorDb/mongo-db-atlas.ts
+++ b/src/vectorDb/mongo-db-atlas.ts
@@ -10,6 +10,7 @@ export class MongoDBAtlas implements BaseDb {
private static readonly INDEX_NAME = "vector_index";
private static readonly EMBEDDING_KEY = "embedding";
private static readonly TEXT_KEY = "text";
+ private static readonly TEXT_INDEX_NAME = "text_index";
private readonly connectionString: string;
private readonly dbName: string;
private readonly collectionName: string;
@@ -17,6 +18,7 @@ export class MongoDBAtlas implements BaseDb {
private readonly embeddingKey: string;
private readonly textKey: string;
private readonly indexName: string;
+ private readonly textIndexName?: string;
private similarityFunction: string;
private collection: any;
private numCandidates: number;
@@ -32,7 +34,7 @@ export class MongoDBAtlas implements BaseDb {
* @param numCandidates The number of candidates to consider during similarity search. Default is 100.
* @param similarityFunction The similarity function to use during similarity search.
*/
- constructor({ connectionString, dbName, collectionName, embeddingKey = MongoDBAtlas.EMBEDDING_KEY, textKey = MongoDBAtlas.TEXT_KEY, indexName = MongoDBAtlas.INDEX_NAME, numCandidates = 100, similarityFunction, minScore = 0.1 }: { connectionString: string; dbName: string; collectionName: string; embeddingKey?: string; textKey?: string; indexName?: string; numCandidates: number; similarityFunction: string; minScore: number; }
+ constructor({ connectionString, dbName, collectionName, embeddingKey = MongoDBAtlas.EMBEDDING_KEY, textKey = MongoDBAtlas.TEXT_KEY, indexName = MongoDBAtlas.INDEX_NAME, numCandidates = 100, similarityFunction, minScore = 0.1, textIndexName = MongoDBAtlas.TEXT_INDEX_NAME }: { connectionString: string; dbName: string; collectionName: string; embeddingKey?: string; textKey?: string; indexName?: string; numCandidates: number; similarityFunction: string; minScore: number; textIndexName?: string;}
) {
this.connectionString = connectionString;
this.dbName = dbName;
@@ -41,6 +43,7 @@ export class MongoDBAtlas implements BaseDb {
this.embeddingKey = embeddingKey;
this.textKey = textKey;
this.indexName = indexName;
+ this.textIndexName = textIndexName;
this.similarityFunction = similarityFunction;
this.numCandidates = numCandidates;
this.minScore = minScore;
@@ -85,12 +88,144 @@ export class MongoDBAtlas implements BaseDb {
const query_object = [await this.getVectorSearchQuery(query, k), { "$project": { "_id": 0, "score": { "$meta": "vectorSearchScore" }, "text": 1, "metadata": 1 } }, {"$match": {"score": {"$gt": this.minScore}}}];
const results = await this.collection.aggregate(query_object).toArray();
+ const result = await results.map((result) => {
+ const pageContent = (result)[this.textKey];
+ delete (result.metadata).pageContent;
+
+ return {
+ score: result.score,
+ pageContent,
+ metadata: result.metadata,
+ }
+ });
+
+ return result
+ }
+
+ /**
+ * Performs a hybrid search using reciprocal-rank-fusion for the given textQuery and vectorQuery.
+ * @see https://www.mongodb.com/docs/atlas/atlas-vector-search/tutorials/reciprocal-rank-fusion/
+ *
+ * @param textQuery The query text for hybrid search.
+ * @param query The query vector.
+ * @param k The number of results to return.
+ * @returns A Promise that resolves to an array of ExtractChunkData objects representing the search results.
+ */
+ async hybridSearch(textQuery: string, query: number[], k: number, vectorWeight = 0.1, fullTextWeight = 0.9): Promise {
+ const query_object = [
+ await this.getVectorSearchQuery(query, k),
+ {
+ "$group": {
+ "_id": null,
+ "docs": {"$push": "$$ROOT"}
+ }
+ },
+ {
+ "$unwind": {
+ "path": "$docs",
+ "includeArrayIndex": "rank"
+ }
+ },
+ {
+ "$addFields": {
+ "_id": "$docs._id",
+ "vs_score": {
+ "$multiply": [
+ vectorWeight, {
+ "$divide": [
+ 1.0, {
+ "$add": ["$rank", 60]
+ }
+ ]
+ }
+ ]
+ }
+ }
+ },
+ {
+ "$unionWith": {
+ "coll": this.collectionName,
+ "pipeline": [
+ {
+ "$search": {
+ "index": this.textIndexName,
+ "text": {
+ "query": textQuery,
+ "path": this.textKey
+ }
+ }
+ }, {
+ "$limit": k
+ }, {
+ "$group": {
+ "_id": null,
+ "docs": {"$push": "$$ROOT"}
+ }
+ }, {
+ "$unwind": {
+ "path": "$docs",
+ "includeArrayIndex": "rank"
+ }
+ }, {
+ "$addFields": {
+ "_id": "$docs._id",
+ "fts_score": {
+ "$multiply": [
+ fullTextWeight, {
+ "$divide": [
+ 1.0, {
+ "$add": ["$rank", 60]
+ }
+ ]
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+ },
+ {
+ "$group": {
+ "_id": "$_id",
+ docs: {
+ $first: "$docs",
+ },
+ "vs_score": {"$max": "$vs_score"},
+ "fts_score": {"$max": "$fts_score"}
+ }
+ },
+ {
+ "$addFields": {
+ "docs.vs_score": {"$ifNull": ["$vs_score", 0]},
+ "docs.fts_score": {"$ifNull": ["$fts_score", 0]}
+ }
+ },
+ {
+ "$addFields": {
+ "docs.score": {"$add": ["$docs.vs_score", "$docs.fts_score"]}
+ }
+ },
+ {
+ $replaceRoot: {newRoot: "$docs"}
+ },
+ {
+ "$sort": {"score": -1}
+ },
+ {
+ "$limit": k
+ }
+ ];
+ const results = await this.collection.aggregate(query_object).toArray();
+
return results.map((result) => {
const pageContent = (result)[this.textKey];
delete (result.metadata).pageContent;
return {
score: result.score,
+ vs_score: result.vs_score,
+ fts_score: result.fts_score,
pageContent,
metadata: result.metadata,
}
@@ -153,7 +288,7 @@ export class MongoDBAtlas implements BaseDb {
*/
async createVectorIndex(numDimensions: number, similarityFunction?: string): Promise {
try {
-
+
this.similarityFunction = similarityFunction ?? "cosine";
const index = {
name: this.indexName,
@@ -176,6 +311,33 @@ export class MongoDBAtlas implements BaseDb {
}
}
+ /**
+ * Creates a text search index in the collection.
+ * @returns A Promise that resolves when the text search index has been created.
+ */
+ async createTextIndex(): Promise {
+ try {
+ const index = {
+ name: this.textIndexName,
+ type: "search",
+ definition: {
+ "mappings": {
+ "dynamic": false,
+ "fields": {
+ "text": [{
+ "type": "string"
+ }]
+ }
+ }
+ }
+ }
+ await this.collection.createSearchIndex(index);
+ console.log("\n-- Text search index created --")
+ } catch (e) {
+ return Promise.reject(e.codeName);
+ }
+ }
+
async docsCount(): Promise {
try {
const docsCount = await this.client.db(this.dbName).collection(this.collectionName).estimatedDocumentCount();
@@ -185,4 +347,4 @@ export class MongoDBAtlas implements BaseDb {
return 0;
}
}
-}
\ No newline at end of file
+}
diff --git a/src/yaml_parser/src/LoadYaml.ts b/src/yaml_parser/src/LoadYaml.ts
index 59e83260..1407a8d7 100644
--- a/src/yaml_parser/src/LoadYaml.ts
+++ b/src/yaml_parser/src/LoadYaml.ts
@@ -34,8 +34,8 @@ function getDataFromYamlFile() {
export function getSystemPrompt() {
const parsedData = getDataFromYamlFile();
- const systemPrompt = readFileSync(parsedData.systemPromptPath, 'utf8');
- return systemPrompt;
+ const systemPrompt = readFileSync(parsedData.systemPromptPath, 'utf8');
+ return systemPrompt;
}
export function getDatabaseConfig() {
@@ -74,9 +74,9 @@ try {
query: query,
jsonSchema: jsonSchema
});
-
+
} catch (error) {
- console.log('Error reading aggregate operator query file:', error);
+ console.log('Error reading aggregate operator query file:', error);
}
}
return aggregateOperatorConfigs;
@@ -110,19 +110,21 @@ export function getConditionOpConfigs(){
export function getVBDConfigInfo() {
const parsedData = getDataFromYamlFile();
const {
- vector_store: { connectionString, dbName, collectionName, vectorSearchIndexName, minScore, numCandidates },
+ vector_store: { connectionString, dbName, collectionName, vectorSearchIndexName, textSearchIndexName, minScore, numCandidates },
} = parsedData;
assert(typeof connectionString === 'string', 'connectionString is required');
assert(typeof dbName === 'string', 'dbName is required');
assert(typeof collectionName === 'string', 'collectionName is required');
assert(typeof vectorSearchIndexName === 'string', 'vectorSearchIndexName is required');
+ assert(typeof textSearchIndexName === 'string', 'textSearchIndexName is required');
return {
connectionString,
dbName,
collectionName,
vectorSearchIndexName,
+ textSearchIndexName,
minScore,
numCandidates
};
@@ -194,7 +196,7 @@ export function getEmbeddingModel() {
case 'Bedrock':
return new BedrockEmbedding({ modelName: parsedData.embedding.model_name, dimension: parsedData.embedding.dimension});
case 'Fireworks':
- return new FireworksEmbedding({ modelName: parsedData.embedding.model_name, dimension: parsedData.embedding.dimension});
+ return new FireworksEmbedding({ modelName: parsedData.embedding.model_name, dimension: parsedData.embedding.dimension});
case 'Nomic-v1':
return new NomicEmbeddingsv1();
case 'Nomic-v1.5':
@@ -367,4 +369,4 @@ export function getStreamOptions() {
return {
stream: parsedData.stream_options.stream ?? false,
};
-}
\ No newline at end of file
+}