Add support for SQL Server Vector Search (#722)

## Motivation and Context (Why the change? What's the scenario?)

Azure SQL Database now provides native Vector Support, currently in EAP
(https://devblogs.microsoft.com/azure-sql/announcing-eap-native-vector-support-in-azure-sql-database).

This PR extends SQL Server Memory DB with a flag that allows to use this
new feature.

> [!IMPORTANT]  
> Remember that, at this time, Vector Support is available only on Azure
SQL Database. On the other hand, the current SQL Server Memory DB
requires a COLUMNSTORE INDEX that, on Azure, is available only on vCore
databases and Standard databases in S3 and above pricing tiers
(https://azure.microsoft.com/en-us/blog/columnstore-support-in-standard-tier-azure-sql-databases).

---------

Co-authored-by: Devis Lucato <[email protected]>

extensions/SQLServer/SQLServer/DependencyInjection.cs

@@ -14,10 +14,10 @@ namespace Microsoft.KernelMemory;
 public static partial class KernelMemoryBuilderExtensions
 {
  /// <summary>
- /// Kernel Memory Builder extension method to add SqlServer memory connector.
+ /// Kernel Memory Builder extension method to add SQL Server memory connector.
  /// </summary>
  /// <param name="builder">KM builder instance</param>
- /// <param name="config">SqlServer configuration</param>
+ /// <param name="config">SQL Server configuration</param>
  public static IKernelMemoryBuilder WithSqlServerMemoryDb(
  this IKernelMemoryBuilder builder,
  SqlServerConfig config)
@@ -27,15 +27,17 @@ public static IKernelMemoryBuilder WithSqlServerMemoryDb(
  }
 
  /// <summary>
- /// Kernel Memory Builder extension method to add SqlServer memory connector.
+ /// Kernel Memory Builder extension method to add SQL Server memory connector.
  /// </summary>
  /// <param name="builder">KM builder instance</param>
- /// <param name="connString">SqlServer connection string</param>
+ /// <param name="connString">SQL Server connection string</param>
+ /// <param name="useNativeVectorSearch">Whether to use native vector search or not</param>
  public static IKernelMemoryBuilder WithSqlServerMemoryDb(
  this IKernelMemoryBuilder builder,
- string connString)
+ string connString,
+ bool useNativeVectorSearch = false)
  {
- builder.Services.AddSqlServerAsMemoryDb(connString);
+ builder.Services.AddSqlServerAsMemoryDb(connString, useNativeVectorSearch);
  return builder;
  }
 }
@@ -46,10 +48,10 @@ public static IKernelMemoryBuilder WithSqlServerMemoryDb(
 public static partial class DependencyInjection
 {
  /// <summary>
- /// Inject SqlServer as the default implementation of IMemoryDb
+ /// Inject SQL Server as the default implementation of IMemoryDb
  /// </summary>
  /// <param name="services">Service collection</param>
- /// <param name="config">Postgres configuration</param>
+ /// <param name="config">SQL Server configuration</param>
  public static IServiceCollection AddSqlServerAsMemoryDb(
  this IServiceCollection services,
  SqlServerConfig config)
@@ -60,15 +62,17 @@ public static IServiceCollection AddSqlServerAsMemoryDb(
  }
 
  /// <summary>
- /// Inject SqlServer as the default implementation of IMemoryDb
+ /// Inject SQL Server as the default implementation of IMemoryDb
  /// </summary>
  /// <param name="services">Service collection</param>
  /// <param name="connString">SQL Server connection string</param>
+ /// <param name="useNativeVectorSearch">Whether to use native vector search or not</param>
  public static IServiceCollection AddSqlServerAsMemoryDb(
  this IServiceCollection services,
- string connString)
+ string connString,
+ bool useNativeVectorSearch = false)
  {
- var config = new SqlServerConfig { ConnectionString = connString };
+ var config = new SqlServerConfig { ConnectionString = connString, UseNativeVectorSearch = useNativeVectorSearch };
  return services.AddSqlServerAsMemoryDb(config);
  }
 }

extensions/SQLServer/SQLServer/QueryProviders/ISqlServerQueryProvider.cs

@@ -0,0 +1,59 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Data.SqlClient;
+
+namespace Microsoft.KernelMemory.MemoryDb.SQLServer.QueryProviders;
+
+public interface ISqlServerQueryProvider
+{
+ /// <summary>
+ /// Return SQL used to create a new index
+ /// </summary>
+ public string PrepareCreateIndexQuery(
+ int sqlServerVersion,
+ string index,
+ int vectorSize);
+
+ /// <summary>
+ /// Return SQL used to get a list of indexes
+ /// </summary>
+ public string PrepareGetIndexesQuery();
+
+ /// <summary>
+ /// Return SQL used to delete an index
+ /// </summary>
+ public string PrepareDeleteIndexQuery(string index);
+
+ /// <summary>
+ /// Return SQL used to delete a memory record
+ /// </summary>
+ public string PrepareDeleteRecordQuery(string index);
+
+ /// <summary>
+ /// Return SQL used to get a list of memory records
+ /// </summary>
+ public string PrepareGetRecordsListQuery(
+ string index,
+ ICollection<MemoryFilter>? filters,
+ bool withEmbedding,
+ SqlParameterCollection parameters);
+
+ /// <summary>
+ /// Return SQL used to get a list of similar memory records
+ /// </summary>
+ public string PrepareGetSimilarRecordsListQuery(
+ string index,
+ ICollection<MemoryFilter>? filters,
+ bool withEmbedding,
+ SqlParameterCollection parameters);
+
+ /// <summary>
+ /// Return SQL used to upsert a batch of memory records
+ /// </summary>
+ public string PrepareUpsertRecordsBatchQuery(string index);
+
+ /// <summary>
+ /// Return SQL used to create all supporting tables
+ /// </summary>
+ public string PrepareCreateAllSupportingTablesQuery();
+}

extensions/SQLServer/SQLServer/QueryProviders/Utils.cs

@@ -0,0 +1,85 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Globalization;
+using System.Text;
+using Microsoft.Data.SqlClient;
+
+namespace Microsoft.KernelMemory.MemoryDb.SQLServer.QueryProviders;
+
+internal static class Utils
+{
+ /// <summary>
+ /// Gets the full table name with schema.
+ /// </summary>
+ /// <param name="config">Server settings</param>
+ /// <param name="tableName">The table name.</param>
+ internal static string GetFullTableName(SqlServerConfig config, string tableName)
+ {
+ return $"[{config.Schema}].[{tableName}]";
+ }
+
+ /// <summary>
+ /// Generates the filters as SQL commands and sets the SQL parameters
+ /// </summary>
+ /// <param name="config">Server settings</param>
+ /// <param name="index">The index name.</param>
+ /// <param name="parameters">The SQL parameters to populate.</param>
+ /// <param name="filters">The filters to apply</param>
+ internal static string GenerateFilters(
+ SqlServerConfig config,
+ string index,
+ SqlParameterCollection parameters,
+ ICollection<MemoryFilter>? filters)
+ {
+ var filterBuilder = new StringBuilder();
+
+ if (filters is null || filters.Count <= 0 || filters.All(f => f.Count <= 0))
+ {
+ return string.Empty;
+ }
+
+ filterBuilder.Append("AND ( ");
+
+ for (int i = 0; i < filters.Count; i++)
+ {
+ var filter = filters.ElementAt(i);
+
+ if (i > 0)
+ {
+ filterBuilder.Append(" OR ");
+ }
+
+ for (int j = 0; j < filter.Pairs.Count(); j++)
+ {
+ var value = filter.Pairs.ElementAt(j);
+
+ if (j > 0)
+ {
+ filterBuilder.Append(" AND ");
+ }
+
+ filterBuilder.Append(" ( ");
+
+ filterBuilder.Append(CultureInfo.CurrentCulture, $@"EXISTS (
+ SELECT
+ 1
+ FROM {GetFullTableName(config, $"{config.TagsTableName}_{index}")} AS [tags]
+ WHERE
+ [tags].[memory_id] = {GetFullTableName(config, config.MemoryTableName)}.[id]
+ AND [name] = @filter_{i}_{j}_name
+ AND [value] = @filter_{i}_{j}_value
+ )
+ ");
+
+ filterBuilder.Append(" ) ");
+
+ parameters.AddWithValue($"@filter_{i}_{j}_name", value.Key);
+ parameters.AddWithValue($"@filter_{i}_{j}_value", value.Value);
+ }
+ }
+
+ filterBuilder.Append(" )");
+
+ return filterBuilder.ToString();
+ }
+}