Clean up tools

KernelMemory.sln

@@ -106,7 +106,6 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{CA49F1A1-C3FA-4E99-ACB3-D7FF33D47976}"
  ProjectSection(SolutionItems) = preProject
  tools\ask.sh = tools\ask.sh
- tools\create-azure-webapp-publish-artifacts.sh = tools\create-azure-webapp-publish-artifacts.sh
  tools\README.md = tools\README.md
  tools\run-elasticsearch.sh = tools\run-elasticsearch.sh
  tools\run-mongodb-atlas.sh = tools\run-mongodb-atlas.sh

tools/AzureBlobUpload/AzureBlobUpload.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <OutputType>Exe</OutputType>
+ <TargetFramework>net8.0</TargetFramework>
+ <RootNamespace />
+ <ImplicitUsings>enable</ImplicitUsings>
+ <NoWarn>$(NoWarn);KMEXP03;CA2000;CA1303;</NoWarn>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <ProjectReference Include="..\..\extensions\AzureBlobs\AzureBlobs.csproj" />
+ </ItemGroup>
+
+</Project>

tools/AzureBlobUpload/Program.cs

@@ -0,0 +1,116 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+/*
+ * Usage: dotnet run [file path]
+ *
+ * Example:
+ * dotnet run file4-KM-Readme.pdf
+ * upload.sh file4-KM-Readme.pdf
+ *
+ * For more advanced features, consider using azcopy: https://learn.microsoft.com/azure/storage/common/storage-ref-azcopy
+ *
+ * Env vars required:
+ * - BLOB_CONN_STRING: Azure blob connection string
+ * - BLOB_CONTAINER: name of the container where files are uploaded
+ * - BLOB_PATH: name of the virtual folder where files are uploaded
+ * - DOCUMENT_ID: name of the document folder containing the files
+ *
+ * You can store env vars under Properties/launchSettings.json, see the example below:{
+ {
+ "profiles": {
+ "run": {
+ "environmentVariables": {
+ "ASPNETCORE_ENVIRONMENT": "Development",
+ "BLOB_CONN_STRING": "DefaultEndpointsProtocol=https;AccountName=...FOO...;AccountKey=...KEY...;EndpointSuffix=core.windows.net",
+ "BLOB_CONTAINER": "...NAME...",
+ "BLOB_PATH": "/",
+ "DOCUMENT_ID": "...NAME..."
+ },
+ "commandName": "Project",
+ "launchBrowser": false
+ }
+ }
+ }
+ */
+
+using Microsoft.KernelMemory;
+using Microsoft.KernelMemory.DocumentStorage.AzureBlobs;
+
+if (args.Length == 0 || string.IsNullOrWhiteSpace(args[0]))
+{
+ Console.WriteLine("File path not specified. Provide the path of the file to upload.");
+ Environment.Exit(-1);
+}
+
+var filePath = args[0];
+if (Directory.Exists(filePath))
+{
+ Console.WriteLine($"The path provided is a directory, not a file: {filePath}");
+ Environment.Exit(-2);
+}
+
+if (!File.Exists(filePath))
+{
+ Console.WriteLine($"File not found: {filePath}");
+ Environment.Exit(-3);
+}
+
+var fileName = filePath.Replace('\\', '/').Split('/').Last();
+
+Console.WriteLine("Uploading...");
+await GetAzureBlobClient().WriteFileAsync(
+ index: GetIndexName(),
+ documentId: GetDocumentId(),
+ fileName: fileName,
+ streamContent: File.OpenRead(filePath)).ConfigureAwait(false);
+
+Console.WriteLine($"File uploaded: {fileName}");
+
+static AzureBlobsStorage GetAzureBlobClient()
+{
+ var blobConnString = Environment.GetEnvironmentVariable("BLOB_CONN_STRING");
+ var blobContainer = Environment.GetEnvironmentVariable("BLOB_CONTAINER");
+
+ if (string.IsNullOrWhiteSpace(blobConnString))
+ {
+ Console.WriteLine("BLOB_CONN_STRING env var not defined. Provide the Azure Blobs connection string.");
+ Environment.Exit(-4);
+ }
+
+ if (string.IsNullOrWhiteSpace(blobContainer))
+ {
+ Console.WriteLine("BLOB_CONTAINER env var not defined. Provide the Azure Blobs container name.");
+ Environment.Exit(-5);
+ }
+
+ return new AzureBlobsStorage(new AzureBlobsConfig
+ {
+ Auth = AzureBlobsConfig.AuthTypes.ConnectionString,
+ ConnectionString = blobConnString,
+ Container = blobContainer,
+ });
+}
+
+static string GetIndexName()
+{
+ var indexName = Environment.GetEnvironmentVariable("BLOB_PATH");
+ if (string.IsNullOrWhiteSpace(indexName))
+ {
+ Console.WriteLine("BLOB_PATH env var not defined. Provide the Azure Blobs container virtual folder name.");
+ Environment.Exit(-6);
+ }
+
+ return indexName;
+}
+
+static string GetDocumentId()
+{
+ var id = Environment.GetEnvironmentVariable("DOCUMENT_ID");
+ if (string.IsNullOrWhiteSpace(id))
+ {
+ Console.WriteLine("DOCUMENT_ID env var not defined. Provide the name of the document folder containing the files.");
+ Environment.Exit(-6);
+ }
+
+ return id;
+}

tools/AzureBlobUpload/Properties/.gitignore

@@ -0,0 +1 @@
+launchSettings.json

tools/AzureBlobUpload/build.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
+cd $HERE
+
+dotnet build -c Release --nologo -v m

tools/AzureBlobUpload/upload.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
+cd $HERE
+
+# if file x doesn't exist, then create it
+if [ ! -f "bin/Release/net8.0/AzureBlobUpload.dll" ]; then
+ echo "Building tool..."
+ dotnet build -c Release --nologo -v q
+fi
+
+dotnet run -c Release --no-build $*

tools/dev/build.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && cd ../.. && pwd)"
+cd $ROOT
+
+clean() {
+ dotnet clean --nologo -v q -c Release KernelMemory.sln
+ dotnet clean --nologo -v q -c Debug KernelMemory.sln
+}
+
+echo "### Release build"
+clean
+dotnet build --nologo -v m -c Release KernelMemory.sln
+
+echo "### Debug build"
+clean
+dotnet build --nologo -v m -c Debug KernelMemory.sln

tools/dev/changes-since-last-release.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && cd ../.. && pwd)"
+cd $ROOT
+
+LAST_TAG="$(git tag -l|grep packages|sort -r|head -n 1)"
+echo "# Last release: $LAST_TAG"
+
+CMD='git log --pretty=oneline ${LAST_TAG}..HEAD -- .'
+
+if [ -z "$(eval $CMD)" ]; then
+ echo "# No changes since last release"
+else
+ echo "# Changes since last release:"
+ eval $CMD
+fi

tools/dev/create-azure-webapp-publish-artifacts.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && cd ../.. && pwd)"
+cd $ROOT
+
+cd service/Service
+
+dotnet publish -c Release -o ./bin/Publish

tools/dev/run-unit-tests.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && cd ../.. && pwd)"
+cd $ROOT
+
+dotnet test KernelMemory.sln -c Debug --nologo --filter Category=UnitTest -v q

tools/run-km-service.sh

@@ -2,9 +2,11 @@
 
 set -e
 
-cd "$(dirname "${BASH_SOURCE[0]:-$0}")"
-cd ../service/Service
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && cd .. && pwd)"
+cd $ROOT
+
+cd service/Service
 
 dotnet clean
-dotnet build -c Debug -p "SolutionName=KernelMemory"
+dotnet build -c Debug
 ASPNETCORE_ENVIRONMENT=Development dotnet run --no-build --no-restore

tools/setup-km-service.sh

@@ -2,9 +2,11 @@
 
 set -e
 
-cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/"
-cd ../service/Service
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && cd .. && pwd)"
+cd $ROOT
+
+cd service/Service
 
 dotnet clean
-dotnet build -c Debug -p "SolutionName=KernelMemory"
+dotnet build -c Debug
 ASPNETCORE_ENVIRONMENT=Development dotnet run setup --no-build --no-restore