diff --git a/KernelMemory.sln b/KernelMemory.sln index 17bd10995..79a5db85a 100644 --- a/KernelMemory.sln +++ b/KernelMemory.sln @@ -106,7 +106,6 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{CA49F1A1-C3FA-4E99-ACB3-D7FF33D47976}" ProjectSection(SolutionItems) = preProject tools\ask.sh = tools\ask.sh - tools\create-azure-webapp-publish-artifacts.sh = tools\create-azure-webapp-publish-artifacts.sh tools\README.md = tools\README.md tools\run-elasticsearch.sh = tools\run-elasticsearch.sh tools\run-mongodb-atlas.sh = tools\run-mongodb-atlas.sh diff --git a/tools/AzureBlobUpload/AzureBlobUpload.csproj b/tools/AzureBlobUpload/AzureBlobUpload.csproj new file mode 100644 index 000000000..cd0a8df28 --- /dev/null +++ b/tools/AzureBlobUpload/AzureBlobUpload.csproj @@ -0,0 +1,15 @@ + + + + Exe + net8.0 + + enable + $(NoWarn);KMEXP03;CA2000;CA1303; + + + + + + + diff --git a/tools/AzureBlobUpload/Program.cs b/tools/AzureBlobUpload/Program.cs new file mode 100644 index 000000000..b829343ab --- /dev/null +++ b/tools/AzureBlobUpload/Program.cs @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft. All rights reserved. + +/* + * Usage: dotnet run [file path] + * + * Example: + * dotnet run file4-KM-Readme.pdf + * upload.sh file4-KM-Readme.pdf + * + * For more advanced features, consider using azcopy: https://learn.microsoft.com/azure/storage/common/storage-ref-azcopy + * + * Env vars required: + * - BLOB_CONN_STRING: Azure blob connection string + * - BLOB_CONTAINER: name of the container where files are uploaded + * - BLOB_PATH: name of the virtual folder where files are uploaded + * - DOCUMENT_ID: name of the document folder containing the files + * + * You can store env vars under Properties/launchSettings.json, see the example below:{ + { + "profiles": { + "run": { + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "BLOB_CONN_STRING": "DefaultEndpointsProtocol=https;AccountName=...FOO...;AccountKey=...KEY...;EndpointSuffix=core.windows.net", + "BLOB_CONTAINER": "...NAME...", + "BLOB_PATH": "/", + "DOCUMENT_ID": "...NAME..." + }, + "commandName": "Project", + "launchBrowser": false + } + } + } + */ + +using Microsoft.KernelMemory; +using Microsoft.KernelMemory.DocumentStorage.AzureBlobs; + +if (args.Length == 0 || string.IsNullOrWhiteSpace(args[0])) +{ + Console.WriteLine("File path not specified. Provide the path of the file to upload."); + Environment.Exit(-1); +} + +var filePath = args[0]; +if (Directory.Exists(filePath)) +{ + Console.WriteLine($"The path provided is a directory, not a file: {filePath}"); + Environment.Exit(-2); +} + +if (!File.Exists(filePath)) +{ + Console.WriteLine($"File not found: {filePath}"); + Environment.Exit(-3); +} + +var fileName = filePath.Replace('\\', '/').Split('/').Last(); + +Console.WriteLine("Uploading..."); +await GetAzureBlobClient().WriteFileAsync( + index: GetIndexName(), + documentId: GetDocumentId(), + fileName: fileName, + streamContent: File.OpenRead(filePath)).ConfigureAwait(false); + +Console.WriteLine($"File uploaded: {fileName}"); + +static AzureBlobsStorage GetAzureBlobClient() +{ + var blobConnString = Environment.GetEnvironmentVariable("BLOB_CONN_STRING"); + var blobContainer = Environment.GetEnvironmentVariable("BLOB_CONTAINER"); + + if (string.IsNullOrWhiteSpace(blobConnString)) + { + Console.WriteLine("BLOB_CONN_STRING env var not defined. Provide the Azure Blobs connection string."); + Environment.Exit(-4); + } + + if (string.IsNullOrWhiteSpace(blobContainer)) + { + Console.WriteLine("BLOB_CONTAINER env var not defined. Provide the Azure Blobs container name."); + Environment.Exit(-5); + } + + return new AzureBlobsStorage(new AzureBlobsConfig + { + Auth = AzureBlobsConfig.AuthTypes.ConnectionString, + ConnectionString = blobConnString, + Container = blobContainer, + }); +} + +static string GetIndexName() +{ + var indexName = Environment.GetEnvironmentVariable("BLOB_PATH"); + if (string.IsNullOrWhiteSpace(indexName)) + { + Console.WriteLine("BLOB_PATH env var not defined. Provide the Azure Blobs container virtual folder name."); + Environment.Exit(-6); + } + + return indexName; +} + +static string GetDocumentId() +{ + var id = Environment.GetEnvironmentVariable("DOCUMENT_ID"); + if (string.IsNullOrWhiteSpace(id)) + { + Console.WriteLine("DOCUMENT_ID env var not defined. Provide the name of the document folder containing the files."); + Environment.Exit(-6); + } + + return id; +} diff --git a/tools/AzureBlobUpload/Properties/.gitignore b/tools/AzureBlobUpload/Properties/.gitignore new file mode 100644 index 000000000..031950aa8 --- /dev/null +++ b/tools/AzureBlobUpload/Properties/.gitignore @@ -0,0 +1 @@ +launchSettings.json diff --git a/tools/AzureBlobUpload/build.sh b/tools/AzureBlobUpload/build.sh new file mode 100755 index 000000000..da042dbc8 --- /dev/null +++ b/tools/AzureBlobUpload/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +cd $HERE + +dotnet build -c Release --nologo -v m diff --git a/tools/AzureBlobUpload/file4-KM-Readme.pdf b/tools/AzureBlobUpload/file4-KM-Readme.pdf new file mode 100644 index 000000000..1cba9d854 Binary files /dev/null and b/tools/AzureBlobUpload/file4-KM-Readme.pdf differ diff --git a/tools/AzureBlobUpload/file5-NASA-news.pdf b/tools/AzureBlobUpload/file5-NASA-news.pdf new file mode 100644 index 000000000..92c129799 Binary files /dev/null and b/tools/AzureBlobUpload/file5-NASA-news.pdf differ diff --git a/tools/AzureBlobUpload/upload.sh b/tools/AzureBlobUpload/upload.sh new file mode 100755 index 000000000..e5ccb5a2d --- /dev/null +++ b/tools/AzureBlobUpload/upload.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -e + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +cd $HERE + +# if file x doesn't exist, then create it +if [ ! -f "bin/Release/net8.0/AzureBlobUpload.dll" ]; then + echo "Building tool..." + dotnet build -c Release --nologo -v q +fi + +dotnet run -c Release --no-build $* diff --git a/tools/create-azure-webapp-publish-artifacts.sh b/tools/create-azure-webapp-publish-artifacts.sh deleted file mode 100755 index eab6616a1..000000000 --- a/tools/create-azure-webapp-publish-artifacts.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -e - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/" -cd "$HERE" - -cd ../dotnet/Service - -dotnet publish -c Release -o ./bin/Publish diff --git a/tools/dev/build.sh b/tools/dev/build.sh new file mode 100755 index 000000000..f21a5e9cb --- /dev/null +++ b/tools/dev/build.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -e + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && cd ../.. && pwd)" +cd $ROOT + +clean() { + dotnet clean --nologo -v q -c Release KernelMemory.sln + dotnet clean --nologo -v q -c Debug KernelMemory.sln +} + +echo "### Release build" +clean +dotnet build --nologo -v m -c Release KernelMemory.sln + +echo "### Debug build" +clean +dotnet build --nologo -v m -c Debug KernelMemory.sln diff --git a/tools/dev/changes-since-last-release.sh b/tools/dev/changes-since-last-release.sh new file mode 100755 index 000000000..def4c8abf --- /dev/null +++ b/tools/dev/changes-since-last-release.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -e + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && cd ../.. && pwd)" +cd $ROOT + +LAST_TAG="$(git tag -l|grep packages|sort -r|head -n 1)" +echo "# Last release: $LAST_TAG" + +CMD='git log --pretty=oneline ${LAST_TAG}..HEAD -- .' + +if [ -z "$(eval $CMD)" ]; then + echo "# No changes since last release" +else + echo "# Changes since last release:" + eval $CMD +fi \ No newline at end of file diff --git a/tools/dev/create-azure-webapp-publish-artifacts.sh b/tools/dev/create-azure-webapp-publish-artifacts.sh new file mode 100755 index 000000000..c3f9ec1fc --- /dev/null +++ b/tools/dev/create-azure-webapp-publish-artifacts.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && cd ../.. && pwd)" +cd $ROOT + +cd service/Service + +dotnet publish -c Release -o ./bin/Publish diff --git a/tools/dev/run-unit-tests.sh b/tools/dev/run-unit-tests.sh new file mode 100755 index 000000000..6668b09af --- /dev/null +++ b/tools/dev/run-unit-tests.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && cd ../.. && pwd)" +cd $ROOT + +dotnet test KernelMemory.sln -c Debug --nologo --filter Category=UnitTest -v q diff --git a/tools/run-km-service-from-source.sh b/tools/run-km-service-from-source.sh deleted file mode 100755 index 8f522217b..000000000 --- a/tools/run-km-service-from-source.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -# Use this script to avoid relying on KM published packages, -# and use the local source code instead, e.g. in case changes -# are being made to Abstractions, Core, etc. - -set -e - -cd "$(dirname "${BASH_SOURCE[0]:-$0}")" -cd ../service/Service - -dotnet clean - -# Build using a different solution name. The same can be done using a KernelMemoryDev.sln file. -# Note: dotnet run doesn't support [ -p "SolutionName=KernelMemoryDev" ] -dotnet build -c Debug -p "SolutionName=KernelMemoryDev" - -# Run the special build, detached from external KM nugets. -dotnet run -c Debug --no-build --no-restore diff --git a/tools/run-km-service.sh b/tools/run-km-service.sh index 6769100a5..6847944cb 100755 --- a/tools/run-km-service.sh +++ b/tools/run-km-service.sh @@ -2,9 +2,11 @@ set -e -cd "$(dirname "${BASH_SOURCE[0]:-$0}")" -cd ../service/Service +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && cd .. && pwd)" +cd $ROOT + +cd service/Service dotnet clean -dotnet build -c Debug -p "SolutionName=KernelMemory" +dotnet build -c Debug ASPNETCORE_ENVIRONMENT=Development dotnet run --no-build --no-restore diff --git a/tools/setup-km-service.sh b/tools/setup-km-service.sh index b29669a47..74816fca7 100755 --- a/tools/setup-km-service.sh +++ b/tools/setup-km-service.sh @@ -2,9 +2,11 @@ set -e -cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/" -cd ../service/Service +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && cd .. && pwd)" +cd $ROOT + +cd service/Service dotnet clean -dotnet build -c Debug -p "SolutionName=KernelMemory" +dotnet build -c Debug ASPNETCORE_ENVIRONMENT=Development dotnet run setup --no-build --no-restore