Updating batch evals

dotnet · May 15, 2024 · cf94ac2 · cf94ac2
1 parent f9c1b2d
commit cf94ac2
Show file tree

Hide file tree

Showing 9 changed files with 38 additions and 23 deletions.
diff --git a/.github/workflows/pr-validation-ai.yml b/.github/workflows/pr-validation-ai.yml
@@ -19,10 +19,10 @@ on:
 
 env:
  ESHOP_AI_MODEL: "phi3"
- ESHOP_TESTS_AI_COMPLETION_TYPE: "openai"
+ ESHOP_TESTS_AI_COMPLETION_TYPE: "azureopenai"
 
 jobs: 
- test:
+ eval:
  runs-on: ubuntu-latest 
  steps:
  - uses: actions/checkout@v4
@@ -47,4 +47,4 @@ jobs:
  ESHOP_AI_ENDPOINT: "http://localhost:11434"
  ESHOP_AI_MODEL: ${{ env.ESHOP_AI_MODEL }}
  ESHOP_AI_KEY: "api"
- run: dotnet test eShop.LLMEvals.slnf
+ run: dotnet test eShop.LLMEvals.slnf
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -41,7 +41,7 @@
  <PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="$(AspnetVersion)" />
  <PackageVersion Include="Microsoft.Extensions.Identity.Stores" Version="$(AspnetVersion)" />
  <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="$(MicrosoftExtensionsVersion)" />
- <PackageVersion Include="Microsoft.SemanticKernel.Core" Version="1.10.0" />
+
  <!-- Version together with EF -->
  <PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.2" />
  <PackageVersion Include="Microsoft.EntityFrameworkCore.Tools" Version="$(EfVersion)" />
@@ -59,7 +59,8 @@
  <!-- Xabaril packages -->
  <PackageVersion Include="AspNetCore.HealthChecks.Uris" Version="8.0.1" />
  <!-- AI -->
- <PackageVersion Include="Microsoft.SemanticKernel" Version="1.10.0" />
+ <PackageVersion Include="Microsoft.SemanticKernel" Version="1.11.1" />
+ <PackageVersion Include="Microsoft.SemanticKernel.Core" Version="1.11.1" />
  <PackageVersion Include="Microsoft.SemanticKernel.Connectors.Onnx" Version="1.10.0-alpha" />
  <!-- Open Telemetry -->
  <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.8.1" />

diff --git a/src/Microsoft.SKEval/BatchEval.cs b/src/Microsoft.SKEval/BatchEval.cs
@@ -37,9 +37,9 @@ public BatchEval<T> WithOutputProcessor(IOutputProcessor outputProcessor)
  return this;
  }
 
- public BatchEval<T> WithCsvOutputProcessor(string filename)
+ public BatchEval<T> WithCsvOutputProcessor()
  {
- return WithOutputProcessor(new CsvOutputProcessor(filename));
+ return WithOutputProcessor(new CsvOutputProcessor());
  }
 
  public BatchEval<T> AddEvaluator(IEvaluator<int> evaluator)

diff --git a/src/Microsoft.SKEval/CsvOutputProcessor.cs b/src/Microsoft.SKEval/CsvOutputProcessor.cs
@@ -2,10 +2,8 @@
 
 namespace Microsoft.SKEval;
 
-public class CsvOutputProcessor(string filePath) : IOutputProcessor
+public class CsvOutputProcessor() : IOutputProcessor
 {
- public string FilePath { get; set; } = filePath;
-
  public Task Init()
  {
  return Task.CompletedTask;

diff --git a/src/Microsoft.SKEval/Evals/PromptScoreEval.cs b/src/Microsoft.SKEval/Evals/PromptScoreEval.cs
@@ -6,6 +6,7 @@
 using System.Diagnostics.Metrics;
 using System.Text;
 using Microsoft.SemanticKernel.Connectors;
+using Microsoft.Extensions.Logging;
 
 namespace Microsoft.SKEval;
 
@@ -15,6 +16,8 @@ public class PromptScoreEval : IEvaluator<int>
 
  private readonly KernelFunction function;
 
+ public ILogger Logger { get; set; } = default!;
+
  public string Id { get; }
 
  public PromptScoreEval(string id, Kernel kernel, KernelFunction function)
@@ -44,11 +47,17 @@ public async Task<int> Eval(ModelOutput modelOutput)
 
  var evalResult = await function.InvokeAsync(kernel, promptArgs);
 
- if (int.TryParse(evalResult.ToString(), out var evalInt))
+ Logger?.LogDebug($"Model Eval Answer: {evalResult}");
+
+ var evalInt = 0;
+
+ try
  {
- return evalInt;
+ evalInt = int.Parse(evalResult.ToString().Trim());
+ } catch (Exception ex) {
+ Logger?.LogError(ex.Message);
  }
 
- return 0;
+ return evalInt;
  }
 }
diff --git a/src/Microsoft.SKEval/JsonOutputProcessor.cs b/src/Microsoft.SKEval/JsonOutputProcessor.cs
@@ -5,11 +5,8 @@ namespace Microsoft.SKEval
 {
  public class JsonOutputProcessor : IOutputProcessor
  {
- public string FilePath { get; set; }
-
- public JsonOutputProcessor(string filePath)
+ public JsonOutputProcessor()
  {
- FilePath = filePath;
  }
 
  public Task Init()

diff --git a/src/Microsoft.SKEval/TsvOutputProcessor.cs b/src/Microsoft.SKEval/TsvOutputProcessor.cs
@@ -6,11 +6,8 @@ namespace Microsoft.SKEval
 {
  public class TsvOutputProcessor : IOutputProcessor
  {
- public string FilePath { get; set; }
-
- public TsvOutputProcessor(string filePath)
+ public TsvOutputProcessor()
  {
- FilePath = filePath;
  }
 
  public Task Init()

diff --git a/tests/AI.BatchEvals/AI.BatchEvals.csproj b/tests/AI.BatchEvals/AI.BatchEvals.csproj
@@ -15,6 +15,7 @@
  </ItemGroup>
 
  <ItemGroup>
+ <ProjectReference Include="..\..\src\Microsoft.SKEval\Microsoft.SKEval.csproj" />
  <ProjectReference Include="..\src\Microsoft.SKEval\Microsoft.SKEval.csproj" />
  <ProjectReference Include="..\src\WebApp\WebApp.csproj" />
 

diff --git a/tests/AI.BatchEvals/Program.cs b/tests/AI.BatchEvals/Program.cs
@@ -53,8 +53,20 @@
  .AddEvaluator(new CoherenceEval(kernel))
  .AddEvaluator(eShop.WebApp.AIBatchEvals.RelevanceEval.GetInstance(kernel))
  .AddEvaluator(new GroundednessEval(kernel));
- 
- batchEval.WithCsvOutputProcessor("results_adversary.csv");
+
+ switch (format.ToLowerInvariant())
+ {
+ case "csv":
+ batchEval.WithOutputProcessor(new CsvOutputProcessor());
+ break;
+ case "tsv":
+ batchEval.WithOutputProcessor(new TsvOutputProcessor());
+ break;
+ default:
+ case "json":
+ batchEval.WithOutputProcessor(new JsonOutputProcessor());
+ break;
+ }
 
  var results = await batchEval
  .WithInputProcessor(chatInputProcessor)