feat: Added ImageToText abstractions and HuggingFace implementation.

examples/LangChain.Samples.HuggingFace/Program.cs

@@ -1,10 +1,29 @@
-using LangChain.Providers.HuggingFace;
+using LangChain.Providers;
+using LangChain.Providers.HuggingFace;
 using LangChain.Providers.HuggingFace.Predefined;
 
 using var client = new HttpClient();
 var provider = new HuggingFaceProvider(apiKey: string.Empty, client);
 var gpt2Model = new Gpt2Model(provider);
 
-var response = await gpt2Model.GenerateAsync("What would be a good company name be for name a company that makes colorful socks?");
+var gp2ModelResponse = await gpt2Model.GenerateAsync("What would be a good company name be for name a company that makes colorful socks?");
 
-Console.WriteLine(response);
+Console.WriteLine("### GP2 Response");
+Console.WriteLine(gp2ModelResponse);
+
+const string imageToTextModel = "Salesforce/blip-image-captioning-base";
+var model = new HuggingFaceImageToTextModel(provider, imageToTextModel);
+
+var path = Path.Combine(Path.GetTempPath(), "solar_system.png");
+var imageData = await File.ReadAllBytesAsync(path);
+var binaryData = new BinaryData(imageData, "image/jpg");
+
+var imageToTextResponse = await model.GenerateTextFromImageAsync(new ImageToTextRequest
+{
+ Image = binaryData
+});
+
+Console.WriteLine("\n\n### ImageToText Response");
+Console.WriteLine(imageToTextResponse.Text);
+
+Console.ReadLine();

src/Core/src/Chains/Chain.cs

@@ -4,6 +4,7 @@
 using LangChain.Chains.StackableChains.Agents.Crew;
 using LangChain.Chains.StackableChains.Files;
 using LangChain.Chains.StackableChains.ImageGeneration;
+using LangChain.Chains.StackableChains.ImageToTextGeneration;
 using LangChain.Chains.StackableChains.ReAct;
 using LangChain.Indexes;
 using LangChain.Memory;
@@ -298,4 +299,19 @@ public static ExtractCodeChain ExtractCode(
  {
  return new ExtractCodeChain(inputKey, outputKey);
  }
+
+ /// <summary>
+ /// 
+ /// </summary>
+ /// <param name="model"></param>
+ /// <param name="image"></param>
+ /// <param name="outputKey"></param>
+ /// <returns></returns>
+ public static ImageToTextGenerationChain GenerateImageToText(
+ IImageToTextModel model,
+ BinaryData image,
+ string outputKey = "text")
+ {
+ return new ImageToTextGenerationChain(model, image, outputKey);
+ }
 }

src/Core/src/Chains/StackableChains/ImageToTextGeneration/ImageToTextGenerationChain.cs

@@ -0,0 +1,40 @@
+using LangChain.Abstractions.Schema;
+using LangChain.Chains.HelperChains;
+using LangChain.Providers;
+
+namespace LangChain.Chains.StackableChains.ImageToTextGeneration;
+
+/// <summary>
+/// 
+/// </summary>
+public class ImageToTextGenerationChain : BaseStackableChain
+{
+ private readonly IImageToTextModel _model;
+ private readonly BinaryData _image;
+
+ /// <summary>
+ /// 
+ /// </summary>
+ /// <param name="model"></param>
+ /// <param name="image"></param>
+ /// <param name="outputKey"></param>
+ public ImageToTextGenerationChain(
+ IImageToTextModel model,
+ BinaryData image,
+ string outputKey = "text")
+ {
+ _model = model;
+ _image = image;
+ OutputKeys = new[] { outputKey };
+ }
+
+ /// <inheritdoc />
+ protected override async Task<IChainValues> InternalCall(IChainValues values)
+ {
+ values = values ?? throw new ArgumentNullException(nameof(values));
+
+ var text = await _model.GenerateTextFromImageAsync(new ImageToTextRequest { Image = _image }).ConfigureAwait(false);
+ values.Value[OutputKeys[0]] = text;
+ return values;
+ }
+}

src/Directory.Packages.props

@@ -54,6 +54,7 @@
  <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
  </PackageVersion>
  <PackageVersion Include="StackExchange.Redis" Version="2.7.20" />
+ <PackageVersion Include="System.Memory.Data" Version="8.0.0" />
  <PackageVersion Include="System.Net.Http" Version="4.3.4" />
  <PackageVersion Include="System.Text.Json" Version="8.0.0" />
  <PackageVersion Include="System.ValueTuple" Version="4.5.0" />

src/Providers/Abstractions/src/Common/Provider.cs

@@ -21,4 +21,7 @@ public abstract class Provider(string id) : Model(id), IProvider
 
  /// <inheritdoc />
  public TextToSpeechSettings? TextToSpeechSettings { get; init; }
+
+ /// <inheritdoc />
+ public ImageToTextSettings? ImageToTextSettings { get; init; }
 }

src/Providers/Abstractions/src/ImageToText/IImageToTextModel.cs

@@ -0,0 +1,19 @@
+namespace LangChain.Providers;
+
+/// <summary>
+/// Defines a large language model that can be used for image to text generation.
+/// </summary>
+public interface IImageToTextModel : IModel<ImageToTextSettings>
+{
+ /// <summary>
+ /// Run the LLM on the given image.
+ /// </summary>
+ /// <param name="request"></param>
+ /// <param name="settings"></param>
+ /// <param name="cancellationToken"></param>
+ /// <returns></returns>
+ public Task<ImageToTextResponse> GenerateTextFromImageAsync(
+ ImageToTextRequest request,
+ ImageToTextSettings? settings = null,
+ CancellationToken cancellationToken = default);
+}

src/Providers/Abstractions/src/ImageToText/IImageToTextModel`2.cs

@@ -0,0 +1,19 @@
+namespace LangChain.Providers;
+
+/// <summary>
+/// Defines a large language model that can be used for image to text generation.
+/// </summary>
+public interface IImageToTextModel<in TRequest, TResponse, in TSettings> : IImageToTextModel
+{
+ /// <summary>
+ /// Run the LLM on the image.
+ /// </summary>
+ /// <param name="request"></param>
+ /// <param name="settings"></param>
+ /// <param name="cancellationToken"></param>
+ /// <returns></returns>
+ public Task<TResponse> GenerateTextFromImageAsync(
+ TRequest request,
+ TSettings? settings = default,
+ CancellationToken cancellationToken = default);
+}

src/Providers/Abstractions/src/ImageToText/ImageToTextGenerationResponse.cs

@@ -0,0 +1,15 @@
+using System.Text.Json.Serialization;
+
+namespace LangChain.Providers;
+
+public class ImageToTextGenerationResponse : List<ImageToTextGenerationResponse.GeneratedTextItem>
+{
+ public sealed class GeneratedTextItem
+ {
+ /// <summary>
+ /// The continuated string
+ /// </summary>
+ [JsonPropertyName("generated_text")]
+ public string? GeneratedText { get; set; }
+ }
+}

src/Providers/Abstractions/src/ImageToText/ImageToTextModel.cs

@@ -0,0 +1,10 @@
+// ReSharper disable once CheckNamespace
+namespace LangChain.Providers;
+
+public abstract class ImageToTextModel(string id) : Model<ImageToTextSettings>(id), IImageToTextModel<ImageToTextRequest, ImageToTextResponse, ImageToTextSettings>
+{
+ public abstract Task<ImageToTextResponse> GenerateTextFromImageAsync(
+ ImageToTextRequest request,
+ ImageToTextSettings? settings = default,
+ CancellationToken cancellationToken = default);
+}

src/Providers/Abstractions/src/ImageToText/ImageToTextRequest.cs

@@ -0,0 +1,13 @@
+// ReSharper disable once CheckNamespace
+namespace LangChain.Providers;
+
+/// <summary>
+/// Base class for image to text requests.
+/// </summary>
+public class ImageToTextRequest
+{
+ /// <summary>
+ /// Image to upload.
+ /// </summary>
+ public required BinaryData Image { get; init; }
+}

src/Providers/Abstractions/src/ImageToText/ImageToTextResponse.cs

@@ -0,0 +1,27 @@
+// ReSharper disable once CheckNamespace
+// ReSharper disable ConditionalAccessQualifierIsNonNullableAccordingToAPIContract
+namespace LangChain.Providers;
+
+#pragma warning disable CA2225
+
+/// <summary>
+/// 
+/// </summary>
+public class ImageToTextResponse
+{
+ /// <summary>
+ /// 
+ /// </summary>
+ public required ImageToTextSettings UsedSettings { get; init; }
+
+ /// <summary>
+ /// 
+ /// </summary>
+ public Usage Usage { get; init; } = Usage.Empty;
+
+
+ /// <summary>
+ /// Generated text
+ /// </summary>
+ public string? Text { get; set; }
+}

src/Providers/Abstractions/src/ImageToText/ImageToTextSettings.cs

@@ -0,0 +1,55 @@
+// ReSharper disable once CheckNamespace
+namespace LangChain.Providers;
+
+/// <summary>
+/// Base class for image to text request settings.
+/// </summary>
+public class ImageToTextSettings
+{
+ public static ImageToTextSettings Default { get; } = new()
+ {
+ User = string.Empty,
+ Endpoint = "https://api-inference.huggingface.co/models/"
+ };
+
+ /// <summary>
+ /// Unique user identifier.
+ /// </summary>
+ public string? User { get; init; }
+
+ /// <summary>
+ /// Endpoint url for api.
+ /// </summary>
+ public string Endpoint { get; set; }
+
+
+ /// <summary>
+ /// Calculate the settings to use for the request.
+ /// </summary>
+ /// <param name="requestSettings"></param>
+ /// <param name="modelSettings"></param>
+ /// <param name="providerSettings"></param>
+ /// <returns></returns>
+ /// <exception cref="InvalidOperationException"></exception>
+ public static ImageToTextSettings Calculate(
+ ImageToTextSettings? requestSettings,
+ ImageToTextSettings? modelSettings,
+ ImageToTextSettings? providerSettings)
+ {
+ return new ImageToTextSettings
+ {
+ User = 
+ requestSettings?.User ??
+ modelSettings?.User ??
+ providerSettings?.User ??
+ Default.User ??
+ throw new InvalidOperationException("Default User is not set."),
+ Endpoint =
+ requestSettings?.Endpoint ??
+ modelSettings?.Endpoint ??
+ providerSettings?.Endpoint ??
+ Default.Endpoint ??
+ throw new InvalidOperationException("Default Endpoint is not set."),
+ };
+ }
+}

src/Providers/Abstractions/src/LangChain.Providers.Abstractions.csproj

@@ -16,6 +16,7 @@
  <PrivateAssets>all</PrivateAssets>
  <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
  </PackageReference>
+ <PackageReference Include="System.Memory.Data" />
  </ItemGroup>
 
 </Project>