Read/write bundles to/from Azure Blob Storage (#2425)

* wip. Add ability to read/write FHIR bundles to/from Azure blob storage

* move client inits inside function to see if it will remove errors

* add tests

* add note to make const global after Azure access

* place source consts in utils and import instead

containers/ecr-viewer/.env

@@ -1,6 +1,8 @@
 DATABASE_URL=postgres://postgres:pw@localhost:5432/ecr_viewer_db
 AWS_REGION=us-east-1
 ECR_BUCKET_NAME=ecr-viewer-files
+AZURE_STORAGE_CONNECTION_STRING=idk
+AZURE_CONTAINER_NAME=idk
 SOURCE=postgres
 NEXT_TELEMETRY_DISABLED=1
 NEXT_PUBLIC_NON_INTEGRATED_VIEWER=true

containers/ecr-viewer/environment.d.ts

@@ -13,6 +13,6 @@ namespace NodeJS {
  NEXT_RUNTIME: string;
  NEXT_PUBLIC_NON_INTEGRATED_VIEWER: "true" | "false";
  NEXTAUTH_SECRET: string;
- SOURCE: "s3" | "postgres";
+ SOURCE: "s3" | "azure" | "postgres";
  }
 }

containers/ecr-viewer/package.json

@@ -19,6 +19,7 @@
  },
  "dependencies": {
  "@aws-sdk/client-s3": "^3.621.0",
+ "@azure/storage-blob": "^12.24.0",
  "@opentelemetry/api": "^1.9.0",
  "@opentelemetry/auto-instrumentations-node": "^0.48.0",
  "@opentelemetry/exporter-jaeger": "^1.25.1",

containers/ecr-viewer/src/app/api/fhir-data/fhir-data-service.ts

@@ -1,6 +1,11 @@
 import { NextRequest, NextResponse } from "next/server";
 import pgPromise from "pg-promise";
 import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
+import {
+ BlobClient,
+ BlobDownloadResponseParsed,
+ BlobServiceClient,
+} from "@azure/storage-blob";
 import { loadYamlConfig, streamToJson } from "../utils";
 import { database } from "@/app/api/fhir-data/db";
 
@@ -70,3 +75,44 @@ export const get_s3 = async (request: NextRequest) => {
  return NextResponse.json({ message: error.message }, { status: 500 });
  }
 };
+
+/**
+ * Retrieves FHIR data from Azure Blob Storage based on eCR ID.
+ * @param request - The NextRequest object containing the request information.
+ * @returns A promise resolving to a NextResponse object.
+ */
+export const get_azure = async (request: NextRequest) => {
+ // TODO: Make this global after we get Azure access
+ const blobClient = BlobServiceClient.fromConnectionString(
+ process.env.AZURE_STORAGE_CONNECTION_STRING!,
+ );
+
+ const params = request.nextUrl.searchParams;
+ const ecr_id = params.get("id");
+
+ if (!process.env.AZURE_CONTAINER_NAME)
+ throw Error("Azure container name not found");
+
+ const containerName = process.env.AZURE_CONTAINER_NAME;
+ const blobName = `${ecr_id}.json`;
+
+ try {
+ const containerClient = blobClient.getContainerClient(containerName);
+ const blockBlobClient: BlobClient = containerClient.getBlobClient(blobName);
+
+ const downloadResponse: BlobDownloadResponseParsed =
+ await blockBlobClient.download();
+ const content = await streamToJson(downloadResponse.readableStreamBody);
+
+ return NextResponse.json(
+ { fhirBundle: content, fhirPathMappings: loadYamlConfig() },
+ { status: 200 },
+ );
+ } catch (error: any) {
+ console.error(
+ "Failed to download the FHIR data from Azure Blob Storage:",
+ error,
+ );
+ return NextResponse.json({ message: error.message }, { status: 500 });
+ }
+};

containers/ecr-viewer/src/app/api/fhir-data/route.ts

@@ -1,8 +1,6 @@
 import { NextRequest, NextResponse } from "next/server";
-import { get_s3, get_postgres } from "./fhir-data-service";
-
-const S3_SOURCE = "s3";
-const POSTGRES_SOURCE = "postgres";
+import { get_s3, get_azure, get_postgres } from "./fhir-data-service";
+import { S3_SOURCE, AZURE_SOURCE, POSTGRES_SOURCE } from "@/app/api/utils";
 
 /**
  * Handles GET requests by fetching data from different sources based on the environment configuration.
@@ -17,6 +15,8 @@ const POSTGRES_SOURCE = "postgres";
 export async function GET(request: NextRequest) {
  if (process.env.SOURCE === S3_SOURCE) {
  return get_s3(request);
+ } else if (process.env.SOURCE === AZURE_SOURCE) {
+ return await get_azure(request);
  } else if (process.env.SOURCE === POSTGRES_SOURCE) {
  return await get_postgres(request);
  } else {

containers/ecr-viewer/src/app/api/save-fhir-data/route.ts

@@ -1,8 +1,10 @@
 import { NextRequest, NextResponse } from "next/server";
-import { saveToS3, saveToPostgres } from "./save-fhir-data-service";
-
-const S3_SOURCE = "s3";
-const POSTGRES_SOURCE = "postgres";
+import {
+ saveToS3,
+ saveToAzure,
+ saveToPostgres,
+} from "./save-fhir-data-service";
+import { S3_SOURCE, AZURE_SOURCE, POSTGRES_SOURCE } from "@/app/api/utils";
 
 /**
  * Handles POST requests and saves the FHIR Bundle to the database.
@@ -50,6 +52,8 @@ export async function POST(request: NextRequest) {
 
  if (saveSource === S3_SOURCE) {
  return saveToS3(fhirBundle, ecrId);
+ } else if (saveSource === AZURE_SOURCE) {
+ return saveToAzure(fhirBundle, ecrId);
  } else if (saveSource === POSTGRES_SOURCE) {
  return await saveToPostgres(fhirBundle, ecrId);
  } else {

containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts

@@ -1,3 +1,4 @@
+import { BlobServiceClient } from "@azure/storage-blob";
 import { NextResponse } from "next/server";
 import pgPromise from "pg-promise";
 import {
@@ -85,3 +86,53 @@ export const saveToS3 = async (fhirBundle: Bundle, ecrId: string) => {
  );
  }
 };
+
+/**
+ * Saves a FHIR bundle to Azure Blob Storage.
+ * @async
+ * @function saveToAzure
+ * @param fhirBundle - The FHIR bundle to be saved.
+ * @param ecrId - The unique ID for the eCR associated with the FHIR bundle.
+ * @returns A promise that resolves when the FHIR bundle is successfully saved to Azure Blob Storage.
+ * @throws {Error} Throws an error if the FHIR bundle cannot be saved to Azure Blob Storage.
+ */
+export const saveToAzure = async (fhirBundle: Bundle, ecrId: string) => {
+ // TODO: Make this global after we get Azure access
+ const blobClient = BlobServiceClient.fromConnectionString(
+ process.env.AZURE_STORAGE_CONNECTION_STRING!,
+ );
+
+ if (!process.env.AZURE_CONTAINER_NAME)
+ throw Error("Azure container name not found");
+
+ const containerName = process.env.AZURE_CONTAINER_NAME;
+ const blobName = `${ecrId}.json`;
+ const body = JSON.stringify(fhirBundle);
+
+ try {
+ const containerClient = blobClient.getContainerClient(containerName);
+ const blockBlobClient = containerClient.getBlockBlobClient(blobName);
+
+ const response = await blockBlobClient.upload(body, body.length, {
+ blobHTTPHeaders: { blobContentType: "application/json" },
+ });
+
+ if (response._response.status !== 201) {
+ throw new Error(`HTTP Status Code: ${response._response.status}`);
+ }
+
+ return NextResponse.json(
+ { message: "Success. Saved FHIR bundle to Azure Blob Storage: " + ecrId },
+ { status: 200 },
+ );
+ } catch (error: any) {
+ return NextResponse.json(
+ {
+ message:
+ "Failed to insert FHIR bundle to Azure Blob Storage. " +
+ error.message,
+ },
+ { status: 400 },
+ );
+ }
+};

containers/ecr-viewer/src/app/api/utils.ts

@@ -3,6 +3,10 @@ import * as path from "path";
 import yaml from "js-yaml";
 import { PathMappings } from "@/app/view-data/utils/utils";
 
+export const S3_SOURCE = "s3";
+export const POSTGRES_SOURCE = "postgres";
+export const AZURE_SOURCE = "azure";
+
 /**
  * Loads the YAML configuration for path mappings from a predefined file location.
  * @returns An object representing the path mappings defined in the YAML configuration file.

containers/ecr-viewer/src/app/tests/fhir-data.test.tsx

@@ -7,34 +7,56 @@ import { mockClient } from "aws-sdk-client-mock";
 import { GET } from "../api/fhir-data/route"; // Adjust the import path to your actual file path
 import { sdkStreamMixin } from "@smithy/util-stream";
 import { NextRequest } from "next/server";
+import { Readable } from "stream";
 
 const s3Mock = mockClient(S3Client);
 const stream = sdkStreamMixin(
  fs.createReadStream("src/app/tests/assets/BundleTravelHistory.json"),
 );
 
-beforeEach(() => {
- s3Mock.reset();
-});
-
 const mockYamlConfig = {}; // Adjust this to match what loadYamlConfig() would return
+const mockData = {
+ resourceType: "Bundle",
+ type: "batch",
+ entry: [
+ {
+ fullUrl: "urn:uuid:1dd10047-2207-4eac-a993-0f706c88be5d",
+ resource: {
+ resourceType: "Composition",
+ id: "1dd10047-2207-4eac-a993-0f706c88be5d",
+ },
+ },
+ ],
+};
+
 jest.mock("../view-data/utils/utils", () => ({
  loadYamlConfig: jest.fn().mockReturnValue(mockYamlConfig),
- streamToJson: jest.fn().mockResolvedValue({
- resourceType: "Bundle",
- type: "batch",
- entry: [
- {
- fullUrl: "urn:uuid:1dd10047-2207-4eac-a993-0f706c88be5d",
- resource: {
- resourceType: "Composition",
- id: "1dd10047-2207-4eac-a993-0f706c88be5d",
- },
- },
- ],
- }),
+ streamToJson: jest.fn().mockResolvedValue(mockData),
+}));
+jest.mock("@azure/storage-blob", () => ({
+ BlobServiceClient: {
+ fromConnectionString: jest.fn(() => mockBlobServiceClient),
+ },
 }));
 
+const mockStream = new Readable({
+ read() {
+ this.push(JSON.stringify(mockData));
+ this.push(null);
+ },
+});
+const mockBlobClient = {
+ download: jest.fn().mockResolvedValue({
+ readableStreamBody: mockStream,
+ }),
+};
+const mockContainerClient = {
+ getBlobClient: jest.fn(() => mockBlobClient),
+};
+const mockBlobServiceClient = {
+ getContainerClient: jest.fn(() => mockContainerClient),
+};
+
 describe("GET API Route", () => {
  it("fetches data from S3 and returns a JSON response", async () => {
  const fakeId = "test-id";
@@ -55,4 +77,16 @@ describe("GET API Route", () => {
  const jsonResponse = await response.json();
  expect(jsonResponse.fhirBundle).toBeDefined();
  });
+
+ it("fetches data from Azure Blob Storage and returns a JSON response", async () => {
+ const fakeId = "test-id";
+ process.env.SOURCE = "azure";
+ const request = new NextRequest(`http://localhost?id=${fakeId}`);
+
+ const response = await GET(request);
+ expect(response.status).toBe(200);
+ const jsonResponse = await response.json();
+ expect(jsonResponse.fhirBundle).toBeDefined();
+ expect(jsonResponse.fhirBundle).toEqual(mockData);
+ });
 });