From 04837faedd6fca95753e46053d26f4eed9fb8921 Mon Sep 17 00:00:00 2001 From: angelathe Date: Thu, 22 Aug 2024 14:39:10 -0500 Subject: [PATCH 1/5] Add scripts to run azure or s3 locally --- containers/ecr-viewer/.env | 9 ++++++-- containers/ecr-viewer/.gitignore | 5 ++++- containers/ecr-viewer/compose.storage.yml | 21 +++++++++++++++++++ containers/ecr-viewer/environment.d.ts | 5 +++++ containers/ecr-viewer/package.json | 6 +++++- .../app/api/fhir-data/fhir-data-service.ts | 6 +++++- .../save-fhir-data/save-fhir-data-service.ts | 6 +++++- 7 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 containers/ecr-viewer/compose.storage.yml diff --git a/containers/ecr-viewer/.env b/containers/ecr-viewer/.env index f5d313ea36..497567a7da 100644 --- a/containers/ecr-viewer/.env +++ b/containers/ecr-viewer/.env @@ -1,8 +1,13 @@ DATABASE_URL=postgres://postgres:pw@localhost:5432/ecr_viewer_db +AWS_CUSTOM_ENDPOINT="http://localhost:4566" AWS_REGION=us-east-1 ECR_BUCKET_NAME=ecr-viewer-files -AZURE_STORAGE_CONNECTION_STRING=idk -AZURE_CONTAINER_NAME=idk +# These are the default credentials for localstack +AWS_ACCESS_KEY_ID=123 +AWS_SECRET_ACCESS_KEY=xyz +# This is the default connection string for Azurite +AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://0.0.0.0:10000/devstoreaccount1;" +AZURE_CONTAINER_NAME=ecr-viewer-files SOURCE=postgres NEXT_TELEMETRY_DISABLED=1 NEXT_PUBLIC_NON_INTEGRATED_VIEWER=true \ No newline at end of file diff --git a/containers/ecr-viewer/.gitignore b/containers/ecr-viewer/.gitignore index 615a1883a8..c0349d84cd 100644 --- a/containers/ecr-viewer/.gitignore +++ b/containers/ecr-viewer/.gitignore @@ -1,3 +1,6 @@ node_modules .next/ -.env.local \ No newline at end of file +.env.local + +# localstack data +volume/ \ No newline at end of file diff --git a/containers/ecr-viewer/compose.storage.yml b/containers/ecr-viewer/compose.storage.yml new file mode 100644 index 0000000000..3117f51115 --- /dev/null +++ b/containers/ecr-viewer/compose.storage.yml @@ -0,0 +1,21 @@ +services: + # Azure Blob Storage emulator + azure-storage: + image: mcr.microsoft.com/azure-storage/azurite + hostname: azurite + restart: always + command: "azurite --blobHost 0.0.0.0 --blobPort 10000 -l data" + ports: + - "10000:10000" + + # AWS S3 Storage emulator + aws-storage: + container_name: "localstack-main" + image: localstack/localstack:s3-latest + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + environment: + - DEBUG=1 + volumes: + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" \ No newline at end of file diff --git a/containers/ecr-viewer/environment.d.ts b/containers/ecr-viewer/environment.d.ts index b02f715dfd..413085ba13 100644 --- a/containers/ecr-viewer/environment.d.ts +++ b/containers/ecr-viewer/environment.d.ts @@ -3,6 +3,11 @@ namespace NodeJS { interface ProcessEnv { APP_ENV: "test" | "middleware" | "prod"; AWS_REGION: string; + AWS_CUSTOM_ENDPOINT: string; + AWS_ACCESS_KEY_ID: string; + AWS_SECRET_ACCESS_KEY: string; + AZURE_STORAGE_CONNECTION_STRING: string; + AZURE_CONTAINER_NAME: string; DATABASE_SCHEMA: "core" | "extended"; DATABASE_TYPE: string; DATABASE_URL: string; diff --git a/containers/ecr-viewer/package.json b/containers/ecr-viewer/package.json index 7dc235c988..afbcb4b8b5 100644 --- a/containers/ecr-viewer/package.json +++ b/containers/ecr-viewer/package.json @@ -6,7 +6,11 @@ "scripts": { "dev": "next dev", "local-dev": "npm run docker-compose-with-logs && npm run dev", + "local-dev-azure": "npm run docker-compose-azure && npm run dev", + "local-dev-aws": "npm run docker-compose-aws && npm run dev", "docker-compose-with-logs": "docker compose up db -d && docker compose logs", + "docker-compose-azure": "docker compose -f docker-compose.yml -f compose.storage.yml up db azure-storage -d && docker compose logs", + "docker-compose-aws": "docker compose -f docker-compose.yml -f compose.storage.yml up aws-storage db -d && docker compose logs", "setup-local-env": "./setup-env.sh", "build": "next build", "start": "next start", @@ -90,4 +94,4 @@ "sass": "^1.69.5", "typescript": "^5.5.2" } -} +} \ No newline at end of file diff --git a/containers/ecr-viewer/src/app/api/fhir-data/fhir-data-service.ts b/containers/ecr-viewer/src/app/api/fhir-data/fhir-data-service.ts index c87625ef5e..a4334a554b 100644 --- a/containers/ecr-viewer/src/app/api/fhir-data/fhir-data-service.ts +++ b/containers/ecr-viewer/src/app/api/fhir-data/fhir-data-service.ts @@ -9,7 +9,11 @@ import { import { loadYamlConfig, streamToJson } from "../utils"; import { database } from "@/app/api/fhir-data/db"; -const s3Client = new S3Client({ region: process.env.AWS_REGION }); +const s3Client = new S3Client({ + region: process.env.AWS_REGION, + endpoint: process.env.AWS_CUSTOM_ENDPOINT, + forcePathStyle: process.env.AWS_CUSTOM_ENDPOINT !== undefined +}); /** * Retrieves FHIR data from PostgreSQL database based on eCR ID. diff --git a/containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts b/containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts index ff6e99549a..6ac0cac27f 100644 --- a/containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts +++ b/containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts @@ -8,7 +8,11 @@ import { } from "@aws-sdk/client-s3"; import { Bundle } from "fhir/r4"; -const s3Client = new S3Client({ region: process.env.AWS_REGION }); +const s3Client = new S3Client({ + region: process.env.AWS_REGION, + endpoint: process.env.AWS_CUSTOM_ENDPOINT, + forcePathStyle: process.env.AWS_CUSTOM_ENDPOINT !== undefined +}); /** * Saves a FHIR bundle to a postgres database. From 5392cb1d762a5edde8b2ee06759aa7aa154ebc4a Mon Sep 17 00:00:00 2001 From: Austin Hall Date: Wed, 28 Aug 2024 16:04:08 -0400 Subject: [PATCH 2/5] Update local-dev script to choose storage based on SOURCE from .env --- containers/ecr-viewer/check-source-env.mjs | 6 ++++++ containers/ecr-viewer/package.json | 6 ++++-- package-lock.json | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 containers/ecr-viewer/check-source-env.mjs diff --git a/containers/ecr-viewer/check-source-env.mjs b/containers/ecr-viewer/check-source-env.mjs new file mode 100644 index 0000000000..7fe6894f1c --- /dev/null +++ b/containers/ecr-viewer/check-source-env.mjs @@ -0,0 +1,6 @@ +import dotenv from "dotenv" + +dotenv.config(); +dotenv.config({ path: ".env.local", override: true }); + +console.log(`local-dev-${process.env.SOURCE}`); \ No newline at end of file diff --git a/containers/ecr-viewer/package.json b/containers/ecr-viewer/package.json index afbcb4b8b5..b5178c772d 100644 --- a/containers/ecr-viewer/package.json +++ b/containers/ecr-viewer/package.json @@ -5,9 +5,10 @@ "private": true, "scripts": { "dev": "next dev", - "local-dev": "npm run docker-compose-with-logs && npm run dev", + "local-dev": "npm run $(node ./check-source-env.mjs)", + "local-dev-postgres": "npm run docker-compose-with-logs && npm run dev", "local-dev-azure": "npm run docker-compose-azure && npm run dev", - "local-dev-aws": "npm run docker-compose-aws && npm run dev", + "local-dev-s3": "npm run docker-compose-aws && npm run dev", "docker-compose-with-logs": "docker compose up db -d && docker compose logs", "docker-compose-azure": "docker compose -f docker-compose.yml -f compose.storage.yml up db azure-storage -d && docker compose logs", "docker-compose-aws": "docker compose -f docker-compose.yml -f compose.storage.yml up aws-storage db -d && docker compose logs", @@ -79,6 +80,7 @@ "autoprefixer": "^10.4.19", "aws-sdk-client-mock": "^3.1.0", "cypress": "^13.6.6", + "dotenv": "^16.4.5", "eslint": "^8.56.0", "eslint-config-next": "14.0.3", "eslint-config-prettier": "^9.1.0", diff --git a/package-lock.json b/package-lock.json index b5969723e4..9bd316b47d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -73,6 +73,7 @@ "autoprefixer": "^10.4.19", "aws-sdk-client-mock": "^3.1.0", "cypress": "^13.6.6", + "dotenv": "^16.4.5", "eslint": "^8.56.0", "eslint-config-next": "14.0.3", "eslint-config-prettier": "^9.1.0", @@ -8267,6 +8268,19 @@ "url": "https://github.com/fb55/domutils?sponsor=1" } }, + "node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "dev": true, From c2c604d9b074388eea00070fe11e02f767a85516 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 20:11:58 +0000 Subject: [PATCH 3/5] [pre-commit.ci] auto fixes from pre-commit hooks --- containers/ecr-viewer/check-source-env.mjs | 4 ++-- containers/ecr-viewer/compose.storage.yml | 4 ++-- .../ecr-viewer/src/app/api/fhir-data/fhir-data-service.ts | 8 ++++---- .../src/app/api/save-fhir-data/save-fhir-data-service.ts | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/containers/ecr-viewer/check-source-env.mjs b/containers/ecr-viewer/check-source-env.mjs index 7fe6894f1c..34ceede4b1 100644 --- a/containers/ecr-viewer/check-source-env.mjs +++ b/containers/ecr-viewer/check-source-env.mjs @@ -1,6 +1,6 @@ -import dotenv from "dotenv" +import dotenv from "dotenv"; dotenv.config(); dotenv.config({ path: ".env.local", override: true }); -console.log(`local-dev-${process.env.SOURCE}`); \ No newline at end of file +console.log(`local-dev-${process.env.SOURCE}`); diff --git a/containers/ecr-viewer/compose.storage.yml b/containers/ecr-viewer/compose.storage.yml index 3117f51115..3fc4344ac8 100644 --- a/containers/ecr-viewer/compose.storage.yml +++ b/containers/ecr-viewer/compose.storage.yml @@ -13,9 +13,9 @@ services: container_name: "localstack-main" image: localstack/localstack:s3-latest ports: - - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4566:4566" # LocalStack Gateway environment: - DEBUG=1 volumes: - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - - "/var/run/docker.sock:/var/run/docker.sock" \ No newline at end of file + - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/containers/ecr-viewer/src/app/api/fhir-data/fhir-data-service.ts b/containers/ecr-viewer/src/app/api/fhir-data/fhir-data-service.ts index a4334a554b..2bdd7b3291 100644 --- a/containers/ecr-viewer/src/app/api/fhir-data/fhir-data-service.ts +++ b/containers/ecr-viewer/src/app/api/fhir-data/fhir-data-service.ts @@ -9,10 +9,10 @@ import { import { loadYamlConfig, streamToJson } from "../utils"; import { database } from "@/app/api/fhir-data/db"; -const s3Client = new S3Client({ - region: process.env.AWS_REGION, - endpoint: process.env.AWS_CUSTOM_ENDPOINT, - forcePathStyle: process.env.AWS_CUSTOM_ENDPOINT !== undefined +const s3Client = new S3Client({ + region: process.env.AWS_REGION, + endpoint: process.env.AWS_CUSTOM_ENDPOINT, + forcePathStyle: process.env.AWS_CUSTOM_ENDPOINT !== undefined, }); /** diff --git a/containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts b/containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts index 6ac0cac27f..b154a8e5f1 100644 --- a/containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts +++ b/containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts @@ -8,10 +8,10 @@ import { } from "@aws-sdk/client-s3"; import { Bundle } from "fhir/r4"; -const s3Client = new S3Client({ - region: process.env.AWS_REGION, - endpoint: process.env.AWS_CUSTOM_ENDPOINT, - forcePathStyle: process.env.AWS_CUSTOM_ENDPOINT !== undefined +const s3Client = new S3Client({ + region: process.env.AWS_REGION, + endpoint: process.env.AWS_CUSTOM_ENDPOINT, + forcePathStyle: process.env.AWS_CUSTOM_ENDPOINT !== undefined, }); /** From 6e04e87e42092b319fbebd3db96115bfeda85c45 Mon Sep 17 00:00:00 2001 From: Austin Hall Date: Thu, 29 Aug 2024 16:44:48 -0400 Subject: [PATCH 4/5] Replace localstack with localstack-persist --- containers/ecr-viewer/compose.storage.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/containers/ecr-viewer/compose.storage.yml b/containers/ecr-viewer/compose.storage.yml index 3fc4344ac8..6bdf6270d3 100644 --- a/containers/ecr-viewer/compose.storage.yml +++ b/containers/ecr-viewer/compose.storage.yml @@ -10,12 +10,8 @@ services: # AWS S3 Storage emulator aws-storage: - container_name: "localstack-main" - image: localstack/localstack:s3-latest + image: gresau/localstack-persist:3 ports: - - "127.0.0.1:4566:4566" # LocalStack Gateway - environment: - - DEBUG=1 + - "4566:4566" volumes: - - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - - "/var/run/docker.sock:/var/run/docker.sock" + - "./volume:/persisted-data" From 133dcf376f316f25e61dcf045080305dd0b52087 Mon Sep 17 00:00:00 2001 From: Austin Hall Date: Thu, 29 Aug 2024 17:01:07 -0400 Subject: [PATCH 5/5] Update .env comments --- containers/ecr-viewer/.env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/containers/ecr-viewer/.env b/containers/ecr-viewer/.env index 497567a7da..6ba49f352b 100644 --- a/containers/ecr-viewer/.env +++ b/containers/ecr-viewer/.env @@ -5,7 +5,8 @@ ECR_BUCKET_NAME=ecr-viewer-files # These are the default credentials for localstack AWS_ACCESS_KEY_ID=123 AWS_SECRET_ACCESS_KEY=xyz -# This is the default connection string for Azurite +# NOT A REAL CONNECTION STRING! This is the default connection string for Azurite +# https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio%2Cblob-storage#well-known-storage-account-and-key AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://0.0.0.0:10000/devstoreaccount1;" AZURE_CONTAINER_NAME=ecr-viewer-files SOURCE=postgres