filemanager: annotate portal run id

.github/workflows/prbuild.yml

@@ -36,6 +36,7 @@ env:
  #
  # And from pyenv docs setting this PYENV_VERSION as env-var would set the python version globally
  PYENV_VERSION: '3.12'
+ GOENV_VERSION: '1.22'
 
 jobs:
  pre-commit-lint-security:
@@ -82,21 +83,24 @@ jobs:
  runs-on: codebuild-orcabus-codebuild-gh-runner-${{ github.run_id }}-${{ github.run_attempt }}
  if: ${{ !github.event.pull_request.draft }}
  steps:
+ - name: Rust installation
+ uses: actions-rust-lang/setup-rust-toolchain@v1
+ with:
+ components: rustfmt
+
  - name: Print toolchain versions
  run: |
  node -v
  python -V
  pip3 -V
+ rustc --version
+ go version
  make --version
+ docker version
 
  - name: Checkout code
  uses: actions/checkout@v4
 
- - name: Rust installation
- uses: actions-rust-lang/setup-rust-toolchain@v1
- with:
- components: rustfmt
-
  - name: Install dependencies
  run: |
  yarn install --immutable

Makefile

@@ -53,6 +53,7 @@ test-stateless-app-suite:
  @(cd lib/workload/stateless/stacks/sequence-run-manager && $(MAKE) test)
  @(cd lib/workload/stateless/stacks/metadata-manager && $(MAKE) test)
  @(cd lib/workload/stateless/stacks/filemanager && $(MAKE) test)
+ @(cd lib/workload/stateless/stacks/fmannotator && $(MAKE) test)
  @(cd lib/workload/stateless/stacks/bclconvert-manager && $(MAKE) test)
  @(cd lib/workload/stateless/stacks/workflow-manager && $(MAKE) test)
 

config/config.ts

@@ -47,6 +47,7 @@ import {
  getRnasumIcav2PipelineManagerStackProps,
  getRnasumIcav2PipelineTableStackProps,
 } from './stacks/rnasumPipelineManager';
+import { getFmAnnotatorProps } from './stacks/fmAnnotator';
 
 interface EnvironmentConfig {
  name: string;
@@ -102,6 +103,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
  bclConvertManagerStackProps: getBclConvertManagerStackProps(stage),
  workflowManagerStackProps: getWorkflowManagerStackProps(stage),
  stackyMcStackFaceProps: getGlueStackProps(stage),
+ fmAnnotatorProps: getFmAnnotatorProps(),
  },
  };
 

config/stacks/fmAnnotator.ts

@@ -0,0 +1,10 @@
+import { FMAnnotatorConfigurableProps } from '../../lib/workload/stateless/stacks/fmannotator/deploy/stack';
+import { eventBusName, jwtSecretName, vpcProps } from '../constants';
+
+export const getFmAnnotatorProps = (): FMAnnotatorConfigurableProps => {
+ return {
+ vpcProps,
+ eventBusName,
+ jwtSecretName,
+ };
+};

lib/pipeline/statelessPipelineStack.ts

@@ -91,6 +91,7 @@ export class StatelessPipelineStack extends cdk.Stack {
  install: {
  'runtime-versions': {
  python: '3.12',
+ golang: '1.22',
  },
  },
  },
@@ -126,6 +127,18 @@ export class StatelessPipelineStack extends cdk.Stack {
  resources: ['*'],
  }),
  ],
+ partialBuildSpec: codebuild.BuildSpec.fromObject({
+ phases: {
+ install: {
+ 'runtime-versions': {
+ // Don't strictly need golang here as CDK can do docker bundling, but `local`
+ // bundling tends to be faster.
+ golang: '1.22',
+ },
+ },
+ },
+ version: '0.2',
+ }),
  });
 
  const pipeline = new pipelines.CodePipeline(this, 'Pipeline', {

lib/workload/components/api-gateway/index.ts

@@ -42,6 +42,7 @@ export interface ApiGatewayConstructProps {
 
 export class ApiGatewayConstruct extends Construct {
  private readonly _httpApi: HttpApi;
+ private readonly _domainName: string;
 
  constructor(scope: Construct, id: string, props: ApiGatewayConstructProps) {
  super(scope, id);
@@ -54,9 +55,9 @@ export class ApiGatewayConstruct extends Construct {
  );
  const hostedZoneId = StringParameter.valueForStringParameter(this, '/hosted_zone/umccr/id');
 
- const domainName = `${props.customDomainNamePrefix}.${hostedDomainName}`;
+ this._domainName = `${props.customDomainNamePrefix}.${hostedDomainName}`;
  const apiGWDomainName = new DomainName(this, 'UmccrDomainName', {
- domainName: `${props.customDomainNamePrefix}.${hostedDomainName}`,
+ domainName: this.domainName,
  certificate: Certificate.fromCertificateArn(this, 'cert', umccrAcmArn),
  });
 
@@ -85,7 +86,7 @@ export class ApiGatewayConstruct extends Construct {
  hostedZoneId,
  zoneName: hostedDomainName,
  }),
- recordName: domainName,
+ recordName: this.domainName,
  target: RecordTarget.fromAlias(
  new ApiGatewayv2DomainProperties(
  apiGWDomainName.regionalDomainName,
@@ -177,4 +178,8 @@ export class ApiGatewayConstruct extends Construct {
  get httpApi(): HttpApi {
  return this._httpApi;
  }
+
+ get domainName(): string {
+ return this._domainName;
+ }
 }

lib/workload/stateless/stacks/filemanager/.env.example

@@ -3,6 +3,7 @@ FILEMANAGER_DATABASE_HOST=0.0.0.0
 FILEMANAGER_DATABASE_PORT=4321
 
 DATABASE_URL=postgresql://filemanager:filemanager@${FILEMANAGER_DATABASE_HOST}:${FILEMANAGER_DATABASE_PORT}/filemanager #pragma: allowlist secret
+API_PORT=8000
 
 FILEMANAGER_LINKS_URL=localhost:8000
 FILEMANAGER_API_CORS_ALLOW_ORIGINS=http://localhost:8000

lib/workload/stateless/stacks/filemanager/Makefile

@@ -40,7 +40,7 @@ docker-find:
  @docker ps --filter name=$(DOCKER_PROJECT_NAME)-postgres --latest --format "{{.ID}}" | xargs -I {} docker port {} | tail -n 1 | awk '{print $$NF}'
 docker-api: docker-postgres
  # Run the local API server in a docker container.
- @docker compose -p "$(DOCKER_PROJECT_NAME)" up api
+ @docker compose -p "$(DOCKER_PROJECT_NAME)" -f compose-api.yml up api
 
 ## Build related commands
 build: docker-postgres

lib/workload/stateless/stacks/filemanager/compose-api.yml

@@ -0,0 +1,20 @@
+services:
+ api:
+ build:
+ context: .
+ args:
+ # The build itself needs access to the database.
+ - DATABASE_URL=${BUILD_DATABASE_URL:-postgresql://filemanager:[email protected]:4321/filemanager} # pragma: allowlist secret
+ environment:
+ # Container database address for running server inside a docker container.
+ - DATABASE_URL=${DATABASE_URL:-postgresql://filemanager:filemanager@postgres:4321/filemanager}
+ - RUST_LOG=debug
+ - FILEMANAGER_API_CORS_ALLOW_ORIGINS=${FILEMANAGER_API_CORS_ALLOW_ORIGINS:-http://localhost:3000}
+ - FILEMANAGER_API_CORS_ALLOW_HEADERS=${FILEMANAGER_API_CORS_ALLOW_HEADERS:-accept,authorization,content-type,user-agent,x-csrftoken,x-requested-with,x-amz-security-token,x-amz-date,content-disposition}
+ - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-access_key_id}
+ - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret_access_key}
+ - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-ap-southeast-2}
+ - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-session_token}
+ ports:
+ - '${API_PORT}:8000'
+ restart: always

lib/workload/stateless/stacks/filemanager/compose.yml

@@ -3,30 +3,9 @@ services:
  build: database
  restart: always
  environment:
- - POSTGRES_DATABASE=filemanager
+ - POSTGRES_DB=filemanager
  - POSTGRES_USER=filemanager
  - POSTGRES_PASSWORD=filemanager
  - PGPORT=4321
  ports:
  - "${FILEMANAGER_DATABASE_HOST}:${FILEMANAGER_DATABASE_PORT}:4321"
- api:
- build:
- context: .
- args:
- # The build itself needs access to the database.
- DATABASE_URL: postgresql://filemanager:[email protected]:4321/filemanager # pragma: allowlist secret
- environment:
- # Container database address for running server inside a docker container.
- - DATABASE_URL=postgresql://filemanager:filemanager@postgres:4321/filemanager
- - RUST_LOG=debug
- - FILEMANAGER_API_CORS_ALLOW_ORIGINS=${FILEMANAGER_API_CORS_ALLOW_ORIGINS:-http://localhost:3000}
- - FILEMANAGER_API_CORS_ALLOW_HEADERS=${FILEMANAGER_API_CORS_ALLOW_HEADERS:-accept,authorization,content-type,user-agent,x-csrftoken,x-requested-with,x-amz-security-token,x-amz-date,content-disposition}
- - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-access_key_id}
- - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret_access_key}
- - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-ap-southeast-2}
- - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-session_token}
- ports:
- - "8000:8000"
- depends_on:
- - postgres
- restart: always

lib/workload/stateless/stacks/filemanager/deploy/stack.ts

@@ -48,6 +48,7 @@ export class Filemanager extends Stack {
  private readonly host: string;
  private readonly securityGroup: ISecurityGroup;
  private readonly queue: IQueue;
+ readonly domainName: string;
 
  constructor(scope: Construct, id: string, props: FilemanagerProps) {
  super(scope, id, props);
@@ -93,8 +94,9 @@ export class Filemanager extends Stack {
  );
 
  this.createIngestFunction(props);
- this.createApiFunction(props);
  this.createInventoryFunction(props);
+
+ this.domainName = this.createApiFunction(props);
  }
 
  private createIngestRole(name: string) {
@@ -130,9 +132,9 @@ export class Filemanager extends Stack {
  }
 
  /**
- * Query function and API Gateway fronting the function.
+ * Query function and API Gateway fronting the function. Returns the configured domain name.
  */
- private createApiFunction(props: FilemanagerProps) {
+ private createApiFunction(props: FilemanagerProps): string {
  let apiLambda = new ApiFunction(this, 'ApiFunction', {
  vpc: this.vpc,
  host: this.host,
@@ -141,13 +143,14 @@ export class Filemanager extends Stack {
  ...props,
  });
 
- const ApiGateway = new ApiGatewayConstruct(this, 'ApiGateway', {
+ const apiGateway = new ApiGatewayConstruct(this, 'ApiGateway', {
  region: this.region,
  apiName: 'FileManager',
  customDomainNamePrefix: 'file',
  ...props,
  });
- const httpApi = ApiGateway.httpApi;
+
+ const httpApi = apiGateway.httpApi;
 
  const apiIntegration = new HttpLambdaIntegration('ApiIntegration', apiLambda.function);
 
@@ -156,5 +159,7 @@ export class Filemanager extends Stack {
  integration: apiIntegration,
  routeKey: HttpRouteKey.with('/{proxy+}', HttpMethod.ANY),
  });
+
+ return apiGateway.domainName;
  }
 }

lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_entities.rs

@@ -17,6 +17,8 @@ use crate::Config;
 pub async fn generate_entities() -> Result<()> {
  let config = Config::load()?;
 
+ println!("{:#?}", config);
+
  let out_dir = config.out_dir;
  let command: &[&_] = &[
  "sea-orm-cli",

lib/workload/stateless/stacks/fmannotator/.gitignore

@@ -0,0 +1 @@
+build

lib/workload/stateless/stacks/fmannotator/Makefile

@@ -0,0 +1,28 @@
+## Default target
+default: help
+
+install: build
+all: build
+
+## Build related commands
+build:
+ @go build -o build
+test:
+ @go test
+lint:
+ @go fmt
+golangci-lint:
+ @golangci-lint run
+check: lint golangci-lint
+
+## Clean
+clean:
+ @go mod tidy
+
+## Help text
+help:
+ @printf "The attribute annotator Makefile.\n\n"
+ @printf "Usage: make <target>\n"
+
+# No targets are files, so they are all PHONY.
+.PHONY: *

lib/workload/stateless/stacks/fmannotator/README.md

@@ -0,0 +1,52 @@
+# FMAnnotator
+
+The FMAnnotator service annotates records using the FileManager API.
+
+## Development
+
+This service is written in Go, which should be [installed][golang]. Please also install [golangci-lint] to run lints on
+the codebase.
+
+This project is organised using the Go Lambda function CDK, and contains Lambda function handlers under [`cmd`][cmd]
+
+Makefile is used to simplify development. Tests can be run by using:
+
+```sh
+make test
+```
+
+Lints and checks can be run using:
+
+```sh
+make check
+```
+
+To update the [go.mod][go-mod] and download any new go modules:
+
+```sh
+make clean
+```
+
+## Project layout
+
+This service has the following structure:
+
+* [cmd]: The Lambda handler `main.go` functions.
+* [deploy]: CDK deployment code.
+* [fixtures]: Database fixtures and test data.
+* [internal]: Internal package containing common test code.
+* [schema]: Generated EventBridge code-bindings.
+
+Top-level `.go` files contain library related code which implements the functionality of the FMAnnotator.
+Tests are defined using the `_test.go` suffix.
+
+[golang]: https://go.dev/doc/install
+[golangci-lint]: https://golangci-lint.run/welcome/install/#local-installation
+[cmd]: cmd
+[api]: api.go
+[config]: config.go
+[handlers]: handlers.go
+[internal]: internal
+[fixtures]: fixtures
+[schema]: schema
+[go-mod]: go.mod