fix(filemanager): make HeadObject optional

config/constants.ts

@@ -161,6 +161,8 @@ export const bclconvertInteropQcIcav2EventSource = 'orcabus.bclconvertinteropqc'
 export const bclconvertInteropQcIcav2EventDetailType = 'WorkflowRunStateChange';
 export const bclconvertInteropQcStateMachinePrefix = 'bclconvertInteropQcSfn';
 
+export const fileManagerIngestRoleName = 'orcabus-file-manager-ingest-role';
+
 /*
 Resources used by the bclConvert InteropQc Pipeline
 */

config/stacks/fileManager.ts

@@ -10,6 +10,8 @@ import {
  cognitoStatusPageAppClientIdParameterName,
  cognitoUserPoolIdParameterName,
  oncoanalyserBucket,
+ icav2PipelineCacheBucket,
+ fileManagerIngestRoleName,
 } from '../constants';
 
 export const getFileManagerStackProps = (stage: AppStage): FilemanagerConfig => {
@@ -24,6 +26,7 @@ export const getFileManagerStackProps = (stage: AppStage): FilemanagerConfig =>
  cognitoStatusPageAppClientIdParameterName: cognitoStatusPageAppClientIdParameterName,
  cognitoUserPoolIdParameterName: cognitoUserPoolIdParameterName,
  inventorySourceBuckets: ['filemanager-inventory-test'],
- eventSourceBuckets: [oncoanalyserBucket[stage]],
+ eventSourceBuckets: [oncoanalyserBucket[stage], icav2PipelineCacheBucket[stage]],
+ fileManagerIngestRoleName: fileManagerIngestRoleName,
  };
 };

lib/workload/components/named-lambda-role/index.ts

@@ -0,0 +1,29 @@
+import { Construct } from 'constructs';
+import { Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam';
+
+/**
+ * Props for the named lambda role construct.
+ */
+export type NamedLambdaRoleProps = {
+ /**
+ * The name of the role, automatically generated if not specified.
+ */
+ name?: string;
+ /**
+ * Description for the role.
+ */
+ description?: string;
+};
+
+/**
+ * A construct which represents a named role that a Lambda function can assume.
+ */
+export class NamedLambdaRole extends Role {
+ constructor(scope: Construct, id: string, props?: NamedLambdaRoleProps) {
+ super(scope, id, {
+ assumedBy: new ServicePrincipal('lambda.amazonaws.com'),
+ description: props?.description ?? 'Lambda execution role',
+ roleName: props?.name,
+ });
+ }
+}

lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/function.ts

@@ -2,14 +2,15 @@ import { Construct } from 'constructs';
 import { Duration } from 'aws-cdk-lib';
 import { ISecurityGroup, IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2';
 import { Architecture, Version } from 'aws-cdk-lib/aws-lambda';
-import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam';
+import { ManagedPolicy, PolicyStatement, Role } from 'aws-cdk-lib/aws-iam';
 import { RustFunction } from 'cargo-lambda-cdk';
 import path from 'path';
 import { exec } from 'cargo-lambda-cdk/lib/util';
 import { randomUUID } from 'node:crypto';
 import { print } from 'aws-cdk/lib/logging';
 import { PostgresManagerStack } from '../../../../postgres-manager/deploy/stack';
 import { FILEMANAGER_SERVICE_NAME } from '../../stack';
+import { NamedLambdaRole } from '../../../../../../components/named-lambda-role';
 
 /**
  * Properties for the database.
@@ -27,7 +28,7 @@ export type DatabaseProps = {
  * The database security group.
  */
  readonly securityGroup: ISecurityGroup;
-}
+};
 
 /**
  * Props for a Rust function without the package.
@@ -41,6 +42,10 @@ export type FunctionPropsNoPackage = {
  * RUST_LOG string, defaults to trace on local crates and info everywhere else.
  */
  readonly rustLog?: string;
+ /**
+ * The role which the Function assumes.
+ */
+ readonly role?: Role;
  /**
  * Vpc for the function.
  */
@@ -50,16 +55,17 @@ export type FunctionPropsNoPackage = {
 /**
  * Props for the Rust function.
  */
-export type FunctionProps = FunctionPropsNoPackage & DatabaseProps & {
- /**
- * The package to build for this function.
- */
- readonly package: string;
- /**
- * Name of the Lambda function resource.
- */
- readonly functionName?: string;
-};
+export type FunctionProps = FunctionPropsNoPackage &
+ DatabaseProps & {
+ /**
+ * The package to build for this function.
+ */
+ readonly package: string;
+ /**
+ * Name of the Lambda function resource.
+ */
+ readonly functionName?: string;
+ };
 
 /**
  * A construct for a Rust Lambda function.
@@ -72,14 +78,13 @@ export class Function extends Construct {
  super(scope, id);
 
  // Lambda role needs SQS execution role.
- this._role = new Role(this, 'Role', {
- assumedBy: new ServicePrincipal('lambda.amazonaws.com'),
- description: 'Lambda execution role for ' + id,
- });
+ this._role = props.role ?? new NamedLambdaRole(this, 'Role');
  // Lambda needs VPC access if it is created in a VPC.
  this.addAwsManagedPolicy('service-role/AWSLambdaVPCAccessExecutionRole');
  // Using RDS IAM credentials, so we add the managed policy created by the postgres manager.
- this.addCustomerManagedPolicy(PostgresManagerStack.formatRdsPolicyName(FILEMANAGER_SERVICE_NAME));
+ this.addCustomerManagedPolicy(
+ PostgresManagerStack.formatRdsPolicyName(FILEMANAGER_SERVICE_NAME)
+ );
 
  // Lambda needs to be able to reach out to access S3, security manager, etc.
  // Could this use an endpoint instead?
@@ -89,7 +94,7 @@ export class Function extends Construct {
  description: 'Security group that allows a filemanager Lambda function to egress out.',
  });
 
- const manifestPath =  path.join(__dirname, '..', '..', '..');
+ const manifestPath = path.join(__dirname, '..', '..', '..');
  const uuid = randomUUID();
  const localDatabaseUrl = this.resolveLocalDatabaseUrl(uuid, manifestPath);
 
@@ -104,7 +109,7 @@ export class Function extends Construct {
  },
  dockerOptions: {
  network: 'host',
- }
+ },
  },
  memorySize: 128,
  timeout: Duration.seconds(28),
@@ -124,7 +129,7 @@ export class Function extends Construct {
  securityGroups: [
  securityGroup,
  // Allow access to database.
- props.securityGroup
+ props.securityGroup,
  ],
  functionName: props.functionName,
  });
@@ -138,12 +143,8 @@ export class Function extends Construct {
  private resolveLocalDatabaseUrl(uuid: string, manifestPath: string) {
  const execMake = (commandArgs: string) => {
  print(`running \`make ${commandArgs}\``);
- return exec(
- 'make',
- [commandArgs],
- { cwd: manifestPath, shell: true }
- );
- }
+ return exec('make', [commandArgs], { cwd: manifestPath, shell: true });
+ };
 
  print('trying to find running filemanager container');
 
@@ -187,19 +188,21 @@ export class Function extends Construct {
  * Add a policy statement to this function's role.
  */
  addToPolicy(policyStatement: PolicyStatement) {
- this._role.addToPolicy(policyStatement)
+ this._role.addToPolicy(policyStatement);
  }
 
  /**
  * Add policies for 's3:List*' and 's3:Get*' on the buckets to this function's role.
  */
  addPoliciesForBuckets(buckets: string[]) {
  buckets.map((bucket) => {
- this.addToPolicy(new PolicyStatement({
- actions: ['s3:ListBucket', 's3:GetObject'],
- resources: [`arn:aws:s3:::${bucket}`, `arn:aws:s3:::${bucket}/*`],
- }));
- })
+ this.addToPolicy(
+ new PolicyStatement({
+ actions: ['s3:ListBucket', 's3:GetObject'],
+ resources: [`arn:aws:s3:::${bucket}`, `arn:aws:s3:::${bucket}/*`],
+ })
+ );
+ });
  }
 
  /**

lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/ingest.ts

@@ -1,7 +1,6 @@
 import { Construct } from 'constructs';
 import { IQueue } from 'aws-cdk-lib/aws-sqs';
 import * as fn from './function';
-import { PolicyStatement } from 'aws-cdk-lib/aws-iam';
 import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources';
 import { DatabaseProps } from './function';
 
@@ -18,7 +17,7 @@ export type EventSourceProps = {
  * 's3:List*' and 's3:Get*'.
  */
  readonly buckets: string[];
-}
+};
 
 /**
  * Props for the ingest function.

lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/inventory.ts

@@ -14,19 +14,25 @@ export type InventoryFunctionConfig = {
  * 's3:List*' and 's3:Get*'.
  */
  readonly buckets: string[];
-}
+};
 
 /**
  * Props for the inventory function.
  */
-export type InventoryFunctionProps = fn.FunctionPropsNoPackage & DatabaseProps & InventoryFunctionConfig;
+export type InventoryFunctionProps = fn.FunctionPropsNoPackage &
+ DatabaseProps &
+ InventoryFunctionConfig;
 
 /**
  * A construct for the Lambda inventory function.
  */
 export class InventoryFunction extends fn.Function {
  constructor(scope: Construct, id: string, props: InventoryFunctionProps) {
- super(scope, id, { package: 'filemanager-inventory-lambda', ...props, functionName: INVENTORY_FUNCTION_NAME });
+ super(scope, id, {
+ package: 'filemanager-inventory-lambda',
+ ...props,
+ functionName: INVENTORY_FUNCTION_NAME,
+ });
 
  this.addPoliciesForBuckets(props.buckets);
  }

lib/workload/stateless/stacks/filemanager/deploy/stack.ts

@@ -12,6 +12,7 @@ import { IQueue, Queue } from 'aws-cdk-lib/aws-sqs';
 import { HttpMethod, HttpRoute, HttpRouteKey } from 'aws-cdk-lib/aws-apigatewayv2';
 import { HttpLambdaIntegration } from 'aws-cdk-lib/aws-apigatewayv2-integrations';
 import { InventoryFunction } from './constructs/functions/inventory';
+import { NamedLambdaRole } from '../../../../components/named-lambda-role';
 
 export const FILEMANAGER_SERVICE_NAME = 'filemanager';
 
@@ -29,7 +30,8 @@ export type FilemanagerConfig = Omit<DatabaseProps, 'host' | 'securityGroup'> &
  cognitoUserPoolIdParameterName: string;
  cognitoPortalAppClientIdParameterName: string;
  cognitoStatusPageAppClientIdParameterName: string;
-}
+ fileManagerIngestRoleName: string;
+};
 
 /**
  * Props for the filemanager stack.
@@ -44,7 +46,7 @@ export class Filemanager extends Stack {
  private readonly host: string;
  private readonly securityGroup: ISecurityGroup;
  private readonly queue: IQueue;
- 
+
  constructor(scope: Construct, id: string, props: FilemanagerProps) {
  super(scope, id, props);
 
@@ -93,16 +95,22 @@ export class Filemanager extends Stack {
  this.createInventoryFunction(props);
  }
 
- /// Lambda function definitions and surrounding infra
- // Ingest function
+ private createIngestRole(name: string) {
+ return new NamedLambdaRole(this, 'IngestFunctionRole', { name })
+ }
+
+ /**
+ * Lambda function definitions and surrounding infrastructure.
+ */
  private createIngestFunction(props: FilemanagerProps) {
  return new IngestFunction(this, 'IngestFunction', {
  vpc: this.vpc,
  host: this.host,
  securityGroup: this.securityGroup,
  eventSources: [this.queue],
  buckets: props.eventSourceBuckets,
- ...props
+ role: this.createIngestRole(props.fileManagerIngestRoleName),
+ ...props,
  });
  }
 
@@ -119,13 +127,15 @@ export class Filemanager extends Stack {
  });
  }
 
- // Query function and API Gateway fronting the function
+ /**
+ * Query function and API Gateway fronting the function.
+ */
  private createQueryFunction(props: FilemanagerProps) {
  let objectsQueryLambda = new QueryFunction(this, 'ObjectsQueryFunction', {
  vpc: this.vpc,
  host: this.host,
  securityGroup: this.securityGroup,
- ...props
+ ...props,
  });
 
  const ApiGateway = new ApiGatewayConstruct(this, 'ApiGateway', {
@@ -137,7 +147,8 @@ export class Filemanager extends Stack {
 
  const apiIntegration = new HttpLambdaIntegration('ApiIntegration', objectsQueryLambda.function);
 
- new HttpRoute(this, 'HttpRoute', { // FIXME: Should not be just proxy but objects/{:id}
+ new HttpRoute(this, 'HttpRoute', {
+ // FIXME: Should not be just proxy but objects/{:id}
  httpApi: httpApi,
  integration: apiIntegration,
  routeKey: HttpRouteKey.with('/{proxy+}', HttpMethod.ANY),

lib/workload/stateless/stacks/filemanager/filemanager/src/error.rs

@@ -23,8 +23,6 @@ pub enum Error {
  MissingEnvironmentVariable(String),
  #[error("Invalid environment variable: `{0}`")]
  InvalidEnvironmentVariable(String),
- #[error("S3 error: `{0}`")]
- S3Error(String),
  #[error("credential generator error: `{0}`")]
  CredentialGeneratorError(String),
  #[error("S3 inventory error: `{0}`")]