Feat (MM): Custom CSV Importer for MM

lib/workload/stateless/stacks/metadata-manager/README.md

@@ -61,7 +61,7 @@ on the model of the record.
 
 ## How things work
 
-### How Syncing The Data Works
+### How Tracking Sheet Syncing Works
 
 In the near future, we might introduce different ways to load data into the application. For the time being, we are
 loading data

lib/workload/stateless/stacks/metadata-manager/app/management/commands/load_from_csv.py

@@ -0,0 +1,22 @@
+import logging
+from django.core.management import BaseCommand
+from libumccr import libjson
+
+from handler.load_custom_metadata_csv import handler
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+
+class Command(BaseCommand):
+ help = "Trigger lambda handler for to sync metadata from csv url"
+
+ def handle(self, *args, **options):
+ event = {
+ "url" :"SOME_URL",
+ }
+
+ print(f"Trigger lambda handler for sync tracking sheet. Event {libjson.dumps(event)}")
+ result = handler(event, {})
+
+ print(f"result: {libjson.dumps(result)}")

lib/workload/stateless/stacks/metadata-manager/app/migrations/0001_initial.py

@@ -81,14 +81,106 @@ class Migration(migrations.Migration):
  },
  ),
  migrations.CreateModel(
- name='Subject',
+ name='HistoricalContact',
  fields=[
  ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])),
  ('subject_id', models.CharField(blank=True, null=True, unique=True)),
  ],
  options={
- 'abstract': False,
+ 'verbose_name': 'historical contact',
+ 'verbose_name_plural': 'historical contacts',
+ 'ordering': ('-history_date', '-history_id'),
+ 'get_latest_by': ('history_date', 'history_id'),
+ },
+ bases=(simple_history.models.HistoricalChanges, models.Model),
+ ),
+ migrations.CreateModel(
+ name='HistoricalIndividual',
+ fields=[
+ ('orcabus_id', models.CharField(db_index=True, editable=False, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])),
+ ('individual_id', models.CharField(blank=True, db_index=True, null=True)),
+ ('source', models.CharField(blank=True, null=True)),
+ ('history_id', models.AutoField(primary_key=True, serialize=False)),
+ ('history_date', models.DateTimeField(db_index=True)),
+ ('history_change_reason', models.CharField(max_length=100, null=True)),
+ ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
+ ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
+ ],
+ options={
+ 'verbose_name': 'historical individual',
+ 'verbose_name_plural': 'historical individuals',
+ 'ordering': ('-history_date', '-history_id'),
+ 'get_latest_by': ('history_date', 'history_id'),
+ },
+ bases=(simple_history.models.HistoricalChanges, models.Model),
+ ),
+ migrations.CreateModel(
+ name='HistoricalLibrary',
+ fields=[
+ ('orcabus_id', models.CharField(db_index=True, editable=False, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])),
+ ('library_id', models.CharField(blank=True, db_index=True, null=True)),
+ ('phenotype', models.CharField(blank=True, choices=[('normal', 'Normal'), ('tumor', 'Tumor'), ('negative-control', 'Negative Control')], null=True)),
+ ('workflow', models.CharField(blank=True, choices=[('clinical', 'Clinical'), ('research', 'Research'), ('qc', 'Qc'), ('control', 'Control'), ('bcl', 'Bcl'), ('manual', 'Manual')], null=True)),
+ ('quality', models.CharField(blank=True, choices=[('very-poor', 'VeryPoor'), ('poor', 'Poor'), ('good', 'Good'), ('borderline', 'Borderline')], null=True)),
+ ('type', models.CharField(blank=True, choices=[('10X', 'Ten X'), ('BiModal', 'Bimodal'), ('ctDNA', 'Ct Dna'), ('ctTSO', 'Ct Tso'), ('exome', 'Exome'), ('MeDIP', 'Me Dip'), ('Metagenm', 'Metagenm'), ('MethylSeq', 'Methyl Seq'), ('TSO-DNA', 'TSO_DNA'), ('TSO-RNA', 'TSO_RNA'), ('WGS', 'Wgs'), ('WTS', 'Wts'), ('other', 'Other')], null=True)),
+ ('assay', models.CharField(blank=True, null=True)),
+ ('coverage', models.FloatField(blank=True, null=True)),
+ ('history_id', models.AutoField(primary_key=True, serialize=False)),
+ ('history_date', models.DateTimeField(db_index=True)),
+ ('history_change_reason', models.CharField(max_length=100, null=True)),
+ ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
+ ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
+ ('sample', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='app.sample')),
+ ],
+ options={
+ 'verbose_name': 'historical library',
+ 'verbose_name_plural': 'historical librarys',
+ 'ordering': ('-history_date', '-history_id'),
+ 'get_latest_by': ('history_date', 'history_id'),
  },
+ bases=(simple_history.models.HistoricalChanges, models.Model),
+ ),
+ migrations.CreateModel(
+ name='HistoricalProject',
+ fields=[
+ ('orcabus_id', models.CharField(db_index=True, editable=False, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])),
+ ('project_id', models.CharField(blank=True, db_index=True, null=True)),
+ ('name', models.CharField(blank=True, null=True)),
+ ('description', models.CharField(blank=True, null=True)),
+ ('history_id', models.AutoField(primary_key=True, serialize=False)),
+ ('history_date', models.DateTimeField(db_index=True)),
+ ('history_change_reason', models.CharField(max_length=100, null=True)),
+ ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
+ ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
+ ],
+ options={
+ 'verbose_name': 'historical project',
+ 'verbose_name_plural': 'historical projects',
+ 'ordering': ('-history_date', '-history_id'),
+ 'get_latest_by': ('history_date', 'history_id'),
+ },
+ bases=(simple_history.models.HistoricalChanges, models.Model),
+ ),
+ migrations.CreateModel(
+ name='HistoricalSample',
+ fields=[
+ ('orcabus_id', models.CharField(db_index=True, editable=False, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])),
+ ('sample_id', models.CharField(blank=True, db_index=True, null=True)),
+ ('external_sample_id', models.CharField(blank=True, null=True)),
+ ('source', models.CharField(blank=True, choices=[('ascites', 'Ascites'), ('blood', 'Blood'), ('bone-marrow', 'BoneMarrow'), ('buccal', 'Buccal'), ('cell-line', 'Cell_line'), ('cfDNA', 'Cfdna'), ('cyst-fluid', 'Cyst Fluid'), ('DNA', 'Dna'), ('eyebrow-hair', 'Eyebrow Hair'), ('FFPE', 'Ffpe'), ('FNA', 'Fna'), ('OCT', 'Oct'), ('organoid', 'Organoid'), ('PDX-tissue', 'Pdx Tissue'), ('plasma-serum', 'Plasma Serum'), ('RNA', 'Rna'), ('tissue', 'Tissue'), ('skin', 'Skin'), ('water', 'Water')], null=True)),
+ ('history_id', models.AutoField(primary_key=True, serialize=False)),
+ ('history_date', models.DateTimeField(db_index=True)),
+ ('history_change_reason', models.CharField(max_length=100, null=True)),
+ ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
+ ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
+ ],
+ options={
+ 'verbose_name': 'historical sample',
+ 'verbose_name_plural': 'historical samples',
+ 'ordering': ('-history_date', '-history_id'),
+ 'get_latest_by': ('history_date', 'history_id'),
+ },
+ bases=(simple_history.models.HistoricalChanges, models.Model),
  ),
  migrations.CreateModel(
  name='HistoricalContact',
@@ -228,6 +320,31 @@ class Migration(migrations.Migration):
  ('project', models.ForeignKey(db_column='project_orcabus_id', on_delete=django.db.models.deletion.CASCADE, to='app.project')),
  ],
  ),
+ migrations.CreateModel(
+ name='HistoricalLibrary_project_set',
+ fields=[
+ ('id', models.BigIntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')),
+ ('m2m_history_id', models.AutoField(primary_key=True, serialize=False)),
+ ('history', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='app.historicallibrary')),
+ ('library', models.ForeignKey(blank=True, db_constraint=False, db_tablespace='', null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='app.library')),
+ ('project', models.ForeignKey(blank=True, db_constraint=False, db_tablespace='', null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='app.project')),
+ ],
+ options={
+ 'verbose_name': 'HistoricalLibrary_project_set',
+ },
+ bases=(simple_history.models.HistoricalChanges, models.Model),
+ ),
+ migrations.CreateModel(
+ name='Subject',
+ fields=[
+ ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])),
+ ('subject_id', models.CharField(blank=True, null=True, unique=True)),
+ ('individual_set', models.ManyToManyField(blank=True, related_name='subject_set', to='app.individual')),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
  migrations.AddField(
  model_name='library',
  name='project_set',

lib/workload/stateless/stacks/metadata-manager/deploy/README.md

@@ -58,3 +58,35 @@ aws lambda invoke \
  --cli-binary-format raw-in-base64-out \
  res.json
 ```
+
+### CustomCsvLambda
+
+- Load tracking sheet data from csv presigned url
+
+To manually trigger the sync, the lambda ARN is stored in the SSM Parameter Store named
+`/orcabus/metadata-manager/load-custom-csv-lambda-arn`.
+
+To query in a local terminal
+
+```sh
+load_custom_csv_lambda_arn=$(aws ssm get-parameter --name '/orcabus/metadata-manager/load-custom-csv-lambda-arn' --with-decryption | jq -r .Parameter.Value)
+```
+
+The lambda handler will accept a json which only accepts a single key `url` which is the presigned url of the csv file.
+
+```json
+{
+ "url": "https://example.com/csv"
+}
+```
+
+Invoking lambda cmd:
+
+```sh
+aws lambda invoke \
+ --function-name load_custom_csv_lambda_arn \
+ --invocation-type Event \
+ --payload '{ "url": "https://the.url.csv" }' \
+ --cli-binary-format raw-in-base64-out \
+ res.json
+```

lib/workload/stateless/stacks/metadata-manager/deploy/construct/lambda-load-custom-csv/index.ts

@@ -0,0 +1,54 @@
+import path from 'path';
+import { Construct } from 'constructs';
+import { Duration } from 'aws-cdk-lib';
+import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha';
+import { ISecret } from 'aws-cdk-lib/aws-secretsmanager';
+import { StringParameter } from 'aws-cdk-lib/aws-ssm';
+import {
+ DockerImageFunction,
+ DockerImageFunctionProps,
+ DockerImageCode,
+} from 'aws-cdk-lib/aws-lambda';
+
+type LambdaProps = {
+ /**
+ * The basic common lambda properties that it should inherit from
+ */
+ basicLambdaConfig: Partial<DockerImageFunctionProps>;
+ /**
+ * The secret for the db connection where the lambda will need access to
+ */
+ dbConnectionSecret: ISecret;
+};
+
+export class LambdaLoadCustomCSVConstruct extends Construct {
+ private readonly lambda: PythonFunction;
+
+ constructor(scope: Construct, id: string, lambdaProps: LambdaProps) {
+ super(scope, id);
+
+ this.lambda = new DockerImageFunction(this, 'LoadCustomCSVLambda', {
+ environment: {
+ ...lambdaProps.basicLambdaConfig.environment,
+ },
+ securityGroups: lambdaProps.basicLambdaConfig.securityGroups,
+ vpc: lambdaProps.basicLambdaConfig.vpc,
+ vpcSubnets: lambdaProps.basicLambdaConfig.vpcSubnets,
+ architecture: lambdaProps.basicLambdaConfig.architecture,
+ code: DockerImageCode.fromImageAsset(path.join(__dirname, '../../../'), {
+ file: 'deploy/construct/lambda-load-custom-csv/lambda.Dockerfile',
+ }),
+ timeout: Duration.minutes(15),
+ memorySize: 4096,
+ });
+
+ lambdaProps.dbConnectionSecret.grantRead(this.lambda);
+
+ // We need to store this lambda ARN somewhere so that we could refer when need to load this manually
+ const ssmParameter = new StringParameter(this, 'LoadCustomCSVLambdaArnParameterStore', {
+ parameterName: '/orcabus/metadata-manager/load-custom-csv-lambda-arn',
+ description: 'The ARN of the lambda that load metadata from a presigned URL CSV file',
+ stringValue: this.lambda.functionArn,
+ });
+ }
+}

lib/workload/stateless/stacks/metadata-manager/deploy/construct/lambda-load-custom-csv/lambda.Dockerfile

@@ -0,0 +1,12 @@
+FROM public.ecr.aws/lambda/python:3.12
+
+WORKDIR ${LAMBDA_TASK_ROOT}
+
+# COPY all files
+COPY . .
+
+# Install the specified packages
+RUN pip install -r deps/requirements-full.txt
+
+# Specify handler
+CMD [ "handler.load_custom_metadata_csv.handler" ]

lib/workload/stateless/stacks/metadata-manager/handler/load_custom_metadata_csv.py

@@ -0,0 +1,35 @@
+import django
+import os
+import logging
+
+from libumccr import libjson
+
+from proc.service.utils import sanitize_lab_metadata_df, warn_drop_duplicated_library
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'app.settings.base')
+django.setup()
+
+from proc.service.load_csv_srv import load_metadata_csv, download_csv_to_pandas
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+
+def handler(event, _context):
+ logger.info(f'event: {libjson.dumps(event)}')
+
+ csv_url = event.get('url', None)
+ if csv_url is None:
+ raise ValueError("URL is required")
+
+ csv_df = download_csv_to_pandas(csv_url)
+ sanitize_df = sanitize_lab_metadata_df(csv_df)
+ duplicate_clean_df = warn_drop_duplicated_library(sanitize_df)
+ result = load_metadata_csv(duplicate_clean_df)
+
+ logger.info(f'persist report: {libjson.dumps(result)}')
+ return result
+
+
+if __name__ == '__main__':
+ handler({}, {})

lib/workload/stateless/stacks/metadata-manager/handler/sync_tracking_sheet.py

@@ -7,8 +7,8 @@
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'app.settings.base')
 django.setup()
 
-from proc.service.tracking_sheet_srv import download_tracking_sheet, sanitize_lab_metadata_df, persist_lab_metadata, \
-  warn_drop_duplicated_library
+from proc.service.tracking_sheet_srv import download_tracking_sheet, persist_lab_metadata
+from proc.service.utils import sanitize_lab_metadata_df, warn_drop_duplicated_library
 
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)