diff --git a/lib/workload/components/dynamodb-icav2-ready-event-handler-sfn/icav2_launch_pipeline_lambda_py/requirements.txt b/lib/workload/components/dynamodb-icav2-ready-event-handler-sfn/icav2_launch_pipeline_lambda_py/requirements.txt index fb4ae1e6a..ee3438956 100644 --- a/lib/workload/components/dynamodb-icav2-ready-event-handler-sfn/icav2_launch_pipeline_lambda_py/requirements.txt +++ b/lib/workload/components/dynamodb-icav2-ready-event-handler-sfn/icav2_launch_pipeline_lambda_py/requirements.txt @@ -1,2 +1,2 @@ -wrapica>=2.27.0.post20240714154223,<2.28.0 +wrapica>=2.27.1.post20240803214221,<2.28.0 boto3>=1.28 diff --git a/lib/workload/components/icav2-copy-files/check_or_launch_job_lambda_py/check_or_launch_job_lambda.py b/lib/workload/components/icav2-copy-files/check_or_launch_job_lambda_py/check_or_launch_job_lambda.py index 112937e72..ca3e742f2 100644 --- a/lib/workload/components/icav2-copy-files/check_or_launch_job_lambda_py/check_or_launch_job_lambda.py +++ b/lib/workload/components/icav2-copy-files/check_or_launch_job_lambda_py/check_or_launch_job_lambda.py @@ -36,7 +36,7 @@ # Wrapica imports from wrapica.job import get_job -from wrapica.project_data import convert_icav2_uri_to_project_data_obj, project_data_copy_batch_handler +from wrapica.project_data import convert_uri_to_project_data_obj, project_data_copy_batch_handler if typing.TYPE_CHECKING: from mypy_boto3_ssm import SSMClient @@ -112,7 +112,7 @@ def submit_copy_job(dest_uri: str, source_uris: List[str]) -> str: # Rerun copy batch process source_data_ids = list( map( - lambda source_uri_iter: convert_icav2_uri_to_project_data_obj( + lambda source_uri_iter: convert_uri_to_project_data_obj( source_uri_iter ).data.id, source_uris diff --git a/lib/workload/components/icav2-copy-files/check_or_launch_job_lambda_py/requirements.txt b/lib/workload/components/icav2-copy-files/check_or_launch_job_lambda_py/requirements.txt index 49185ffc6..f44e15d2e 100644 --- a/lib/workload/components/icav2-copy-files/check_or_launch_job_lambda_py/requirements.txt +++ b/lib/workload/components/icav2-copy-files/check_or_launch_job_lambda_py/requirements.txt @@ -1,2 +1,2 @@ -wrapica==2.27.0 +wrapica==2.27.1 boto3>=1.34 \ No newline at end of file diff --git a/lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/lambdas/set_outputs_json_py/requirements.txt b/lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/lambdas/set_outputs_json_py/requirements.txt index 389038c8c..993360ac4 100644 --- a/lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/lambdas/set_outputs_json_py/requirements.txt +++ b/lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/lambdas/set_outputs_json_py/requirements.txt @@ -1,2 +1,2 @@ -wrapica==2.27.0 +wrapica==2.27.1 boto3>=1.34.0 \ No newline at end of file diff --git a/lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py b/lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py index f66f7e2c2..c9773c6ce 100644 --- a/lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py +++ b/lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py @@ -33,10 +33,10 @@ from mypy_boto3_secretsmanager.client import SecretsManagerClient # ICA imports -from wrapica.enums import DataType +from wrapica.enums import DataType, UriType from wrapica.libica_models import ProjectData from wrapica.project_data import ( - convert_icav2_uri_to_project_data_obj, convert_project_data_obj_to_icav2_uri, + convert_uri_to_project_data_obj, convert_project_data_obj_to_uri, list_project_data_non_recursively ) @@ -78,7 +78,7 @@ def handler(events, context): analysis_uri = events.get("analysis_output_uri") # Convert analysis uri to project folder object - analysis_project_data_obj = convert_icav2_uri_to_project_data_obj(analysis_uri) + analysis_project_data_obj = convert_uri_to_project_data_obj(analysis_uri) # Analysis list analysis_top_level_data_list = list_project_data_non_recursively( @@ -128,9 +128,9 @@ def handler(events, context): ) return { - "interop_output_dir": convert_project_data_obj_to_icav2_uri(interop_data_obj), - "multiqc_html_report": convert_project_data_obj_to_icav2_uri(multiqc_html_data_obj), - "multiqc_output_dir": convert_project_data_obj_to_icav2_uri(multiqc_data_obj) + "interop_output_dir": convert_project_data_obj_to_uri(interop_data_obj, uri_type=UriType.S3), + "multiqc_html_report": convert_project_data_obj_to_uri(multiqc_html_data_obj, uri_type=UriType.S3), + "multiqc_output_dir": convert_project_data_obj_to_uri(multiqc_data_obj, uri_type=UriType.S3) } diff --git a/lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/layers/pyproject.toml b/lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/layers/pyproject.toml index cfd46221f..34491234d 100644 --- a/lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/layers/pyproject.toml +++ b/lib/workload/stateless/stacks/bclconvert-interop-qc-pipeline-manager/layers/pyproject.toml @@ -19,7 +19,7 @@ boto3 = "^1.28" botocore = "^1.31" aws_requests_auth = "^0.4.3" v2_samplesheet_maker = "^4.2.4" -wrapica = "2.27.0" +wrapica = "2.27.1" uuid6 = "^2024.1.12" [tool.poetry.group.dev] diff --git a/lib/workload/stateless/stacks/bclconvert-manager/deps/requirements.txt b/lib/workload/stateless/stacks/bclconvert-manager/deps/requirements.txt index 732678d46..522f19438 100644 --- a/lib/workload/stateless/stacks/bclconvert-manager/deps/requirements.txt +++ b/lib/workload/stateless/stacks/bclconvert-manager/deps/requirements.txt @@ -1,6 +1,6 @@ uuid6>=2024.1.12 v2-samplesheet-maker>=4.2.4.post20240528161300 -wrapica==2.27.0 +wrapica==2.27.1 xmltodict==0.13.0 mypy_boto3_ssm>=1.16.0 mypy_boto3_secretsmanager>=1.16.0 \ No newline at end of file diff --git a/lib/workload/stateless/stacks/bs-runs-upload-manager/layers/poetry.lock b/lib/workload/stateless/stacks/bs-runs-upload-manager/layers/poetry.lock index 9e0e22eae..c3f198786 100644 --- a/lib/workload/stateless/stacks/bs-runs-upload-manager/layers/poetry.lock +++ b/lib/workload/stateless/stacks/bs-runs-upload-manager/layers/poetry.lock @@ -48,17 +48,17 @@ lxml = ["lxml"] [[package]] name = "boto3" -version = "1.34.143" +version = "1.34.153" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.143-py3-none-any.whl", hash = "sha256:0d16832f23e6bd3ae94e35ea8e625529850bfad9baccd426de96ad8f445d8e03"}, - {file = "boto3-1.34.143.tar.gz", hash = "sha256:b590ce80c65149194def43ebf0ea1cf0533945502507837389a8d22e3ecbcf05"}, + {file = "boto3-1.34.153-py3-none-any.whl", hash = "sha256:ff9af9206fb235605cb65922f9090fe60f78ea89b4adc463f8f6391b30a3df03"}, + {file = "boto3-1.34.153.tar.gz", hash = "sha256:db9f2ac64582d847003a71720cd28dfffff61e2882e5d3db8e0c1fe1902ebb5b"}, ] [package.dependencies] -botocore = ">=1.34.143,<1.35.0" +botocore = ">=1.34.153,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -67,13 +67,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.143" +version = "1.34.153" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.143-py3-none-any.whl", hash = "sha256:094aea179e8aaa1bc957ad49cc27d93b189dd3a1f3075d8b0ca7c445a2a88430"}, - {file = "botocore-1.34.143.tar.gz", hash = "sha256:059f032ec05733a836e04e869c5a15534420102f93116f3bc9a5b759b0651caf"}, + {file = "botocore-1.34.153-py3-none-any.whl", hash = "sha256:9fc2ad40be8c103ab9bfcb48b97b117d299d0b3a542cdd30134ee2935bee827a"}, + {file = "botocore-1.34.153.tar.gz", hash = "sha256:1634a00f996cfff67f0fd4d0ddc436bc3318b2202dfd82ad0bc11c7169694092"}, ] [package.dependencies] @@ -343,13 +343,13 @@ files = [ [[package]] name = "libica" -version = "2.4.0" +version = "2.5.0" description = "Python SDK for Illumina Connected Analytics" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "libica-2.4.0-py3-none-any.whl", hash = "sha256:c80927a1f4aa3f60b2c9faf6e8c22f72000bfdf096429d065ebabfce9928120a"}, - {file = "libica-2.4.0.tar.gz", hash = "sha256:436098a5d5c5e843428e9b1592db33bcd2dbc09b47ed83b9ca1d533f65803189"}, + {file = "libica-2.5.0-py3-none-any.whl", hash = "sha256:8998a3bcf80cd3fcba2d20dca44a83795d98fae021b5c2148add59809101f1da"}, + {file = "libica-2.5.0.tar.gz", hash = "sha256:d1175a40f79d1b7657629da03e6429d5e98a149912d015edde746148420d9a9e"}, ] [package.dependencies] @@ -455,13 +455,13 @@ files = [ [[package]] name = "mypy-boto3-secretsmanager" -version = "1.34.128" -description = "Type annotations for boto3.SecretsManager 1.34.128 service generated with mypy-boto3-builder 7.24.0" +version = "1.34.145" +description = "Type annotations for boto3.SecretsManager 1.34.145 service generated with mypy-boto3-builder 7.25.0" optional = false python-versions = ">=3.8" files = [ - {file = "mypy_boto3_secretsmanager-1.34.128-py3-none-any.whl", hash = "sha256:7ce9815d116fa1749971691355b1e1c8f462d46e7eaa9d84133b8db96dd3515f"}, - {file = "mypy_boto3_secretsmanager-1.34.128.tar.gz", hash = "sha256:ae2b398efa1a32214c3eddb6901efa67cfc24a893b113f549a06bb70bb43b402"}, + {file = "mypy_boto3_secretsmanager-1.34.145-py3-none-any.whl", hash = "sha256:986511caa6626edfed7eb11b63c929801e9468c58e15927dc6fc0339c4eb34cb"}, + {file = "mypy_boto3_secretsmanager-1.34.145.tar.gz", hash = "sha256:e5a82c05cce68168a3709e5f0d35066cf250961db1d8670f0111da66206814c7"}, ] [[package]] @@ -477,13 +477,13 @@ files = [ [[package]] name = "mypy-boto3-stepfunctions" -version = "1.34.137" -description = "Type annotations for boto3.SFN 1.34.137 service generated with mypy-boto3-builder 7.25.0" +version = "1.34.149" +description = "Type annotations for boto3.SFN 1.34.149 service generated with mypy-boto3-builder 7.25.0" optional = false python-versions = ">=3.8" files = [ - {file = "mypy_boto3_stepfunctions-1.34.137-py3-none-any.whl", hash = "sha256:c78a6abaaf6b3c9eaee183b4cc59cb0d6aa838e2f20698be3d6ad46788a64945"}, - {file = "mypy_boto3_stepfunctions-1.34.137.tar.gz", hash = "sha256:c568275ab1b2d9600352ec26e9915335526d39fc6e7192a664428118f6c7799f"}, + {file = "mypy_boto3_stepfunctions-1.34.149-py3-none-any.whl", hash = "sha256:dbe3c90af541178826f0992c58ef3699d5f1f42d0f51ff5b761d4c8036768726"}, + {file = "mypy_boto3_stepfunctions-1.34.149.tar.gz", hash = "sha256:155f6d7aa4b61799166e5beae85386cf5393612e4cbabe7a8426225325e3c12b"}, ] [[package]] @@ -807,19 +807,19 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pyjwt" -version = "2.8.0" +version = "2.9.0" description = "JSON Web Token implementation in Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, - {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, + {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, + {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, ] [package.extras] crypto = ["cryptography (>=3.4.0)"] -dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] -docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] @@ -1091,48 +1091,53 @@ crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] [[package]] name = "schema-salad" -version = "8.6.20240710082410" +version = "8.7.20240718183047" description = "Schema Annotations for Linked Avro Data (SALAD)" optional = false python-versions = "<3.13,>=3.8" files = [ - {file = "schema_salad-8.6.20240710082410-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:76e916baa5b73daddef8ae3369cb9bc18dc0ddd1a84670e7c996e7ceb04820b3"}, - {file = "schema_salad-8.6.20240710082410-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20e00917d53e08596a122734b1f3700058b9d863f08b86bfc4639af004b6ae1a"}, - {file = "schema_salad-8.6.20240710082410-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23fec1d39cf1dd1b7217974af5ffff5efd6d56aaabf3432d8eb28c215f4629dd"}, - {file = "schema_salad-8.6.20240710082410-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e006e391b6dcc7c5c1ce18de74ff4ba7bf233f1ad745d83a4ce86272a2e75692"}, - {file = "schema_salad-8.6.20240710082410-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:07e9df7acdb85ce47a19dae55e659793e99efcd914ddc3198f219ed8883da630"}, - {file = "schema_salad-8.6.20240710082410-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:827d78b91e17803224c77981df0afa64573f1b3f887804cafb2b82bf30a92eb0"}, - {file = "schema_salad-8.6.20240710082410-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:14e5eefba4fa3658ae05f683c541c6a99f86d93bfcc4c63363add7352719e52e"}, - {file = "schema_salad-8.6.20240710082410-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c865a160c66e09bed2a77551ebe569735945b249e404ca37c49db43d5285f781"}, - {file = "schema_salad-8.6.20240710082410-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79961a6ce0b2dee9b66affad73883dd02da3d56576f89893acb06a815b72d44e"}, - {file = "schema_salad-8.6.20240710082410-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0624c287f6cd5072e0c3fa72a0493963f51a452a53378808271c22a64eac028a"}, - {file = "schema_salad-8.6.20240710082410-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c2b9b00c1b7ff6de2dae2cef4496a07e75cd0dc72766941f4ed97da59c141a19"}, - {file = "schema_salad-8.6.20240710082410-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4b20799deb1552e9358ad407d3bf84a25e45c34c168be1b268f6b0a35393fab2"}, - {file = "schema_salad-8.6.20240710082410-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a5bb572c2a29304b0d02c6f1d8756400ace624507a76be0ae847a75dd5d59f4a"}, - {file = "schema_salad-8.6.20240710082410-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7e6d20fc3ab4d42d72f272b82dad2d52b9543a5aa1e57ee7a078e01a10f25b90"}, - {file = "schema_salad-8.6.20240710082410-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:edaa4f11c58336d807040378dde97d82d21af72c349b30a257f46db0f5d2ccbb"}, - {file = "schema_salad-8.6.20240710082410-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f398392eda9fd302999250bc950e35cb50d5d9e804192ba9223ddf51a223d9a7"}, - {file = "schema_salad-8.6.20240710082410-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13d92ef4bba0c4fbdeeec6cc70538532826db818711156cd8da271a401e5df9e"}, - {file = "schema_salad-8.6.20240710082410-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:671ba89a7a34e9c59b34459580c3581a15bc647aaa3ed565756e573808d61347"}, - {file = "schema_salad-8.6.20240710082410-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:85431930b431f0c1289d1385b38f509abcb42f42675bfa2f22baf14898063408"}, - {file = "schema_salad-8.6.20240710082410-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2dfc0c52a08b623edadd614f44c222397f645d7e3e523e735860fdbb85777582"}, - {file = "schema_salad-8.6.20240710082410-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d5a76aad8d54484c51bdd749a123a5c144191e0691bec2cbf97a985b0f1b9bbc"}, - {file = "schema_salad-8.6.20240710082410-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e0ea9e606b34f5695f384ca3f11baccf2579aa3486d42190f24f7d8c172e5d61"}, - {file = "schema_salad-8.6.20240710082410-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13b8ef75d5cd63cc27e791184401270a69ae75b210a61381b313dfdbf181eac4"}, - {file = "schema_salad-8.6.20240710082410-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f342835ff739521ca79e0b4ece1fffe02eb15fea29b3c89fb846d7d0c6ffcf2a"}, - {file = "schema_salad-8.6.20240710082410-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5301ba938a129b6673695f855e6b447ed31e9666e21daa867a20692379d7c385"}, - {file = "schema_salad-8.6.20240710082410-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b9270691a706ca91c1ca51dd89ea0699e98baa43f86a1eb19ec9ccd61d385430"}, - {file = "schema_salad-8.6.20240710082410-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:06b51bfbeb4b367cd61b692ebb6d70b5c7222547ae31e74756570b91f1e3696e"}, - {file = "schema_salad-8.6.20240710082410-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:813c744f2016688e3b7d95691def54116597b66ded2b664d5ba1ec3a0963ad06"}, - {file = "schema_salad-8.6.20240710082410-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d1395567e945d1a5fa8154aab6abc9d7072d7cc4f12b184afd714c8eb86ed17f"}, - {file = "schema_salad-8.6.20240710082410-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:614686b9832a51797df2aa145a12f2eef9a47a4540d5b980ce1db1b4941b29b5"}, - {file = "schema_salad-8.6.20240710082410-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20bcc33a0f4219afaa9b363490597cd6757fe6487529141d816a754d33735ec0"}, - {file = "schema_salad-8.6.20240710082410-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8850efe1d8edfff215b04387e4cd420ad07a106da0a3f3aeb15b82f1be0c0d3a"}, - {file = "schema_salad-8.6.20240710082410-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8915b987d81f64026da18aa91867a6eac464e1bad20b7ce2cbaf9ae61372c10e"}, - {file = "schema_salad-8.6.20240710082410-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9a922070b63c2c9855f0d2dbc946beecb7341ce889a5dfdd31902e5c01ccbccf"}, - {file = "schema_salad-8.6.20240710082410-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4aec0a36019b8eebc3bfc2267102a41177f6fcfb08dfdab008fa7c76725874c3"}, - {file = "schema_salad-8.6.20240710082410-py3-none-any.whl", hash = "sha256:96bedcb7ba8d31c5357465ba417a508c5699693dffb88426b470666d86f5c656"}, - {file = "schema_salad-8.6.20240710082410.tar.gz", hash = "sha256:65a497f91994967a46206535eac5ff3d27b00f318b8b50ebc8dea43be73dd369"}, + {file = "schema_salad-8.7.20240718183047-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8fe989d25111f06a45b336957cac03059ed1f06db8877c63f633c28a9a6f37a7"}, + {file = "schema_salad-8.7.20240718183047-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f00869da9e803c5a6514083801976a4d498be32b9e034bb6da46d30c3b4787cf"}, + {file = "schema_salad-8.7.20240718183047-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9d700c4bb83098589fc2bb83a460d65a23a3f7f3ebe0845cc07b21741b79c4d0"}, + {file = "schema_salad-8.7.20240718183047-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ead8c6e8b9e1a9c8fac509e11702df5d48da7015c27de33338bcfe17ea629e20"}, + {file = "schema_salad-8.7.20240718183047-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4290377a09cc8d1261d6aa50cbc271bcfff3c2fe01eced0812253bc6efdf836a"}, + {file = "schema_salad-8.7.20240718183047-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:04cd13b0360372ae8d249f4d7117a90c346f2460fe90fa7b466837c540ca515a"}, + {file = "schema_salad-8.7.20240718183047-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:81c167d8642b68b2b7b140fd47b74fe43cbfea85707bae3866546136a3c0885a"}, + {file = "schema_salad-8.7.20240718183047-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1d3536e68c21008c074a2aa8af8eb32a7669a6ad47f7cb584090b1573b5b3435"}, + {file = "schema_salad-8.7.20240718183047-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:76f289207b759e559cf5bb613e481b47039a9e06dd4fa0812ffb58cd1583977c"}, + {file = "schema_salad-8.7.20240718183047-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6173cd6163d60426323e2295ab13e4378660f130a3a2260236e4720b7362aeee"}, + {file = "schema_salad-8.7.20240718183047-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2823c9bd246ec95171c9f3e79805859a0d49d9a100323567b9114fc762005593"}, + {file = "schema_salad-8.7.20240718183047-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d2f61016c3dc146d64706b91a0516defbeae73a5cf77ecf9ddb15a1f7b08e0d"}, + {file = "schema_salad-8.7.20240718183047-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fce752e4d932bd90c7e56039bbb4137dfd85c5efde3ade868a76596815603e7b"}, + {file = "schema_salad-8.7.20240718183047-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4466c48837c59fd05378b33588bea4be85c53f2588db337144b852ba0c09a298"}, + {file = "schema_salad-8.7.20240718183047-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:27c54152c54d86a903e9774c09bdf87a4b9bd69417221e01f5e48363201b3ef8"}, + {file = "schema_salad-8.7.20240718183047-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:eb1548906c31dbde163961a210e544af723170cc4504c221c47aa34f4a133d65"}, + {file = "schema_salad-8.7.20240718183047-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:76972921fcdfdcad37665b5d72bb070a83a72fc24fc8cf9155049311e4b3d80f"}, + {file = "schema_salad-8.7.20240718183047-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4981c905494883155416ed4145786ebb9f34ed547259e4dc864fcbe23930fc84"}, + {file = "schema_salad-8.7.20240718183047-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c96597906783d54c153f884f2408248b98dde72e94699a25c6b09554f1358af"}, + {file = "schema_salad-8.7.20240718183047-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92f4051997dad05d55d35db575d1333e5c5ac1528c910e79a048db0f4257ca35"}, + {file = "schema_salad-8.7.20240718183047-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:caa60ff25d95725e43fc0230761458df5839b3e89402f109fd94ccee61a05f1c"}, + {file = "schema_salad-8.7.20240718183047-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:99a9868eaf44811711007cf7314cd67c1cb89a01ebfc4adaa4dc5f451048e358"}, + {file = "schema_salad-8.7.20240718183047-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bba87b473019b72653ad501026aea2a1e31a918a96c00e7ee8ae844529ee7b99"}, + {file = "schema_salad-8.7.20240718183047-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:936ab67593cff799827cf418be00160145b52d136928f6702cba6266f70f5381"}, + {file = "schema_salad-8.7.20240718183047-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b5eef332f97d80e34e07a511e53a7651936df202114457339f3bb2651a19252c"}, + {file = "schema_salad-8.7.20240718183047-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:569b3a48280490597ac2f4af297593df1433b82aca8a26d339579850d56336d9"}, + {file = "schema_salad-8.7.20240718183047-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9accfab6b214e8b969e5a39d34afb1b7af750d4f0738b4f296bcbed97b867bf4"}, + {file = "schema_salad-8.7.20240718183047-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:790c02122c7f34d7fb3199d56ff53621e61b37aba0dae718c3b811b77dee7d6e"}, + {file = "schema_salad-8.7.20240718183047-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d67640cbf44f6238fbb499cd96a522b0408b075c511a133fdb239f16d808fedd"}, + {file = "schema_salad-8.7.20240718183047-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:abc28092856c6596bbe248e00a8a65a47197f8152d78ee3e272a091fd6d6a88d"}, + {file = "schema_salad-8.7.20240718183047-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:a2fbf5ab7edcc6948a5f4980b2dacd889bdd48652d43fe787c14d18edbc1dfcc"}, + {file = "schema_salad-8.7.20240718183047-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4e92e816913dd3c395148c27696e7aa285ebf7631a61f17fe3967f0ac1e8e4ff"}, + {file = "schema_salad-8.7.20240718183047-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:265aded79c28c9f70332163285486be7aa0c51581fcd4210df92b53aa55974f9"}, + {file = "schema_salad-8.7.20240718183047-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8e6c3707bd7ace73512ace08fd67528e2e0d39c60aafee70de8e22913f837cb3"}, + {file = "schema_salad-8.7.20240718183047-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6afc02b425e19a293054d016b71e49fee16b89dcc1c7a8be9152559d4793680"}, + {file = "schema_salad-8.7.20240718183047-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e6b4aa572365126872d36c9596dc3052b9e5ef0608ad103855a26e205bbfd8fa"}, + {file = "schema_salad-8.7.20240718183047-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8ba1eb33fd11aae603ecfabb3dde7ef6a959ec7881d166df35b82842c40b2981"}, + {file = "schema_salad-8.7.20240718183047-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f745700ce6b31c7e74d130bc0578581f8014dfe3b30ffe74228e4955b5304412"}, + {file = "schema_salad-8.7.20240718183047-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1509fdc5e46f2baa601d40a8dae3e61c5fa1a0f51382a17bec00e43104d06701"}, + {file = "schema_salad-8.7.20240718183047-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0cbc8e077bb55c33dcd65932b357cd882c113f1599f4b9b4bbaf2040f806aaab"}, + {file = "schema_salad-8.7.20240718183047-py3-none-any.whl", hash = "sha256:7a191c1c5449545ba078a1b74718f89d3c0dc12c74b6ebc5ec270e2e68f0c848"}, + {file = "schema_salad-8.7.20240718183047.tar.gz", hash = "sha256:8174c62a6b108bd89a9391cc0dd5e8404f24a0773f3461cfd18983a61f1fe074"}, ] [package.dependencies] @@ -1149,18 +1154,19 @@ pycodegen = ["black"] [[package]] name = "setuptools" -version = "70.3.0" +version = "72.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.3.0-py3-none-any.whl", hash = "sha256:fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc"}, - {file = "setuptools-70.3.0.tar.gz", hash = "sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5"}, + {file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"}, + {file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"}, ] [package.extras] +core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -1276,13 +1282,13 @@ test = ["websockets"] [[package]] name = "wrapica" -version = "2.27.0" +version = "2.27.1" description = "Secondary level functions for ICAv2 based off libica" optional = false python-versions = ">=3.12" files = [ - {file = "wrapica-2.27.0-py3-none-any.whl", hash = "sha256:e5a39ecf4a44d630bd5276a710ee00e0fe182d50f603625ab7a95a4185878fbb"}, - {file = "wrapica-2.27.0.tar.gz", hash = "sha256:e717390e23e105fa91ab96c3e2c746993b1f6a270b88738e7d45edfda576f7f6"}, + {file = "wrapica-2.27.1-py3-none-any.whl", hash = "sha256:ba9dd1e743e864e71edbc1ab5860853af250037aa94d3cd6d3acad2df5a222a8"}, + {file = "wrapica-2.27.1.tar.gz", hash = "sha256:0e90e9d3138c4d509894fa673ab7e68cf133950a7b7091e3d449c09a7ea67dd8"}, ] [package.dependencies] @@ -1316,4 +1322,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.12, <3.13" -content-hash = "19f31df44070c41a62b76a5fd7ea16b5d942f52fa2f1cd38c384a85725dbd5fc" +content-hash = "221490e2552a143a35f54cc1fb33e7c2a8ff6bda051ad0f1e084f1c256afc74c" diff --git a/lib/workload/stateless/stacks/bs-runs-upload-manager/layers/pyproject.toml b/lib/workload/stateless/stacks/bs-runs-upload-manager/layers/pyproject.toml index 6edc66fa2..47d064762 100644 --- a/lib/workload/stateless/stacks/bs-runs-upload-manager/layers/pyproject.toml +++ b/lib/workload/stateless/stacks/bs-runs-upload-manager/layers/pyproject.toml @@ -23,7 +23,7 @@ aws_requests_auth = "^0.4.3" xmltodict = "^0.13.0" pandas = "^2.2.0" v2_samplesheet_maker = "^4.2.4" -wrapica = "2.27.0" +wrapica = "2.27.1" [tool.poetry.group.dev] optional = true diff --git a/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/lambdas/query_bclconvert_outputs_handler_py/query_bclconvert_outputs_handler.py b/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/lambdas/query_bclconvert_outputs_handler_py/query_bclconvert_outputs_handler.py index 2200671c9..a8621024f 100644 --- a/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/lambdas/query_bclconvert_outputs_handler_py/query_bclconvert_outputs_handler.py +++ b/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/lambdas/query_bclconvert_outputs_handler_py/query_bclconvert_outputs_handler.py @@ -29,16 +29,16 @@ from typing import Dict from urllib.parse import urlparse import pandas as pd -from wrapica.enums import DataType import logging # Wrapica imports +from wrapica.enums import DataType from wrapica.libica_models import ProjectData from wrapica.project_data import ( get_project_data_obj_by_id, read_icav2_file_contents_to_string, get_project_data_folder_id_from_project_id_and_path, - convert_project_id_and_data_path_to_icav2_uri + convert_project_id_and_data_path_to_uri ) # Local imports @@ -179,7 +179,7 @@ def handler(event, context): # Generate the manifest output for all files in the output directory # to link to the standard output path from the run id logger.info("Generating the run manifest file") - run_root_uri = convert_project_id_and_data_path_to_icav2_uri( + run_root_uri = convert_project_id_and_data_path_to_uri( project_id, bcl_convert_output_path, DataType.FOLDER @@ -237,13 +237,13 @@ def handler(event, context): interops_as_uri = dict( map( lambda interop_iter: ( - convert_project_id_and_data_path_to_icav2_uri( + convert_project_id_and_data_path_to_uri( project_id, Path(interop_iter.data.details.path), data_type=DataType.FILE ), [ - convert_project_id_and_data_path_to_icav2_uri( + convert_project_id_and_data_path_to_uri( dest_project_id, ( dest_folder_path / @@ -275,7 +275,7 @@ def handler(event, context): ) except StopIteration: # Add 'IndexMetricsOut.bin' from reports directory to the interop files - index_metrics_uri = convert_project_id_and_data_path_to_icav2_uri( + index_metrics_uri = convert_project_id_and_data_path_to_uri( project_id, Path(bcl_convert_output_obj.data.details.path) / "Reports" / "IndexMetricsOut.bin", data_type=DataType.FILE @@ -283,13 +283,15 @@ def handler(event, context): # Append InterOp directory to list of outputs for IndexMetricsOut.bin run_manifest[index_metrics_uri].append( - convert_project_id_and_data_path_to_icav2_uri( + convert_project_id_and_data_path_to_uri( dest_project_id, dest_folder_path / "InterOp", data_type=DataType.FOLDER ) ) + logger.info("Outputting the manifest and fastq list rows") + return { "manifest_b64gz": compress_dict(run_manifest), "fastq_list_rows_b64gz": compress_dict(fastq_list_rows_df_list), @@ -302,8 +304,8 @@ def handler(event, context): # handler( # { # "project_id": "a7c67a80-c8f2-4348-adec-3a5a073d1d55", -# "analysis_id": "2800e302-4a12-4bf0-9028-f74e4a83479f", -# "output_uri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary_data/240424_A01052_0193_BH7JMMDRX5/202406066dd81cc9/" +# "analysis_id": "47936e52-21fd-457b-a1dc-97139d571441", +# "output_uri": "icav2://development/primary_data/240424_A01052_0193_BH7JMMDRX5/202406066dd81cc9/" # }, # None # ), @@ -312,6 +314,6 @@ def handler(event, context): # ) # # # { -# # "manifest_b64gz": "H4sIAH+KVmYC/+2dX2/bNhTFv0qQ58kSr0T96VviJouBBGmjFh0wDAQt0Y4wWdIkOYs77LuPdhu2RdrELhiPou9D4hPkwZc/H55LXcvwP8dFxu/gletOwZ9NKQmdPKbgBElMHZ5nmRP7JCMg4iwHcItyUTm84uWqE50LgQeQsBPPI77HPIhjdnpBL67gdfrbmMVBkFDCgkyQBJzTrBzX1Z1o+6O7gAGLHAiiTEyzxJnybOoE3ix3OPEzh8pfs4j4wg+nbr3sm2XvXtbzzj1r27rtRmU9P3519LsqPKIJFfEMHB9y3wmyKHS4D4GThTymORdxNKWbwlnTFgverp6u2wVP/p9CnPPQCzxCPj358R+/HA2I1Tnv+r/G9aIpRS9G/X2PyJ5D9oG3VVHN0WDP0boRTd32nXuS86YXLbsSfVtk3Sjr7vbL7aGQ4aF7LRbLsi/k7rxnac97hLftLp1Usxp36JY2e7vkZdGvcIfuji7l696Z3grRI7btsU2qXEbaRd00spWycb2sMNt24XezrDYJd78oEdlux5DxKisFRt3uAGfrawVWFh0m3U8e4d4V0nl4jvuZXvF5v14v+9G0qBDdduje1Q17X/1Z1X9X7JS3WZ0LtN4O/NrN4+i2xza767XEJuuwy27N79OFROde8koweSUpn9kjHijBUsIupWI3hMmH0aYbj+Yf98v1R1XayBmQs27OoacES31T/SyrtJEzIGftnEEJllJjOQ8/N0IlWBobmxuhlZwBOe8lNwjmhm7ONFGCpWAqZ1nl0P38RbA0MDY3iJWcCXLeS24A5oY+zu+rXPSiXRSVyL/5g6Weeb7+plqbmQMyf5Hc9pVgaWhsbvtWcgbkrIvzw2VLEinBUrIZmIJBhv6qTCtBA4LWnhyBEiyNjE3owErOgJy1BwdVgqWJsQFNB885VEIGtGcs6NBKQwMaej+GBjS0ZtDgx0pI0L6hybEuc7Cgnxt4mMXbhoHH0+YGNLd20IkSEnRgbIokVqcIYIrsz9yA5tYOmighQYOxKULsBA0IWhdo/8eDU98gR/uDH5w+DRoQtHbQjwYgRhp66AMQ+aOENDQ1tRcGnp2gAUFrT47HIz0joyO0MqIBI1oz5+9Nlww09ICnS08nB2By7MfRgI7WB/q5MZ5ZCWLDGG8r5oDMXyRQHo9OjWyRiZ2gAUHrAv1ww1hClJCgN7eOBQY5+qsyrQQNCFp7dDx+M8DIjB66oyEgSkjQiaHRsS7TStCAoLWDVtNJYCl4xjoa7AQNCFo76EQJCZoY6+jETtCAoHWDpp4SEjSY6mjq2XmOBjxH78fRgI7WDpooIUH7xkYHsRM0IGjtoEEJCTow1tFgJ2hA0NpB+0pI0NRYR/t2ggYErfsc/Z0b8kycRwfDP959EdLRobHREQweNFVCgo6MBU3tjA7A6NhPdABGh3bQkRIStLHvGdLITtCAoPfTDAGboXbQoRISdGxsdIR2ggYErfuWsJAqIY93pl6wrMu0EjQgaO03OT7+2LKRd5NGdoIGBK0P9HOfsjDL2DZ8ymIr5oDMXyRQYiVkoMTGJndsJ2iwB3S7rDp32nW3jDcdON2t07R1zvw4loVQd1LJLXPduCdlMa8Wour/z+8gfijmZZd6yjuR8bIsqrn9iz1r27o9gGUumkIukZdvbnl3EC/suG5bkfUinxzCnj2774Vs7/n6C5ftX+1106+9fFXnorR/tZMFnx/Ai5ouF+vibpaV/Wt9c/5rW+QHEUstz/qiPoDX9O3nJZ6u1idTixd6GD3m/MPF1WFs0gfjgud7B7BMm+P27iAuUG/EvOhkYzmMvjKR5/p7M5b573/PZmjGApoAAA==", # pragma: allowlist secret -# # "fastq_list_rows_b64gz": "H4sIAH+KVmYC/9Wb7WvcRhDG/xVzn2OdtC96ybfNhmwKzpdoC4FShGpfwoEdUqcU2tL/vTO6y8aNnltz92kWbN8gGfthfszsPKvVL/9s3oefXm9eXm2CczH6EKsYXQyOgmbz4opuj+/49o0ydd3U6nDt5tWP127mzzu61vDt3XzXvNnf735+3POv7W/nP9XL7bazg931H9W1Vnf62tx27fWslbm+befe3s27vvvNbvf3D5+nL4/7h/nxry39faWGydF/0fVUq76fXr21b9+p1+MHv1U13beqv5vb2tRNsx3nhy/3u69b1jI122/yUjCNzXRD0fS+meij+jh//eP36tPfm6NoJVu0+r/of19cPaXH/EKIzC94+vLBIYJ2WBM8XBNL0A4pmEZVCEEoOkeQAIbgnfO+8t5F+oouIIJtvSZ4uCaWYFunYBp1IQSh6BzBJ9iewEQEG0CwkU3wezCNphSCSHS+i0bmxq0zMM1AfRTXIFgHW9nrYKtSMI22FIJIdLaLLmtgcD5SDdLHghMS1ICglk1Qp2Aa21IIItE5gjR9ctX5yOtgpBbKECFBAwga2QRNCqaxK4UgEp0lGIkbdU/n2UsQvZOTTAsItrIJtimYxr4Ugkj0M1300Dg9m0KqwUqt6Q12Te9w7UhPiaH3bRUZbAqmceBEKLn0sqKz9Lht0ihD/ZMaKVtCRA/U3tDKptemgExxXQo+qDo7hUai5sLSPePiIRC/DvDrZPPrUkCZaErhB1Vn1z8faMGLgX08fXpUf0qvPeDxmlR+JC8FlAlVBr8TqrP1x8VH3MgDOq5ED/n1gF8vm1+fAsqELoUfVP2cB2TzXtHaR90z4vobAL9BNr8hBZQJUwo/qDpbfzx10o+K1j6aXuD6R99rfqYWzc/UKaBM2EL4YdXnuQd9gXvQYvjpk4O4losvK/o896AvcA8C6a3n8BLwQdXnuQd9gXsQyG89h5fAD6o+zz3oC9yDOH5oDpfP74Tq89yDvsA9COS3nsNL4AdVn+ce9AXuQSC/9RxeAj+o+jz3oC9wD/L4gTm8AH5YdX7+DFx1PHgeAdIgUxnw9Ag4iPapgzBiGKYnaDYFlI3laaiRyzCv+pkaPCx9/CTXcQD4DeAUzNCI5jc0KaBMdIXww6rP8xDmAg8hkN96Gi+BH1SdPYkW+RgMeXguQo4hvx7w62Xz61NAmehL4QdV5+svsvuL7CGWc9mAnzLAAxrR/ZPkpYAyMZTB74Tq/AyzVJ2v0lYo4qcAPyWbX9oUVtOo6lL4QdX5s9iR34Vwywk0z5uhiB/wgGaQzW9IAWWiKYUfVJ0/x8sV6Kl/Rr8cJkT8LPCAthbNz9YpoEyoQvhh1Xn/4P1xD3Sx87D+LFj/rOz1zzYpoEzoUvhB1dk97OXktednSMtudoD8wPpnZa9/VqWAMmFK4QdVZ/0Dv8hC7Gh+4TcivIP8NOCnZfPTKaBM2FL4QdV5/8f2PbqKjcTyDBfxM4Cfkc3ve0CZaEvhB1U/8x4SNU5qm46LjwBCfhbws7L52RRQJrpS+EHVWf/Ab614nl+WR0kn1r8W8Gtl82tTQJnoS+EHVef3X8Ly7KGKyyCK/Z/tAL9ONr8uBZSJUvZfsOof+f36H1n1Xz7MQAAA" # pragma: allowlist secret +# # "manifest_b64gz": "H4sIABfArGYC/+2db2/aSBDGv0qU12dsj//3XUKSC6egNrhVK51Oq429AeuM7bMNl9zpvvsttNm2StoEytJlNFIknigRnvn54dn1YMS/x0XGl/DKtnmUhRGPHSuLb8HyPT+2eC4yy+MBdyIvd/MgsItyXlm84uV9JzobfAcgYSeO43oOcyCO2ellcDmGs/TDkHk85BlnWRxxT1inWTmsq6Vo+6Olz4BFlgd+kCVhaPHMcyw/z6Ryb1wriF3hQRT5fh7Y9aJvFr19VU87+7xt67YblPX0+NXR78edJ6tuikaURSWsXCytjGczYQWOl0QRREHoymdsrE4+x0zwrrfAbtpiztt7lvOer8r3wZflu04AzHETT5Yf/TYen00+BDY48u+hE4Z5HrtZlnys4fiPX44OCNl73lZFNSVom0C7kG3/NaznTSl6MejveiL3QnKj6rYmq70M2EQ0ddt39pmYL8q+kGa7Y2nP+26QdcufAvChosNjeJLzphctG4u+LTIiuDnBUZVL/13WTSOXCzasFxUZ8QeMOLzPSkF23Jrj7WoJZmXR9QRvY3iTRbVeiO/mJZHbiFzKV5u+dCYE+W5zetcLXhb9PcXe1gTf1g17V/1Z1X9X7JS3WZ0LwrjlZuaTC18v+sFNURHBrS9K3hZyK0NXJtuBbNePg1lPa/GWq8nafrSkbIrx41ams694JZhrX8kjy15BCZa67EoqNnGZfBisd9yD6T8/Be+3isWIGwi3JtyhowRLPcPdLYvFiBsIty7coARLA9NxH36YhEqwNDY9TEKUuIFwa8IdJEqwFAx3tywWY3a7lN26cH8WLPVNx+2ixA2Ee5/ZDZTdO8f9rspFL9q5xJd/9QtLHWND5auiMaMHQq8z0z0lWBqavoR6KHED4d4x7odteBIpwVJ3PZkF8+z9RbUoeQPx1hUnvhIsjUxPbx8lbiDcutIkUIKlienhHRw87lAJGd6O6bxDlPYGsvde7Q1kbz28wYuVkLw9s+NkVS1K3kC8d877uaGVkTbHMLRS5kmUkFb3TY+WBLXVgay+d6sDWV0Xb1cJyRtMjxYXJ28g3jvm7X17QuuZ52/v4Ce03+cNxFtTnviOEpJ3YHh++87B+/vREMvkOAlw2hvI3rrs/XhoaLK/Q5RxAhQnenA/NTM0194HPDP8fpwAxcle/Q3k753zfm5QaGSsYBgUvgg9EHqdKfN4RmvyKprg5A3Ee8e8H+52S1wlJO/1fW++ef7+olqUvIF468qTx+9BmJzfh+5v8F0lJO/E7DxZVYuSNxBvXbzV/BNYCo7p/gacvIF46+KdKCF5u6b7O8HJG4i3Jt6Bo4TkDYb7O3Bw7r+B9t979TeQv3XxdpWQvD3T88TFyRuIty7eoITk7Zvub8DJG4i3Lt6eEpJ3YLq/PZy8gXhr2n8/cVOhwfNv//D3g5+F9Hdoep74OP0N5G9d/g6UkP6OTPd3gDNPgPJEF+9ICcnb9PcvgwgnbyDee81voPzWxTtUQvKOTc+TECdvIN6a7mcLAyXk/tvw651VtSh5A/HWdb/m4894m3x/bISTNxDvnfN+7qMlRtocw0dLXoQeCL3OlImVkCkTm57qMU7egI53u6g6+6brZow3HVjdzGraOme+k8gf1x5V8gX0urFPymJazUXVP/H91g8Hlb2Lsm5W//VDHT4cU29Hp7wTGS/lWZui6em8besWTzd3vZAJnq++ihlNU6+bXj53Oa5zUeI5U/OmaFdtvZnxDtML6vpTJ6f3q7g9/H7eXPzaFjma0zOs21ZkvchHiBYmVHE3mvOpQBcH4HjO4Xdz8f5yjCoOrjFtfVqe9UVdoWkpXcxXFU4WeFqaiGnRyROF6jydLzFd5Y3kBcTdXrv5738WyW3/3JoAAA==", # pragma: allowlist secret +# # "fastq_list_rows_b64gz": "H4sIABfArGYC/9Wbb2vjRhDGv0rw67Ms7T9Z925vj25bkjfRFgqlCBO7PUPumiZpoS397p2RfXvp6fEG+5UGkniQAn6YHzM7z2r10z+L2/jd+8Xbq0X0PqUQU5WST9FT0CzeXNHt/oZvXytT102tDteu33197XrzaUfXGr6922ybb/b3ux8e9/xvT/rtavWwf9jd7z/tltvdn8u7zd2H3dLWumtb1VrXuOXmYfn02x/PH3abp+elWj087j9uHv8atpvnzYq+xigzePouq4a66fTw7tv2+5ub97c/2pWq6b6rndtu183dXbfqNx8f7ndPK5Y0NKvPKnMw9M1wTdFw2wz0Uf1CX/l79evfi6N2JUK7+r/2f99cvWTJNGNMTDMG+gnRI562m/I8XJs7T9vlYOiVLJ5Qe4kn4YwxeB9CFYJP9JN8RDxdPeV5uDZ3nq7OwdBrWTyh9hLPFxBfoEU8G8CzEcHzSzD0RhhPpL3cbxNT5CYbmW2kjovrE6yfTsT66VQOht4K44m0F/vtuHZGHxLVJ32McCFPDXhqETx1DobeCeOJtJd40kTLFRkSr5+Jmi0jhTwN4GlE8DQ5GPpWGE+kvcgzEUXqsz6wWyGWJ+chB3g6ETxdDoZ+LYwn0v5Kvz202MAmlOqzUlOWnZ2yPFw7slRzY/l52elsDoa+43yo2bMsai+y5AZLAxF1Wmq5bEERS1CXnRPB0uWAvHgtDCYUX5xsEzH0ceyzaXQpiGYLaLYiaLY5oIQ0wmhC8cV1M0RaKFPkXQT6DKg2lZ56zuO1mdMklTmghChRNE+IL9YmFyZRJM/puUoDpLkGNNciaK5zQAnRwmhC8a95Tt46qGjNpD6bcG12gGYngmaXA0qIEUYTii/WJk+y9KeiNZNmILhu0u+Upqkl0DR1DighVhZNLP48f6Iv8Cd6bjT1yRlfzx5mUft5/kRf4E/my3I64guCCcWf50/0Bf5kvjSnI74gmlD8ef5EX+BP5koTjfhiaJ4Qf54/0Rf4k/nSnI74gmhC8ef5E32BP5kvzemIL4gmFH+eP9EX+JPZ0gQjvhyaWHx5po1ckTzMHnHSOFQZ8DwMeBT30qOYuRHNjwZtDigp40NfM3uiZfGv1OdhyeTn1p4DQLMDp4O6RgLNrskBJaSVRROLP8+lmAtcynxpTgd9QTSh+OLpvcTHg3yquEA5hjTXgOZaBM11Digha2E0ofhybSZ2m4ldyngGHtBUBnhOI6HTksocUEI6UTRPiC9PQmNFhipv2SKaCtBUImjmPWw19KoWRhOKL597T/xOih9P7QXetEU0gec0nQiaXQ4oIY0wmlB8+ZQ0V2egTpvCeBwT0bTAc9paAk1b54ASomTRxOLLDiWE417tuJkAa9OCddOKWDdtkwNKiBZGE4ov7ryPp9wDPxUb9+AjpAnWTSti3bQqB5QQI4wmFF90KPx6EZGkKYjfTAke0tSAphZBU+eAEmKF0YTiy36TNw+Sr9iqjE+sEU0DaBoRNL8ElBAnjCYU/8q7YtRiqcF6LkzCCWlaQNOKoGlzQAlphdGE4osOhd8lCjwFjQ/HTqybDtB0Imi6HFBC1sJoQvHlvaA4Pj+p0jjcYr9pW0CzFUGzzQElRNheEBb/Nc2f/wMHcuL20kIAAA==" # pragma: allowlist secret # # } diff --git a/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/layers/pyproject.toml b/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/layers/pyproject.toml index 1cd2e4a4a..847ac8e5b 100644 --- a/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/layers/pyproject.toml +++ b/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/layers/pyproject.toml @@ -23,7 +23,7 @@ aws_requests_auth = "^0.4.3" xmltodict = "^0.13.0" pandas = "^2.2.0" v2_samplesheet_maker = "^4.2.4" -wrapica = "2.27.0" +wrapica = "2.27.1" [tool.poetry.group.dev] optional = true diff --git a/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/layers/src/bssh_manager_tools/utils/manifest_helper.py b/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/layers/src/bssh_manager_tools/utils/manifest_helper.py index d72baa2c8..704622438 100644 --- a/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/layers/src/bssh_manager_tools/utils/manifest_helper.py +++ b/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/layers/src/bssh_manager_tools/utils/manifest_helper.py @@ -11,7 +11,9 @@ # Wrapica imports from wrapica.libica_models import ProjectData -from wrapica.project_data import convert_project_data_obj_to_icav2_uri +from wrapica.project_data import convert_project_data_obj_to_uri +from wrapica.storage_configuration import convert_icav2_uri_to_s3_uri +from wrapica.enums import UriType def get_destination_uri_path( @@ -35,7 +37,7 @@ def get_destination_uri_path( # Get the output path (which is the parent directory) dest_uri_path = str( urlunparse(( - "icav2", + UriType.ICAV2.value, output_project_id, str(output_folder_path.joinpath(relative_file_path).absolute().parent) + "/", None, None, None @@ -66,14 +68,14 @@ def get_dest_uri_from_src_uri( # Get the output path (which is the parent directory) dest_uri_path = str( urlunparse(( - "icav2", + UriType.ICAV2.value, dest_project_id, str(dest_folder_path.joinpath(relative_file_path).absolute().parent) + "/", None, None, None )) ) - return dest_uri_path + return convert_icav2_uri_to_s3_uri(dest_uri_path) def generate_run_manifest( @@ -100,7 +102,9 @@ def generate_run_manifest( for file_obj_iter in project_data_list: # Get the source file uri - source_file_uri = convert_project_data_obj_to_icav2_uri(file_obj_iter) + # The source file uri is the path to the file in the input project + # The source file will not be on byob storage. + source_file_uri = convert_project_data_obj_to_uri(file_obj_iter, uri_type=UriType.ICAV2) # Get the output path (which is the parent directory) dest_uri_path = get_destination_uri_path( @@ -110,6 +114,9 @@ def generate_run_manifest( file_path=file_obj_iter.data.details.path ) + # However we convert the dest uri path to an s3 path + dest_uri_path = convert_icav2_uri_to_s3_uri(dest_uri_path) + # Add the source file uri to the run manifest dictionary run_manifest_dict[source_file_uri] = [ dest_uri_path diff --git a/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/layers/src/bssh_manager_tools/utils/sample_helper.py b/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/layers/src/bssh_manager_tools/utils/sample_helper.py index a54967f5a..dcaded223 100644 --- a/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/layers/src/bssh_manager_tools/utils/sample_helper.py +++ b/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/layers/src/bssh_manager_tools/utils/sample_helper.py @@ -6,9 +6,9 @@ from pathlib import Path # Wrapica imports -from wrapica.enums import DataType +from wrapica.enums import DataType, UriType from wrapica.project_data import ( - convert_project_id_and_data_path_to_icav2_uri + convert_project_id_and_data_path_to_uri ) @@ -250,19 +250,21 @@ def get_fastq_list_paths_from_bssh_output_and_fastq_list_csv( # Update the fastqs to contain the full path fastq_list_pd["Read1FileUriSrc"] = fastq_list_pd.apply( - lambda row: convert_project_id_and_data_path_to_icav2_uri( - project_id, - run_output_path / row["sample_prefix"] / row["Read1File"], - DataType.FILE + lambda row: convert_project_id_and_data_path_to_uri( + project_id=project_id, + data_path=run_output_path / row["sample_prefix"] / row["Read1File"], + data_type=DataType.FILE, + uri_type=UriType.ICAV2 ), axis="columns" ) fastq_list_pd["Read2FileUriSrc"] = fastq_list_pd.apply( - lambda row: convert_project_id_and_data_path_to_icav2_uri( - project_id, - run_output_path / row["sample_prefix"] / row["Read2File"], - DataType.FILE + lambda row: convert_project_id_and_data_path_to_uri( + project_id=project_id, + data_path=run_output_path / row["sample_prefix"] / row["Read2File"], + data_type=DataType.FILE, + uri_type=UriType.ICAV2 ), axis="columns" ) diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/compress_icav2_vcf/compress_icav2_vcf.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/compress_icav2_vcf/compress_icav2_vcf.py index a09841375..4e143dd0f 100644 --- a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/compress_icav2_vcf/compress_icav2_vcf.py +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/compress_icav2_vcf/compress_icav2_vcf.py @@ -18,7 +18,7 @@ from wrapica.project_data import ( write_icav2_file_contents, read_icav2_file_contents, - convert_icav2_uri_to_project_data_obj, delete_project_data + convert_uri_to_project_data_obj, delete_project_data ) if typing.TYPE_CHECKING: @@ -83,7 +83,7 @@ def write_icav2_vcf_file_to_compressed_output(icav2_uri: str, output_path: Path) :return: """ # Download decompressed file - project_data_obj = convert_icav2_uri_to_project_data_obj(icav2_uri) + project_data_obj = convert_uri_to_project_data_obj(icav2_uri) # Write to output file read_icav2_file_contents(project_data_obj.project_id, project_data_obj.data.id, output_path.with_suffix('')) @@ -128,7 +128,7 @@ def compress_icav2_vcf_and_upload(icav2_uri): :param icav2_uri: :return: """ - project_data_obj = convert_icav2_uri_to_project_data_obj(icav2_uri) + project_data_obj = convert_uri_to_project_data_obj(icav2_uri) with TemporaryDirectory() as temp_dir: # Initialise parameters diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/delete_cache_uri_py/delete_cache_uri.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/delete_cache_uri_py/delete_cache_uri.py index a882d78fb..3a8837bcb 100644 --- a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/delete_cache_uri_py/delete_cache_uri.py +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/delete_cache_uri_py/delete_cache_uri.py @@ -15,9 +15,8 @@ """ from wrapica.enums import DataType -from wrapica.libica_exceptions import ApiException from wrapica.project_data import ( - convert_icav2_uri_to_project_data_obj, list_project_data_non_recursively, delete_project_data + convert_uri_to_project_data_obj, list_project_data_non_recursively, delete_project_data ) from cttso_v2_pipeline_manager_tools.utils.aws_ssm_helpers import set_icav2_env_vars import logging @@ -52,7 +51,7 @@ def handler(event, context): # Part 1 - check that in the cache uri, only the sample_id directory exists along with the file SampleSheet.csv try: - cache_obj = convert_icav2_uri_to_project_data_obj(cache_uri) + cache_obj = convert_uri_to_project_data_obj(cache_uri) except NotADirectoryError as e: logger.info("Cache directory has already been deleted") return None diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/find_all_vcf_files_py/find_all_vcf_files.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/find_all_vcf_files_py/find_all_vcf_files.py index 02f5b617b..ae1cfea8b 100644 --- a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/find_all_vcf_files_py/find_all_vcf_files.py +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/find_all_vcf_files_py/find_all_vcf_files.py @@ -6,6 +6,8 @@ # Standard imports import typing +from os import environ + import boto3 from typing import List @@ -15,8 +17,8 @@ from wrapica.project_data import ( find_project_data_bulk, - convert_icav2_uri_to_project_data_obj, - convert_project_data_obj_to_icav2_uri + convert_uri_to_project_data_obj, + convert_project_data_obj_to_uri ) # Globals @@ -82,7 +84,7 @@ def handler(event, context): icav2_uri = event.get("icav2_uri") - data_obj: ProjectData = convert_icav2_uri_to_project_data_obj(icav2_uri) + data_obj: ProjectData = convert_uri_to_project_data_obj(icav2_uri) all_project_data: List[ProjectData] = find_project_data_bulk( project_id=data_obj.project_id, @@ -93,7 +95,7 @@ def handler(event, context): return { "vcf_icav2_uri_list": list( map( - lambda project_data_iter: convert_project_data_obj_to_icav2_uri(project_data_iter), + lambda project_data_iter: convert_project_data_obj_to_uri(project_data_iter), filter( lambda project_data_iter: ( # Is a vcf file diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/find_all_vcf_files_py/requirements.txt b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/find_all_vcf_files_py/requirements.txt index d6dad23ce..68c845fa9 100644 --- a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/find_all_vcf_files_py/requirements.txt +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/find_all_vcf_files_py/requirements.txt @@ -1 +1 @@ -wrapica==2.27.0 \ No newline at end of file +wrapica==2.27.1 \ No newline at end of file diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/generate_copy_manifest_dict_py/generate_copy_manifest_dict.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/generate_copy_manifest_dict_py/generate_copy_manifest_dict.py index a1018da6b..6771ab792 100644 --- a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/generate_copy_manifest_dict_py/generate_copy_manifest_dict.py +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/generate_copy_manifest_dict_py/generate_copy_manifest_dict.py @@ -33,8 +33,8 @@ from urllib.parse import urlparse # Wrapica imports -from wrapica.enums import DataType -from wrapica.project_data import convert_project_id_and_data_path_to_icav2_uri +from wrapica.enums import DataType, UriType +from wrapica.project_data import convert_project_id_and_data_path_to_uri def handler(event, context): @@ -71,10 +71,11 @@ def handler(event, context): raise ValueError("Fastq list rows are required") # Generate the manifest list - fastq_cache_path = convert_project_id_and_data_path_to_icav2_uri( + fastq_cache_path = convert_project_id_and_data_path_to_uri( project_id=project_id, data_path=Path(cache_path) / sample_id, - data_type=DataType.FOLDER + data_type=DataType.FOLDER, + uri_type=UriType.ICAV2 ) # Filter fastq list rows by RGSM (match sample_id) diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py index 95e86e363..86a07d756 100644 --- a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py @@ -31,12 +31,12 @@ # ICA imports # Wrapica imports -from wrapica.enums import DataType +from wrapica.enums import DataType, UriType from wrapica.libica_models import ProjectData from wrapica.project_data import ( - convert_icav2_uri_to_project_data_obj, + convert_uri_to_project_data_obj, list_project_data_non_recursively, read_icav2_file_contents_to_string, - convert_project_data_obj_to_icav2_uri + convert_project_data_obj_to_uri ) # Local imports @@ -55,7 +55,7 @@ def handler(events, context): analysis_output_uri = events.get("analysis_output_uri") # Get analysis uri as an object - analysis_project_data_obj = convert_icav2_uri_to_project_data_obj(analysis_output_uri) + analysis_project_data_obj = convert_uri_to_project_data_obj(analysis_output_uri) # Top level list analysis_output_list = list_project_data_non_recursively( @@ -132,9 +132,9 @@ def handler(events, context): sample_passed = "MetricsOutput" in passing_sample_steps_dict[sample_id] return { - "results_dir": convert_project_data_obj_to_icav2_uri(results_dir_data_obj), - "logs_intermediates_dir": convert_project_data_obj_to_icav2_uri(logs_intermediates_dir_data_obj), - "nextflow_logs_dir": convert_project_data_obj_to_icav2_uri(nextflow_logs_dir_data_obj), + "results_dir": convert_project_data_obj_to_uri(results_dir_data_obj, uri_type=UriType.S3), + "logs_intermediates_dir": convert_project_data_obj_to_uri(logs_intermediates_dir_data_obj, uri_type=UriType.S3), + "nextflow_logs_dir": convert_project_data_obj_to_uri(nextflow_logs_dir_data_obj, uri_type=UriType.S3), "sample_passed": sample_passed } diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/upload_samplesheet_to_cache_dir_py/upload_samplesheet_to_cache_dir.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/upload_samplesheet_to_cache_dir_py/upload_samplesheet_to_cache_dir.py index 491462e10..ba80289d8 100644 --- a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/upload_samplesheet_to_cache_dir_py/upload_samplesheet_to_cache_dir.py +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/upload_samplesheet_to_cache_dir_py/upload_samplesheet_to_cache_dir.py @@ -29,7 +29,7 @@ # Wrapica imports from wrapica.project_data import ( write_icav2_file_contents, - convert_project_data_obj_to_icav2_uri, + convert_project_data_obj_to_uri, get_project_data_obj_by_id ) @@ -81,7 +81,7 @@ def handler(event, context): ) # Get the uri for the samplesheet file - samplesheet_file_uri = convert_project_data_obj_to_icav2_uri( + samplesheet_file_uri = convert_project_data_obj_to_uri( get_project_data_obj_by_id(project_id, samplesheet_file_id) ) diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/pyproject.toml b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/pyproject.toml index 35ca6dd8d..5f2cba48c 100644 --- a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/pyproject.toml +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/layers/pyproject.toml @@ -19,7 +19,7 @@ boto3 = "^1.28" botocore = "^1.31" aws_requests_auth = "^0.4.3" v2_samplesheet_maker = "^4.2.4" -wrapica = "2.27.0" +wrapica = "2.27.1" [tool.poetry.group.dev] optional = true diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/lambdas/generate_event_data_objects_py/generate_event_data_objects.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/lambdas/generate_event_data_objects_py/generate_event_data_objects.py index 6182fefd8..6bad6e286 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/lambdas/generate_event_data_objects_py/generate_event_data_objects.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/lambdas/generate_event_data_objects_py/generate_event_data_objects.py @@ -47,7 +47,7 @@ def generate_fastq_list_row_event(fastq_list_row: Dict, instrument_run_id: str) fastq_list_row_id = '.'.join( [ new_fastq_list_row_dict["rgid"], - instrument_run_id,new_fastq_list_row_dict["rgsm"], + instrument_run_id, new_fastq_list_row_dict["rgsm"], ] ) diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/collect_qc_metrics_from_alignment_directory.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/collect_qc_metrics_from_alignment_directory.py index fe0624012..aa1e979fc 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/collect_qc_metrics_from_alignment_directory.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/collect_qc_metrics_from_alignment_directory.py @@ -41,7 +41,7 @@ ) from wrapica.project_data import ( list_project_data_non_recursively, - read_icav2_file_contents_to_string, convert_icav2_uri_to_project_data_obj + read_icav2_file_contents_to_string, convert_uri_to_project_data_obj ) # Globals @@ -198,7 +198,7 @@ def handler(event, context): # Get analysis output uri from the event # Get analysis output folder as a ProjectData object - analysis_project_data_obj: ProjectData = convert_icav2_uri_to_project_data_obj(alignment_output_uri) + analysis_project_data_obj: ProjectData = convert_uri_to_project_data_obj(alignment_output_uri) all_output_files: List[ProjectData] = list_project_data_non_recursively( project_id=analysis_project_data_obj.project_id, diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/requirements.txt b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/requirements.txt index 474329bee..3cad6309b 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/requirements.txt +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/requirements.txt @@ -1,3 +1,3 @@ -wrapica==2.27.0 +wrapica==2.27.1 pandas>=2.2.2 boto3>=1.34.0 \ No newline at end of file diff --git a/lib/workload/stateless/stacks/transcriptome-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py b/lib/workload/stateless/stacks/transcriptome-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py index c05662109..c32b6e0ad 100644 --- a/lib/workload/stateless/stacks/transcriptome-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py +++ b/lib/workload/stateless/stacks/transcriptome-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py @@ -29,11 +29,11 @@ from typing import Dict, List # ICA imports -from wrapica.enums import DataType +from wrapica.enums import DataType, UriType from wrapica.libica_models import ProjectData from wrapica.project_data import ( - convert_icav2_uri_to_project_data_obj, - list_project_data_non_recursively, convert_project_data_obj_to_icav2_uri + convert_uri_to_project_data_obj, + list_project_data_non_recursively, convert_project_data_obj_to_uri ) # IDE imports only @@ -181,7 +181,7 @@ def handler(event, context): output_prefix = event["output_prefix"] # Get the analysis output uri as a project data object - analysis_output_obj = convert_icav2_uri_to_project_data_obj(analysis_output_uri) + analysis_output_obj = convert_uri_to_project_data_obj(analysis_output_uri) # FInd the dragen germline output top_dir_list: List[ProjectData] = list_project_data_non_recursively( @@ -231,7 +231,7 @@ def handler(event, context): # Every value in the file outputs dict is a project data object, convert each value to a uri outputs_as_uri = dict( map( - lambda key_val: (key_val[0], convert_project_data_obj_to_icav2_uri(key_val[1])), + lambda key_val: (key_val[0], convert_project_data_obj_to_uri(key_val[1], uri_type=UriType.S3)), file_outputs_dict.items() ) ) diff --git a/lib/workload/stateless/stacks/tumor-normal-pipeline-manager/lambdas/set_outputs_json_py/requirements.txt b/lib/workload/stateless/stacks/tumor-normal-pipeline-manager/lambdas/set_outputs_json_py/requirements.txt index fe9fccaa4..b92b0dd2d 100644 --- a/lib/workload/stateless/stacks/tumor-normal-pipeline-manager/lambdas/set_outputs_json_py/requirements.txt +++ b/lib/workload/stateless/stacks/tumor-normal-pipeline-manager/lambdas/set_outputs_json_py/requirements.txt @@ -1,2 +1,2 @@ -wrapica>=2.27.0.post20240714154223,<2.28.0 +wrapica>=2.27.1,<2.28.0 boto3>=1.34 diff --git a/lib/workload/stateless/stacks/tumor-normal-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py b/lib/workload/stateless/stacks/tumor-normal-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py index 1f9f301e0..ab2c11486 100644 --- a/lib/workload/stateless/stacks/tumor-normal-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py +++ b/lib/workload/stateless/stacks/tumor-normal-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py @@ -29,11 +29,11 @@ from typing import Dict, List # ICA imports -from wrapica.enums import DataType +from wrapica.enums import DataType, UriType from wrapica.libica_models import ProjectData from wrapica.project_data import ( - convert_icav2_uri_to_project_data_obj, - list_project_data_non_recursively, convert_project_data_obj_to_icav2_uri + convert_uri_to_project_data_obj, + list_project_data_non_recursively, convert_project_data_obj_to_uri ) # IDE imports only @@ -231,7 +231,7 @@ def handler(event, context): germline_output_prefix = event["germline_output_prefix"] # Get the analysis output uri as a project data object - analysis_output_obj = convert_icav2_uri_to_project_data_obj(analysis_output_uri) + analysis_output_obj = convert_uri_to_project_data_obj(analysis_output_uri) # FInd the dragen germline output top_dir_list: List[ProjectData] = list_project_data_non_recursively( @@ -283,7 +283,7 @@ def handler(event, context): map( lambda key_val: ( key_val[0], ( - convert_project_data_obj_to_icav2_uri(key_val[1]) + convert_project_data_obj_to_uri(key_val[1], uri_type=UriType.S3) if key_val[1] is not None else None ) diff --git a/lib/workload/stateless/stacks/wgts-alignment-qc-pipeline-manager/lambdas/set_outputs_json_py/requirements.txt b/lib/workload/stateless/stacks/wgts-alignment-qc-pipeline-manager/lambdas/set_outputs_json_py/requirements.txt index 7be7ed2f2..6733977af 100644 --- a/lib/workload/stateless/stacks/wgts-alignment-qc-pipeline-manager/lambdas/set_outputs_json_py/requirements.txt +++ b/lib/workload/stateless/stacks/wgts-alignment-qc-pipeline-manager/lambdas/set_outputs_json_py/requirements.txt @@ -1,2 +1,2 @@ -wrapica>=2.27.0.post20240714154223,<2.28.0 +wrapica>=2.27.1,<2.28.0 boto3 \ No newline at end of file diff --git a/lib/workload/stateless/stacks/wgts-alignment-qc-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py b/lib/workload/stateless/stacks/wgts-alignment-qc-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py index a806281e6..5ecb972cd 100644 --- a/lib/workload/stateless/stacks/wgts-alignment-qc-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py +++ b/lib/workload/stateless/stacks/wgts-alignment-qc-pipeline-manager/lambdas/set_outputs_json_py/set_outputs_json.py @@ -30,10 +30,10 @@ import logging # ICA imports -from wrapica.enums import DataType +from wrapica.enums import DataType, UriType from wrapica.libica_models import ProjectData from wrapica.project_data import ( - convert_icav2_uri_to_project_data_obj, convert_project_data_obj_to_icav2_uri, + convert_uri_to_project_data_obj, convert_project_data_obj_to_uri, list_project_data_non_recursively ) @@ -85,7 +85,7 @@ def handler(events, context): analysis_uri = events.get("analysis_output_uri") # Convert analysis uri to project folder object - analysis_project_data_obj = convert_icav2_uri_to_project_data_obj(analysis_uri) + analysis_project_data_obj = convert_uri_to_project_data_obj(analysis_uri) # Analysis list analysis_top_level_data_list = list_project_data_non_recursively( @@ -151,10 +151,10 @@ def handler(events, context): ) return { - "alignment_output_uri": convert_project_data_obj_to_icav2_uri(alignment_data_obj), - "bam_file_uri": convert_project_data_obj_to_icav2_uri(bam_file_obj) if bam_file_obj else None, - "multiqc_html_report": convert_project_data_obj_to_icav2_uri(multiqc_html_data_obj), - "multiqc_output_uri": convert_project_data_obj_to_icav2_uri(multiqc_data_obj) + "alignment_output_uri": convert_project_data_obj_to_uri(alignment_data_obj, uri_type=UriType.S3), + "bam_file_uri": convert_project_data_obj_to_uri(bam_file_obj, uri_type=UriType.S3) if bam_file_obj else None, + "multiqc_html_report": convert_project_data_obj_to_uri(multiqc_html_data_obj, uri_type=UriType.S3), + "multiqc_output_uri": convert_project_data_obj_to_uri(multiqc_data_obj, uri_type=UriType.S3) } # if __name__ == "__main__":