diff --git a/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/function.ts index 0e4962baf..6ca5ed426 100644 --- a/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/function.ts @@ -116,7 +116,7 @@ export class Function extends Construct { }, }, memorySize: 128, - timeout: Duration.seconds(300), + timeout: Duration.seconds(28), environment: { // No password here, using RDS IAM to generate credentials. PGHOST: props.host, diff --git a/lib/workload/stateless/stacks/filemanager/docs/API_GUIDE.md b/lib/workload/stateless/stacks/filemanager/docs/API_GUIDE.md index f09c03283..41c92d047 100644 --- a/lib/workload/stateless/stacks/filemanager/docs/API_GUIDE.md +++ b/lib/workload/stateless/stacks/filemanager/docs/API_GUIDE.md @@ -25,7 +25,7 @@ The API is designed to have a standard set of REST routes which can be used to q For example, to query a single record, use the `s3_object_id` in the path, which returns the JSON record: ```sh -curl -H "Authorization: Bearer $TOKEN" "https://file.dev.umccr.org/api/v1/s3_objects/01912c56-c458-797f-9d3f-b803e093b7cf" | jq +curl -H "Authorization: Bearer $TOKEN" "https://file.dev.umccr.org/api/v1/s3_objects/0190465f-68fa-76e4-9c36-12bdf1a1571d" | jq ``` Multiple records can be reached using the same route, which returns an array of JSON records: @@ -90,14 +90,14 @@ and `_` to match one character. These queries get converted to postgres `like` q on a key prefix: ```sh -curl --get -H "Authorization: Bearer $TOKEN" --data-urlencode "key=temp_data%" \ -"https://file.dev.umccr.org/api/v1/s3_objects/01912c56-c458-797f-9d3f-b803e093b7cf" | jq +curl --get -H "Authorization: Bearer $TOKEN" --data-urlencode "key=temp\_data%" \ +"https://file.dev.umccr.org/api/v1/s3_objects" | jq ``` Case-insensitive wildcard matching, which gets converted to a postgres `ilike` statement, is supported by using `case_sensitive`: ```sh -curl --get -H "Authorization: Bearer $TOKEN" --data-urlencode "key=temp_data%" \ +curl --get -H "Authorization: Bearer $TOKEN" --data-urlencode "key=temp\_data%" \ "https://file.dev.umccr.org/api/v1/s3_objects?case_sensitive=false" | jq ``` @@ -118,18 +118,17 @@ Attributes are update using [JSON patch][json-patch]. For example, update attributes on a single record: ```sh -curl --patch -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \ - --data "{ "attributes": [ { "op": "add", "path": "/portal_run_id", "value": "202405212aecb782" } ] }" \ -"https://file.dev.umccr.org/api/v1/s3_objects/01912c56-c458-797f-9d3f-b803e093b7cf" | jq +curl -X PATCH -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \ +--data '{ "attributes": [ { "op": "add", "path": "/portal_run_id", "value": "202405212aecb782" } ] }' \ +"https://file.dev.umccr.org/api/v1/s3_objects/0190465f-68fa-76e4-9c36-12bdf1a1571d" | jq ``` Or, update attributes for multiple records with the same key prefix: ```sh -curl --patch -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \ - --data "{ "attributes": [ { "op": "add", "path": "/attribute_id", "value": "attribute_id" } ] }" \ - --data-urlencode "key=temp_data%" \ -"https://file.dev.umccr.org/api/v1/s3_objects" | jq +curl -X PATCH -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \ +--data '{ "attributes": [ { "op": "add", "path": "/portal_run_id", "value": "202405212aecb782" } ] }' \ +"https://file.dev.umccr.org/api/v1/s3_objects?key=%25202405212aecb782%25" | jq ``` ## Count objects diff --git a/lib/workload/stateless/stacks/filemanager/docs/ATTRIBUTE_LINKING.md b/lib/workload/stateless/stacks/filemanager/docs/ATTRIBUTE_LINKING.md index c00e2dede..fa3632158 100644 --- a/lib/workload/stateless/stacks/filemanager/docs/ATTRIBUTE_LINKING.md +++ b/lib/workload/stateless/stacks/filemanager/docs/ATTRIBUTE_LINKING.md @@ -1,3 +1,7 @@ +> [!IMPORTANT] +> This document is a discussion on potential designs for filemanager attribute linking. The current implementation +> isn't exactly like this, although it will probably contain components of the design here. + # FileManager Attribute Linking The filemanager needs to be able to store data from other microservices on `s3_object` records in order to perform some diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/src/queries/list.rs b/lib/workload/stateless/stacks/filemanager/filemanager/src/queries/list.rs index c7df7a303..7de989f8b 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/src/queries/list.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager/src/queries/list.rs @@ -943,10 +943,7 @@ pub(crate) mod tests { false, ) .await; - assert_eq!( - result, - &s3_entries[0..2] - ); + assert_eq!(result, &s3_entries[0..2]); } #[sqlx::test(migrator = "MIGRATOR")] diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/src/queries/update.rs b/lib/workload/stateless/stacks/filemanager/filemanager/src/queries/update.rs index 00c92d658..58fd98048 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/src/queries/update.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager/src/queries/update.rs @@ -4,15 +4,14 @@ use json_patch::patch; use sea_orm::prelude::{Expr, Json}; use sea_orm::sea_query::{ - Alias, Asterisk, CommonTableExpression, PostgresQueryBuilder, Query, SelectStatement, - SimpleExpr, WithClause, WithQuery, + Alias, Asterisk, CommonTableExpression, Query, SelectStatement, SimpleExpr, WithClause, + WithQuery, }; use sea_orm::{ ColumnTrait, ConnectionTrait, EntityTrait, FromQueryResult, Iterable, ModelTrait, QueryFilter, QueryTrait, StatementBuilder, Value, }; use serde_json::json; -use tracing::trace; use uuid::Uuid; use crate::database::entities::{object, s3_object}; @@ -293,7 +292,6 @@ where fn trace_query(&self, message: &str) { self.select_to_update.trace_query(message); - trace!("{message}: {}", self.update.to_string(PostgresQueryBuilder)); } } diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/filter/mod.rs b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/filter/mod.rs index 908e246fb..1a5630759 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/filter/mod.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/filter/mod.rs @@ -16,19 +16,19 @@ use utoipa::{IntoParams, ToSchema}; #[serde(default)] #[into_params(parameter_in = Query)] pub struct S3ObjectsFilter { - #[param(required = false)] - /// Query by event type. + #[param(required = false, value_type = Wildcard)] + /// Query by event type. Supports wildcards. pub(crate) event_type: Option>, #[param(required = false)] - /// Query by bucket. + /// Query by bucket. Supports wildcards. pub(crate) bucket: Option, #[param(required = false)] - /// Query by key. + /// Query by key. Supports wildcards. pub(crate) key: Option, #[param(required = false)] - /// Query by version_id. + /// Query by version_id. Supports wildcards. pub(crate) version_id: Option, - #[param(required = false)] + #[param(required = false, value_type = Wildcard)] /// Query by date. Supports wildcards. pub(crate) date: Option>, #[param(required = false)] @@ -37,14 +37,14 @@ pub struct S3ObjectsFilter { #[param(required = false)] /// Query by the sha256 checksum. pub(crate) sha256: Option, - #[param(required = false)] - /// Query by the last modified date. + #[param(required = false, value_type = Wildcard)] + /// Query by the last modified date. Supports wildcards. pub(crate) last_modified_date: Option>, #[param(required = false)] /// Query by the e_tag. pub(crate) e_tag: Option, - #[param(required = false)] - /// Query by the storage class. + #[param(required = false, value_type = Wildcard)] + /// Query by the storage class. Supports wildcards. pub(crate) storage_class: Option>, #[param(required = false)] /// Query by the object delete marker. @@ -54,7 +54,7 @@ pub struct S3ObjectsFilter { /// fields, e.g. `attributes[attribute_id]=...`. This only deserializes /// into string fields, and does not support other JSON types. E.g. /// `attributes[attribute_id]=1` converts to `{ "attribute_id" = "1" }` - /// rather than `{ "attribute_id" = 1 }`. + /// rather than `{ "attribute_id" = 1 }`. Supports wildcards. pub(crate) attributes: Option, } @@ -70,6 +70,6 @@ pub struct ObjectsFilter { /// fields, e.g. `attributes[attribute_id]=...`. This only deserializes /// into string fields, and does not support other JSON types. E.g. /// `attributes[attribute_id]=1` converts to `{ "attribute_id" = "1" }` - /// rather than `{ "attribute_id" = 1 }`. + /// rather than `{ "attribute_id" = 1 }`. Supports wildcards. pub(crate) attributes: Option, } diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/filter/wildcard.rs b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/filter/wildcard.rs index cf06085b8..bdcaaf254 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/filter/wildcard.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/filter/wildcard.rs @@ -6,7 +6,7 @@ use utoipa::ToSchema; /// An enum which deserializes into a concrete type or a wildcard. This is used for better /// type support when non-string filter parameters such as `StorageClass` or `EventType`. -#[derive(Serialize, Deserialize, Debug, ToSchema, Eq, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] #[serde(untagged)] pub enum WildcardEither { Or(T), @@ -37,7 +37,7 @@ impl WildcardEither { } /// A wildcard type represents a filter to match arbitrary characters. Use '%' for multiple characters -/// and '_' for a single character. Use '\' to escape these characters. Wildcards are converted to +/// and '_' for a single character. Use '\\' to escape these characters. Wildcards are converted to /// postgres `like` or `ilike` queries. #[derive(Serialize, Deserialize, Debug, Default, ToSchema, Eq, PartialEq)] #[serde(default)]