Skip to content

Commit

Permalink
fix(filemanager): docs and OpenAPI definitions
Browse files Browse the repository at this point in the history
  • Loading branch information
mmalenic committed Aug 8, 2024
1 parent 418bb29 commit bce0135
Show file tree
Hide file tree
Showing 7 changed files with 32 additions and 34 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ export class Function extends Construct {
},
},
memorySize: 128,
timeout: Duration.seconds(300),
timeout: Duration.seconds(28),
environment: {
// No password here, using RDS IAM to generate credentials.
PGHOST: props.host,
Expand Down
21 changes: 10 additions & 11 deletions lib/workload/stateless/stacks/filemanager/docs/API_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ The API is designed to have a standard set of REST routes which can be used to q
For example, to query a single record, use the `s3_object_id` in the path, which returns the JSON record:

```sh
curl -H "Authorization: Bearer $TOKEN" "https://file.dev.umccr.org/api/v1/s3_objects/01912c56-c458-797f-9d3f-b803e093b7cf" | jq
curl -H "Authorization: Bearer $TOKEN" "https://file.dev.umccr.org/api/v1/s3_objects/0190465f-68fa-76e4-9c36-12bdf1a1571d" | jq
```

Multiple records can be reached using the same route, which returns an array of JSON records:
Expand Down Expand Up @@ -90,14 +90,14 @@ and `_` to match one character. These queries get converted to postgres `like` q
on a key prefix:

```sh
curl --get -H "Authorization: Bearer $TOKEN" --data-urlencode "key=temp_data%" \
"https://file.dev.umccr.org/api/v1/s3_objects/01912c56-c458-797f-9d3f-b803e093b7cf" | jq
curl --get -H "Authorization: Bearer $TOKEN" --data-urlencode "key=temp\_data%" \
"https://file.dev.umccr.org/api/v1/s3_objects" | jq
```

Case-insensitive wildcard matching, which gets converted to a postgres `ilike` statement, is supported by using `case_sensitive`:

```sh
curl --get -H "Authorization: Bearer $TOKEN" --data-urlencode "key=temp_data%" \
curl --get -H "Authorization: Bearer $TOKEN" --data-urlencode "key=temp\_data%" \
"https://file.dev.umccr.org/api/v1/s3_objects?case_sensitive=false" | jq
```

Expand All @@ -118,18 +118,17 @@ Attributes are update using [JSON patch][json-patch].
For example, update attributes on a single record:

```sh
curl --patch -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \
--data "{ "attributes": [ { "op": "add", "path": "/portal_run_id", "value": "202405212aecb782" } ] }" \
"https://file.dev.umccr.org/api/v1/s3_objects/01912c56-c458-797f-9d3f-b803e093b7cf" | jq
curl -X PATCH -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \
--data '{ "attributes": [ { "op": "add", "path": "/portal_run_id", "value": "202405212aecb782" } ] }' \
"https://file.dev.umccr.org/api/v1/s3_objects/0190465f-68fa-76e4-9c36-12bdf1a1571d" | jq
```

Or, update attributes for multiple records with the same key prefix:

```sh
curl --patch -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \
--data "{ "attributes": [ { "op": "add", "path": "/attribute_id", "value": "attribute_id" } ] }" \
--data-urlencode "key=temp_data%" \
"https://file.dev.umccr.org/api/v1/s3_objects" | jq
curl -X PATCH -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \
--data '{ "attributes": [ { "op": "add", "path": "/portal_run_id", "value": "202405212aecb782" } ] }' \
"https://file.dev.umccr.org/api/v1/s3_objects?key=%25202405212aecb782%25" | jq
```

## Count objects
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
> [!IMPORTANT]
> This document is a discussion on potential designs for filemanager attribute linking. The current implementation
> isn't exactly like this, although it will probably contain components of the design here.
# FileManager Attribute Linking

The filemanager needs to be able to store data from other microservices on `s3_object` records in order to perform some
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -943,10 +943,7 @@ pub(crate) mod tests {
false,
)
.await;
assert_eq!(
result,
&s3_entries[0..2]
);
assert_eq!(result, &s3_entries[0..2]);
}

#[sqlx::test(migrator = "MIGRATOR")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,14 @@
use json_patch::patch;
use sea_orm::prelude::{Expr, Json};
use sea_orm::sea_query::{
Alias, Asterisk, CommonTableExpression, PostgresQueryBuilder, Query, SelectStatement,
SimpleExpr, WithClause, WithQuery,
Alias, Asterisk, CommonTableExpression, Query, SelectStatement, SimpleExpr, WithClause,
WithQuery,
};
use sea_orm::{
ColumnTrait, ConnectionTrait, EntityTrait, FromQueryResult, Iterable, ModelTrait, QueryFilter,
QueryTrait, StatementBuilder, Value,
};
use serde_json::json;
use tracing::trace;
use uuid::Uuid;

use crate::database::entities::{object, s3_object};
Expand Down Expand Up @@ -293,7 +292,6 @@ where

fn trace_query(&self, message: &str) {
self.select_to_update.trace_query(message);
trace!("{message}: {}", self.update.to_string(PostgresQueryBuilder));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,19 @@ use utoipa::{IntoParams, ToSchema};
#[serde(default)]
#[into_params(parameter_in = Query)]
pub struct S3ObjectsFilter {
#[param(required = false)]
/// Query by event type.
#[param(required = false, value_type = Wildcard)]
/// Query by event type. Supports wildcards.
pub(crate) event_type: Option<WildcardEither<EventType>>,
#[param(required = false)]
/// Query by bucket.
/// Query by bucket. Supports wildcards.
pub(crate) bucket: Option<Wildcard>,
#[param(required = false)]
/// Query by key.
/// Query by key. Supports wildcards.
pub(crate) key: Option<Wildcard>,
#[param(required = false)]
/// Query by version_id.
/// Query by version_id. Supports wildcards.
pub(crate) version_id: Option<Wildcard>,
#[param(required = false)]
#[param(required = false, value_type = Wildcard)]
/// Query by date. Supports wildcards.
pub(crate) date: Option<WildcardEither<DateTimeWithTimeZone>>,
#[param(required = false)]
Expand All @@ -37,14 +37,14 @@ pub struct S3ObjectsFilter {
#[param(required = false)]
/// Query by the sha256 checksum.
pub(crate) sha256: Option<String>,
#[param(required = false)]
/// Query by the last modified date.
#[param(required = false, value_type = Wildcard)]
/// Query by the last modified date. Supports wildcards.
pub(crate) last_modified_date: Option<WildcardEither<DateTimeWithTimeZone>>,
#[param(required = false)]
/// Query by the e_tag.
pub(crate) e_tag: Option<String>,
#[param(required = false)]
/// Query by the storage class.
#[param(required = false, value_type = Wildcard)]
/// Query by the storage class. Supports wildcards.
pub(crate) storage_class: Option<WildcardEither<StorageClass>>,
#[param(required = false)]
/// Query by the object delete marker.
Expand All @@ -54,7 +54,7 @@ pub struct S3ObjectsFilter {
/// fields, e.g. `attributes[attribute_id]=...`. This only deserializes
/// into string fields, and does not support other JSON types. E.g.
/// `attributes[attribute_id]=1` converts to `{ "attribute_id" = "1" }`
/// rather than `{ "attribute_id" = 1 }`.
/// rather than `{ "attribute_id" = 1 }`. Supports wildcards.
pub(crate) attributes: Option<Json>,
}

Expand All @@ -70,6 +70,6 @@ pub struct ObjectsFilter {
/// fields, e.g. `attributes[attribute_id]=...`. This only deserializes
/// into string fields, and does not support other JSON types. E.g.
/// `attributes[attribute_id]=1` converts to `{ "attribute_id" = "1" }`
/// rather than `{ "attribute_id" = 1 }`.
/// rather than `{ "attribute_id" = 1 }`. Supports wildcards.
pub(crate) attributes: Option<Json>,
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use utoipa::ToSchema;

/// An enum which deserializes into a concrete type or a wildcard. This is used for better
/// type support when non-string filter parameters such as `StorageClass` or `EventType`.
#[derive(Serialize, Deserialize, Debug, ToSchema, Eq, PartialEq)]
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
#[serde(untagged)]
pub enum WildcardEither<T> {
Or(T),
Expand Down Expand Up @@ -37,7 +37,7 @@ impl<T> WildcardEither<T> {
}

/// A wildcard type represents a filter to match arbitrary characters. Use '%' for multiple characters
/// and '_' for a single character. Use '\' to escape these characters. Wildcards are converted to
/// and '_' for a single character. Use '\\' to escape these characters. Wildcards are converted to
/// postgres `like` or `ilike` queries.
#[derive(Serialize, Deserialize, Debug, Default, ToSchema, Eq, PartialEq)]
#[serde(default)]
Expand Down

0 comments on commit bce0135

Please sign in to comment.