bird-house · fmigneault · Feb 10, 2023 · Sep 12, 2022 · Nov 17, 2022 · Dec 14, 2022
@@ -14,7 +14,45 @@
 [Unreleased](https:/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes:
+
+- secure-data-proxy: add new [`secure-data-proxy`][secure-data-proxy] optional component.
+
+ When enabled, this component will enforce authentication and authorization to be resolved against the `/wpsoutputs`
+ endpoint prior to accessing the results produced by WPS executions. A Magpie service named `secure-data-proxy` is
+ created to define the resource and permission hierarchy of directories and files the users and groups can access.
+ When disabled, the original behavior to provide open access to `/wpsoutputs` is employed.
+
+ A variable named `SECURE_DATA_PROXY_AUTH_INCLUDE` is dynamically assigned based on the activation or not of this
+ component. Corresponding validation of optional/mandatory/delayed-eval variables used by this component are also
+ applied dynamically, as well as mounting the necessary `nginx` and `docker-compose` extended configurations.
+
+- Weaver: adjust user-context output directory hooks and permissions for [`secure-data-proxy`][secure-data-proxy].
+
+ When a process defined in Weaver (either a WPS provider or a local definition) is executed by a user that was granted
+ authorization to run a job, the corresponding user-context directory under `/wpsoutputs/users/{user-id}` will be used
+ for storing the execution outputs and will have the appropriate permissions set for that user to grant them access to
+ those outputs.
+
+## Fixes:
+
+- Magpie/Twitcher: update minimum version `magpie>=3.31.0` to employ `twitcher>=0.8.0` in `MapgieAdatepr`.
+
+ - Resolve an issue where `response.request` references were not set in OWS proxy responses when handled by Twitcher.
+ This caused `MapgieAdatepr` response hooks to fail, which in turn caused failing requests for any non-WPS
+ service that defined any proxy request hook, such as in the case of [`weaver`][weaver-component] component.
+
+ - Adds the Twitcher ``/ows/verify/{service_name}[/{extra_path}`` endpoint employed for validating authorized access
+ to Magpie service/resources, in the same fashion as the protected proxy endpoint, but without performing the proxied
+ request toward the target service. This is mandatory for using the new [`secure-data-proxy`][secure-data-proxy] 
+ optional component, otherwise the proxy endpoint triggers data download twice, once for authorization and another
+ for actually accessing the data.
+
+ See also [Ouranosinc/Magpie#571](https:/Ouranosinc/Magpie/pull/571)
+ and [bird-house/twitcher#118](https:/bird-house/twitcher/pull/118).
+
+[secure-data-proxy]: birdhouse/optional-components/secure-data-proxy
+[weaver-component]: birdhouse/components/weaver
 
 [1.22.11](https:/bird-house/birdhouse-deploy/tree/1.22.11) (2023-02-03)
 ------------------------------------------------------------------------------------------------------------------

@@ -50,12 +50,16 @@ sync_permissions:
  # the `user` variable name would be matched with `user_xyz` and `synced_file`, with `file_abc`.
  # Also, this key would need to sync permissions with the `thredds_workspace` resource key, considering the
  # `permissions_mapping` defined below. The `thredds_workspace` would be deduced to the resource path
- # `/catalog/workspaces/user_xyz/dir1/dir2/subdir/file_abc`.
+ # `/thredds/catalog/workspaces/user_xyz/dir1/dir2/subdir/file_abc`.
  # The types of each segment of this target resource path would be deduced
  # from the `thredds_workspace` config below.
  thredds_workspace:
- - name: catalog
+ - name: thredds
  type: service
+ # not a resource in Magpie
+ # 'catalog' is the file/view format specifier for the rest of the path
+ # - name: catalog
+ # type: directory
  - name: workspaces
  type: directory
  - name: "{user}"
@@ -102,13 +106,17 @@ sync_permissions:
  - "geoserver_workspace : createStoredQuery <-> thredds_workspace : write"
  weaver_outputs:
  services:
- api:
+ weaver:
  process_description:
+ - name: weaver
+ type: service
  - name: processes
  type: route
  - name: "{processID}"
  type: route
  process_job_status:
+ - name: weaver
+ type: service
  - name: processes
  type: route
  - name: "{processID}"
@@ -118,28 +126,106 @@ sync_permissions:
  - name: "{jobID}"
  type: route
  job_status:
+ - name: weaver
+ type: service
  - name: jobs
  type: route
  - name: "{jobID}"
  type: route
  job_outputs:
+ - name: weaver
+ type: service
  - name: jobs
  type: route
  - name: "{jobID}"
  type: route
  - name: outputs
  type: route
+ job_output_single:
+ - name: weaver
+ type: service
+ - name: jobs
+ type: route
+ - name: "{jobID}"
+ type: route
+ - name: outputs
+ type: route
+ - name: "{outputID}"
+ type: route
+ # see 'optional-components/secure-data-proxy' for more details on protected WPS-outputs
+ wps_outputs:
+ # /wpsoutputs/weaver/{public|<user-id>}/{job-id}
  weaver_wps_outputs:
+ - name: secure-data-proxy
+ type: service
  - name: wpsoutputs
  type: route
  - name: weaver
  type: route
+ - name: "{user_context_dir}"
+ type: route
  - name: "{jobID}"
  type: route
+ # /wpsoutputs/weaver/{public|<user-id>}/{job-id}/{output-file}
+ weaver_wps_output_single:
+ - name: secure-data-proxy
+ type: service
+ - name: wpsoutputs
+ type: route
+ - name: weaver
+ type: route
+ - name: "{user_context_dir}"
+ type: route
+ - name: "{jobID}"
+ type: route
+ - name: "{outputID}"
+ type: route
+ thredds:
+ # /twitcher/ows/proxy/thredds/catalog/birdhouse/wps_outputs/weaver/catalog.html
+ # /twitcher/ows/proxy/thredds/catalog/birdhouse/wps_outputs/weaver/{public|<user-id>}/catalog.html
+ # /twitcher/ows/proxy/thredds/catalog/birdhouse/wps_outputs/weaver/{public|<user-id>}/{job-id}/catalog.html
+ # /twitcher/ows/proxy/thredds/catalog/birdhouse/wps_outputs/weaver/{public|<user-id>}/{job-id}/{output-file}
+ # note: paths start after ows-proxy portion extracted when Twitcher/Magpie resolve between each other
+ thredds_wps_outputs:
+ - name: thredds
+ type: service
+ # not a resource in Magpie
+ # 'catalog' is the file/view format specifier for the rest of the path
+ # - name: catalog
+ # type: directory
+ - name: birdhouse
+ type: directory
+ - name: wps_outputs
+ type: directory
+ - name: weaver
+ type: directory
+ - name: "{user_context_dir}"
+ type: directory
+ - name: "{jobID}"
+ type: directory
+ thredds_wps_output_single:
+ - name: thredds
+ type: service
+ # not a resource in Magpie
+ # 'catalog' is the file/view format specifier for the rest of the path
+ # - name: catalog
+ # type: directory
+ - name: birdhouse
+ type: directory
+ - name: wps_outputs
+ type: directory
+ - name: weaver
+ type: directory
+ - name: "{user_context_dir}"
+ type: directory
+ - name: "{jobID}"
+ type: directory
+ - name: "{outputID}"
+ type: file
  permissions_mapping:
  # When user is granted access to an output (either side),
  # output retrieval is allowed from both endpoints (wps-outputs/weaver).
- - "weaver_wps_outputs : read <-> job_outputs : read"
+ - "weaver_wps_outputs : read -> job_outputs : read"
  # When output can be retrieved, access to details about the process and
  # the job are also provided (to understand what each output represents),
  # but getting read access to a process description should not grant
@@ -149,7 +235,16 @@ sync_permissions:
  - "weaver_wps_outputs : read -> job_status : read"
  # process-prefixed items can be only one-way since wps-outputs does not
  # encode the 'processID' information ('jobID' directly the top-level dir)
- - "process_job_status : read -> weaver_wps_outputs : read"
  - "process_job_status : read -> job_status : read"
+ # NOTE:
+ # missing 'user_context_dir' information not defined in path of process execution request
+ # this permission must be set using the magpie/twitcher pre/post request hook to extract the authorized user
+ ###- "process_job_status : read -> weaver_wps_outputs : read"
  # different permission (match), otherwise all jobs/outputs become available.
  - "process_job_status : read -> process_description : read-match"
+ # corresponding outputs retrieved under wps-outputs or thredds share access
+ - "weaver_wps_outputs : read <-> thredds_wps_outputs : read"
+ # permissions if outputs are shared one-by-one in case of multiple files produced by the process
+ - "weaver_wps_output_single : read <-> thredds_wps_output_single : read"
+ - "weaver_wps_output_single : read -> job_output_single : read"
+ - "thredds_wps_output_single : read -> job_output_single : read"
@@ -4,9 +4,9 @@
 
 # All env in this default.env can be overridden by env.local.
 
-# All env in this default.env must NOT depend on any other env. If they do,
-# must use single quote to avoid early expansion before overrides in local.env
-# are applied and must add to the list of DELAYED_EVAL.
+# All env in this default.env must NOT depend on any other env. If they do, they
+# must use single quotes to avoid early expansion before overrides in env.local
+# are applied and must be added to the list of DELAYED_EVAL.
 
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
 # single quotes are important in below list to keep variable names intact until 'pavics-compose' parses them

@@ -73,10 +73,15 @@ services:
 
  # extend Magpie permissions to grant access to Cowbird API via secured Twitcher proxy
  magpie:
+ links:
+ # must have link to send webhook requests directly though internal network
+ - cowbird
  volumes:
  # NOTE:
- # Although file uses the "config.yml" format, it is very important to pass it as independent/duplicate reference
- # provider/permissions config files. This is because 'MAGPIE_CONFIG_PATH' is not used to allow parsing multiple
- # config files for each extendable service, using loading of all configuration files found in mount directories.
+ # Although the file uses the combined "config.yml" format, it is very important to pass it as independent and
+ # duplicate references for providers/permissions/webhooks config files. This is because 'MAGPIE_CONFIG_PATH' is
+ # not used to allow parsing additive per-component config files for each extendable service, using loading of
+ # all configuration files found in mounted directories.
  - ./components/cowbird/config/magpie/config.yml:/opt/local/src/magpie/config/permissions/cowbird.yml:ro
  - ./components/cowbird/config/magpie/config.yml:/opt/local/src/magpie/config/providers/cowbird.yml:ro
+ - ./components/cowbird/config/magpie/config.yml:/opt/local/src/magpie/config/webhooks/cowbird.yml:ro
@@ -1,8 +1,8 @@
 # All env in this default.env can be overridden by env.local.
 
-# All env in this default.env must NOT depend on any other env. If they do,
-# must use single quote to avoid early expansion before overrides in local.env
-# are applied and must add to the list of DELAYED_EVAL.
+# All env in this default.env must NOT depend on any other env. If they do, they
+# must use single quotes to avoid early expansion before overrides in env.local
+# are applied and must be added to the list of DELAYED_EVAL.
 
 export GRAFANA_ADMIN_PASSWORD="changeme!"
 export ALERTMANAGER_ADMIN_EMAIL_RECEIVER="" # "[email protected],[email protected]"

@@ -19,18 +19,33 @@ providers:
  # see also:
  # - https://pavics-weaver.readthedocs.io/en/latest/processes.html?highlight=x-wps-output-context#outputs-location
  # each path below are equivalents, but with more or less specific reference to the requested service/process
+ # for job execution, 2 endpoints exist for older and newer OGC API - Processes specification
  - type: request
- path: "/providers/[\\w_-]+/processes/[\\w_-]+/jobs"
+ path: "/providers/[\\w_-]+/processes/[\\w_-:]+/(jobs|execution)"
  method: POST
  target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:add_x_wps_output_context
  - type: request
- path: "/processes/[\\w_-]+/jobs"
+ path: "/processes/[\\w_-:]+/(jobs|execution)"
  method: POST
  target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:add_x_wps_output_context
  - type: request
- path: "/jobs"
+ path: "/(jobs|execution)"
  method: POST
  target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:add_x_wps_output_context
+ # apply relevant permissions to allow the user executing a process to retrieve its outputs
+ - type: response
+ path: "/providers/[\\w_-]+/processes/[\\w_-:]+/(jobs|execution)"
+ method: POST
+ target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:allow_user_execute_outputs
+ - type: response
+ path: "/processes/[\\w_-:]+/(jobs|execution)"
+ method: POST
+ target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:allow_user_execute_outputs
+ - type: response
+ path: "/(jobs|execution)"
+ method: POST
+ target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:allow_user_execute_outputs
+ # apply relevant permissions such that the user can access its deployed process
  - type: response
  path: "/processes"
  method: GET