Force rotation: adds an integration test to verify that the JWT autho… (

#5583)

* Force rotation: adds an integration test to verify that the JWT authority correctly handles forced rotation. Ensures that JWT tokens are invalidated and reissued as expected.

Signed-off-by: Marcos Yacob <[email protected]>

test/integration/suites/force-rotation-jwt-authority/00-setup

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+"${ROOTDIR}/setup/x509pop/setup.sh" conf/server conf/agent
+

test/integration/suites/force-rotation-jwt-authority/01-start-server

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker-up spire-server

test/integration/suites/force-rotation-jwt-authority/02-bootstrap-agent

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+log-debug "bootstrapping agent..."
+docker compose exec -T spire-server \
+ /opt/spire/bin/spire-server bundle show > conf/agent/bootstrap.crt

test/integration/suites/force-rotation-jwt-authority/03-start-agent

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker-up spire-agent

test/integration/suites/force-rotation-jwt-authority/04-create-workload-entry

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+log-debug "creating registration entry..."
+docker compose exec -T spire-server \
+ /opt/spire/bin/spire-server entry create \
+ -parentID "spiffe://domain.test/spire/agent/x509pop/$(fingerprint conf/agent/agent.crt.pem)" \
+ -spiffeID "spiffe://domain.test/workload" \
+ -selector "unix:uid:0" \
+ -x509SVIDTTL 0
+check-synced-entry "spire-agent" "spiffe://domain.test/workload"
+
+log-info "checking X509-SVID"
+docker compose exec -T spire-agent \
+ /opt/spire/bin/spire-agent api fetch x509 || fail-now "SVID check failed"

test/integration/suites/force-rotation-jwt-authority/05-prepare-jwt-authority

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Initial check for x509 authorities in spire-server
+jwt_authorities=$(docker compose exec -T spire-server \
+ /opt/spire/bin/spire-server bundle show -output json | jq '.jwt_authorities' -c)
+
+amount_authorities=$(echo "$jwt_authorities" | jq length)
+
+# Ensure only one JWT authority is present at the start
+if [[ $amount_authorities -ne 1 ]]; then
+ fail-now "Only one JWT authority expected at start"
+fi
+
+# Prepare authority
+prepared_authority_id=$(docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation spire-server \
+ /opt/spire/bin/spire-server localauthority jwt prepare -output json | jq -r .prepared_authority.authority_id)
+
+# Verify that the prepared authority is logged
+searching="JWT key prepared|local_authority_id=${prepared_authority_id}"
+check-log-line spire-server "$searching"
+
+# Check for updated x509 authorities in spire-server
+# Check for updated JWT authorities in spire-server
+jwt_authorities=$(docker compose exec -T spire-server \
+ /opt/spire/bin/spire-server bundle show -output json | jq '.jwt_authorities' -c)
+amount_authorities=$(echo "$jwt_authorities" | jq length)
+
+# Ensure two JWT authorities are present after preparation
+if [[ $amount_authorities -ne 2 ]]; then
+ fail-now "Two JWT authorities expected after prepare"
+fi
+
+# Ensure the prepared authority is present
+if ! echo "$jwt_authorities" | jq -e ".[] | select(.key_id == \"$prepared_authority_id\")" > /dev/null; then
+ fail-now "Prepared authority not found"
+fi

test/integration/suites/force-rotation-jwt-authority/06-fetch-jwt-svid

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+prepared_authority=$(docker compose exec -t -e SPIRE_SERVER_FFLAGS=forced_rotation spire-server \
+ /opt/spire/bin/spire-server \
+ localauthority jwt show -output json | jq -r .active.authority_id) || fail-now "Failed to fetch prepared JWT authority ID"
+
+svid_json=$(docker compose exec spire-agent ./bin/spire-agent \
+ api fetch jwt -audience aud -output json) || fail-now "Failed to fetch JWT SVID"
+
+jwt_svid=$(echo $svid_json | jq -c '.[0].svids[0].svid') || fail-now "Failed to parse JWT SVID"
+
+# Store JWT SVID for the next steps
+echo $jwt_svid > conf/agent/jwt_svid
+
+# Extract key ID from JWT SVID
+skid=$(echo "$jwt_svid" | jq -r 'split(".") | .[0] | @base64d | fromjson | .kid')
+
+# Check if the key ID matches the prepared authority ID
+if [[ $skid != $prepared_authority ]]; then
+ fail-now "JWT SVID key ID does not match the prepared authority ID, got $skid, expected $prepared_authority"
+fi
+
+keys=$(echo $svid_json | jq -c '.[1].bundles["spiffe://domain.test"] | @base64d | fromjson')
+
+retry_count=0
+max_retries=20
+success=false
+
+while [[ $retry_count -lt $max_retries ]]; do
+ keysLen=$(echo $keys | jq -c '.keys | length')
+ if [[ $keysLen -eq 2 ]]; then
+ success=true
+ break
+ else
+ echo "Retrying... ($((retry_count+1))/$max_retries)"
+ retry_count=$((retry_count+1))
+ sleep 2
+ # Re-fetch the JWT SVID and keys
+ svid_json=$(docker compose exec spire-agent ./bin/spire-agent \
+ api fetch jwt -audience aud -output json) || fail-now "Failed to re-fetch JWT SVID"
+ jwt_svid=$(echo $svid_json | jq -c '.[0].svids[0].svid') || fail-now "Failed to parse re-fetched JWT SVID"
+ keys=$(echo $svid_json | jq -c '.[1].bundles["spiffe://domain.test"] | @base64d | fromjson')
+ fi
+done
+
+if [[ $success == false ]]; then
+ fail-now "Expected one key in JWT SVID bundle, got $keysLen after $max_retries retries"
+fi
+
+echo $keys | jq --arg kid $prepared_authority -e '.keys[] | select(.kid == $kid)' > /dev/null || fail-now "Prepared authority not found in JWT SVID bundle"

test/integration/suites/force-rotation-jwt-authority/07-activate-jwt-authority

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Fetch the prepared authority ID
+prepared_authority=$(docker compose exec -t -e SPIRE_SERVER_FFLAGS=forced_rotation spire-server \
+ /opt/spire/bin/spire-server \
+ localauthority jwt show -output json | jq -r .prepared.authority_id) || fail-now "Failed to fetch prepared JWT authority ID"
+
+# Activate the authority
+activated_authority=$(docker compose exec -t -e SPIRE_SERVER_FFLAGS=forced_rotation spire-server \
+ /opt/spire/bin/spire-server \
+ localauthority jwt activate -authorityID "${prepared_authority}" \
+ -output json | jq -r .activated_authority.authority_id) || fail-now "Failed to activate JWT authority"
+
+log-info "Activated authority: ${activated_authority}"
+
+# Check logs for specific lines
+check-log-line spire-server "JWT key activated|local_authority_id=${prepared_authority}"
+

test/integration/suites/force-rotation-jwt-authority/08-taint-jwt-authority

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+check-logs() {
+ local component=$1
+ shift
+ for log in "$@"; do
+ check-log-line "$component" "$log"
+ done
+}
+
+# Fetch old authority ID
+old_jwt_authority=$(docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation spire-server \
+ /opt/spire/bin/spire-server \
+ localauthority jwt show -output json | jq -r .old.authority_id) || fail-now "Failed to fetch old authority ID"
+
+log-debug "Old authority: $old_jwt_authority"
+
+# Taint the old authority
+docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation spire-server \
+ /opt/spire/bin/spire-server \
+ localauthority jwt taint -authorityID "${old_jwt_authority}" || fail-now "Failed to taint old authority"
+
+# check Server logs
+check-logs spire-server \
+ "JWT authority tainted successfully|local_authority_id=${old_jwt_authority}"
+
+# Check Agent logs
+check-logs spire-agent \
+ "JWT-SVIDs were removed from the JWT cache because they were issued by a tainted authority|count_jwt_svids=1|jwt_authority_key_ids=${old_jwt_authority}"
+

test/integration/suites/force-rotation-jwt-authority/09-verify-svid-rotation

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+active_authority=$(docker compose exec -t -e SPIRE_SERVER_FFLAGS=forced_rotation spire-server \
+ /opt/spire/bin/spire-server \
+ localauthority jwt show -output json | jq -r .active.authority_id) || fail-now "Failed to fetch active JWT authority ID"
+
+jwt_svid=$(docker compose exec spire-agent ./bin/spire-agent \
+ api fetch jwt -audience aud -output json | jq -c '.[0].svids[0].svid') || fail-now "Failed to fetch JWT SVID"
+
+oldJWT=$(cat conf/agent/jwt_svid)
+if [[ $oldJWT == $jwt_svid ]]; then
+ fail-now "JWT SVID did not rotate"
+fi
+
+# Extract key ID from JWT SVID
+skid=$(echo "$jwt_svid" | jq -r 'split(".") | .[0] | @base64d | fromjson | .kid')
+
+# Check if the key ID matches the active authority ID
+if [[ $skid != $active_authority ]]; then
+ fail-now "JWT SVID key ID does not match the active authority ID, got $skid, expected $active_authority"
+fi

test/integration/suites/force-rotation-jwt-authority/10-revoke-jwt-authority

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+old_jwt_authority=$(docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation spire-server \
+ /opt/spire/bin/spire-server \
+ localauthority jwt show -output json | jq -r .old.authority_id) || fail-now "Failed to fetch old authority ID"
+
+log-debug "Old authority: $old_jwt_authority"
+
+jwt_authorities_count=$(docker compose exec -T spire-server \
+ /opt/spire/bin/spire-server bundle \
+ show -output json | jq '.jwt_authorities | length')
+
+if [ $jwt_authorities_count -eq 2 ]; then
+ log-debug "Two JWT Authorities found"
+else
+ fail-now "Expected to be two JWT Authorities. Found $jwt_authorities_count."
+fi
+
+tainted_found=$(docker compose exec -T spire-server /opt/spire/bin/spire-server bundle show -output json | jq '.jwt_authorities[] | select(.tainted == true)')
+
+if [[ -z "$tainted_found" ]]; then
+ fail-now "Tainted JWT authority expected"
+fi
+
+docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation spire-server \
+ /opt/spire/bin/spire-server localauthority jwt \
+ revoke -authorityID $old_jwt_authority -output json || fail-now "Failed to revoke JWT authority"
+
+check-log-line spire-server "JWT authority revoked successfully|local_authority_id=$old_jwt_authority"
+

test/integration/suites/force-rotation-jwt-authority/11-verify-revoked-jwt-authority

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+for i in {1..20}; do
+ active_jwt_authority=$(docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation spire-server \
+ /opt/spire/bin/spire-server \
+ localauthority jwt show -output json | jq -r .active.authority_id) || fail-now "Failed to fetch old jwt authority ID"
+
+ log-debug "Active old authority: $active_jwt_authority"
+
+ svid_json=$(docker compose exec spire-agent ./bin/spire-agent \
+ api fetch jwt -audience aud -output json)
+
+ keys=$(echo $svid_json | jq -c '.[1].bundles["spiffe://domain.test"] | @base64d | fromjson')
+
+ keysLen=$(echo $keys | jq -c '.keys | length')
+ if [[ $keysLen -eq 1 ]]; then
+ break
+ fi
+
+ if [[ $i -eq 20 ]]; then
+ fail-now "Expected one key in JWT SVID bundle, got $keysLen after 20 attempts"
+ fi
+
+ sleep 2s
+done
+
+echo $keys | jq --arg kid $active_jwt_authority -e '.keys[] | select(.kid == $kid)' > /dev/null || fail-now "Active authority not found in JWT SVID bundle"
+

test/integration/suites/force-rotation-jwt-authority/README.md

@@ -0,0 +1,12 @@
+# Force rotation with JWT Authority Test Suite
+
+## Description
+
+This test suite configures a single SPIRE Server and Agent to validate the forced rotation and revocation of JWT authorities.
+
+## Test steps
+
+1. **Prepare a new JWT authority**: Verify that a new JWT authority is successfully created.
+2. **Activate the new JWT authority**: Ensure that the new JWT authority becomes the active authority.
+3. **Taint the old JWT authority**: Confirm that the old JWT authority is marked as tainted, and verify that the taint instruction is propagated to the agent, triggering the deletion of any JWT-SVID signed by tainted authority.
+4. **Revoke the tainted JWT authority**: Validate that the revocation instruction is propagated to the agent and that all the JWT-SVIDs have the revoked authority removed.

test/integration/suites/force-rotation-jwt-authority/conf/agent/agent.conf

@@ -0,0 +1,26 @@
+agent {
+ data_dir = "/opt/spire/data/agent"
+ log_level = "DEBUG"
+ server_address = "spire-server"
+ server_port = "8081"
+ trust_bundle_path = "/opt/spire/conf/agent/bootstrap.crt"
+ trust_domain = "domain.test"
+}
+
+plugins {
+ NodeAttestor "x509pop" {
+ plugin_data {
+ private_key_path = "/opt/spire/conf/agent/agent.key.pem"
+ certificate_path = "/opt/spire/conf/agent/agent.crt.pem"
+ }
+ }
+ KeyManager "disk" {
+ plugin_data {
+ directory = "/opt/spire/data/agent"
+ }
+ }
+ WorkloadAttestor "unix" {
+ plugin_data {
+ }
+ }
+}

test/integration/suites/force-rotation-jwt-authority/conf/server/server.conf

@@ -0,0 +1,29 @@
+server {
+ bind_address = "0.0.0.0"
+ bind_port = "8081"
+ trust_domain = "domain.test"
+ data_dir = "/opt/spire/data/server"
+ log_level = "DEBUG"
+ ca_ttl = "24h"
+ default_jwt_svid_ttl = "8h"
+ experimental {
+ feature_flags = ["forced_rotation"] 
+ }
+}
+
+plugins {
+ DataStore "sql" {
+ plugin_data {
+ database_type = "sqlite3"
+ connection_string = "/opt/spire/data/server/datastore.sqlite3"
+ }
+ }
+ NodeAttestor "x509pop" {
+ plugin_data {
+ ca_bundle_path = "/opt/spire/conf/server/agent-cacert.pem"
+ }
+ }
+ KeyManager "memory" {
+ plugin_data = {}
+ }
+}

test/integration/suites/force-rotation-jwt-authority/docker-compose.yaml

@@ -0,0 +1,14 @@
+services:
+ spire-server:
+ image: spire-server:latest-local
+ hostname: spire-server
+ volumes:
+ - ./conf/server:/opt/spire/conf/server
+ command: ["-config", "/opt/spire/conf/server/server.conf"]
+ spire-agent:
+ image: spire-agent:latest-local
+ hostname: spire-agent
+ depends_on: ["spire-server"]
+ volumes:
+ - ./conf/agent:/opt/spire/conf/agent
+ command: ["-config", "/opt/spire/conf/agent/agent.conf"]

test/integration/suites/force-rotation-jwt-authority/teardown

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+if [ -z "$SUCCESS" ]; then
+ docker compose logs
+fi
+docker-down