[Feature] Upgrade Rudolph backend to support 2022-24 schema/sync ch…

…anges (#41)

* update types and tests

* update ClientMode tests and create SyncType type

* Add SyncType to configs and update EventUploadEvents

* updates configuration models

* refactor preflight state and update models

* update tests

* update API gateway schemas

* upgrade some go deps and remove legacy deps

* upgrade all remaining AWS libs/deps

* update build scripts to include local deployment CLI and macos cross-compatible

update on the scripts

* upgrade lambda pkgs to v2, cleanup, and postflight updates

* updated providers and lambda memory/publish changes

* update the CLI paths to match the TF folders

update rudolph-cli env config file finder

* add symbolic link for non-Darwin build pipelines

deployments/terraform_modules/_providers.tf

@@ -0,0 +1,8 @@
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "5.15.0"
+ }
+ }
+}

deployments/terraform_modules/santa_api/lambda.tf

@@ -81,6 +81,7 @@ module "preflight_function" {
  lambda_source_key = aws_s3_bucket_object.santa_api_source.key
  lambda_source_hash = local.lambda_source_hash
  endpoint = "preflight"
+ lambda_memory_size = 256
  api_gateway_execution_arn = aws_api_gateway_rest_api.api_gateway.execution_arn
 
  env_vars = {
@@ -100,6 +101,7 @@ module "eventupload_function" {
  lambda_source_key = aws_s3_bucket_object.santa_api_source.key
  lambda_source_hash = local.lambda_source_hash
  endpoint = "eventupload"
+ lambda_memory_size = 256
  api_gateway_execution_arn = aws_api_gateway_rest_api.api_gateway.execution_arn
 
  env_vars = {
@@ -122,6 +124,7 @@ module "ruledownload_function" {
  lambda_source_key = aws_s3_bucket_object.santa_api_source.key
  lambda_source_hash = local.lambda_source_hash
  endpoint = "ruledownload"
+ lambda_memory_size = 512
  api_gateway_execution_arn = aws_api_gateway_rest_api.api_gateway.execution_arn
 
  env_vars = {

deployments/terraform_modules/santa_api/models.tf

@@ -16,7 +16,11 @@ resource "aws_api_gateway_model" "machine_config" {
  "batch_size": { "type": "number" },
  "enable_bundles": { "type": "boolean" },
  "enable_transitive_rules": { "type": "boolean" },
- "clean_sync": { "type": "boolean" },
+ "full_sync_interval": { "type": "number" },
+ "upload_logs_url": { "type": "string" },
+ "block_usb_mount": { "type": "boolean" },
+ "remount_usb_mode": { "type": "string" },
+ "sync_type": { "type": "string" },
  "upload_logs_url": { "type": "string" }
  },
  "required": ["client_mode", "batch_size"]
@@ -85,7 +89,11 @@ resource "aws_api_gateway_model" "preflight_request" {
  "primary_user": { "type": "string" },
  "compiler_rule_count": { "type": "number" },
  "transitive_rule_count": { "type": "number" },
- "request_clean_sync": { "type": "boolean" }
+ "cdhash_rule_count": { "type": "number" },
+ "teamid_rule_count": { "type": "number" },
+ "signingid_rule_count": { "type": "number" },
+ "request_clean_sync": { "type": "boolean" },
+ "model_identifier": { "type": "string" }
  },
  "required": ["santa_version", "serial_num", "primary_user"]
 }

deployments/terraform_modules/santa_api/modules/lambda/api-handler/_variables.tf

@@ -50,3 +50,15 @@ variable "api_gateway_execution_arn" {
  type = string
  description = "Execution ARN of the API gateway"
 }
+
+variable "lambda_memory_size" {
+ type = number
+ description = "Lambda function runtime memory size in MB. Valid value between 128 MB to 10,240 MB (10 GB), in 64 MB increments."
+ default = 128
+}
+
+variable "lambda_timeout_seconds" {
+ type = number
+ description = "Lambda function timeout in seconds. Default lambda's timeout is intentionally short right now at 10 seconds."
+ default = 10
+}

deployments/terraform_modules/santa_api/modules/lambda/api-handler/lambda.tf

@@ -13,12 +13,13 @@ resource "aws_lambda_function" "api_handler" {
  role = aws_iam_role.api_handler_role.arn
  handler = local.handler
  runtime = local.runtime
- publish = true
+ publish = false
  architectures = ["arm64"]
 
  s3_bucket = var.lambda_source_bucket
  s3_key = var.lambda_source_key
  source_code_hash = var.lambda_source_hash
+ memory_size = var.lambda_memory_size
 
  # The lambda's timeout is intentionally short right now.
  # Most HTTP API requests should not take a huge amount of time, or they will cause the UI to hang which is a
@@ -28,7 +29,7 @@ resource "aws_lambda_function" "api_handler" {
  # 2) Use POST and return a created resource via http 201 or 202. Then, have the server asynchronously process
  # this resource until it reaches a consistent state. Meanwhile, the client can poll with GET requests
  # until the resource reaches a consistent state.
- timeout = 5
+ timeout = var.lambda_timeout_seconds
 
  dynamic "environment" {
  for_each = length(var.env_vars) == 0 ? [] : [1]

go.mod

@@ -1,19 +1,59 @@
 module github.com/airbnb/rudolph
 
-go 1.15
+go 1.22
 
 require (
- github.com/aws/aws-lambda-go v1.23.0
- github.com/aws/aws-sdk-go v1.37.32
- github.com/aws/aws-sdk-go-v2 v1.7.1
- github.com/aws/aws-sdk-go-v2/config v1.3.0
- github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.1.3
- github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.1.3
- github.com/aws/aws-sdk-go-v2/service/dynamodb v1.4.1
- github.com/google/uuid v1.2.0
- github.com/pkg/errors v0.9.1
- github.com/spf13/cobra v1.2.1
- github.com/spf13/viper v1.8.1
- github.com/stretchr/objx v0.1.1 // indirect
- github.com/stretchr/testify v1.7.0
+ github.com/aws/aws-lambda-go v1.46.0
+ github.com/aws/aws-sdk-go v1.51.6
+ github.com/aws/aws-sdk-go-v2 v1.26.0
+ github.com/aws/aws-sdk-go-v2/config v1.27.9
+ github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.13.11
+ github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.7.11
+ github.com/aws/aws-sdk-go-v2/service/dynamodb v1.31.0
+ github.com/aws/aws-sdk-go-v2/service/lambda v1.53.3
+ github.com/google/uuid v1.6.0
+ github.com/spf13/cobra v1.8.0
+ github.com/spf13/viper v1.18.2
+ github.com/stretchr/testify v1.9.0
+)
+
+require (
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
+ github.com/aws/aws-sdk-go-v2/credentials v1.17.9 // indirect
+ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+ github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.20.3 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.5 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect
+ github.com/aws/smithy-go v1.20.1 // indirect
+ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+ github.com/fsnotify/fsnotify v1.7.0 // indirect
+ github.com/hashicorp/hcl v1.0.0 // indirect
+ github.com/inconshreveable/mousetrap v1.1.0 // indirect
+ github.com/jmespath/go-jmespath v0.4.0 // indirect
+ github.com/magiconair/properties v1.8.7 // indirect
+ github.com/mitchellh/mapstructure v1.5.0 // indirect
+ github.com/pelletier/go-toml/v2 v2.2.0 // indirect
+ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+ github.com/sagikazarmark/locafero v0.4.0 // indirect
+ github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+ github.com/sourcegraph/conc v0.3.0 // indirect
+ github.com/spf13/afero v1.11.0 // indirect
+ github.com/spf13/cast v1.6.0 // indirect
+ github.com/spf13/pflag v1.0.5 // indirect
+ github.com/stretchr/objx v0.5.2 // indirect
+ github.com/subosito/gotenv v1.6.0 // indirect
+ go.uber.org/multierr v1.11.0 // indirect
+ golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect
+ golang.org/x/sys v0.18.0 // indirect
+ golang.org/x/text v0.14.0 // indirect
+ gopkg.in/ini.v1 v1.67.0 // indirect
+ gopkg.in/yaml.v2 v2.4.0 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
 )