From 37db5400548d7ee8a2479d362ca83f68559a993f Mon Sep 17 00:00:00 2001 From: Damani <37876601+DamaniN@users.noreply.github.com> Date: Mon, 19 Aug 2024 23:07:33 -0700 Subject: [PATCH] PCR for AWS Scritp Updte (#128) * Added PCR for AWS script * Fixed linting errors * formatting fix * Added support for AWS Profiles * Removed Rubrik Python module from required modules. * Fixed improper bundle tagging. * Added steps to install Docker. * Added updated bundle version number output. * Removed option to set AWS profile. Get profile from environment. * Added option to set PCR in RSC * Added option to use non-ECR PCR * Documentation Update * Support updating PCR config on every run. * Support multiple versions of EKS. * Download only if bundle version has changed. * Clarify authentication requirements * Add additional logging and debugging. * Support repos with / in the name. * Fixed handling of bundle approval. --- sample/pcr-aws/README.MD | 3 +- sample/pcr-aws/pcr.py | 221 +++++++++++++++++++++++++++++---------- 2 files changed, 167 insertions(+), 57 deletions(-) diff --git a/sample/pcr-aws/README.MD b/sample/pcr-aws/README.MD index 00321ae..f5fde85 100644 --- a/sample/pcr-aws/README.MD +++ b/sample/pcr-aws/README.MD @@ -12,4 +12,5 @@ This script downloads the images that are used for Rubrik Security Cloud Exocomp 1. [Install the Docker engine](https://docs.docker.com/engine/install/). 1. Run `docker run hello-world` to verify that the Docker engine is working properly. 1. Make sure that you have enough space on your local system to download the images. Images will be downloaded to the default local Docker registry. -1. Run `python3 ./pcr.py --keyfile --pcrFqdn --profile ` \ No newline at end of file +1. If required create a [service account](https://docs.rubrik.com/en-us/saas/saas/service_account.html) in RSC and save the credential file (key file). +1. Run `python3 ./pcr.py --keyfile --pcrFqdn --profile -a --eksVersion ` \ No newline at end of file diff --git a/sample/pcr-aws/pcr.py b/sample/pcr-aws/pcr.py index f1ce3ba..80352bd 100644 --- a/sample/pcr-aws/pcr.py +++ b/sample/pcr-aws/pcr.py @@ -15,22 +15,29 @@ #logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) parser = argparse.ArgumentParser() -parser.add_argument('-p', '--password', dest='password', help="Polaris Password", default=None) -parser.add_argument('-u', '--username', dest='username', help="Polaris UserName", default=None) +parser.add_argument('-a', '--awsAccountid', dest='awsAccountId', help="AWS Account ID used to download images from Rubrik ECR", default=None,required=True) parser.add_argument('-d', '--domain', dest='domain', help="Polaris Domain", default=None) parser.add_argument('-k', '--keyfile', dest='json_keyfile', help="JSON Keyfile", default=None) +parser.add_argument('-p', '--password', dest='password', help="Polaris Password", default=None) parser.add_argument('-r', '--root', dest='root_domain', help="Polaris Root Domain", default=None) -parser.add_argument('--profile', dest='profile', help="AWS Profile", default=None, required=True) +parser.add_argument('-u', '--username', dest='username', help="Polaris UserName", default=None) +parser.add_argument('-v', '--verbose', help="Be verbose", action="store_const", dest="loglevel", const=logging.INFO) +parser.add_argument('--debug', help="Print lots of debugging statements", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.WARNING) +parser.add_argument('--eksVersion', dest='eksVersion', help='Version of EKS cluster being used for Exocompute', default='1.27', required=True) parser.add_argument('--insecure', help='Deactivate SSL Verification', action="store_true") +parser.add_argument('--pcrAuth', dest='pcrAuth', help='Set to "ECR" to use ECR based private container registry. Set to "PWD" to use username/password based private container registry', default="ECR", required=False, choices=['ECR', 'PWD']) parser.add_argument('--pcrFqdn', dest='pcrFqdn', help='Private Container Registry URL', default=None, required=True) -parser.add_argument('--debug', help="Print lots of debugging statements", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.WARNING) -parser.add_argument('-v', '--verbose', help="Be verbose", action="store_const", dest="loglevel", const=logging.INFO) +parser.add_argument('--pcrPassword', dest='pcrPassword', help='Password for the private container registry.', default=None, required=False) +parser.add_argument('--pcrUsername', dest='pcrUsername', help='Username for the private container registry.', default=None, required=False) args = parser.parse_args() pcrFqdn = args.pcrFqdn logging.basicConfig(level=args.loglevel) +if args.pcrAuth == "PWD" and not (args.pcrPassword and args.pcrUsername): + parser.error('Username/Password authentication to private container registry specified (--pcrAuth PWD), however, --pcrPassword or --pcrUsername not specified.') + if not (args.json_keyfile or (args.username and args.password and args.domain)): parser.error('Login credentials not specified. You must specify either a JSON keyfile or a username, password, and domain.') @@ -48,22 +55,79 @@ print(err) sys.exit(1) -# Login to AWS ECR +# Set Private Container Registry (white list AWS account to download images) -rscEcrSession = boto3.Session(profile_name=args.profile) -rscEcrClient = rscEcrSession.client('ecr', region_name="us-east-1") +variables = { +"awsNativeAccountIdOrNamePrefix": "" +} -# Setup Docker client +try: + allAwsExocomputeConfigs = rubrik._query_raw(raw_query='query ($awsNativeAccountIdOrNamePrefix: String!) {allAwsExocomputeConfigs(awsNativeAccountIdOrNamePrefix: $awsNativeAccountIdOrNamePrefix) {awsCloudAccount {id nativeId accountName message seamlessFlowEnabled cloudType}}}', + operation_name=None, + variables=variables, + timeout=60) +except Exception as err: + print("Error: Unable to retrieve the AWS account details.") + print(err) + sys.exit(1) -dockerClient = docker.from_env() -docker_api_client = docker.APIClient(base_url='unix://var/run/docker.sock') +logging.debug(json.dumps(allAwsExocomputeConfigs, indent=2)) + +# Make sure that all repos are authorized to pull images from Rubrik ECR. +for awsCloudAccount in allAwsExocomputeConfigs['data']['allAwsExocomputeConfigs']: + if awsCloudAccount['awsCloudAccount']['nativeId'] == args.awsAccountId: + print("Setting private container registry for AWS account " + awsCloudAccount['awsCloudAccount']['accountName']) + variables = { + "input": { + "exocomputeAccountId": awsCloudAccount['awsCloudAccount']['id'], + "registryUrl": pcrFqdn, + "pcrAwsImagePullDetails": { + "awsNativeId": args.awsAccountId + } + } + } + + try: + setPrivateContainerRegistry = rubrik._query_raw(raw_query='mutation SetPrivateContainerRegistry($input: SetPrivateContainerRegistryInput!) {setPrivateContainerRegistry(input: $input)}', + operation_name=None, + variables=variables, + timeout=60) + except Exception as err: + print("Error: Unable to set the private container registry.") + print(err) + sys.exit(1) + + print("Getting currently approved PCR bundle version numbers") + variables = { + "input": { + "exocomputeAccountId": awsCloudAccount['awsCloudAccount']['id'] + } + } + + try: + privateContainerRegistry = rubrik._query_raw(raw_query='query PrivateContainerRegistry($input: PrivateContainerRegistryInput!) {privateContainerRegistry(input: $input) {pcrDetails {registryUrl imagePullDetails {... on PcrAwsImagePullDetails {awsNativeId}}} pcrLatestApprovedBundleVersion}}', + operation_name=None, + variables=variables, + timeout=60) + + except Exception as err: + print("Error: Unable to get the private container registry information for exocompute account: " + awsCloudAccount['awsCloudAccount']['accountName']) + print(err) + sys.exit(1) + print("Current approved bundle version for AWS account " + awsCloudAccount['awsCloudAccount']['accountName'] + "is: " + privateContainerRegistry['data']['privateContainerRegistry']['pcrLatestApprovedBundleVersion']) # Get Exocompute Bundle (containers) +variables = { + "input": { + "eksVersion": args.eksVersion + } +} + try: exoTaskImageBundle = rubrik._query_raw(raw_query='query ExotaskImageBundle { exotaskImageBundle {bundleVersion repoUrl bundleImages {name tag sha}}}', operation_name=None, - variables={}, + variables=variables, timeout=60) except Exception as err: print("Error: Unable to retrieve exotaskImageBundle") @@ -74,6 +138,13 @@ logging.debug(json.dumps(exoTaskImageBundle, indent=2)) logging.debug("") +print("New bundle version is: " + exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion']) + +# Exit if new bundle version is the same or lower than the current approved bundle version +if privateContainerRegistry['data']['privateContainerRegistry']['pcrLatestApprovedBundleVersion'] >= exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion']: + print("New bundle version is the same or lower than the current approved bundle version. Exiting.") + sys.exit(0) + region = exoTaskImageBundle['data']['exotaskImageBundle']['repoUrl'].split('.')[3] print("") print("Region: " + region) @@ -83,6 +154,17 @@ print("PCR Region: " + pcrRegion) print("") + +# Login to AWS ECR + +rscEcrSession = boto3.Session() +rscEcrClient = rscEcrSession.client('ecr', region_name=region) + +# Setup Docker client + +dockerClient = docker.from_env() +docker_api_client = docker.APIClient(base_url='unix://var/run/docker.sock') + # Login to RSC ECR # Requires that the RSC setPrivateContainerRegistry GraphQL mutation has been run to set the registry URL in RSC. # This step would have been done as part of the RSC setup process. @@ -116,7 +198,7 @@ try: for line in docker_api_client.pull(rscRepoFqdn + '/' + bundleImages['name'], tag=bundleImages['tag'], stream=True, auth_config=rsc_auth_config_payload, decode=True): print(line) - logging.info(json.dumps(line, indent=2)) + logging.debug(json.dumps(line, indent=2)) except Exception as err: print("Error: Image pull failed for " + bundleImages['name'] + " with tag " + bundleImages['tag']) @@ -128,7 +210,7 @@ try: for line in docker_api_client.pull(rscRepoFqdn + '/' + bundleImages['name'], tag="sha256:" + bundleImages['sha'], stream=True, auth_config=rsc_auth_config_payload, decode=True): print(line) - logging.info(json.dumps(line, indent=2)) + logging.debug(json.dumps(line, indent=2)) except Exception as err: print("Error: Image pull failed for " + bundleImages['name'] + " with sha " + bundleImages['sha']) print(err) @@ -163,53 +245,73 @@ sys.exit(1) print("") -#Login to customer PCR -customerEcrSession = boto3.Session(profile_name=args.profile) -customerEcrClient = customerEcrSession.client('ecr', region_name=pcrRegion) -# Get customer PCR token -# CLI Example "aws ecr get-authorization-token --region " -try: - customerEcrToken = customerEcrClient.get_authorization_token(registryIds=[pcrFqdn.split('.')[0]]) -except Exception as err: - print("Error: Unable to get customer PCR token.") - print(err) - sys.exit(1) - -# CLI Example "aws ecr get-login-password --region | docker login --username AWS --password-stdin " -try: - username, password = base64.b64decode(customerEcrToken['authorizationData'][0]['authorizationToken']).decode('utf-8').split(":") - customer_auth_config_payload = { 'username': username, 'password': password } - customerEcr = dockerClient.login(username=username, password=password, registry=customerEcrToken['authorizationData'][0]['proxyEndpoint'].replace("https://", ""), reauth=True) -except Exception as err: - print("Error: Unable to login to customer PCR") - print(err) - sys.exit(1) +#Login to customer PCR on ECR if configured +if args.pcrAuth == "ECR": + customerEcrSession = boto3.Session() + customerEcrClient = customerEcrSession.client('ecr', region_name=pcrRegion) + # Get customer PCR token + # CLI Example "aws ecr get-authorization-token --region " + try: + customerEcrToken = customerEcrClient.get_authorization_token(registryIds=[pcrFqdn.split('.')[0]]) + except Exception as err: + print("Error: Unable to get customer PCR token.") + print(err) + sys.exit(1) + + # CLI Example "aws ecr get-login-password --region | docker login --username AWS --password-stdin " + try: + username, password = base64.b64decode(customerEcrToken['authorizationData'][0]['authorizationToken']).decode('utf-8').split(":") + customer_auth_config_payload = { 'username': username, 'password': password } + customerEcr = dockerClient.login(username=username, password=password, registry=customerEcrToken['authorizationData'][0]['proxyEndpoint'].replace("https://", ""), reauth=True) + except Exception as err: + print("Error: Unable to login to customer PCR on ECR") + print(err) + sys.exit(1) +elif args.pcrAuth == "PWD": + # Login to customer PCR on non ECR + customer_auth_config_payload = { 'username': args.pcrUsername, 'password': args.pcrPassword } + try: + customerEcr = dockerClient.login(username=args.pcrUsername, password=args.pcrPassword, registry=pcrFqdn, reauth=True) + except Exception as err: + print("Error: Unable to login to customer PCR on non-ECR") + print(err) + sys.exit(1) # Create Repos, Tag and push images to customer PCR # Determine if repository exists and create if it does not. # CLI example: "aws ecr describe-repositories --region " -pcrRepositories = customerEcrClient.describe_repositories() for bundleImages in exoTaskImageBundle['data']['exotaskImageBundle']['bundleImages']: print("") - repoExists = False - for repo in pcrRepositories['repositories']: - if repo['repositoryName'] == bundleImages['name']: - print("Repository " + bundleImages['name'] + " already exists in " + pcrFqdn + ". Skipping create" ) - repoExists = True - break -# If repo does not exist, create it. - if not repoExists: - # CLI example: "aws ecr create-repository --repository-name --region --image-scanning-configuration scanOnPush=true --encryption-configuration encryptionType=AES256 --image-tag-mutability IMMUTABLE" - print("Creating repository " + bundleImages['name'] + " in " + pcrFqdn) - customerEcrClient.create_repository(repositoryName=bundleImages['name'], - imageScanningConfiguration={'scanOnPush': True}, - encryptionConfiguration={'encryptionType': 'AES256'}, - imageTagMutability='IMMUTABLE') + if not pcrFqdn.partition('/')[2]: + pcrRepoName = bundleImages['name'] + else: + pcrRepoName = pcrFqdn.partition('/')[2] + '/' + bundleImages['name'] + logging.debug("pcrAuth is " + args.pcrAuth) + if args.pcrAuth == "ECR": + pcrRepositories = customerEcrClient.describe_repositories() + repoExists = False + logging.debug("PCR Repositories:") + # logging.debug(pp.pprint(pcrRepositories)) + for repo in pcrRepositories['repositories']: + logging.debug("repo name: " + repo['repositoryName']) + logging.debug("bundleImages name:" + bundleImages['name']) + if repo['repositoryUri'] == pcrFqdn + '/' + bundleImages['name']: + print("Repository " + pcrFqdn + '/' + bundleImages['name'] + " already exists in " + pcrFqdn + ". Skipping create" ) + repoExists = True + break + # If repo does not exist, create it. + if not repoExists: + # CLI example: "aws ecr create-repository --repository-name --region --image-scanning-configuration scanOnPush=true --encryption-configuration encryptionType=AES256 --image-tag-mutability IMMUTABLE" + print("Creating repository: " + pcrRepoName) + customerEcrClient.create_repository(repositoryName=pcrRepoName, + imageScanningConfiguration={'scanOnPush': True}, + encryptionConfiguration={'encryptionType': 'AES256'}, + imageTagMutability='IMMUTABLE') if bundleImages['tag']: - print("Tagging and pushing " + bundleImages['name'] + " with tag " + bundleImages['tag'] + " to " + bundleImages['name'] + " with version tag " + exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion']) + print("Tagging and pushing " + bundleImages['name'] + " with tag " + bundleImages['tag'] + " to " + pcrFqdn + '/' + bundleImages['name'] + " with version tag " + exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion']) # CLI Example "docker image tag .dkr.ecr.us-east-1.amazonaws.com/:/:" try: docker_api_client.tag(rscRepoFqdn + '/' + bundleImages['name'] + ":" + bundleImages['tag'], pcrFqdn + '/' + bundleImages['name'] + ":" + exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion']) @@ -223,13 +325,13 @@ try: for line in docker_api_client.push(pcrFqdn + '/' + bundleImages['name'], tag=exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion'], stream=True, auth_config=customer_auth_config_payload, decode=True): print(line) - logging.info(json.dumps(line, indent=2)) + logging.debug(json.dumps(line, indent=2)) except Exception as err: print("Error: Image push failed for " + bundleImages['name'] + " with tag " + exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion']) print(err) sys.exit(1) elif bundleImages['sha']: - print("Tagging and pushing " + bundleImages['name'] + " with sha " + bundleImages['sha'] + " to " + bundleImages['name'] + " with version tag " + exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion']) + print("Tagging and pushing " + bundleImages['name'] + " with sha " + bundleImages['sha'] + " to " + pcrFqdn + '/' + bundleImages['name'] + " with version tag " + exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion']) # CLI Example "docker image tag .dkr.ecr.us-east-1.amazonaws.com/@sha256: /:" try: docker_api_client.tag(rscRepoFqdn + '/' + bundleImages['name'] + "@sha256:" + bundleImages['sha'], pcrFqdn + '/' + bundleImages['name'] + ":" + exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion']) @@ -243,12 +345,14 @@ try: for line in docker_api_client.push(pcrFqdn + '/' + bundleImages['name'], tag=exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion'], stream=True, auth_config=customer_auth_config_payload, decode=True): print(line) - logging.info(json.dumps(line, indent=2)) + logging.debug(json.dumps(line, indent=2)) except Exception as err: print("Error: Image push failed for " + bundleImages['name'] + " with sha " + bundleImages['sha']) print(err) sys.exit(1) +logging.debug(json.dumps(exoTaskImageBundle, indent=2)) + #Accept Container Bundle variables = { "input": { @@ -256,8 +360,13 @@ "bundleVersion": "{}".format(exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion']) } } -exoTaskImageBundle = rubrik._query_raw(raw_query='mutation SetBundleApprovalStatus($input: SetBundleApprovalStatusInput!) {setBundleApprovalStatus(input: $input)}', +SetBundleApprovalStatus = rubrik._query_raw(raw_query='mutation SetBundleApprovalStatus($input: SetBundleApprovalStatusInput!) {setBundleApprovalStatus(input: $input)}', operation_name=None, -# variables={'"input": {"approvalStatus": "ACCEPTED","bundleVersion": {}}'.format(exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion'])}, variables=variables, timeout=60) + +logging.debug(json.dumps(exoTaskImageBundle, indent=2)) + +print() +print() +print("Bundle " + exoTaskImageBundle['data']['exotaskImageBundle']['bundleVersion'] + " has been accepted.") \ No newline at end of file