-
Notifications
You must be signed in to change notification settings - Fork 2
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Infra cost reduction (Move from ECS to EC2) #387
Merged
Merged
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
383c640
Move from ECS to EC2
LDMGN d44918e
Remove NAT gateway from VPC
LDMGN f8e5e8c
Add comments to infrastructure
LDMGN b69d728
Split Elastic IP to separate stack
LDMGN dc14835
Add documentation on updating certificates
LDMGN 9f3b27d
Add TODO on database port in security group
LDMGN 5d0e4a4
Format imports
LDMGN File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,17 +3,21 @@ import { | |
| AmazonLinuxCpuType, | ||
| AmazonLinuxGeneration, | ||
| AmazonLinuxImage, | ||
| CfnEIP, CfnEIPAssociation, CfnKeyPair, | ||
| CfnEIP, | ||
| CfnEIPAssociation, | ||
| Instance, | ||
| InstanceClass, | ||
| InstanceSize, | ||
| InstanceType, Peer, Port, | ||
| InstanceType, | ||
| KeyPair, | ||
| Peer, | ||
| Port, | ||
| SecurityGroup, | ||
| SubnetType, UserData, | ||
| SubnetType, | ||
| UserData, | ||
| Vpc | ||
| } from 'aws-cdk-lib/aws-ec2'; | ||
| import { CfnAccessKey, Effect, Policy, PolicyStatement, Role, User } from 'aws-cdk-lib/aws-iam'; | ||
| import { LogGroup } from 'aws-cdk-lib/aws-logs'; | ||
| import { Effect, Policy, PolicyStatement } from 'aws-cdk-lib/aws-iam'; | ||
| import { Construct } from 'constructs'; | ||
| import { LittilEnvironmentSettings } from './littil-environment-settings'; | ||
| import { LoggingStack } from './logging-stack'; | ||
|
|
@@ -24,8 +28,15 @@ export interface ApiEc2StackProps extends StackProps { | |
| littil: LittilEnvironmentSettings; | ||
|
|
||
| apiVpc: Vpc; | ||
| elasticIp: CfnEIP; | ||
| ecrRepository: { | ||
| awsAccount: string; | ||
| name: string; | ||
| }; | ||
| database: { | ||
| host: string; | ||
| port: string; | ||
| name: string; | ||
| securityGroup: { | ||
| id: string; | ||
| }; | ||
|
|
@@ -43,61 +54,94 @@ export class ApiEc2Stack extends Stack { | |
| }); | ||
| ec2SecurityGroup.addIngressRule(Peer.anyIpv4(), Port.tcp(443), 'HTTPS over IPv4'); | ||
| ec2SecurityGroup.addIngressRule(Peer.anyIpv6(), Port.tcp(443), 'HTTPS over IPv6'); | ||
| ec2SecurityGroup.addIngressRule(Peer.anyIpv4(), Port.tcp(80), 'HTTP over IPv4'); | ||
| ec2SecurityGroup.addIngressRule(Peer.anyIpv6(), Port.tcp(80), 'HTTP over IPv6'); | ||
| ec2SecurityGroup.addIngressRule(Peer.anyIpv4(), Port.tcp(22), 'SSH access'); | ||
| ec2SecurityGroup.addIngressRule(Peer.anyIpv6(), Port.tcp(22), 'SSH access'); | ||
|
|
||
| const elasticIp = new CfnEIP(this, 'ApiIP', { | ||
| tags: [ | ||
| { | ||
| key: 'Name', | ||
| value: 'API EC2', | ||
| } | ||
| ] | ||
| }); | ||
|
|
||
| const keypair = new CfnKeyPair(this, 'ApiEc2Keypair', { | ||
| keyName: 'EC2 Keypair', | ||
| publicKeyMaterial: '---- BEGIN SSH2 PUBLIC KEY ----\n' + | ||
| 'Comment: "rsa-key-20230917"\n' + | ||
| 'AAAAB3NzaC1yc2EAAAADAQABAAABAQCbNh2l09RvvYyDtIXjtqnJG/nFYtV44Gwx\n' + | ||
| 'TjYteFvwyK3wSFlgA0qFIjoUxrh5KLGsVYzoz7JmcD3thg7YdbcCy3agZ8EIK8ds\n' + | ||
| '8ly37dGn5D1u7re5AU+7Y+LPsw31lxjusZCFPJEElKexryuhIP043EAe/pWXDfM6\n' + | ||
| '6urIhgXGKRhxu3prw43xX8STTGGUaropESaEnudAxMlgHu/nNI8DauQhf5LZWboT\n' + | ||
| 'YjnB2X8lpDY9Vsab6e0OINUcXgHvH9A9r/twBPt1Hx8MXWjmDTEiU5+vuDOcws+g\n' + | ||
| '4/TsVUxuJ/MqhBkJj/ou2cVkCcBuzbreQdd9zlc5J5CJSfReg4dl\n' + | ||
| '---- END SSH2 PUBLIC KEY ----\n', | ||
| const keypair = new KeyPair(this, 'ApiEc2Keypair', { | ||
| keyPairName: 'EC2 Keypair', | ||
| publicKeyMaterial: 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN5OhXLM34h3omM+5AoaYgUktcBMUBrC5awrPoItmf3S [email protected]', | ||
| }); | ||
|
|
||
| const apiDomain = 'api.' + (props.littil.environment === 'production' ? '' : props.littil.environment + '.') + 'littil.org'; | ||
|
|
||
| const littilServerConf = fs.readFileSync('lib/nginx/serverconfiguration') | ||
| .toString('utf-8') | ||
| .replaceAll('%ENVIRONMENT%', props.littil.environment); | ||
| .replaceAll('%API_DOMAIN%', apiDomain); | ||
|
|
||
| /* Logging. */ | ||
| const logGroupName = 'BackendApiEc2Logs'; | ||
| const apiEc2LoggingStack = new LoggingStack(this, 'ApiEc2LoggingStack', { | ||
| littil: props.littil, | ||
| logGroupName, | ||
| }); | ||
|
|
||
| const dockerTag = '1.3.1'; | ||
| const dockerImage = props.ecrRepository.awsAccount + '.dkr.ecr.eu-west-1.amazonaws.com/' + props.ecrRepository.name + ':' + dockerTag; | ||
|
|
||
| const littilOidcSecretName = 'littil/backend/' + props.littil.environment + '/oidc'; | ||
| const littilSmtpSecretName = 'littil/backend/' + props.littil.environment + '/smtp'; | ||
| // TODO: Find by tags | ||
| const littilBackendDatabaseSecretName = 'ApiDatabaseStackLittilApiDa-ia57olJcscCP'; | ||
|
|
||
| const userData = UserData.forLinux(); | ||
| userData.addCommands( | ||
| 'sudo su', | ||
| 'yum update', | ||
| 'yum update -y', | ||
|
|
||
| 'yum install docker -y', | ||
| 'systemctl start docker.service', | ||
| 'docker run -p 8081:80 -P -d nginxdemos/hello', | ||
| 'aws ecr get-login-password --region ' + this.region + ' | docker login --username AWS --password-stdin ' + this.account + '.dkr.ecr.' + this.region + '.amazonaws.com', | ||
| 'docker pull ' + this.account + '.dkr.ecr.eu-west-1.amazonaws.com/littil-backend:latest', | ||
|
|
||
| 'aws ecr get-login-password --region ' + this.region + ' | docker login --username AWS --password-stdin ' + props.ecrRepository.awsAccount + '.dkr.ecr.' + this.region + '.amazonaws.com', | ||
| 'docker pull ' + dockerImage, | ||
|
|
||
| 'echo DATASOURCE_HOST="' + props.database.host + '" >> littil.env', | ||
| 'echo DATASOURCE_PORT=' + props.database.port + ' >> littil.env', | ||
| 'echo DATASOURCE_DATABASE=' + props.database.name + ' >> littil.env', | ||
| 'echo QUARKUS_HTTP_CORS_ORIGINS="' + props.littil.httpCorsOrigin + '" >> littil.env', | ||
| 'echo QUARKUS_LOG_CLOUDWATCH_ACCESS_KEY_ID="' + apiEc2LoggingStack.loggingAccessKey.ref + '" >> littil.env', | ||
| 'echo QUARKUS_LOG_CLOUDWATCH_ACCESS_KEY_SECRET="' + apiEc2LoggingStack.loggingAccessKey.attrSecretAccessKey + '" >> littil.env', | ||
| 'echo QUARKUS_LOG_CLOUDWATCH_LOG_GROUP="' + apiEc2LoggingStack.cloudwatchLogGroup.logGroupName + '" >> littil.env', | ||
| 'echo QUARKUS_LOG_CLOUDWATCH_REGION=' + this.region + ' >> littil.env', | ||
| 'echo QUARKUS_LOG_CLOUDWATCH_LOG_STREAM_NAME=' + logGroupName + ' >> littil.env', | ||
| 'echo [email protected] >> littil.env', | ||
|
|
||
| 'yum install jq -y', | ||
| // TODO: Pass these secret values to the docker process in a more secure way than storing them in a file | ||
| 'echo DATASOURCE_USERNAME=$(aws secretsmanager get-secret-value --region ' + this.region + ' --secret-id ' + littilBackendDatabaseSecretName + ' | jq --raw-output \'.SecretString\' | jq -r .username) >> littil.env', | ||
| 'echo DATASOURCE_PASSWORD=$(aws secretsmanager get-secret-value --region ' + this.region + ' --secret-id ' + littilBackendDatabaseSecretName + ' | jq --raw-output \'.SecretString\' | jq -r .password) >> littil.env', | ||
| 'echo OIDC_CLIENT_ID=$(aws secretsmanager get-secret-value --region ' + this.region + ' --secret-id ' + littilOidcSecretName + ' | jq --raw-output \'.SecretString\' | jq -r .oidcClientId) >> littil.env', | ||
| 'echo OIDC_CLIENT_SECRET=$(aws secretsmanager get-secret-value --region ' + this.region + ' --secret-id ' + littilOidcSecretName + ' | jq --raw-output \'.SecretString\' | jq -r .oidcClientSecret) >> littil.env', | ||
| 'echo OIDC_TENANT=$(aws secretsmanager get-secret-value --region ' + this.region + ' --secret-id ' + littilOidcSecretName + ' | jq --raw-output \'.SecretString\' | jq -r .oidcTenant) >> littil.env', | ||
| 'echo M2M_CLIENT_ID=$(aws secretsmanager get-secret-value --region ' + this.region + ' --secret-id ' + littilOidcSecretName + ' | jq --raw-output \'.SecretString\' | jq -r .m2mClientId) >> littil.env', | ||
| 'echo M2M_CLIENT_SECRET=$(aws secretsmanager get-secret-value --region ' + this.region + ' --secret-id ' + littilOidcSecretName + ' | jq --raw-output \'.SecretString\' | jq -r .m2mClientSecret) >> littil.env', | ||
| 'echo SMTP_HOST=$(aws secretsmanager get-secret-value --region ' + this.region + ' --secret-id ' + littilSmtpSecretName + ' | jq --raw-output \'.SecretString\' | jq -r .smtpHost) >> littil.env', | ||
| 'echo SMTP_USERNAME=$(aws secretsmanager get-secret-value --region ' + this.region + ' --secret-id ' + littilSmtpSecretName + ' | jq --raw-output \'.SecretString\' | jq -r .smtpUsername) >> littil.env', | ||
| 'echo SMTP_PASSWORD=$(aws secretsmanager get-secret-value --region ' + this.region + ' --secret-id ' + littilSmtpSecretName + ' | jq --raw-output \'.SecretString\' | jq -r .smtpPassword) >> littil.env', | ||
|
|
||
| 'docker run -p 8080:8080 --env-file littil.env -d ' + dockerImage, | ||
| // TODO: Test whether we can immediately delete the file | ||
| // 'rm littil.env', | ||
|
|
||
| /* Install Nginx. */ | ||
| 'amazon-linux-extras install nginx1', | ||
| 'systemctl stop nginx', | ||
|
|
||
| /* Nginx configuration. */ | ||
| 'echo test > /etc/nginx/conf.d/test', | ||
| 'echo ' + Buffer.from(littilServerConf).toString('base64') + ' > /etc/nginx/conf.d/test2', | ||
| 'echo ' + Buffer.from(littilServerConf).toString('base64') + ' | base64 -d > /etc/nginx/conf.d/api.' + props.littil.environment + '.littil.org.conf', | ||
| 'echo ' + Buffer.from(littilServerConf).toString('base64') + ' | base64 -d > /etc/nginx/conf.d/' + apiDomain + '.conf', | ||
|
|
||
| /* Certbot. */ | ||
| 'amazon-linux-extras install epel', | ||
| 'yum update', | ||
| 'yum update -y', | ||
| 'yum install certbot -y', | ||
| 'letsencrypt certonly --standalone -d api.' + props.littil.environment + '.littil.org -m [email protected] --agree-tos --no-eff-email', | ||
| // Will fail when there's no DNS A record pointing to the above created Elastic IP. Should be interactively waiting or perhaps put the Elastic IP in a separate stack that needs to be created first. | ||
| 'letsencrypt certonly --standalone -d ' + apiDomain + ' -m [email protected] --agree-tos --no-eff-email', | ||
|
|
||
| 'systemctl enable docker', | ||
| 'systemctl enable nginx', | ||
| 'systemctl start nginx', | ||
| ); | ||
|
|
||
| const pullPolicyStatement = new PolicyStatement({ | ||
|
|
@@ -116,41 +160,47 @@ export class ApiEc2Stack extends Stack { | |
| '*', | ||
| ] | ||
| }); | ||
| const pullPolicy = new Policy(this, 'PullPolicy'); | ||
| pullPolicy.addStatements( | ||
| const readSecretsPolicyStatment = new PolicyStatement({ | ||
| effect: Effect.ALLOW, | ||
| actions: [ | ||
| 'secretsmanager:GetSecretValue', | ||
| ], | ||
| resources: [ | ||
| '*', | ||
| ] | ||
| }); | ||
| const littilApiPolicy = new Policy(this, 'PullPolicy'); | ||
| littilApiPolicy.addStatements( | ||
| pullPolicyStatement, | ||
| readSecretsPolicyStatment, | ||
| ); | ||
|
|
||
| /**/ | ||
| const ec2Instance = new Instance(this, 'ApiInstance', { | ||
| vpc: props.apiVpc, | ||
| instanceType: InstanceType.of(InstanceClass.T4G, InstanceSize.NANO), | ||
| instanceType: InstanceType.of(InstanceClass.T3A, InstanceSize.MICRO), | ||
| machineImage: new AmazonLinuxImage({ | ||
| generation: AmazonLinuxGeneration.AMAZON_LINUX_2, | ||
| cpuType: AmazonLinuxCpuType.ARM_64, | ||
| cpuType: AmazonLinuxCpuType.X86_64, | ||
| }), | ||
| vpcSubnets: {subnetType: SubnetType.PUBLIC}, | ||
| securityGroup: ec2SecurityGroup, | ||
| keyName: keypair.keyName, | ||
| keyPair: keypair, | ||
| userData, | ||
| }); | ||
|
|
||
| pullPolicy.attachToRole(ec2Instance.role); | ||
| littilApiPolicy.attachToRole(ec2Instance.role); | ||
|
|
||
| new CfnEIPAssociation(this, 'Ec2EipAssociation', { | ||
| eip: elasticIp.ref, | ||
| eip: props.elasticIp.ref, | ||
| instanceId: ec2Instance.instanceId, | ||
| }); | ||
| /**/ | ||
|
|
||
| /* Logging. */ | ||
| const apiEc2LoggingStack = new LoggingStack(this, 'ApiEc2LoggingStack', { | ||
| littil: props.littil, | ||
| logGroupName: 'BackendApiEc2Logs', | ||
| }); | ||
|
|
||
| /* Database access. */ | ||
| const databaseSecurityGroup = SecurityGroup.fromSecurityGroupId(this, 'DatabaseSecurityGroup', props.database.securityGroup.id); | ||
| databaseSecurityGroup.connections.allowFrom(ec2SecurityGroup, Port.tcp(parseInt(props.database.port))); | ||
| // TODO: Uncomment, test and remove the allTcp rule | ||
| // databaseSecurityGroup.connections.allowFrom(ec2SecurityGroup, Port.tcp(parseInt(props.database.port))); | ||
| databaseSecurityGroup.connections.allowFrom(ec2SecurityGroup, Port.allTcp()); | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| import { Stack, StackProps } from 'aws-cdk-lib'; | ||
| import { CfnEIP } from 'aws-cdk-lib/aws-ec2'; | ||
| import { Construct } from 'constructs'; | ||
|
|
||
| export class ApiElasticIpStack extends Stack { | ||
| public readonly elasticIp: CfnEIP; | ||
|
|
||
| constructor(scope: Construct, | ||
| id: string, | ||
| props: StackProps) { | ||
| super(scope, id, props); | ||
|
|
||
| this.elasticIp = new CfnEIP(this, 'ApiIP', { | ||
| tags: [ | ||
| { | ||
| key: 'Name', | ||
| value: 'API EC2', | ||
| } | ||
| ] | ||
| }); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
do we need to keep this as comment ?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes and no. It should be as limited as possible, but I don't have time to test it right now. So for now at least I'll add a TODO.