Merge pull request #338 from AdTechMedia/stage

Stage to master (full deployment cycle testing)

.recink.yml

@@ -9,7 +9,7 @@ $:
  '$.cache.options.1.secretAccessKey': 'eval'
  '$.codeclimate.token': 'eval'
  '$.snyk.token': 'eval'
-# '$.e2e.wait.uri.0': 'eval'
+ '$.e2e.wait.uri.0': 'eval'
  '$.pagespeed.uri': 'eval'
  cache:
  driver: 's3'
@@ -24,7 +24,6 @@ $:
  # - 'compile-travis'
  dependencies:
  # chai: 'latest'
- chance: 'latest'
  emit:
  pattern:
  - /^src.es6\/lib\/.+\.js$/i
@@ -42,13 +41,13 @@ $:
  ignore: ~
  e2e:
  pattern:
- - /.+\.e2e\.js$/i # Files to include
- ignore:
-  - assert-social-links.e2e.js
- browsers:  # Browsers to run the tests against
- - nightmare
- # - firefox
- # - chrome
+ - /.+\.e2e\.js$/i  # Files to include
+ ignore: ~
+ browsers: 
+  - chrome # Browsers to run the tests against
+  # - firefox
+  # - nightmare 
+  # - safari
  wait:
  interval: 500 # Interval of running uri checks
  timeout: 60000 # Timeout to wait for uri's available
@@ -86,5 +85,7 @@ $:
 ### Add other modules here...
 #email-send:
 # root: src/adtechmedia-website/backend/src/email/send
-#website:
-# root: test/e2e
+website:
+ root: test/e2e
+ dependencies:
+ chance: 'latest'

.travis.yml

@@ -12,6 +12,7 @@ branches:
  - master
  - stage
  - test
+ - build
  - dev
 env:
  global:
@@ -51,22 +52,21 @@ jobs:
  after_failure:
  - deepify undeploy src --loglevel=debug
 
-## Temporarily turned off (Electron issue)
-# - stage: "Run e2e Tests :clipboard:"
-# node_js: 8
-# script:
-# - if [ ${ALLOW_DEPLOY} == 1 ]; then recink run e2e; else echo "Skipping e2e tests"; fi
-# addons:
-# firefox: 'latest'
-# chrome: 'beta'
-# apt:
-# packages:
-# - xvfb
-# install:
-# - 'export DISPLAY='':99.0'''
-# - 'Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &'
+ - stage: "Run e2e Tests :clipboard:"
+ node_js: 8
+ script:
+ - if [ ${ALLOW_DEPLOY} == 1 ]; then recink run e2e; else echo "Skipping e2e tests"; fi
+ addons:
+ firefox: 'latest'
+ chrome: 'beta'
+ apt:
+ packages:
+ - xvfb
+ install:
+ - 'export DISPLAY='':99.0'''
+ - 'Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &'
 
  - stage: "Page speed :rocket:"
  node_js: 6
  script:
- - if [ ${ALLOW_DEPLOY} == 1 ]; then recink run unit -s emit -c recink-pagespeed; else echo "Skipping pagespeed"; fi
+ - if [ ${ALLOW_DEPLOY} == 1 ]; then recink run unit -s emit -c recink-pagespeed; else echo "Skipping pagespeed"; fi

bin/deploy/compile.js

@@ -12,28 +12,26 @@ const deepifyRegexp = /\d{2}:\d{2}:\d{2}/;
 const env = process.env.DEPLOY_ENV || 'test';
 const srcPath = path.join(__dirname, '../../', 'src');
 const awsh = new AwsHelper('atm-deploy-caches');
-const s3Prefix = `atm-website/lambdas/${env}`;
+const mainPrefix = `atm-website/lambdas`;
 const compareBranch = process.env.DEPLOY_ENV ? `origin/${env}` : '';
 
-let deployBackend = false;
-
 /**
  * Compile microservise
  * @param microApp
  * @returns {Promise}
  */
 function compileMicroservice(microApp) {
- return awsh.listS3Objects(`${s3Prefix}/${microApp}`).then(res => {
+ return awsh.listS3Objects(`${cacheFrom()}/${microApp}`).then(res => {
  if (res.KeyCount === 0) {
  return Promise.resolve(true);
+ } else if (['master', 'stage'].includes(env) && (process.env.HOTFIX === 0)) {
+ return Promise.resolve(false);
  } else {
  return checkForBackendChanges(microApp).then(res => {
  return Promise.resolve(res);
  });
  }
  }).then(compileBackend => {
- deployBackend = compileBackend;
-
  return compileBackend
  ? runChildCmd(`cd ${srcPath} && deepify compile prod ${microApp}`, deepifyRegexp)
  : reuseCompiledLambdas(microApp);
@@ -48,39 +46,31 @@ exports.compileMicroservice = compileMicroservice;
  * @returns {Promise}
  */
 function cacheMicroserviceLambdas(microApp) {
- return Promise.all(
- findLambdasByMicroAppName(microApp).map(lambdaPath => {
- let stream = fs.createReadStream(lambdaPath);
- return awsh.uploadZipToS3(lambdaPath.replace(srcPath, s3Prefix), stream);
- })
- );
+ if (cacheTo()) {
+ return Promise.all(
+ findLambdasByMicroAppName(microApp).map(lambdaPath => {
+ let stream = fs.createReadStream(lambdaPath);
+ return awsh.uploadZipToS3(lambdaPath.replace(srcPath, cacheTo()), stream);
+ })
+ );
+ }
+
+ return Promise.resolve();
 }
 
 exports.cacheMicroserviceLambdas = cacheMicroserviceLambdas;
 
-/**
- * Deploy frontend and/or backend
- * @returns {Promise}
- */
-function deployApplication() {
- let deployCommand = deployBackend ? 'deepify deploy' : 'deepify deploy --frontend';
-
- return runChildCmd(`cd ${srcPath} && ${deployCommand}`, deepifyRegexp);
-}
-
-exports.deployApplication = deployApplication;
-
 /**
  * Download compiled lambdas
  * @param microApp
  * @returns {Promise}
  */
 function reuseCompiledLambdas(microApp) {
- return awsh.listS3Objects(`${s3Prefix}/${microApp}`).then(res => {
+ return awsh.listS3Objects(`${cacheFrom()}/${microApp}`).then(res => {
  let keys = res.Contents.map(item => item.Key);
 
  return Promise.all(keys.map(key => {
- return awsh.getAndSaveS3Object(key, key.replace(s3Prefix, srcPath));
+ return awsh.getAndSaveS3Object(key, key.replace(cacheFrom(), srcPath));
  }));
  });
 }
@@ -94,15 +84,17 @@ function findLambdasByMicroAppName(microApp) {
  let lambdas = [];
  let searchDir = `${srcPath}/${microApp}/backend/src`;
 
- fs.readdirSync(searchDir).map(item => {
- let lambdaDir = `${searchDir}/${item}`;
+ if (fs.existsSync(searchDir)) {
+ fs.readdirSync(searchDir).map(item => {
+ let lambdaDir = `${searchDir}/${item}`;
 
- if (fs.lstatSync(lambdaDir).isDirectory()) {
- lambdas = lambdas.concat(
- fs.readdirSync(lambdaDir).filter(item => /.*\.zip$/.test(item)).map(item => `${lambdaDir}/${item}`)
- )
- }
- });
+ if (fs.lstatSync(lambdaDir).isDirectory()) {
+ lambdas = lambdas.concat(
+ fs.readdirSync(lambdaDir).filter(item => /.*\.zip$/.test(item)).map(item => `${lambdaDir}/${item}`)
+ )
+ }
+ });
+ }
 
  return lambdas;
 }
@@ -119,18 +111,46 @@ function checkForBackendChanges(microApp) {
  return reject(error);
  }
 
- let hasChanges = false;
  let files = stdout.split('\n').filter(item => item.trim());
  let regExp = new RegExp(`${microApp}/backend`, 'gi');
 
  for (let i = 0, len = files.length; i < len; i++) {
  if (regExp.test(files[i])) {
- hasChanges = true;
- break;
+ return resolve(true);
  }
  }
 
- resolve(hasChanges);
+ resolve(false);
  });
  });
 }
+
+/**
+ * S3 path to fetch cache
+ * @returns {string}
+ */
+function cacheFrom() {
+ if (('master' === env) || (process.env.HOTFIX === 1)) {
+ return `${mainPrefix}/regression`;
+ }
+
+ return `${mainPrefix}/functional`;
+}
+
+/**
+ * S3 path to upload cache
+ * @returns {*}
+ */
+function cacheTo() {
+ if (process.env.HOTFIX === 1) {
+ return false;
+ }
+
+ if ('test' === env) {
+ return `${mainPrefix}/functional`;
+ } else if ('stage' === env) {
+ return `${mainPrefix}/regression`;
+ }
+
+ return false;
+}

bin/deploy/index.js

@@ -15,9 +15,9 @@ const env = process.env.DEPLOY_ENV || 'test';
 const appPath = path.join(__dirname, '../../');
 const srcPath = path.join(appPath, 'src');
 const logPath = path.join(appPath, 'docs/deploy.log');
+const awsh = new AwsHelper('atm-deploy-assets');
 const forked = fork('bin/deploy/cache.js', { cwd: appPath });
 const timerId = setInterval(() => { console.log('.'); }, 60000);
-const awsh = new AwsHelper('atm-deploy-assets');
 
 let newAppInfo = {};
 let cacheInfo = {};
@@ -54,8 +54,7 @@ isEnvironmentLocked().then(isLocked => {
 
  console.log('Updating .parameters.json');
  return awsh.getAndSaveS3Object(
- `atm-website/${env}/.parameters.json`,
- `${srcPath}/adtechmedia-website/.parameters.json`
+ `atm-website/${env}/.parameters.json`, `${srcPath}/adtechmedia-website/.parameters.json`
  );
 
 }).then(() => {
@@ -66,18 +65,17 @@ isEnvironmentLocked().then(isLocked => {
 }).then(() => {
 
  console.log('Deploying application');
- return runChildCmd(`cd ${srcPath} && deepify deploy`, deepifyRegexp).then(() => {
+ return runChildCmd(`cd ${srcPath} && DEEP_CONFIRMATION_REFUSE=1 deepify deploy`, deepifyRegexp).then(() => {
  newAppInfo = getNewApplicationInfo();
  return Promise.resolve();
  });
 
 }).then(() => {
 
- let promises = [];
- promises.push(awsh.waitForDistributionIsDeployed(newAppInfo.cloudfrontId));
-
  console.log('Mark old distributions with REMOVE mark');
  console.log('Waiting freshly deployed CloudFront will get status deployed');
+ let promises = [awsh.waitForDistributionIsDeployed(newAppInfo.cloudfrontId)];
+
  return getOldDistributionIds().then(ids => {
  promises.push(ids.map(id => {
  return handleOldDistribution(id);
@@ -89,29 +87,11 @@ isEnvironmentLocked().then(isLocked => {
 }).then(() => {
 
  console.log('Configuring freshly deployed CloudFront');
- return awsh.getDistributionById(newAppInfo.cloudfrontId).then(distInfo => {
- const id = distInfo.Distribution.Id;
- const etag = distInfo.ETag;
- const config = distInfo.Distribution.DistributionConfig;
-
- config.Aliases = {
- Quantity: 1,
- Items: [getDomain()]
- };
- config.ViewerCertificate = {
- SSLSupportMethod: 'sni-only',
- ACMCertificateArn: `arn:aws:acm:us-east-1:${process.env.AWS_ACCOUNT_ID}:certificate/${process.env.ATM_CERTIFICATE}`,
- MinimumProtocolVersion: 'TLSv1',
- Certificate: `arn:aws:acm:us-east-1:${process.env.AWS_ACCOUNT_ID}:certificate/${process.env.ATM_CERTIFICATE}`,
- CertificateSource: 'acm'
- };
-
- return awsh.updateDistributionAndWait(id, config, etag);
- });
+ return configureNewDistribution();
 
 }).then(() => {
 
- console.log('Repointing Route53 to freshly deployed CloudFront');
+ console.log('Repointing Route53 to a freshly deployed CloudFront');
  return awsh.getResourceRecordByName(getDomain()).then(recordSet => {
  recordSet.ResourceRecords[0].Value = newAppInfo.cloudfrontDomain;
  return awsh.updateResourceRecord(recordSet);
@@ -120,15 +100,15 @@ isEnvironmentLocked().then(isLocked => {
 }).then(() => {
 
  console.log('Checkout to dev branch');
- return runChildCmd('git fetch origin dev && git checkout . && git checkout dev');
+ return runChildCmd('git checkout . && git checkout dev', /.*Switched.*/);
 
 }).then(() => {
 
  console.log('Updating deploy.log');
  updateDeployLog();
 
  console.log('Committing deploy.log changes');
- return runChildCmd(`git commit -m "#ATM continuous deployment logger [skip ci]" -- ${logPath}`);
+ return runChildCmd(`git commit -m "#ATM continuous deployment logger [skip ci]" -- ${logPath}`, /.*#ATM.*/);
 
 }).then(() => {
 
@@ -262,6 +242,33 @@ function updateDeeployJson() {
  }
 }
 
+/**
+ * Assign alias and certificate for cloudfront
+ * @returns {Promise}
+ */
+function configureNewDistribution() {
+ return awsh.getDistributionById(newAppInfo.cloudfrontId).then(distInfo => {
+ const id = distInfo.Distribution.Id;
+ const etag = distInfo.ETag;
+ const config = distInfo.Distribution.DistributionConfig;
+
+ config.Aliases = {
+ Quantity: 1,
+ Items: [getDomain()]
+ };
+ config.ViewerCertificate = {
+ SSLSupportMethod: 'sni-only',
+ ACMCertificateArn: `arn:aws:acm:us-east-1:${process.env.AWS_ACCOUNT_ID}:certificate/${process.env.ATM_CERTIFICATE}`,
+ MinimumProtocolVersion: 'TLSv1',
+ Certificate: `arn:aws:acm:us-east-1:${process.env.AWS_ACCOUNT_ID}:certificate/${process.env.ATM_CERTIFICATE}`,
+ CertificateSource: 'acm'
+ };
+
+ return awsh.updateDistributionAndWait(id, config, etag);
+ });
+}
+
+
 /**
  * Update deploy log with freshly deployed cloudfront info
  */