From 3cd104829c60c52ac315e21e951e415b9f031dd1 Mon Sep 17 00:00:00 2001 From: abetomo Date: Fri, 8 Dec 2017 10:50:24 +0900 Subject: [PATCH 1/5] Add S3Events to event_sources.json --- lib/event_sources.json.example | 16 ++++++++++++- lib/main.js | 9 +++++-- test/main.js | 43 ++++++++++++++++++++++++++++++---- 3 files changed, 61 insertions(+), 7 deletions(-) diff --git a/lib/event_sources.json.example b/lib/event_sources.json.example index d95bd4a9..c3aae42f 100644 --- a/lib/event_sources.json.example +++ b/lib/event_sources.json.example @@ -18,5 +18,19 @@ "key2": "value" } } - ] + ], + "S3Events": [{ + "Bucket": "BUCKET_NAME", + "Events": [ + "s3:ObjectCreated:*" + ], + "Filter": { + "Key": { + "FilterRules": [{ + "Name": "prefix", + "Value": "STRING_VALUE" + }] + } + } + }] } diff --git a/lib/main.js b/lib/main.js index 7d943a4d..51351dab 100644 --- a/lib/main.js +++ b/lib/main.js @@ -226,7 +226,8 @@ so you can easily test run multiple events. if (!program.eventSourceFile) { return { EventSourceMappings: null, - ScheduleEvents: null + ScheduleEvents: null, + S3Events: null } } const list = (() => { @@ -241,7 +242,8 @@ so you can easily test run multiple events. // backward-compatible return { EventSourceMappings: list, - ScheduleEvents: [] + ScheduleEvents: [], + S3Events: [] } } if (!list.EventSourceMappings) { @@ -250,6 +252,9 @@ so you can easily test run multiple events. if (!list.ScheduleEvents) { list.ScheduleEvents = [] } + if (!list.S3Events) { + list.S3Events = [] + } return list } diff --git a/test/main.js b/test/main.js index e85f0a6e..669ff8a4 100644 --- a/test/main.js +++ b/test/main.js @@ -781,7 +781,11 @@ describe('lib/main', function () { program.eventSourceFile = '' assert.deepEqual( lambda._eventSourceList(program), - { EventSourceMappings: null, ScheduleEvents: null } + { + EventSourceMappings: null, + ScheduleEvents: null, + S3Events: null + } ) }) @@ -803,18 +807,23 @@ describe('lib/main', function () { fs.writeFileSync('only_ScheduleEvents.json', JSON.stringify({ ScheduleEvents: [{ test: 2 }] })) + fs.writeFileSync('only_S3Events.json', JSON.stringify({ + S3Events: [{ test: 3 }] + })) }) after(() => { fs.unlinkSync('only_EventSourceMappings.json') fs.unlinkSync('only_ScheduleEvents.json') + fs.unlinkSync('only_S3Events.json') }) it('only EventSourceMappings', () => { program.eventSourceFile = 'only_EventSourceMappings.json' const expected = { EventSourceMappings: [{ test: 1 }], - ScheduleEvents: [] + ScheduleEvents: [], + S3Events: [] } assert.deepEqual(lambda._eventSourceList(program), expected) }) @@ -823,7 +832,18 @@ describe('lib/main', function () { program.eventSourceFile = 'only_ScheduleEvents.json' const expected = { EventSourceMappings: [], - ScheduleEvents: [{ test: 2 }] + ScheduleEvents: [{ test: 2 }], + S3Events: [] + } + assert.deepEqual(lambda._eventSourceList(program), expected) + }) + + it('only S3Events', () => { + program.eventSourceFile = 'only_S3Events.json' + const expected = { + EventSourceMappings: [], + ScheduleEvents: [], + S3Events: [{ test: 3 }] } assert.deepEqual(lambda._eventSourceList(program), expected) }) @@ -845,6 +865,20 @@ describe('lib/main', function () { key1: 'value', key2: 'value' } + }], + S3Events: [{ + Bucket: 'BUCKET_NAME', + Events: [ + 's3:ObjectCreated:*' + ], + Filter: { + Key: { + FilterRules: [{ + Name: 'prefix', + Value: 'STRING_VALUE' + }] + } + } }] } assert.deepEqual(lambda._eventSourceList(program), expected) @@ -867,7 +901,8 @@ describe('lib/main', function () { program.eventSourceFile = fileName const expected = { EventSourceMappings: oldStyleValue, - ScheduleEvents: [] + ScheduleEvents: [], + S3Events: [] } assert.deepEqual(lambda._eventSourceList(program), expected) }) From 58dcaefc07e4051b209b594fd30e0003ae921fc1 Mon Sep 17 00:00:00 2001 From: abetomo Date: Fri, 8 Dec 2017 11:00:00 +0900 Subject: [PATCH 2/5] Modify to promise chain --- lib/main.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/main.js b/lib/main.js index 51351dab..0db1407f 100644 --- a/lib/main.js +++ b/lib/main.js @@ -800,6 +800,8 @@ so you can easily test run multiple events. this._uploadExisting(lambda, params).then((results) => { console.log('=> Zip file(s) done uploading. Results follow: ') console.log(results) + return results + }).then(results => { return this._updateScheduleEvents( scheduleEvents, results.FunctionArn, From 4fa3c032c201d98efa22a94f374383a131ae43da Mon Sep 17 00:00:00 2001 From: abetomo Date: Fri, 8 Dec 2017 14:12:45 +0900 Subject: [PATCH 3/5] Add _updateS3Events() --- lib/main.js | 17 ++++++++++++++++ test/main.js | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/lib/main.js b/lib/main.js index 0db1407f..30893f30 100644 --- a/lib/main.js +++ b/lib/main.js @@ -706,6 +706,23 @@ so you can easily test run multiple events. }) } + _updateS3Events (s3Events, functionArn, s3EventsList) { + if (s3EventsList == null) return Promise.resolve([]) + + const paramsList = s3EventsList.map(s3event => + Object.assign(s3event, { FunctionArn: functionArn })) + + // series + return paramsList.map(params => { + return s3Events.add(params) + }).reduce((a, b) => { + return a.then(b) + }, Promise.resolve()).then(() => { + // Since it is similar to _updateScheduleEvents, it returns meaningful values + return paramsList + }) + } + _setLogsRetentionPolicy (cloudWatchLogs, program, functionName) { const days = parseInt(program.retentionInDays) if (!Number.isInteger(days)) return Promise.resolve({}) diff --git a/test/main.js b/test/main.js index 669ff8a4..3c860ee0 100644 --- a/test/main.js +++ b/test/main.js @@ -97,6 +97,9 @@ const _mockSetting = () => { awsMock.mock('CloudWatchLogs', 'putRetentionPolicy', (params, callback) => { callback(null, {}) }) + awsMock.mock('S3', 'putBucketNotificationConfiguration', (params, callback) => { + callback(null, {}) + }) Object.keys(lambdaMockSettings).forEach((method) => { awsMock.mock('Lambda', method, (params, callback) => { @@ -110,6 +113,7 @@ const _mockSetting = () => { const _awsRestore = () => { awsMock.restore('CloudWatchEvents') awsMock.restore('CloudWatchLogs') + awsMock.restore('S3') awsMock.restore('Lambda') } @@ -1050,6 +1054,58 @@ describe('lib/main', function () { }) }) + describe('_updateS3Events', () => { + const S3Events = require(path.join('..', 'lib', 's3_events')) + const eventSourcesJsonValue = { + S3Events: [{ + Bucket: 'node-lambda-test-bucket', + Events: ['s3:ObjectCreated:*'], + Filter: null + }] + } + + let s3Events = null + + before(() => { + fs.writeFileSync( + 'event_sources.json', + JSON.stringify(eventSourcesJsonValue) + ) + s3Events = new S3Events(aws) + }) + + after(() => fs.unlinkSync('event_sources.json')) + + it('program.eventSourceFile is empty value', () => { + program.eventSourceFile = '' + const eventSourceList = lambda._eventSourceList(program) + return lambda._updateS3Events( + s3Events, + '', + eventSourceList.S3Events + ).then(results => { + assert.deepEqual(results, []) + }) + }) + + it('simple test with mock', () => { + program.eventSourceFile = 'event_sources.json' + const eventSourceList = lambda._eventSourceList(program) + const functionArn = 'arn:aws:lambda:us-west-2:XXX:function:node-lambda-test-function' + return lambda._updateS3Events( + s3Events, + functionArn, + eventSourceList.S3Events + ).then(results => { + const expected = [Object.assign( + eventSourcesJsonValue.S3Events[0], + { FunctionArn: functionArn } + )] + assert.deepEqual(results, expected) + }) + }) + }) + describe('_uploadNew', () => { it('simple test with mock', () => { const params = lambda._params(program, null) From 15c17aff3f69d0f5a2115d2d477bfd09cc9d2bf4 Mon Sep 17 00:00:00 2001 From: abetomo Date: Fri, 8 Dec 2017 14:13:08 +0900 Subject: [PATCH 4/5] Add _updateS3Events() call --- lib/main.js | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/main.js b/lib/main.js index 30893f30..a7981f38 100644 --- a/lib/main.js +++ b/lib/main.js @@ -13,6 +13,7 @@ const archiver = require('archiver') const dotenv = require('dotenv') const proxy = require('proxy-agent') const ScheduleEvents = require(path.join(__dirname, 'schedule_events')) +const S3Events = require(path.join(__dirname, 's3_events')) const CloudWatchLogs = require(path.join(__dirname, 'cloudwatch_logs')) const maxBufferSize = 50 * 1024 * 1024 @@ -803,6 +804,7 @@ so you can easily test run multiple events. const lambda = new aws.Lambda({ apiVersion: '2015-03-31' }) const scheduleEvents = new ScheduleEvents(aws) + const s3Events = new S3Events(aws) const cloudWatchLogs = new CloudWatchLogs(aws) // Checking function @@ -819,11 +821,18 @@ so you can easily test run multiple events. console.log(results) return results }).then(results => { - return this._updateScheduleEvents( - scheduleEvents, - results.FunctionArn, - eventSourceList.ScheduleEvents - ) + return Promise.all([ + this._updateScheduleEvents( + scheduleEvents, + results.FunctionArn, + eventSourceList.ScheduleEvents + ), + this._updateS3Events( + s3Events, + results.FunctionArn, + eventSourceList.S3Events + ) + ]) }), this._updateEventSources( lambda, @@ -860,6 +869,11 @@ so you can easily test run multiple events. results.FunctionArn, eventSourceList.ScheduleEvents ), + this._updateS3Events( + s3Events, + results.FunctionArn, + eventSourceList.S3Events + ), this._setLogsRetentionPolicy( cloudWatchLogs, program, From 6fc55df6d10bbebe6388a10476ec5bacd795eedf Mon Sep 17 00:00:00 2001 From: abetomo Date: Fri, 8 Dec 2017 14:13:55 +0900 Subject: [PATCH 5/5] Improve display of deploy results Since the structure of results has become complicated, I added a method --- lib/main.js | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/main.js b/lib/main.js index a7981f38..bc8729da 100644 --- a/lib/main.js +++ b/lib/main.js @@ -889,6 +889,20 @@ so you can easily test run multiple events. }) } + _printDeployResults (results, isFirst) { + if (!Array.isArray(results)) { + if (results == null) return + console.log(results) + return + } + if (results.length === 0) return + + if (isFirst === true) console.log('=> All tasks done. Results follow:') + results.forEach(result => { + this._printDeployResults(result) + }) + } + deploy (program) { const regions = program.region.split(',') return this._archive(program).then((buffer) => { @@ -897,16 +911,8 @@ so you can easily test run multiple events. return Promise.all(regions.map((region) => { return this._deployToRegion(program, params, region) - })).then((results) => { - const resultsIsEmpty = results.filter((result) => { - return result.filter((res) => { - return res.length > 0 - }).length > 0 - }).length === 0 - if (!resultsIsEmpty) { - console.log('=> All tasks done. Results follow: ') - console.log(JSON.stringify(results, null, ' ')) - } + })).then(results => { + this._printDeployResults(results, true) }) }).catch((err) => { process.exitCode = 1