From bbb0f901b37af1ca21d594c34e7936228cb43ace Mon Sep 17 00:00:00 2001 From: Mark Lee Date: Tue, 3 Dec 2019 09:41:40 -0800 Subject: [PATCH] refactor(publisher-s3): replace the s3 package with just aws-sdk (#1318) --- package.json | 1 - packages/publisher/s3/package.json | 5 +- packages/publisher/s3/src/PublisherS3.ts | 92 ++++++++++-------------- yarn.lock | 91 ++--------------------- 4 files changed, 47 insertions(+), 142 deletions(-) diff --git a/package.json b/package.json index 090bf1e2f3..64db8aa480 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,6 @@ "parse-author": "^2.0.0", "pretty-ms": "^5.0.0", "resolve-package": "^1.0.1", - "s3": "^4.4.0", "semver": "^6.3.0", "source-map-support": "^0.5.13", "sudo-prompt": "^9.1.1", diff --git a/packages/publisher/s3/package.json b/packages/publisher/s3/package.json index e6718109b3..040510c7a8 100644 --- a/packages/publisher/s3/package.json +++ b/packages/publisher/s3/package.json @@ -19,7 +19,6 @@ "@electron-forge/publisher-base": "6.0.0-beta.45", "@electron-forge/shared-types": "6.0.0-beta.45", "aws-sdk": "^2.472.0", - "debug": "^4.1.0", - "s3": "^4.4.0" + "debug": "^4.1.0" } -} +} \ No newline at end of file diff --git a/packages/publisher/s3/src/PublisherS3.ts b/packages/publisher/s3/src/PublisherS3.ts index 12ed3ca4d9..676a24b842 100644 --- a/packages/publisher/s3/src/PublisherS3.ts +++ b/packages/publisher/s3/src/PublisherS3.ts @@ -1,22 +1,21 @@ import PublisherBase, { PublisherOptions } from '@electron-forge/publisher-base'; import { asyncOra } from '@electron-forge/async-ora'; -import AWS from 'aws-sdk'; import debug from 'debug'; +import fs from 'fs'; import path from 'path'; +import S3 from 'aws-sdk/clients/s3'; import { PublisherS3Config } from './Config'; -// FIXME: Drop usage of s3 module in favor of AWS-sdk -const s3 = require('s3'); - const d = debug('electron-forge:publish:s3'); -(AWS as any).util.update(AWS.S3.prototype, { - addExpect100Continue: function addExpect100Continue() { - // Hack around large upload issue: https://github.com/andrewrk/node-s3-client/issues/74 - }, -}); +type S3Artifact = { + path: string; + keyPrefix: string; + platform: string; + arch: string; +}; export default class PublisherS3 extends PublisherBase { name = 's3'; @@ -25,12 +24,7 @@ export default class PublisherS3 extends PublisherBase { makeResults, }: PublisherOptions) { const { config } = this; - const artifacts: { - path: string; - keyPrefix: string; - platform: string; - arch: string; - }[] = []; + const artifacts: S3Artifact[] = []; for (const makeResult of makeResults) { artifacts.push(...makeResult.artifacts.map((artifact) => ({ @@ -41,7 +35,7 @@ export default class PublisherS3 extends PublisherBase { }))); } - const s3Client = new AWS.S3({ + const s3Client = new S3({ accessKeyId: config.accessKeyId || process.env.AWS_ACCESS_KEY_ID, secretAccessKey: config.secretAccessKey || process.env.AWS_SECRET_ACCESS_KEY, }); @@ -52,48 +46,40 @@ export default class PublisherS3 extends PublisherBase { d('creating s3 client with options:', config); - const client = s3.createClient({ - s3Client, - }); - client.s3.addExpect100Continue = () => {}; - let uploaded = 0; - await asyncOra(`Uploading Artifacts ${uploaded}/${artifacts.length}`, async (uploadSpinner) => { - const updateSpinner = () => { - uploadSpinner.text = `Uploading Artifacts ${uploaded}/${artifacts.length}`; - }; - - await Promise.all(artifacts.map((artifact) => new Promise((resolve, reject) => { - const done = (err?: Error) => { - if (err) return reject(err); - uploaded += 1; - updateSpinner(); - return resolve(); - }; - - const uploader = client.uploadFile({ - localFile: artifact.path, - s3Params: { - Bucket: config.bucket, - Key: this.config.keyResolver - ? this.config.keyResolver( - path.basename(artifact.path), - artifact.platform, - artifact.arch, - ) - : `${artifact.keyPrefix}/${path.basename(artifact.path)}`, - ACL: config.public ? 'public-read' : 'private', - }, - }); + const spinnerText = () => `Uploading Artifacts ${uploaded}/${artifacts.length}`; + + await asyncOra(spinnerText(), async (uploadSpinner) => { + await Promise.all(artifacts.map(async (artifact) => { + const uploader = s3Client.upload({ + Body: fs.createReadStream(artifact.path), + Bucket: config.bucket, + Key: this.keyForArtifact(artifact), + ACL: config.public ? 'public-read' : 'private', + } as S3.PutObjectRequest); d('uploading:', artifact.path); - uploader.on('error', (err: Error) => done(err)); - uploader.on('progress', () => { - const p = `${Math.round((uploader.progressAmount / uploader.progressTotal) * 100)}%`; + uploader.on('httpUploadProgress', (progress) => { + const p = `${Math.round((progress.loaded / progress.total) * 100)}%`; d(`Upload Progress (${path.basename(artifact.path)}) ${p}`); }); - uploader.on('end', () => done()); - }))); + + await uploader.promise(); + uploaded += 1; + uploadSpinner.text = spinnerText(); + })); }); } + + keyForArtifact(artifact: S3Artifact): string { + if (this.config.keyResolver) { + return this.config.keyResolver( + path.basename(artifact.path), + artifact.platform, + artifact.arch, + ); + } + + return `${artifact.keyPrefix}/${path.basename(artifact.path)}`; + } } diff --git a/yarn.lock b/yarn.lock index de8affda44..547fd85f73 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1873,14 +1873,6 @@ aws-sdk@^2.472.0: uuid "3.3.2" xml2js "0.4.19" -aws-sdk@~2.0.31: - version "2.0.31" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.0.31.tgz#e72cf1fdc69015bd9fd2bdf3d3b88c16507d268e" - integrity sha1-5yzx/caQFb2f0r3z07iMFlB9Jo4= - dependencies: - xml2js "0.2.6" - xmlbuilder "0.4.2" - aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -3946,13 +3938,6 @@ fast-levenshtein@~2.0.6: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= -fd-slicer@~1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.0.1.tgz#8b5bcbd9ec327c5041bf9ab023fd6750f1177e65" - integrity sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU= - dependencies: - pend "~1.2.0" - fetch-mock@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/fetch-mock/-/fetch-mock-8.0.0.tgz#b0f5c9d639f9f01d1e312597f60ad34a9866aa71" @@ -4097,11 +4082,6 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" -findit2@~2.2.3: - version "2.2.3" - resolved "https://registry.yarnpkg.com/findit2/-/findit2-2.2.3.tgz#58a466697df8a6205cdfdbf395536b8bd777a5f6" - integrity sha1-WKRmaX34piBc39vzlVNri9d3pfY= - findup-sync@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-3.0.0.tgz#17b108f9ee512dfb7a5c7f3c8b27ea9e1a9c08d1" @@ -4590,7 +4570,7 @@ got@^9.6.0: to-readable-stream "^1.0.0" url-parse-lax "^3.0.0" -graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9, graceful-fs@^4.2.0, graceful-fs@^4.2.2: +graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.2: version "4.2.3" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.3.tgz#4a12ff1b60376ef09862c2093edd908328be8423" integrity sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ== @@ -4600,12 +4580,10 @@ graceful-fs@^4.1.15: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.2.tgz#6f0952605d0140c1cfdb138ed005775b92d67b02" integrity sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q== -graceful-fs@~3.0.5: - version "3.0.11" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-3.0.11.tgz#7613c778a1afea62f25c630a086d7f3acbbdd818" - integrity sha1-dhPHeKGv6mLyXGMKCG1/Osu92Bg= - dependencies: - natives "^1.1.0" +graceful-fs@^4.1.9: + version "4.2.0" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.0.tgz#8d8fdc73977cb04104721cb53666c1ca64cd328b" + integrity sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg== growl@1.10.5: version "1.10.5" @@ -6112,11 +6090,6 @@ mime@^2.4.4: resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.4.tgz#bd7b91135fc6b01cde3e9bae33d659b63d8857e5" integrity sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA== -mime@~1.2.11: - version "1.2.11" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" - integrity sha1-WCA+7Ybjpe8XrtK32evUfwpg3RA= - mimic-fn@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022" @@ -6210,7 +6183,7 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -mkdirp@0.5.1, mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0: +mkdirp@0.5.1, mkdirp@^0.5.0, mkdirp@^0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" integrity sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM= @@ -6326,11 +6299,6 @@ nanomatch@^1.2.9: snapdragon "^0.8.1" to-regex "^3.0.1" -natives@^1.1.0: - version "1.1.6" - resolved "https://registry.yarnpkg.com/natives/-/natives-1.1.6.tgz#a603b4a498ab77173612b9ea1acdec4d980f00bb" - integrity sha512-6+TDFewD4yxY14ptjKaS63GVdtKiES1pTPyxn9Jb0rBqPMZ7VcCiooEhPNsr+mqHtMGxa/5c/HhcC4uPEUw/nA== - natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" @@ -7126,11 +7094,6 @@ pbkdf2@^3.0.3: safe-buffer "^5.0.1" sha.js "^2.4.8" -pend@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" - integrity sha1-elfrVQpng/kRUzH89GY9XI4AelA= - performance-now@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" @@ -7822,11 +7785,6 @@ rimraf@^3.0.0: dependencies: glob "^7.1.3" -rimraf@~2.2.8: - version "2.2.8" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.2.8.tgz#e439be2aaee327321952730f99a8929e4fc50582" - integrity sha1-5Dm+Kq7jJzIZUnMPmaiSnk/FBYI= - ripemd160@^2.0.0, ripemd160@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.2.tgz#a1c1a6f624751577ba5d07914cbc92850585890c" @@ -7875,21 +7833,6 @@ rxjs@^6.3.1, rxjs@^6.4.0: dependencies: tslib "^1.9.0" -s3@^4.4.0: - version "4.4.0" - resolved "https://registry.yarnpkg.com/s3/-/s3-4.4.0.tgz#56a4f775515a7b6b9c8e5c6b1ab51f9037669f1f" - integrity sha1-VqT3dVFae2ucjlxrGrUfkDdmnx8= - dependencies: - aws-sdk "~2.0.31" - fd-slicer "~1.0.0" - findit2 "~2.2.3" - graceful-fs "~3.0.5" - mime "~1.2.11" - mkdirp "~0.5.0" - pend "~1.2.0" - rimraf "~2.2.8" - streamsink "~1.2.0" - safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" @@ -7919,11 +7862,6 @@ sanitize-filename@^1.6.0, sanitize-filename@^1.6.2: dependencies: truncate-utf8-bytes "^1.0.0" -sax@0.4.2: - version "0.4.2" - resolved "https://registry.yarnpkg.com/sax/-/sax-0.4.2.tgz#39f3b601733d6bec97105b242a2a40fd6978ac3c" - integrity sha1-OfO2AXM9a+yXEFskKipA/Wl4rDw= - sax@1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" @@ -8335,11 +8273,6 @@ stream-shift@^1.0.0: resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.0.tgz#d5c752825e5367e786f78e18e445ea223a155952" integrity sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI= -streamsink@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/streamsink/-/streamsink-1.2.0.tgz#efafee9f1e22d3591ed7de3dcaa95c3f5e79f73c" - integrity sha1-76/unx4i01ke1949yqlcP1559zw= - string-width@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" @@ -9428,13 +9361,6 @@ xdg-basedir@^3.0.0: resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-3.0.0.tgz#496b2cc109eca8dbacfe2dc72b603c17c5870ad4" integrity sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ= -xml2js@0.2.6: - version "0.2.6" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.2.6.tgz#d209c4e4dda1fc9c452141ef41c077f5adfdf6c4" - integrity sha1-0gnE5N2h/JxFIUHvQcB39a399sQ= - dependencies: - sax "0.4.2" - xml2js@0.4.19: version "0.4.19" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" @@ -9443,11 +9369,6 @@ xml2js@0.4.19: sax ">=0.6.0" xmlbuilder "~9.0.1" -xmlbuilder@0.4.2: - version "0.4.2" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-0.4.2.tgz#1776d65f3fdbad470a08d8604cdeb1c4e540ff83" - integrity sha1-F3bWXz/brUcKCNhgTN6xxOVA/4M= - xmlbuilder@^9.0.7, xmlbuilder@~9.0.1: version "9.0.7" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"