diff --git a/lib/archiveManager.js b/lib/archiveManager.js new file mode 100644 index 00000000..2a1e3e7d --- /dev/null +++ b/lib/archiveManager.js @@ -0,0 +1,95 @@ +/** + * @module Contains functions for working with archives + */ + +const yauzl = require('yauzl'); +const fs = require('fs'); +const path = require('path'); +const { promisify } = require('util'); + +const { readFromStream } = require('./utils/asyncUtils'); + +/** + * @param {object} options + * @param {string} options.zipPath + * @param {string} [options.fileToExtract] - filename to download only + * @param {string[]} [options.exclude] - paths of files and directories to exclude + * @param {object} [options.outputNames] - new names for some files. Format: { 'oldName1': 'newName1', ...} + * @returns {Promise} + */ +async function extractZipFiles({ zipPath, fileToExtract, exclude, outputNames = {}}) { + let foundMatch = false; + + const zipFile = await promisify(yauzl.open)(zipPath, { lazyEntries: true }); + + await new Promise((resolve, reject) => { + zipFile.on('entry', async entry => { + try { + const readableStream = await promisify(zipFile.openReadStream.bind(zipFile))(entry); + + if (fileToExtract) { + if (fileToExtract !== entry.fileName) { + return zipFile.readEntry(); + } + foundMatch = true; + } else if (exclude && exclude.length) { + // Do not process any file or directory within the exclude option + for (const excludeItem of exclude) { + if (entry.fileName.startsWith(excludeItem)) { + return zipFile.readEntry(); + } + } + } + + // If file is a directory, then move to next + if (/\/$/.test(entry.fileName)) { + return zipFile.readEntry(); + } + + // Create a directory if the parent directory does not exists + const parsedPath = path.parse(entry.fileName); + + if (parsedPath.dir && !fs.existsSync(parsedPath.dir)) { + fs.mkdirSync(parsedPath.dir, { recursive: true }); + } + + let fileData = await readFromStream(readableStream); + if (entry.fileName.endsWith('.json')) { + // Make sure that the JSON file if valid + fileData = JSON.stringify(JSON.parse(fileData), null, 2); + } + + const outputFileName = outputNames[entry.fileName] || entry.fileName; + await promisify(fs.writeFile)(outputFileName, fileData, { flag: 'w+' }); + + // Close read if the requested file is found + if (fileToExtract) { + zipFile.close(); + return resolve(); + } + + zipFile.readEntry(); + } catch (err) { + return reject(err); + } + }); + + zipFile.readEntry(); + + zipFile.once('end', function () { + zipFile.close(); + + if (!foundMatch && fileToExtract) { + return reject(new Error(`${fileToExtract} not found`)); + } + + resolve(); + }); + }); +} + + +module.exports = { + extractZipFiles, +}; + diff --git a/lib/archiveManager.spec.js b/lib/archiveManager.spec.js new file mode 100644 index 00000000..5e0623e7 --- /dev/null +++ b/lib/archiveManager.spec.js @@ -0,0 +1,83 @@ +const fs = require('fs'); +const Path = require('path'); +const yauzl = require('yauzl'); + +const { extractZipFiles } = require('./archiveManager'); + +describe('archiveManager', () => { + describe('extractZipFiles', () => { + // We run tests for a real archive containing 2 files: + // - config.json + // - schema.json + let zipPath = Path.join(process.cwd(), 'test', '_mocks', 'themes', 'valid', 'mock-theme.zip'); + let fsWriteSub; + let yauzlOpenSpy; + + beforeEach(() => { + jest.spyOn(console, 'log').mockImplementation(jest.fn()); + + fsWriteSub = jest.spyOn(fs, 'writeFile').mockImplementation((name, config, options, callback) => { + callback(false); + }); + }); + + afterEach(() => { + jest.resetAllMocks(); + jest.restoreAllMocks(); + }); + + beforeEach(() => { + yauzlOpenSpy = jest.spyOn(yauzl, 'open'); + }); + + it('should call yauzl.open with the passed zipPath', async () => { + await extractZipFiles({ zipPath }); + + expect(yauzlOpenSpy).toHaveBeenCalledTimes(1); + }); + + it('should save all the files from the zip archive taking into account options.outputNames', async () => { + const newConfigName = 'config2.json'; + const outputNames = { 'config.json': newConfigName }; + await extractZipFiles({ zipPath, outputNames }); + + expect(fsWriteSub).toHaveBeenCalledTimes(2); + expect(fsWriteSub).toHaveBeenCalledWith( + 'schema.json', expect.anything(), expect.objectContaining({ flag: 'w+' }), expect.any(Function), + ); + expect(fsWriteSub).toHaveBeenCalledWith( + newConfigName, expect.anything(), expect.objectContaining({ flag: 'w+' }), expect.any(Function), + ); + }); + + it('should not save files specified in options.exclude', async () => { + const exclude = ['config.json']; + await extractZipFiles({ zipPath, exclude }); + + expect(fsWriteSub).toHaveBeenCalledTimes(1); + expect(fsWriteSub).toHaveBeenCalledWith( + 'schema.json', expect.anything(), expect.objectContaining({ flag: 'w+' }), expect.any(Function), + ); + }); + + it('should save the file specified in options.fileToExtract only', async () => { + const fileToExtract = 'config.json'; + await extractZipFiles({ zipPath, fileToExtract }); + + expect(fsWriteSub).toHaveBeenCalledTimes(1); + expect(fsWriteSub).toHaveBeenCalledWith( + fileToExtract, expect.anything(), expect.objectContaining({ flag: 'w+' }), expect.any(Function), + ); + }); + + it('should throw an error when the file with name options.fileToExtract was not found', async () => { + const fileToExtract = 'I dont exist.txt'; + + await expect( + extractZipFiles({ zipPath, fileToExtract }), + ).rejects.toThrow(/not found/); + + expect(fsWriteSub).toHaveBeenCalledTimes(0); + }); + }); +}); diff --git a/lib/stencil-download.utils.js b/lib/stencil-download.utils.js index 5dea71e6..f05b568e 100644 --- a/lib/stencil-download.utils.js +++ b/lib/stencil-download.utils.js @@ -1,123 +1,28 @@ -const fetch = require('node-fetch'); -const yauzl = require('yauzl'); -const fs = require('fs'); -const tmp = require('tmp'); -const path = require('path'); +const tmp = require('tmp-promise'); +const { extractZipFiles } = require('./archiveManager'); +const { fetchFile } = require('./utils/networkUtils'); + const utils = {}; module.exports = utils; -utils.downloadThemeFiles = (options, callback) => { - tmp.file(function _tempFileCreated(err, tempThemePath, fd, cleanupCallback) { - if (err) { - callback(err); - } - - Promise.resolve() - .then(() => fetch(options.downloadUrl)) - .then(response => new Promise((resolve, reject) => { - if (!response.ok) { - reject(`Unable to download theme files from ${options.downloadUrl}: ${response.statusText}`); - } - - response.body.pipe(fs.createWriteStream(tempThemePath)) - .on('finish', () => resolve(tempThemePath)) - .on('error', reject); - })) - .then(tempThemePath => new Promise((resolve, reject) => { - let foundMatch = false; - - console.log('ok'.green + ' -- Theme files downloaded'); - console.log('ok'.green + ' -- Extracting theme files'); - - yauzl.open(tempThemePath, {lazyEntries: true}, (error, zipFile) => { - if (error) { - return reject(error); - } - - zipFile.on('entry', entry => { - zipFile.openReadStream(entry, (readStreamError, readStream) => { - if (readStreamError) { - return reject(readStreamError); - } - - let configFileData = ''; - - if (options.file && options.file.length) { - if (options.file !== entry.fileName) { - zipFile.readEntry(); - return; - } - foundMatch = true; - } else if (options.exclude && options.exclude.length) { - // Do not process any file or directory within the exclude option - for (const excludeItem of options.exclude) { - if (entry.fileName.startsWith(excludeItem)) { - zipFile.readEntry(); - return; - } - } - } - - // Create a directory if the parent directory does not exists - const parsedPath = path.parse(entry.fileName); - - if (parsedPath.dir && !fs.existsSync(parsedPath.dir)) { - fs.mkdirSync(parsedPath.dir, {recursive: true}); - } - - // If file is a directory, then move to next - if (/\/$/.test(entry.fileName)) { - zipFile.readEntry(); - return; - } - - readStream.on('end', () => { - if (entry.fileName.endsWith('.json')) { - configFileData = JSON.stringify(JSON.parse(configFileData), null, 2); - } +utils.downloadThemeFiles = async options => { + const { path: tempThemePath, cleanup } = await tmp.file(); - fs.writeFile(entry.fileName, configFileData, {flag: 'w+'}, error => { - if (error) { - reject(error); - } + try { + await fetchFile(options.downloadUrl, tempThemePath); + } catch (err) { + throw new Error(`Unable to download theme files from ${options.downloadUrl}: ${err.message}`); + } - // Close read if file requested is found - if (options.file && options.file.length) { - console.log('ok'.green + ' -- Theme files extracted'); - zipFile.close(); - resolve(options); - } else { - zipFile.readEntry(); - } - }); - }); + console.log('ok'.green + ' -- Theme files downloaded'); + console.log('ok'.green + ' -- Extracting theme files'); - readStream.on('data', chunk => { - configFileData += chunk; - }); - }); - }); + await extractZipFiles({ zipPath: tempThemePath, fileToExtract: options.file, exclude: options.exclude }); - zipFile.readEntry(); + console.log('ok'.green + ' -- Theme files extracted'); - zipFile.once('end', function () { - if (!foundMatch && (options.file && options.file.length)) { - console.log('Warning'.yellow + ` -- ${options.file} not found!`); - reject(`${options.file} not found`); - return; - } + await cleanup(); - console.log('ok'.green + ' -- Theme files extracted'); - zipFile.close(); - resolve(options); - }); - }); - })) - .then(() => { - cleanupCallback(); - callback(null, options); - }) - .catch(callback); - }); + return options; }; diff --git a/lib/stencil-download.utils.spec.js b/lib/stencil-download.utils.spec.js deleted file mode 100644 index ee4c979d..00000000 --- a/lib/stencil-download.utils.spec.js +++ /dev/null @@ -1,138 +0,0 @@ -'use strict'; - -require('colors'); -const fs = require('fs'); -const Path = require('path'); -const { promisify } = require('util'); -const yauzl = require('yauzl'); -const fetch = require('node-fetch'); -const tmp = require('tmp'); - -const { downloadThemeFiles } = require('./stencil-download.utils'); - -jest.mock('node-fetch'); - -describe('ThemeDownloader', function () { - let archiveMockUrl = Path.join(process.cwd(), 'test', '_mocks', 'themes', 'valid', 'mock-theme.zip'); - let themeCallback; - let options = {}; - let fsWriteSub; - let fsCreateWriteStreamStub; - let zipOpenSpy; - - beforeEach(() => { - options = { downloadUrl: archiveMockUrl }; - themeCallback = () => {}; - - jest.spyOn(console, 'log').mockImplementation(jest.fn()); - - fetch.mockImplementation(async() => ({ - ok: true, - body: { - pipe: function responseBodyStreamStub () { - return { - pipe: () => { - return this; - }, - on: (event, optionCallback) => { - if (event === 'finish') { - optionCallback(); - } - return this; - }, - }; - }, - }, - })); - - fsWriteSub = jest.spyOn(fs, 'writeFile').mockImplementation(writeFileStub); - - function writeFileStub(name, config, options, callback) { - callback(false); - } - - fsCreateWriteStreamStub = jest.spyOn(fs, 'createWriteStream').mockImplementation(tempPath => { - fs.writeFileSync(tempPath, fs.readFileSync(options.downloadUrl)); - }); - }); - - afterEach(() => { - jest.resetAllMocks(); - jest.restoreAllMocks(); - }); - - describe("Verify till Zip opens", () => { - beforeEach(() => { - zipOpenSpy = jest.spyOn(yauzl, 'open'); - zipOpenSpy(archiveMockUrl, {lazyEntries: true}, themeCallback); - }); - - it('should verify that the tmp.file() is called', async () => { - const tmpFileSpy = jest.spyOn(tmp, 'file'); - - const promise = promisify(downloadThemeFiles)(options); - zipOpenSpy.mock.calls[0][2](); - await promise; - - expect(tmpFileSpy).toHaveBeenCalledTimes(1); - }); - - it('should return a callback error', async () => { - await expect( - promisify(downloadThemeFiles)(null), - ).rejects.toThrow('Cannot read property \'downloadUrl\' of null'); - }); - - it('should verify request is called with downloadUrl', async () => { - const promise = promisify(downloadThemeFiles)(options); - zipOpenSpy.mock.calls[0][2](); - await promise; - - expect(fetch).toHaveBeenCalledTimes(1); - }); - - it('should verify createWriteStream is also called within the request', async () => { - const promise = promisify(downloadThemeFiles)(options); - zipOpenSpy.mock.calls[0][2](); - await promise; - - expect(fsCreateWriteStreamStub).toHaveBeenCalledTimes(1); - }); - - it('should verify that the yauzl zip module is called', async () => { - const promise = promisify(downloadThemeFiles)(options); - zipOpenSpy.mock.calls[0][2](); - await promise; - - expect(yauzl.open).toHaveBeenCalledTimes(2); - }); - - it('should call the zip open callback with zipFile', async () => { - const zip = await promisify(zipOpenSpy)(archiveMockUrl, {lazyEntries: true}); - - expect(zip.fileSize).toBeGreaterThan(100); - }); - }); - - describe("Verify After zip opens", () => { - it('should write the two files inside the zip archive', async () => { - await promisify(downloadThemeFiles)(options); - - expect(fsWriteSub).toHaveBeenCalledTimes(2); - }); - - it('should exclude config.json from files to write', async () => { - options.exclude = ['config.json']; - await promisify(downloadThemeFiles)(options); - - expect(fsWriteSub).toHaveBeenCalledTimes(1); - }); - - it('should write config.json only', async () => { - options.file = 'config.json'; - await promisify(downloadThemeFiles)(options); - - expect(fsWriteSub).toHaveBeenCalledTimes(1); - }); - }); -}); diff --git a/lib/stencil-pull.utils.js b/lib/stencil-pull.utils.js index 2fa6fd61..a3025908 100644 --- a/lib/stencil-pull.utils.js +++ b/lib/stencil-pull.utils.js @@ -1,9 +1,8 @@ const themeApiClient = require('./theme-api-client'); -const fetch = require('node-fetch'); -const yauzl = require('yauzl'); -const fs = require('fs'); -const path = require('path'); -const tmp = require('tmp'); +const tmp = require('tmp-promise'); +const { extractZipFiles } = require('./archiveManager'); +const { fetchFile } = require('./utils/networkUtils'); + const utils = {}; module.exports = utils; @@ -30,71 +29,26 @@ utils.startThemeDownloadJob = async options => { }; }; -utils.downloadThemeConfig = (options, callback) => { - tmp.file(function _tempFileCreated(err, tempThemePath, fd, cleanupCallback) { - if (err) { - callback(err); - } - - Promise.resolve() - .then(() => fetch(options.downloadUrl)) - .then(response => new Promise((resolve, reject) => { - if (!response.ok) { - reject(`Unable to download theme config from ${options.downloadUrl}: ${response.statusText}`); - } +utils.downloadThemeConfig = async options => { + const { path: tempThemePath, cleanup } = await tmp.file(); - response.body.pipe(fs.createWriteStream(tempThemePath)) - .on('finish', () => resolve(tempThemePath)) - .on('error', reject); - })) - .then(tempThemePath => new Promise((resolve, reject) => - yauzl.open(tempThemePath, { lazyEntries: true }, (error, zipFile) => { - if (error) { - return reject(error); - } + try { + await fetchFile(options.downloadUrl, tempThemePath); + } catch (err) { + throw new Error(`Unable to download theme config from ${options.downloadUrl}: ${err.message}`); + } - zipFile.readEntry(); - zipFile.on('entry', entry => { - if (!/config\.json/.test(entry.fileName)) { - zipFile.readEntry(); - return; - } + console.log('ok'.green + ' -- Theme files downloaded'); + console.log('ok'.green + ' -- Extracting theme config'); - zipFile.openReadStream(entry, (readStreamError, readStream) => { - if (readStreamError) { - return reject(readStreamError); - } + const outputNames = { + 'config.json': options.saveConfigName, + }; + await extractZipFiles({ zipPath: tempThemePath, fileToExtract: 'config.json', outputNames }); - let configFileData = ''; + console.log('ok'.green + ' -- Theme config extracted'); - readStream.on('end', () => { - resolve(JSON.parse(configFileData)); - zipFile.close(); - }); - readStream.on('data', chunk => { - configFileData += chunk; - }); - }); - }); - }), - )) - .then( - liveStencilConfig => - new Promise( - (resolve, reject) => - fs.writeFile(path.resolve(options.saveConfigName), JSON.stringify(liveStencilConfig, null, 2), error => { - if (error) { - reject(error); - } + await cleanup(); - resolve(); - }), - ), - ) - .then(() => { - cleanupCallback(); - callback(null, options); - }) - .catch(callback); - }); + return options; }; diff --git a/lib/utils/asyncUtils.js b/lib/utils/asyncUtils.js new file mode 100644 index 00000000..fd3f5add --- /dev/null +++ b/lib/utils/asyncUtils.js @@ -0,0 +1,21 @@ +/** + * @module Contains helpers functions for working with async stuff and streams + */ + +/** + * @param {ReadableStream} stream + * @returns {Promise} + */ +function readFromStream(stream) { + return new Promise((resolve, reject) => { + let data = ""; + + stream.on("data", chunk => data += chunk); + stream.on("end", () => resolve(data)); + stream.on("error", error => reject(error)); + }); +} + +module.exports = { + readFromStream, +}; diff --git a/lib/utils/networkUtils.js b/lib/utils/networkUtils.js new file mode 100644 index 00000000..7efd3325 --- /dev/null +++ b/lib/utils/networkUtils.js @@ -0,0 +1,28 @@ +/** + * @module Contains helpers functions for working with network requests + */ + +const fetch = require('node-fetch'); +const fs = require('fs'); + +/** + * @param {string} url + * @param {string} outputPath + * @returns {Promise} + */ +async function fetchFile(url, outputPath) { + const response = await fetch(url); + if (!response.ok) { + throw new Error(response.statusText); + } + + return await new Promise((resolve, reject) => { + response.body.pipe(fs.createWriteStream(outputPath)) + .on('finish', resolve) + .on('error', reject); + }); +} + +module.exports = { + fetchFile, +}; diff --git a/package-lock.json b/package-lock.json index f3f75c98..0df08f43 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18395,11 +18395,29 @@ "dev": true }, "tmp": { - "version": "0.0.26", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.26.tgz", - "integrity": "sha1-nvqCDOKhD4H4l5VVus4/FVJs4fI=", + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "requires": { + "rimraf": "^3.0.0" + }, + "dependencies": { + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "requires": { + "glob": "^7.1.3" + } + } + } + }, + "tmp-promise": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.2.tgz", + "integrity": "sha512-OyCLAKU1HzBjL6Ev3gxUeraJNlbNingmi8IrHHEsYH8LTmEuhvYfqvhn2F/je+mjf4N58UmZ96OMEy1JanSCpA==", "requires": { - "os-tmpdir": "~1.0.0" + "tmp": "^0.2.0" } }, "tmpl": { diff --git a/package.json b/package.json index 42b6ef43..23db5f08 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,7 @@ "semver": "^7.3.2", "simple-git": "^2.20.1", "tarjan-graph": "^2.0.0", - "tmp": "0.0.26", + "tmp-promise": "^3.0.2", "upath": "^1.2.0", "uuid4": "^2.0.2", "yauzl": "^2.10.0" diff --git a/server/lib/utils.js b/server/lib/utils.js index e8bc2a7c..e68e2950 100644 --- a/server/lib/utils.js +++ b/server/lib/utils.js @@ -71,25 +71,10 @@ function uuid2int(uuid) { return match ? parseInt(match[1], 10) : 0; } -/** - * @param {ReadableStream} stream - * @returns {Promise} - */ -function readStream(stream) { - return new Promise((resolve, reject) => { - let data = ""; - - stream.on("data", chunk => data += chunk); - stream.on("end", () => resolve(data)); - stream.on("error", error => reject(error)); - }); -} - module.exports = { stripDomainFromCookies, normalizeRedirectUrl, int2uuid, uuid2int, uuidRegExp, - readStream, }; diff --git a/server/plugins/renderer/renderer.module.js b/server/plugins/renderer/renderer.module.js index 8c9b3ec4..cb247d69 100644 --- a/server/plugins/renderer/renderer.module.js +++ b/server/plugins/renderer/renderer.module.js @@ -14,6 +14,7 @@ const { PACKAGE_INFO } = require('../../../constants'); const responses = require('./responses/responses'); const templateAssembler = require('../../../lib/template-assembler'); const utils = require('../../lib/utils'); +const { readFromStream } = require('../../../lib/utils/asyncUtils'); const internals = { options: {}, @@ -76,7 +77,7 @@ internals.getResponse = async function (request) { }, // Fetch will break if request body is Stream and server response is redirect, // so we need to read the data first and then send the request - body: request.payload ? await utils.readStream(request.payload) : request.payload, + body: request.payload ? await readFromStream(request.payload) : request.payload, method: 'post', };