diff --git a/.eslintrc b/.eslintrc index aa244a6f2..4b7b0ac09 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,4 +1,5 @@ { + // note you must disable the base rule as it can report incorrect errors "extends": ["airbnb-base", "prettier"], "rules": { "no-bitwise": "off", @@ -10,8 +11,10 @@ "consistent-return": "off", "no-restricted-syntax": "off", "import/prefer-default-export": "off", - "camelcase": "off" - }, + "camelcase": "off", + "no-shadow": "off", + "@typescript-eslint/no-shadow": ["error"] + }, "overrides": [ { "files": ["*.ts"], diff --git a/README.md b/README.md index 741dc877f..47914b3c1 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,16 @@ option to `pkg`. First ensure your computer meets the requirements to compile original Node.js: [BUILDING.md](https://github.com/nodejs/node/blob/master/BUILDING.md) +### Compression + +Pass `--compress Brolti` or `--compress GZip` to `pkg` to compress further the content of the files store in the exectable. + +This option can reduce the size of the embedded file system by up to 60%. + +The startup time of the application might be reduced slightly. + +`-C` can be used as a shortcut for `--compress `. + ### Environment | Var | Description | diff --git a/lib/compress_type.ts b/lib/compress_type.ts new file mode 100644 index 000000000..42b0558f0 --- /dev/null +++ b/lib/compress_type.ts @@ -0,0 +1,6 @@ + +export enum CompressType { + None=0, + GZip= 1, + Brolti= 2 +}; \ No newline at end of file diff --git a/lib/help.ts b/lib/help.ts index 059996b1f..00759ea2f 100644 --- a/lib/help.ts +++ b/lib/help.ts @@ -17,6 +17,7 @@ export default function help() { -d, --debug show more information during packaging process [off] -b, --build don't download prebuilt base binaries, build them --public speed up and disclose the sources of top-level project + -C, --compress [default=None] compression algorithm = Brotli or GZip ${chalk.dim('Examples:')} @@ -30,5 +31,9 @@ export default function help() { ${chalk.cyan('$ pkg -t node12-linux,node14-linux,node14-win index.js')} ${chalk.gray('–')} Bakes '--expose-gc' into executable ${chalk.cyan('$ pkg --options expose-gc index.js')} + ${chalk.gray( + '–' + )} reduce size of the data packed inside the executable with GZip + ${chalk.cyan('$ pkg --compress GZip index.js')} `); } diff --git a/lib/index.ts b/lib/index.ts index 26d987278..f69a54aaa 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -1,6 +1,6 @@ /* eslint-disable require-atomic-updates */ -import { mkdirp, readFile, remove, stat, readFileSync } from 'fs-extra'; +import { existsSync, mkdirp, readFile, remove, stat, readFileSync } from 'fs-extra'; import { need, system } from 'pkg-fetch'; import assert from 'assert'; import minimist from 'minimist'; @@ -16,6 +16,7 @@ import refine from './refiner'; import { shutdown } from './fabricator'; import walk, { Marker, WalkerParams } from './walker'; import { Target, NodeTarget, SymLinks } from './types'; +import { CompressType } from './compress_type'; const { version } = JSON.parse( readFileSync(path.join(__dirname, '../package.json'), 'utf-8') @@ -25,15 +26,6 @@ function isConfiguration(file: string) { return isPackageJson(file) || file.endsWith('.config.json'); } -async function exists(file: string) { - try { - await stat(file); - return true; - } catch (error) { - return false; - } -} - // http://www.openwall.com/lists/musl/2012/12/08/4 const { @@ -231,6 +223,8 @@ export async function exec(argv2: string[]) { 't', 'target', 'targets', + 'C', + 'compress', ], default: { bytecode: true }, }); @@ -258,7 +252,31 @@ export async function exec(argv2: string[]) { const forceBuild = argv.b || argv.build; - // _ + // doCompress + const algo = argv.C || argv.compress || 'None'; + + + let doCompress: CompressType = CompressType.None; + switch (algo.toLowerCase()) { + case 'brotli': + case 'br': + doCompress = CompressType.Brolti; + break; + case 'gzip': + case 'gz': + doCompress = CompressType.GZip; + break; + case 'none': + break; + default: + // eslint-disable-next-line no-console + throw wasReported(`Invalid compression algorithm ${algo} ( should be None, Brolti or Gzip)`); + + } + if (doCompress !== CompressType.None) { + // eslint-disable-next-line no-console + console.log('compression: ', CompressType[doCompress]); + } if (!argv._.length) { throw wasReported('Entry file/directory is expected', [ @@ -274,14 +292,13 @@ export async function exec(argv2: string[]) { let input = path.resolve(argv._[0]); - if (!(await exists(input))) { + if (!existsSync(input)) { throw wasReported('Input file does not exist', [input]); } if ((await stat(input)).isDirectory()) { input = path.join(input, 'package.json'); - - if (!(await exists(input))) { + if (!existsSync(input)) { throw wasReported('Input file does not exist', [input]); } } @@ -316,7 +333,7 @@ export async function exec(argv2: string[]) { } } inputBin = path.resolve(path.dirname(input), bin); - if (!(await exists(inputBin))) { + if (!existsSync(inputBin)) { throw wasReported( 'Bin file does not exist (taken from package.json ' + "'bin' property)", @@ -348,8 +365,7 @@ export async function exec(argv2: string[]) { if (config) { config = path.resolve(config); - - if (!(await exists(config))) { + if (!existsSync(config)) { throw wasReported('Config file does not exist', [config]); } @@ -589,7 +605,7 @@ export async function exec(argv2: string[]) { log.debug('Targets:', JSON.stringify(targets, null, 2)); for (const target of targets) { - if (target.output && (await exists(target.output))) { + if (target.output && existsSync(target.output)) { if ((await stat(target.output)).isFile()) { await remove(target.output); } else { @@ -601,14 +617,8 @@ export async function exec(argv2: string[]) { await mkdirp(path.dirname(target.output)); } - await producer({ - backpack, - bakes, - slash: target.platform === 'win' ? '\\' : '/', - target: target as Target, - symLinks, - }); - + const slash = target.platform === 'win' ? '\\' : '/'; + await producer({ backpack, bakes, slash, target: target as Target, symLinks, doCompress }); if (target.platform !== 'win' && target.output) { await plusx(target.output); } diff --git a/lib/packer.ts b/lib/packer.ts index aee46781b..3a8b945da 100644 --- a/lib/packer.ts +++ b/lib/packer.ts @@ -1,8 +1,8 @@ /* eslint-disable complexity */ import assert from 'assert'; -import fs from 'fs-extra'; -import path from 'path'; +import * as fs from 'fs-extra'; +import * as path from 'path'; import { STORE_BLOB, @@ -157,7 +157,7 @@ export default function packer({ } } const prelude = - `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS) { + `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS, DICT, DOCOMPRESS) { ${bootstrapText}${ log.debugMode ? diagnosticText : '' }\n})(function (exports) {\n${commonText}\n},\n` + @@ -166,6 +166,10 @@ export default function packer({ `%DEFAULT_ENTRYPOINT%` + `\n,\n` + `%SYMLINKS%` + + '\n,\n' + + '%DICT%' + + '\n,\n' + + '%DOCOMPRESS%' + `\n);`; return { prelude, entrypoint, stripes }; diff --git a/lib/producer.ts b/lib/producer.ts index a37852cdc..dc6db9e9a 100644 --- a/lib/producer.ts +++ b/lib/producer.ts @@ -1,3 +1,4 @@ +import { createBrotliCompress, createGzip } from 'zlib'; import Multistream from 'multistream'; import assert from 'assert'; import { execSync } from 'child_process'; @@ -12,6 +13,7 @@ import { log, wasReported } from './log'; import { fabricateTwice } from './fabricator'; import { platform, SymLinks, Target } from './types'; import { Stripe } from './packer'; +import { CompressType } from './compress_type'; interface NotFound { notFound: true; @@ -242,14 +244,34 @@ interface ProducerOptions { slash: string; target: Target; symLinks: SymLinks; + doCompress: CompressType; } +const fileDictionary: {[key:string]: string} = {}; +let counter = 0; +function replace(k: string) { + let existingKey = fileDictionary[k]; + if (!existingKey) { + const newkey = counter; + counter += 1; + existingKey = newkey.toString(36); + fileDictionary[k] = existingKey; + } + return existingKey; +} +const separator = '$'; + +function makeKey(filename: string, slash: string): string { + const a = filename.split(slash).map(replace).join(separator); + return a; +} export default function producer({ backpack, bakes, slash, target, symLinks, + doCompress }: ProducerOptions) { return new Promise((resolve, reject) => { if (!Buffer.alloc) { @@ -268,10 +290,8 @@ export default function producer({ for (const stripe of stripes) { let { snap } = stripe; snap = snapshotify(snap, slash); - - if (!vfs[snap]) { - vfs[snap] = {}; - } + const vfsKey = makeKey(snap, slash); + if (!vfs[vfsKey]) vfs[vfsKey] = {}; } const snapshotSymLinks: SymLinks = {}; @@ -279,7 +299,8 @@ export default function producer({ for (const [key, value] of Object.entries(symLinks)) { const k = snapshotify(key, slash); const v = snapshotify(value, slash); - snapshotSymLinks[k] = v; + const vfsKey = makeKey(k, slash); + snapshotSymLinks[vfsKey] = makeKey(v, slash); } let meter: streamMeter.StreamMeter; @@ -289,6 +310,15 @@ export default function producer({ meter = streamMeter(); return s.pipe(meter); } + function pipeMayCompressToNewMeter(s: Readable): streamMeter.StreamMeter { + if (doCompress === CompressType.GZip) { + return pipeToNewMeter(s.pipe(createGzip())); + } + if (doCompress === CompressType.Brolti) { + return pipeToNewMeter(s.pipe(createBrotliCompress())); + } + return pipeToNewMeter(s); + } function next(s: Readable) { count += 1; @@ -321,7 +351,8 @@ export default function producer({ const { store } = prevStripe; let { snap } = prevStripe; snap = snapshotify(snap, slash); - vfs[snap][store] = [track, meter.bytes]; + const vfsKey = makeKey(snap, slash); + vfs[vfsKey][store] = [track, meter.bytes]; track += meter.bytes; } @@ -347,15 +378,14 @@ export default function producer({ return cb(null, intoStream(Buffer.alloc(0))); } - cb( - null, - pipeToNewMeter(intoStream(buffer || Buffer.from(''))) - ); + cb(null, pipeMayCompressToNewMeter(intoStream(buffer || Buffer.from('')))); } ); } - - return cb(null, pipeToNewMeter(intoStream(stripe.buffer))); + return cb( + null, + pipeMayCompressToNewMeter(intoStream(stripe.buffer)) + ); } if (stripe.file) { @@ -378,15 +408,17 @@ export default function producer({ if (fs.existsSync(platformFile)) { return cb( null, - pipeToNewMeter(fs.createReadStream(platformFile)) + pipeMayCompressToNewMeter(fs.createReadStream(platformFile)) ); } } catch (err) { log.debug(`prebuild-install failed[${stripe.file}]:`, err); } } - - return cb(null, pipeToNewMeter(fs.createReadStream(stripe.file))); + return cb( + null, + pipeMayCompressToNewMeter(fs.createReadStream(stripe.file)) + ); } assert(false, 'producer: bad stripe'); @@ -401,15 +433,23 @@ export default function producer({ replaceDollarWise( replaceDollarWise( replaceDollarWise( - prelude, - '%VIRTUAL_FILESYSTEM%', - JSON.stringify(vfs) + replaceDollarWise( + replaceDollarWise( + prelude, + '%VIRTUAL_FILESYSTEM%', + JSON.stringify(vfs) + ), + '%DEFAULT_ENTRYPOINT%', + JSON.stringify(entrypoint) + ), + '%SYMLINKS%', + JSON.stringify(snapshotSymLinks) ), - '%DEFAULT_ENTRYPOINT%', - JSON.stringify(entrypoint) + '%DICT%', + JSON.stringify(fileDictionary) ), - '%SYMLINKS%', - JSON.stringify(snapshotSymLinks) + '%DOCOMPRESS%', + JSON.stringify(doCompress) ) ) ) diff --git a/lib/walker.ts b/lib/walker.ts index e42ef1670..428bc9288 100644 --- a/lib/walker.ts +++ b/lib/walker.ts @@ -497,6 +497,7 @@ class Walker { assets = expandFiles(assets, base); for (const asset of assets) { + log.debug(' Adding asset : .... ', asset); const stat = await fs.stat(asset); if (stat.isFile()) { diff --git a/package.json b/package.json index 8c67960fb..8b3e136eb 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "lint": "eslint lib prelude test", "lint:fix": "npm run lint -- --fix", "prepare": "npm run build", + "prepublishOnly": "npm run lint", "test": "npm run build && node test/test.js node14 no-npm && node test/test.js node12 no-npm && node test/test.js node10 no-npm && node test/test.js host only-npm" }, "greenkeeper": { diff --git a/prelude/bootstrap.js b/prelude/bootstrap.js index 676d7a45c..6b4d9b76f 100644 --- a/prelude/bootstrap.js +++ b/prelude/bootstrap.js @@ -14,14 +14,23 @@ /* global REQUIRE_COMMON */ /* global VIRTUAL_FILESYSTEM */ /* global DEFAULT_ENTRYPOINT */ +/* global DICT */ +/* global DOCOMPRESS */ /* global SYMLINKS */ 'use strict'; -const { normalize } = require('path'); -const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const { sep } = require('path'); +const { + gunzip, + gunzipSync, + brotliDecompressSync, + brotliDecompress, +} = require('zlib'); -var common = {}; +const common = {}; REQUIRE_COMMON(common); const { @@ -36,9 +45,9 @@ const { removeUplevels, } = common; -var FLAG_ENABLE_PROJECT = false; -var NODE_VERSION_MAJOR = process.version.match(/^v(\d+)/)[1] | 0; -var NODE_VERSION_MINOR = process.version.match(/^v\d+.(\d+)/)[1] | 0; +let FLAG_ENABLE_PROJECT = false; +const NODE_VERSION_MAJOR = process.version.match(/^v(\d+)/)[1] | 0; +const NODE_VERSION_MINOR = process.version.match(/^v\d+.(\d+)/)[1] | 0; // ///////////////////////////////////////////////////////////////// // ENTRYPOINT ////////////////////////////////////////////////////// @@ -46,9 +55,9 @@ var NODE_VERSION_MINOR = process.version.match(/^v\d+.(\d+)/)[1] | 0; // set ENTRYPOINT and ARGV0 here because // they can be altered during process run -var ARGV0 = process.argv[0]; -var EXECPATH = process.execPath; -var ENTRYPOINT = process.argv[1]; +const ARGV0 = process.argv[0]; +const EXECPATH = process.execPath; +let ENTRYPOINT = process.argv[1]; if (process.env.PKG_EXECPATH === 'PKG_INVOKE_NODEJS') { return { undoPatch: true }; @@ -80,7 +89,7 @@ delete process.env.PKG_EXECPATH; // EXECSTAT //////////////////////////////////////////////////////// // ///////////////////////////////////////////////////////////////// -var EXECSTAT = require('fs').statSync(EXECPATH); +const EXECSTAT = fs.statSync(EXECPATH); EXECSTAT.atimeMs = EXECSTAT.atime.getTime(); EXECSTAT.mtimeMs = EXECSTAT.mtime.getTime(); @@ -91,35 +100,37 @@ EXECSTAT.birthtimeMs = EXECSTAT.birthtime.getTime(); // MOUNTPOINTS ///////////////////////////////////////////////////// // ///////////////////////////////////////////////////////////////// -var mountpoints = []; +const mountpoints = []; function insideMountpoint(f) { if (!insideSnapshot(f)) return null; - var file = normalizePath(f); - var found = mountpoints + + const file = normalizePath(f); + const found = mountpoints .map((mountpoint) => { - var { interior } = mountpoint; - var { exterior } = mountpoint; + const { interior } = mountpoint; + const { exterior } = mountpoint; if (interior === file) return exterior; - var left = interior + require('path').sep; + const left = interior + sep; if (file.slice(0, left.length) !== left) return null; return exterior + file.slice(left.length - 1); }) .filter((result) => result); + if (found.length >= 2) throw new Error('UNEXPECTED-00'); if (found.length === 0) return null; return found[0]; } -function readdirMountpoints(path) { +function readdirMountpoints(filename) { return mountpoints .map((mountpoint) => mountpoint.interior) - .filter((interior) => require('path').dirname(interior) === path) - .map((interior) => require('path').basename(interior)); + .filter((interior) => path.dirname(interior) === filename) + .map((interior) => path.basename(interior)); } function translate(f) { - var result = insideMountpoint(f); + const result = insideMountpoint(f); if (!result) throw new Error('UNEXPECTED-05'); return result; } @@ -129,7 +140,7 @@ function cloneArgs(args_) { } function translateNth(args_, index, f) { - var args = cloneArgs(args_); + const args = cloneArgs(args_); args[index] = translate(f); return args; } @@ -165,46 +176,63 @@ console.log(translateNth(["", "r+"], 0, "d:\\snapshot\\countly\\plugins-ext")); console.log(translateNth(["", "rw"], 0, "d:\\snapshot\\countly\\plugins-ext\\")); console.log(translateNth(["", "a+"], 0, "d:\\snapshot\\countly\\plugins-ext\\1234")); */ +const separator = '$'; +function replace(k) { + return DICT[k]; +} +function makeKey(filename, slash) { + const a = filename.split(slash).map(replace).join(separator); + return a || filename; +} +const dictRev = {}; +Object.entries(DICT).forEach(([k, v]) => { + dictRev[v] = k; +}); + +function toOriginal(fShort) { + return fShort + .split('$') + .map((x) => dictRev[x]) + .join(sep); +} -const win32 = process.platform === 'win32'; -const slash = win32 ? '\\' : '/'; -function realpathFromSnapshot(path_) { - path_ = normalize(path_); - let count = 0; - let needToSubstitute = true; - while (needToSubstitute) { - needToSubstitute = false; - for (const [k, v] of Object.entries(SYMLINKS)) { - if (path_.startsWith(k + slash) || path_ === k) { - path_ = path_.replace(k, v); - needToSubstitute = true; +const symlinksEntries = Object.entries(SYMLINKS); +function normalizePathAndFollowLink(f) { + f = normalizePath(f); + f = makeKey(f, sep); + let needToSubstitue = true; + while (needToSubstitue) { + needToSubstitue = false; + for (const [k, v] of symlinksEntries) { + if (f.startsWith(`${k}${separator}`) || f === k) { + f = f.replace(k, v); + needToSubstitue = true; break; } } - count += 1; } - assert(count === 1 || count === 2); - return path_; + return f; +} +function realpathFromSnapshot(f) { + const realPath = toOriginal(normalizePathAndFollowLink(f)); + return realPath; } -function normalizePathAndFollowLink(path_) { - path_ = normalizePath(path_); - var path = realpathFromSnapshot(path_); - return path; +function findVirtualFileSystemEntry(f) { + const fShort = normalizePathAndFollowLink(f); + return VIRTUAL_FILESYSTEM[fShort]; } // ///////////////////////////////////////////////////////////////// // PROJECT ///////////////////////////////////////////////////////// // ///////////////////////////////////////////////////////////////// +const xpdn = path.dirname(EXECPATH); +const maxUplevels = xpdn.split(sep).length; function projectToFilesystem(f) { - var xpdn = require('path').dirname(EXECPATH); - - var relatives = []; + const relatives = []; relatives.push( - removeUplevels( - require('path').relative(require('path').dirname(DEFAULT_ENTRYPOINT), f) - ) + removeUplevels(path.relative(path.dirname(DEFAULT_ENTRYPOINT), f)) ); if (relatives[0].slice(0, 'node_modules'.length) === 'node_modules') { @@ -212,45 +240,40 @@ function projectToFilesystem(f) { relatives.push(relatives[0].slice('node_modules'.length + 1)); } - var uplevels = []; - var maxUplevels = xpdn.split(require('path').sep).length; - for (var i = 0, u = ''; i < maxUplevels; i += 1) { + const uplevels = []; + for (let i = 0, u = ''; i < maxUplevels; i += 1) { uplevels.push(u); u += '/..'; } - var results = []; + const results = []; uplevels.forEach((uplevel) => { relatives.forEach((relative) => { - results.push(require('path').join(xpdn, uplevel, relative)); + results.push(path.join(xpdn, uplevel, relative)); }); }); return results; } function projectToNearby(f) { - return require('path').join( - require('path').dirname(EXECPATH), - require('path').basename(f) - ); + return path.join(xpdn, path.basename(f)); } - -function findNativeAddonSyncFreeFromRequire(path) { - if (!insideSnapshot(path)) throw new Error('UNEXPECTED-10'); - if (path.slice(-5) !== '.node') return null; // leveldown.node.js +function findNativeAddonSyncFreeFromRequire(f) { + if (!insideSnapshot(f)) throw new Error(`UNEXPECTED-10 ${f}`); + if (f.slice(-5) !== '.node') return null; // leveldown.node.js // check mearby first to prevent .node tampering - var projector = projectToNearby(path); - if (require('fs').existsSync(projector)) return projector; - var projectors = projectToFilesystem(path); - for (var i = 0; i < projectors.length; i += 1) { - if (require('fs').existsSync(projectors[i])) return projectors[i]; + const projector = projectToNearby(f); + if (fs.existsSync(projector)) return projector; + const projectors = projectToFilesystem(f); + for (let i = 0; i < projectors.length; i += 1) { + if (fs.existsSync(projectors[i])) return projectors[i]; } return null; } -function findNativeAddonSyncUnderRequire(path) { +function findNativeAddonSyncUnderRequire(f) { if (!FLAG_ENABLE_PROJECT) return null; - return findNativeAddonSyncFreeFromRequire(path); + return findNativeAddonSyncFreeFromRequire(f); } // ///////////////////////////////////////////////////////////////// @@ -264,13 +287,13 @@ function asap(cb) { function dezalgo(cb) { if (!cb) return cb; - var sync = true; + let sync = true; asap(() => { sync = false; }); return function zalgoSafe() { - var args = arguments; + const args = arguments; if (sync) { asap(() => { cb.apply(undefined, args); @@ -295,7 +318,7 @@ if (typeof PAYLOAD_POSITION !== 'number' || typeof PAYLOAD_SIZE !== 'number') { } function readPayload(buffer, offset, length, position, callback) { - require('fs').read( + fs.read( EXECPATH_FD, buffer, offset, @@ -306,7 +329,7 @@ function readPayload(buffer, offset, length, position, callback) { } function readPayloadSync(buffer, offset, length, position) { - return require('fs').readSync( + return fs.readSync( EXECPATH_FD, buffer, offset, @@ -323,12 +346,12 @@ function payloadCopyUni( sourceEnd, cb ) { - var cb2 = cb || rethrow; + const cb2 = cb || rethrow; if (sourceStart >= source[1]) return cb2(null, 0); if (sourceEnd >= source[1]) [, sourceEnd] = source; - var payloadPos = source[0] + sourceStart; - var targetPos = targetStart; - var targetEnd = targetStart + sourceEnd - sourceStart; + const payloadPos = source[0] + sourceStart; + const targetPos = targetStart; + const targetEnd = targetStart + sourceEnd - sourceStart; if (cb) { readPayload(target, targetPos, targetEnd - targetPos, payloadPos, cb); } else { @@ -342,9 +365,9 @@ function payloadCopyUni( } function payloadCopyMany(source, target, targetStart, sourceStart, cb) { - var payloadPos = source[0] + sourceStart; - var targetPos = targetStart; - var targetEnd = targetStart + source[1] - sourceStart; + const payloadPos = source[0] + sourceStart; + let targetPos = targetStart; + const targetEnd = targetStart + source[1] - sourceStart; readPayload( target, targetPos, @@ -364,12 +387,11 @@ function payloadCopyMany(source, target, targetStart, sourceStart, cb) { } function payloadCopyManySync(source, target, targetStart, sourceStart) { - var payloadPos = source[0] + sourceStart; - var targetPos = targetStart; - var targetEnd = targetStart + source[1] - sourceStart; - var chunkSize; + let payloadPos = source[0] + sourceStart; + let targetPos = targetStart; + const targetEnd = targetStart + source[1] - sourceStart; while (true) { - chunkSize = readPayloadSync( + const chunkSize = readPayloadSync( target, targetPos, targetEnd - targetPos, @@ -381,17 +403,39 @@ function payloadCopyManySync(source, target, targetStart, sourceStart) { } } +const GZIP = 1; +const BROTLI = 2; function payloadFile(pointer, cb) { - var target = Buffer.alloc(pointer[1]); + const target = Buffer.alloc(pointer[1]); payloadCopyMany(pointer, target, 0, 0, (error) => { if (error) return cb(error); - cb(null, target); + if (DOCOMPRESS === GZIP) { + gunzip(target, (error2, target2) => { + if (error2) return cb(error2); + cb(null, target2); + }); + } else if (DOCOMPRESS === BROTLI) { + brotliDecompress(target, (error2, target2) => { + if (error2) return cb(error2); + cb(null, target2); + }); + } else { + return cb(null, target); + } }); } function payloadFileSync(pointer) { - var target = Buffer.alloc(pointer[1]); + const target = Buffer.alloc(pointer[1]); payloadCopyManySync(pointer, target, 0, 0); + if (DOCOMPRESS === GZIP) { + const target1 = gunzipSync(target); + return target1; + } + if (DOCOMPRESS === BROTLI) { + const target1 = brotliDecompressSync(target); + return target1; + } return target; } @@ -410,12 +454,11 @@ function payloadFileSync(pointer) { // ///////////////////////////////////////////////////////////////// // PATH.RESOLVE REPLACEMENT //////////////////////////////////////// // ///////////////////////////////////////////////////////////////// -(() => { - var path = require('path'); +(() => { process.pkg.path = {}; - process.pkg.path.resolve = function resolve() { - var args = cloneArgs(arguments); + process.pkg.path.resolve = () => { + const args = cloneArgs(arguments); args.unshift(path.dirname(ENTRYPOINT)); return path.resolve.apply(path, args); // eslint-disable-line prefer-spread }; @@ -424,9 +467,9 @@ function payloadFileSync(pointer) { // ///////////////////////////////////////////////////////////////// // PATCH FS //////////////////////////////////////////////////////// // ///////////////////////////////////////////////////////////////// + (() => { - var fs = require('fs'); - var ancestor = {}; + const ancestor = {}; ancestor.openSync = fs.openSync; ancestor.open = fs.open; ancestor.readSync = fs.readSync; @@ -456,12 +499,12 @@ function payloadFileSync(pointer) { ancestor.mkdirSync = fs.mkdirSync; ancestor.mkdir = fs.mkdir; - var windows = process.platform === 'win32'; + const windows = process.platform === 'win32'; - var docks = {}; - var ENOTDIR = windows ? 4052 : 20; - var ENOENT = windows ? 4058 : 2; - var EISDIR = windows ? 4068 : 21; + const docks = {}; + const ENOTDIR = windows ? 4052 : 20; + const ENOENT = windows ? 4058 : 2; + const EISDIR = windows ? 4068 : 21; function assertEncoding(encoding) { if (encoding && !Buffer.isEncoding(encoding)) { @@ -474,36 +517,36 @@ function payloadFileSync(pointer) { return typeof cb === 'function' ? cb : rethrow; } - function error_ENOENT(fileOrDirectory, path) { + function error_ENOENT(fileOrDirectory, f) { // eslint-disable-line camelcase var error = new Error( - `${fileOrDirectory} '${stripSnapshot(path)}' ` + + `${fileOrDirectory} '${stripSnapshot(f)}' ` + `was not included into executable at compilation stage. ` + `Please recompile adding it as asset or script.` ); error.errno = -ENOENT; error.code = 'ENOENT'; - error.path = path; + error.path = f; error.pkg = true; return error; } - function error_EISDIR(path) { + function error_EISDIR(f) { // eslint-disable-line camelcase var error = new Error('EISDIR: illegal operation on a directory, read'); error.errno = -EISDIR; error.code = 'EISDIR'; - error.path = path; + error.path = f; error.pkg = true; return error; } - function error_ENOTDIR(path) { + function error_ENOTDIR(f) { // eslint-disable-line camelcase - var error = new Error(`ENOTDIR: not a directory, scandir '${path}'`); + var error = new Error(`ENOTDIR: not a directory, scandir '${f}'`); error.errno = -ENOTDIR; error.code = 'ENOTDIR'; - error.path = path; + error.path = f; error.pkg = true; return error; } @@ -512,13 +555,13 @@ function payloadFileSync(pointer) { // open ////////////////////////////////////////////////////////// // /////////////////////////////////////////////////////////////// - function openFromSnapshot(path_, cb) { - var cb2 = cb || rethrow; - var path = normalizePathAndFollowLink(path_); - var entity = VIRTUAL_FILESYSTEM[path]; - if (!entity) return cb2(error_ENOENT('File or directory', path)); - var dock = { path, entity, position: 0 }; - var nullDevice = windows ? '\\\\.\\NUL' : '/dev/null'; + function openFromSnapshot(f, cb) { + const cb2 = cb || rethrow; + const entity = findVirtualFileSystemEntry(f); + + if (!entity) return cb2(error_ENOENT('File or directory', f)); + const dock = { path: f, entity, position: 0 }; + const nullDevice = windows ? '\\\\.\\NUL' : '/dev/null'; if (cb) { ancestor.open.call(fs, nullDevice, 'r', (error, fd) => { if (error) return cb(error); @@ -526,33 +569,32 @@ function payloadFileSync(pointer) { cb(null, fd); }); } else { - var fd = ancestor.openSync.call(fs, nullDevice, 'r'); + const fd = ancestor.openSync.call(fs, nullDevice, 'r'); docks[fd] = dock; return fd; } } - fs.openSync = function openSync(path) { - if (!insideSnapshot(path)) { + fs.openSync = function openSync(f) { + if (!insideSnapshot(f)) { return ancestor.openSync.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.openSync.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.openSync.apply(fs, translateNth(arguments, 0, f)); } - - return openFromSnapshot(path); + return openFromSnapshot(f); }; - fs.open = function open(path) { - if (!insideSnapshot(path)) { + fs.open = function open(f) { + if (!insideSnapshot(f)) { return ancestor.open.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.open.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.open.apply(fs, translateNth(arguments, 0, f)); } - var callback = dezalgo(maybeCallback(arguments)); - openFromSnapshot(path, callback); + const callback = dezalgo(maybeCallback(arguments)); + openFromSnapshot(f, callback); }; // /////////////////////////////////////////////////////////////// @@ -568,7 +610,7 @@ function payloadFileSync(pointer) { position, cb ) { - var p; + let p; if (position !== null && position !== undefined) { p = position; } else { @@ -588,7 +630,7 @@ function payloadFileSync(pointer) { } ); } else { - var bytesRead = payloadCopyUni( + const bytesRead = payloadCopyUni( entityContent, buffer, offset, @@ -601,7 +643,7 @@ function payloadFileSync(pointer) { } function readFromSnapshot(fd, buffer, offset, length, position, cb) { - var cb2 = cb || rethrow; + const cb2 = cb || rethrow; if (offset < 0 && NODE_VERSION_MAJOR >= 14) return cb2( new Error( @@ -637,11 +679,11 @@ function payloadFileSync(pointer) { if (offset + length > buffer.length) return cb2(new Error('Length extends beyond buffer')); - var dock = docks[fd]; - var { entity } = dock; - var entityLinks = entity[STORE_LINKS]; + const dock = docks[fd]; + const { entity } = dock; + const entityLinks = entity[STORE_LINKS]; if (entityLinks) return cb2(error_EISDIR(dock.path)); - var entityContent = entity[STORE_CONTENT]; + const entityContent = entity[STORE_CONTENT]; if (entityContent) return readFromSnapshotSub( entityContent, @@ -668,7 +710,7 @@ function payloadFileSync(pointer) { return ancestor.read.apply(fs, arguments); } - var callback = dezalgo(maybeCallback(arguments)); + const callback = dezalgo(maybeCallback(arguments)); readFromSnapshot(fd, buffer, offset, length, position, callback); }; @@ -677,7 +719,7 @@ function payloadFileSync(pointer) { // /////////////////////////////////////////////////////////////// function writeToSnapshot(cb) { - var cb2 = cb || rethrow; + const cb2 = cb || rethrow; return cb2(new Error('Cannot write to packaged file')); } @@ -694,7 +736,7 @@ function payloadFileSync(pointer) { return ancestor.write.apply(fs, arguments); } - var callback = dezalgo(maybeCallback(arguments)); + const callback = dezalgo(maybeCallback(arguments)); writeToSnapshot(callback); }; @@ -724,7 +766,7 @@ function payloadFileSync(pointer) { return ancestor.close.apply(fs, arguments); } - var callback = dezalgo(maybeCallback(arguments)); + const callback = dezalgo(maybeCallback(arguments)); closeFromSnapshot(fd, callback); }; @@ -753,16 +795,19 @@ function payloadFileSync(pointer) { } } - function readFileFromSnapshot(path_, cb) { - var cb2 = cb || rethrow; - var path = normalizePath(path_); - var entity = VIRTUAL_FILESYSTEM[path]; - if (!entity) return cb2(error_ENOENT('File', path)); - var entityLinks = entity[STORE_LINKS]; - if (entityLinks) return cb2(error_EISDIR(path)); - var entityContent = entity[STORE_CONTENT]; + function readFileFromSnapshot(filename, cb) { + const cb2 = cb || rethrow; + + const entity = findVirtualFileSystemEntry(filename); + if (!entity) return cb2(error_ENOENT('File', filename)); + + const entityLinks = entity[STORE_LINKS]; + if (entityLinks) return cb2(error_EISDIR(filename)); + + const entityContent = entity[STORE_CONTENT]; if (entityContent) return readFileFromSnapshotSub(entityContent, cb); - var entityBlob = entity[STORE_BLOB]; + + const entityBlob = entity[STORE_BLOB]; if (entityBlob) return cb2(null, Buffer.from('source-code-not-available')); // why return empty buffer? @@ -780,51 +825,51 @@ function payloadFileSync(pointer) { return cb2(new Error('UNEXPECTED-20')); } - fs.readFileSync = function readFileSync(path, options_) { - if (path === 'dirty-hack-for-testing-purposes') { - return path; + fs.readFileSync = function readFileSync(f, options_) { + if (f === 'dirty-hack-for-testing-purposes') { + return f; } - if (!insideSnapshot(path)) { + if (!insideSnapshot(f)) { return ancestor.readFileSync.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.readFileSync.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.readFileSync.apply(fs, translateNth(arguments, 0, f)); } - var options = readFileOptions(options_, false); + const options = readFileOptions(options_, false); if (!options) { return ancestor.readFileSync.apply(fs, arguments); } - var { encoding } = options; + const { encoding } = options; assertEncoding(encoding); - var buffer = readFileFromSnapshot(path); + let buffer = readFileFromSnapshot(f); if (encoding) buffer = buffer.toString(encoding); return buffer; }; - fs.readFile = function readFile(path, options_) { - if (!insideSnapshot(path)) { + fs.readFile = function readFile(f, options_) { + if (!insideSnapshot(f)) { return ancestor.readFile.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.readFile.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.readFile.apply(fs, translateNth(arguments, 0, f)); } - var options = readFileOptions(options_, true); + const options = readFileOptions(options_, true); if (!options) { return ancestor.readFile.apply(fs, arguments); } - var { encoding } = options; + const { encoding } = options; assertEncoding(encoding); var callback = dezalgo(maybeCallback(arguments)); - readFileFromSnapshot(path, (error, buffer) => { + readFileFromSnapshot(f, (error, buffer) => { if (error) return callback(error); if (encoding) buffer = buffer.toString(encoding); callback(null, buffer); @@ -877,10 +922,10 @@ function payloadFileSync(pointer) { Dirent.prototype.isSymbolicLink = (fileOrFolderName) => Boolean(SYMLINKS[fileOrFolderName]); - function getFileTypes(path_, entries) { + function getFileTypes(f, entries) { return entries.map((entry) => { - var path = require('path').join(path_, entry); - var entity = VIRTUAL_FILESYSTEM[path]; + const ff = path.join(f, entry); + const entity = findVirtualFileSystemEntry(ff); if (entity[STORE_BLOB] || entity[STORE_CONTENT]) return new Dirent(entry, 1); if (entity[STORE_LINKS]) return new Dirent(entry, 2); @@ -888,90 +933,89 @@ function payloadFileSync(pointer) { }); } - function readdirRoot(path, cb) { + function readdirRoot(file, cb) { if (cb) { - ancestor.readdir(path, (error, entries) => { + ancestor.readdir(file, (error, entries) => { if (error) return cb(error); entries.push('snapshot'); cb(null, entries); }); } else { - var entries = ancestor.readdirSync(path); + const entries = ancestor.readdirSync(file); entries.push('snapshot'); return entries; } } - function readdirFromSnapshotSub(entityLinks, path, cb) { + function readdirFromSnapshotSub(entityLinks, file, cb) { if (cb) { payloadFile(entityLinks, (error, buffer) => { if (error) return cb(error); - cb(null, JSON.parse(buffer).concat(readdirMountpoints(path))); + cb(null, JSON.parse(buffer).concat(readdirMountpoints(file))); }); } else { - var buffer = payloadFileSync(entityLinks); - return JSON.parse(buffer).concat(readdirMountpoints(path)); + const buffer = payloadFileSync(entityLinks); + return JSON.parse(buffer).concat(readdirMountpoints(file)); } } - function readdirFromSnapshot(path_, isRoot, cb) { + function readdirFromSnapshot(folder, isRoot, cb) { var cb2 = cb || rethrow; - if (isRoot) return readdirRoot(path_, cb); - - var path = normalizePathAndFollowLink(path_); - - var entity = VIRTUAL_FILESYSTEM[path]; - if (!entity) return cb2(error_ENOENT('Directory', path)); - var entityBlob = entity[STORE_BLOB]; - if (entityBlob) return cb2(error_ENOTDIR(path)); - var entityContent = entity[STORE_CONTENT]; - if (entityContent) return cb2(error_ENOTDIR(path)); - var entityLinks = entity[STORE_LINKS]; - if (entityLinks) return readdirFromSnapshotSub(entityLinks, path, cb); + if (isRoot) return readdirRoot(folder, cb); + + const entity = findVirtualFileSystemEntry(folder); + + if (!entity) return cb2(error_ENOENT('Directory', folder)); + const entityBlob = entity[STORE_BLOB]; + if (entityBlob) return cb2(error_ENOTDIR(folder)); + const entityContent = entity[STORE_CONTENT]; + if (entityContent) return cb2(error_ENOTDIR(folder)); + const entityLinks = entity[STORE_LINKS]; + if (entityLinks) return readdirFromSnapshotSub(entityLinks, folder, cb); return cb2(new Error('UNEXPECTED-25')); } - fs.readdirSync = function readdirSync(path, options_) { - var isRoot = isRootPath(path); + fs.readdirSync = function readdirSync(f, options_) { + const isRoot = isRootPath(f); - if (!insideSnapshot(path) && !isRoot) { + if (!insideSnapshot(f) && !isRoot) { return ancestor.readdirSync.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.readdirSync.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.readdirSync.apply(fs, translateNth(arguments, 0, f)); } - var options = readdirOptions(options_, false); + const options = readdirOptions(options_, false); if (!options || options.withFileTypes) { return ancestor.readdirSync.apply(fs, arguments); } - var entries = readdirFromSnapshot(path, isRoot); - if (options.withFileTypes) entries = getFileTypes(path, entries); + let entries = readdirFromSnapshot(f, isRoot); + if (options.withFileTypes) entries = getFileTypes(f, entries); return entries; }; - fs.readdir = function readdir(path, options_) { - var isRoot = isRootPath(path); + fs.readdir = function readdir(f, options_) { + const isRoot = isRootPath(f); - if (!insideSnapshot(path) && !isRoot) { + if (!insideSnapshot(f) && !isRoot) { return ancestor.readdir.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.readdir.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.readdir.apply(fs, translateNth(arguments, 0, f)); } - var options = readdirOptions(options_, true); + const options = readdirOptions(options_, true); if (!options || options.withFileTypes) { return ancestor.readdir.apply(fs, arguments); } - var callback = dezalgo(maybeCallback(arguments)); - readdirFromSnapshot(path, isRoot, (error, entries) => { + const callback = dezalgo(maybeCallback(arguments)); + readdirFromSnapshot(f, isRoot, (error, entries) => { if (error) return callback(error); - if (options.withFileTypes) entries = getFileTypes(path, entries); + if (options.withFileTypes) entries = getFileTypes(f, entries); callback(null, entries); }); }; @@ -980,27 +1024,30 @@ function payloadFileSync(pointer) { // realpath ////////////////////////////////////////////////////// // /////////////////////////////////////////////////////////////// - fs.realpathSync = function realpathSync(path) { - if (!insideSnapshot(path)) { + fs.realpathSync = function realpathSync(f) { + if (!insideSnapshot(f)) { return ancestor.realpathSync.apply(fs, arguments); } - if (insideMountpoint(path)) { + if (insideMountpoint(f)) { // app should not know real file name + return f; } - return realpathFromSnapshot(path); + const realPath = realpathFromSnapshot(f); + return realPath; }; - fs.realpath = function realpath(path) { - if (!insideSnapshot(path)) { + fs.realpath = function realpath(f) { + if (!insideSnapshot(f)) { return ancestor.realpath.apply(fs, arguments); } - if (insideMountpoint(path)) { + if (insideMountpoint(f)) { // app should not know real file name + return f; } - var callback = dezalgo(maybeCallback(arguments)); - callback(null, realpathFromSnapshot(path)); + const callback = dezalgo(maybeCallback(arguments)); + callback(null, realpathFromSnapshot(f)); }; // /////////////////////////////////////////////////////////////// @@ -1056,10 +1103,10 @@ function payloadFileSync(pointer) { return s; } - function findNativeAddonForStat(path, cb) { - var cb2 = cb || rethrow; - var foundPath = findNativeAddonSyncUnderRequire(path); - if (!foundPath) return cb2(error_ENOENT('File or directory', path)); + function findNativeAddonForStat(f, cb) { + const cb2 = cb || rethrow; + const foundPath = findNativeAddonSyncUnderRequire(f); + if (!foundPath) return cb2(error_ENOENT('File or directory', f)); if (cb) { ancestor.stat.call(fs, foundPath, cb); } else { @@ -1074,69 +1121,68 @@ function payloadFileSync(pointer) { cb(null, restore(JSON.parse(buffer))); }); } else { - var buffer = payloadFileSync(entityStat); + const buffer = payloadFileSync(entityStat); return restore(JSON.parse(buffer)); } } - function statFromSnapshot(path_, cb) { - var cb2 = cb || rethrow; - var path = normalizePathAndFollowLink(path_); - var entity = VIRTUAL_FILESYSTEM[path]; - if (!entity) return findNativeAddonForStat(path, cb); - var entityStat = entity[STORE_STAT]; + function statFromSnapshot(filename, cb) { + const cb2 = cb || rethrow; + const entity = findVirtualFileSystemEntry(filename); + if (!entity) return findNativeAddonForStat(filename, cb); + const entityStat = entity[STORE_STAT]; if (entityStat) return statFromSnapshotSub(entityStat, cb); return cb2(new Error('UNEXPECTED-35')); } - fs.statSync = function statSync(path) { - if (!insideSnapshot(path)) { + fs.statSync = function statSync(f) { + if (!insideSnapshot(f)) { return ancestor.statSync.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.statSync.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.statSync.apply(fs, translateNth(arguments, 0, f)); } - return statFromSnapshot(path); + return statFromSnapshot(f); }; - fs.stat = function stat(path) { - if (!insideSnapshot(path)) { + fs.stat = function stat(f) { + if (!insideSnapshot(f)) { return ancestor.stat.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.stat.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.stat.apply(fs, translateNth(arguments, 0, f)); } - var callback = dezalgo(maybeCallback(arguments)); - statFromSnapshot(path, callback); + const callback = dezalgo(maybeCallback(arguments)); + statFromSnapshot(f, callback); }; // /////////////////////////////////////////////////////////////// // lstat ///////////////////////////////////////////////////////// // /////////////////////////////////////////////////////////////// - fs.lstatSync = function lstatSync(path) { - if (!insideSnapshot(path)) { + fs.lstatSync = function lstatSync(f) { + if (!insideSnapshot(f)) { return ancestor.lstatSync.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.lstatSync.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.lstatSync.apply(fs, translateNth(arguments, 0, f)); } - return statFromSnapshot(path); + return statFromSnapshot(f); }; - fs.lstat = function lstat(path) { - if (!insideSnapshot(path)) { + fs.lstat = function lstat(f) { + if (!insideSnapshot(f)) { return ancestor.lstat.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.lstat.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.lstat.apply(fs, translateNth(arguments, 0, f)); } - var callback = dezalgo(maybeCallback(arguments)); - statFromSnapshot(path, callback); + const callback = dezalgo(maybeCallback(arguments)); + statFromSnapshot(f, callback); }; // /////////////////////////////////////////////////////////////// @@ -1144,9 +1190,9 @@ function payloadFileSync(pointer) { // /////////////////////////////////////////////////////////////// function fstatFromSnapshot(fd, cb) { - var cb2 = cb || rethrow; - var { entity } = docks[fd]; - var entityStat = entity[STORE_STAT]; + const cb2 = cb || rethrow; + const { entity } = docks[fd]; + const entityStat = entity[STORE_STAT]; if (entityStat) return statFromSnapshotSub(entityStat, cb); return cb2(new Error('UNEXPECTED-40')); } @@ -1164,7 +1210,7 @@ function payloadFileSync(pointer) { return ancestor.fstat.apply(fs, arguments); } - var callback = dezalgo(maybeCallback(arguments)); + const callback = dezalgo(maybeCallback(arguments)); fstatFromSnapshot(fd, callback); }; @@ -1172,75 +1218,74 @@ function payloadFileSync(pointer) { // exists //////////////////////////////////////////////////////// // /////////////////////////////////////////////////////////////// - function findNativeAddonForExists(path) { - var foundPath = findNativeAddonSyncFreeFromRequire(path); + function findNativeAddonForExists(f) { + const foundPath = findNativeAddonSyncFreeFromRequire(f); if (!foundPath) return false; return ancestor.existsSync.call(fs, foundPath); } - function existsFromSnapshot(path_) { - var path = normalizePathAndFollowLink(path_); - var entity = VIRTUAL_FILESYSTEM[path]; - if (!entity) return findNativeAddonForExists(path); + function existsFromSnapshot(f) { + const fShort = normalizePathAndFollowLink(f); + const entity = VIRTUAL_FILESYSTEM[fShort]; + if (!entity) return findNativeAddonForExists(f); return true; } - fs.existsSync = function existsSync(path) { - if (!insideSnapshot(path)) { + fs.existsSync = function existsSync(f) { + if (!insideSnapshot(f)) { return ancestor.existsSync.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.existsSync.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.existsSync.apply(fs, translateNth(arguments, 0, f)); } - return existsFromSnapshot(path); + return existsFromSnapshot(f); }; - fs.exists = function exists(path) { - if (!insideSnapshot(path)) { + fs.exists = function exists(f) { + if (!insideSnapshot(f)) { return ancestor.exists.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.exists.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.exists.apply(fs, translateNth(arguments, 0, f)); } - var callback = dezalgo(maybeCallback(arguments)); - callback(existsFromSnapshot(path)); + const callback = dezalgo(maybeCallback(arguments)); + callback(existsFromSnapshot(f)); }; // /////////////////////////////////////////////////////////////// // access //////////////////////////////////////////////////////// // /////////////////////////////////////////////////////////////// - function accessFromSnapshot(path_, cb) { - var cb2 = cb || rethrow; - var path = normalizePathAndFollowLink(path_); - var entity = VIRTUAL_FILESYSTEM[path]; - if (!entity) return cb2(error_ENOENT('File or directory', path)); + function accessFromSnapshot(filename, cb) { + const cb2 = cb || rethrow; + const entity = findVirtualFileSystemEntry(filename); + if (!entity) return cb2(error_ENOENT('File or directory', filename)); return cb2(null, undefined); } - fs.accessSync = function accessSync(path) { - if (!insideSnapshot(path)) { + fs.accessSync = function accessSync(f) { + if (!insideSnapshot(f)) { return ancestor.accessSync.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.accessSync.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.accessSync.apply(fs, translateNth(arguments, 0, f)); } - return accessFromSnapshot(path); + return accessFromSnapshot(f); }; - fs.access = function access(path) { - if (!insideSnapshot(path)) { + fs.access = function access(f) { + if (!insideSnapshot(f)) { return ancestor.access.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.access.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(f)) { + return ancestor.access.apply(fs, translateNth(arguments, 0, f)); } - var callback = dezalgo(maybeCallback(arguments)); - accessFromSnapshot(path, callback); + const callback = dezalgo(maybeCallback(arguments)); + accessFromSnapshot(f, callback); }; // /////////////////////////////////////////////////////////////// @@ -1254,27 +1299,30 @@ function payloadFileSync(pointer) { ); } - fs.mkdirSync = function mkdirSync(path) { - if (!insideSnapshot(path)) { + fs.mkdirSync = function mkdirSync(folderName) { + if (!insideSnapshot(folderName)) { return ancestor.mkdirSync.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.mkdirSync.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(folderName)) { + return ancestor.mkdirSync.apply( + fs, + translateNth(arguments, 0, folderName) + ); } - return mkdirFailInSnapshot(path); + return mkdirFailInSnapshot(folderName); }; - fs.mkdir = function mkdir(path) { - if (!insideSnapshot(path)) { + fs.mkdir = function mkdir(folderName) { + if (!insideSnapshot(folderName)) { return ancestor.mkdir.apply(fs, arguments); } - if (insideMountpoint(path)) { - return ancestor.mkdir.apply(fs, translateNth(arguments, 0, path)); + if (insideMountpoint(folderName)) { + return ancestor.mkdir.apply(fs, translateNth(arguments, 0, folderName)); } var callback = dezalgo(maybeCallback(arguments)); - mkdirFailInSnapshot(path, callback); + mkdirFailInSnapshot(folderName, callback); }; // /////////////////////////////////////////////////////////////// @@ -1300,7 +1348,7 @@ function payloadFileSync(pointer) { // /////////////////////////////////////////////////////////////// function makeLong(f) { - return require('path')._makeLong(f); + return path._makeLong(f); } function revertMakingLong(f) { @@ -1308,10 +1356,10 @@ function payloadFileSync(pointer) { return f; } - function findNativeAddonForInternalModuleStat(path_) { - var path = findNativeAddonSyncUnderRequire(path_); - if (!path) return -ENOENT; - return process.binding('fs').internalModuleStat(makeLong(path)); + function findNativeAddonForInternalModuleStat(f) { + const fNative = findNativeAddonSyncUnderRequire(f); + if (!fNative) return -ENOENT; + return process.binding('fs').internalModuleStat(makeLong(fNative)); } fs.internalModuleStat = function internalModuleStat(long) { @@ -1320,25 +1368,23 @@ function payloadFileSync(pointer) { // a file, 1 when it's a directory or < 0 on error (usually -ENOENT). // The speedup comes from not creating thousands of Stat and Error objects. - var path = revertMakingLong(long); + const f = revertMakingLong(long); - if (!insideSnapshot(path)) { + if (!insideSnapshot(f)) { return process.binding('fs').internalModuleStat(long); } - if (insideMountpoint(path)) { - return process - .binding('fs') - .internalModuleStat(makeLong(translate(path))); + if (insideMountpoint(f)) { + return process.binding('fs').internalModuleStat(makeLong(translate(f))); } - path = normalizePathAndFollowLink(path); - var entity = VIRTUAL_FILESYSTEM[path]; - if (!entity) return findNativeAddonForInternalModuleStat(path); - var entityBlob = entity[STORE_BLOB]; + const entity = findVirtualFileSystemEntry(f); + + if (!entity) return findNativeAddonForInternalModuleStat(f); + const entityBlob = entity[STORE_BLOB]; if (entityBlob) return 0; - var entityContent = entity[STORE_CONTENT]; + const entityContent = entity[STORE_CONTENT]; if (entityContent) return 0; - var entityLinks = entity[STORE_LINKS]; + const entityLinks = entity[STORE_LINKS]; if (entityLinks) return 1; return -ENOENT; }; @@ -1356,23 +1402,28 @@ function payloadFileSync(pointer) { (NODE_VERSION_MAJOR === 14 && NODE_VERSION_MINOR >= 5) || NODE_VERSION_MAJOR >= 15; - const path = revertMakingLong(long); + const f = revertMakingLong(long); const bindingFs = process.binding('fs'); const readFile = ( bindingFs.internalModuleReadFile || bindingFs.internalModuleReadJSON ).bind(bindingFs); - if (!insideSnapshot(path)) { + if (!insideSnapshot(f)) { return readFile(long); } - if (insideMountpoint(path)) { - return readFile(makeLong(translate(path))); + if (insideMountpoint(f)) { + return readFile(makeLong(translate(f))); + } + + const entity = findVirtualFileSystemEntry(f); + + if (!entity) { + return returnArray ? [undefined, false] : undefined; } - const path2 = normalizePathAndFollowLink(path); - const entity = VIRTUAL_FILESYSTEM[path2]; - if (!entity) return returnArray ? [undefined, false] : undefined; const entityContent = entity[STORE_CONTENT]; - if (!entityContent) return returnArray ? [undefined, false] : undefined; + if (!entityContent) { + return returnArray ? [undefined, false] : undefined; + } return returnArray ? [payloadFileSync(entityContent).toString(), true] : payloadFileSync(entityContent).toString(); @@ -1384,22 +1435,23 @@ function payloadFileSync(pointer) { // ///////////////////////////////////////////////////////////////// // PATCH MODULE //////////////////////////////////////////////////// // ///////////////////////////////////////////////////////////////// + (() => { - var Module = require('module'); - var ancestor = {}; + const Module = require('module'); + const ancestor = {}; ancestor.require = Module.prototype.require; ancestor._compile = Module.prototype._compile; ancestor._resolveFilename = Module._resolveFilename; ancestor.runMain = Module.runMain; - Module.prototype.require = function require(path) { + Module.prototype.require = function require(f) { try { return ancestor.require.apply(this, arguments); } catch (error) { if ( (error.code === 'ENOENT' || error.code === 'MODULE_NOT_FOUND') && - !insideSnapshot(path) && - !require('path').isAbsolute(path) + !insideSnapshot(f) && + !path.isAbsolute(f) ) { if (!error.pkg) { error.pkg = true; @@ -1419,13 +1471,13 @@ function payloadFileSync(pointer) { } }; - var im; - var makeRequireFunction; + let im; + let makeRequireFunction; if (NODE_VERSION_MAJOR === 0) { makeRequireFunction = (self) => { - function rqfn(path) { - return self.require(path); + function rqfn(f) { + return self.require(f); } rqfn.resolve = function resolve(request) { return Module._resolveFilename(request, self); @@ -1457,37 +1509,36 @@ function payloadFileSync(pointer) { return ancestor._compile.apply(this, arguments); } - var filename = normalizePathAndFollowLink(filename_); - var entity = VIRTUAL_FILESYSTEM[filename]; + const entity = findVirtualFileSystemEntry(filename_); if (!entity) { // let user try to "_compile" a packaged file return ancestor._compile.apply(this, arguments); } - var entityBlob = entity[STORE_BLOB]; - var entityContent = entity[STORE_CONTENT]; + const entityBlob = entity[STORE_BLOB]; + const entityContent = entity[STORE_CONTENT]; if (entityBlob) { - var options = { - filename, + const options = { + filename: filename_, lineOffset: 0, displayErrors: true, cachedData: payloadFileSync(entityBlob), sourceless: !entityContent, }; - var { Script } = require('vm'); - var code = entityContent + const { Script } = require('vm'); + const code = entityContent ? require('module').wrap(payloadFileSync(entityContent)) : undefined; - var script = new Script(code, options); - var wrapper = script.runInThisContext(options); + const script = new Script(code, options); + const wrapper = script.runInThisContext(options); if (!wrapper) process.exit(4); // for example VERSION_MISMATCH - var dirname = require('path').dirname(filename); - var rqfn = makeRequireFunction(this); - var args = [this.exports, rqfn, this, filename, dirname]; + const dirname = path.dirname(filename_); + const rqfn = makeRequireFunction(this); + const args = [this.exports, rqfn, this, filename_, dirname]; return wrapper.apply(this.exports, args); } @@ -1502,16 +1553,15 @@ function payloadFileSync(pointer) { }; Module._resolveFilename = function _resolveFilename() { - var filename; - var flagWasOn = false; - + let filename; + let flagWasOn = false; try { filename = ancestor._resolveFilename.apply(this, arguments); } catch (error) { if (error.code !== 'MODULE_NOT_FOUND') throw error; FLAG_ENABLE_PROJECT = true; - var savePathCache = Module._pathCache; + const savePathCache = Module._pathCache; Module._pathCache = Object.create(null); try { filename = ancestor._resolveFilename.apply(this, arguments); @@ -1521,7 +1571,6 @@ function payloadFileSync(pointer) { FLAG_ENABLE_PROJECT = false; } } - if (!insideSnapshot(filename)) { return filename; } @@ -1532,7 +1581,7 @@ function payloadFileSync(pointer) { if (flagWasOn) { FLAG_ENABLE_PROJECT = true; try { - var found = findNativeAddonSyncUnderRequire(filename); + const found = findNativeAddonSyncUnderRequire(filename); if (found) filename = found; } finally { FLAG_ENABLE_PROJECT = false; @@ -1568,7 +1617,7 @@ function payloadFileSync(pointer) { pos += 1; args.splice(pos, 0, {}); } - var opts = args[pos]; + const opts = args[pos]; if (!opts.env) opts.env = require('util')._extend({}, process.env); if (opts.env.PKG_EXECPATH === 'PKG_INVOKE_NODEJS') return; opts.env.PKG_EXECPATH = EXECPATH; @@ -1630,8 +1679,8 @@ function payloadFileSync(pointer) { args[1] = args[1].filter((a) => a.slice(0, 13) !== '--debug-port='); } } else { - for (var i = 1; i < args[1].length; i += 1) { - var mbc = args[1][i - 1]; + for (let i = 1; i < args[1].length; i += 1) { + const mbc = args[1][i - 1]; if (mbc === '-c' || mbc === '/c') { modifyLong(args[1], i); } @@ -1697,9 +1746,9 @@ function payloadFileSync(pointer) { // ///////////////////////////////////////////////////////////// Object.defineProperty(require('fs').exists, custom, { - value(path) { + value(f) { return new Promise((resolve) => { - require('fs').exists(path, (exists) => { + require('fs').exists(f, (exists) => { resolve(exists); }); }); @@ -1756,8 +1805,7 @@ function payloadFileSync(pointer) { // PATCH PROCESS /////////////////////////////////////////////////// // ///////////////////////////////////////////////////////////////// (() => { - const fs = require('fs'); - var ancestor = {}; + const ancestor = {}; ancestor.dlopen = process.dlopen; function revertMakingLong(f) { @@ -1768,12 +1816,12 @@ function payloadFileSync(pointer) { process.dlopen = function dlopen() { const args = cloneArgs(arguments); const modulePath = revertMakingLong(args[1]); - const moduleDirname = require('path').dirname(modulePath); + const moduleDirname = path.dirname(modulePath); if (insideSnapshot(modulePath)) { // Node addon files and .so cannot be read with fs directly, they are loaded with process.dlopen which needs a filesystem path // we need to write the file somewhere on disk first and then load it const moduleContent = fs.readFileSync(modulePath); - const moduleBaseName = require('path').basename(modulePath); + const moduleBaseName = path.basename(modulePath); const hash = require('crypto') .createHash('sha256') .update(moduleContent) @@ -1805,7 +1853,7 @@ function payloadFileSync(pointer) { const moduleName = e.message.match(unknownModuleErrorRegex)[1]; const importModulePath = `${moduleDirname}/${moduleName}`; const moduleContent = fs.readFileSync(importModulePath); - const moduleBaseName = require('path').basename(importModulePath); + const moduleBaseName = path.basename(importModulePath); const tmpModulePath = `${require('os').tmpdir()}/${moduleBaseName}`; try { fs.statSync(tmpModulePath); diff --git a/prelude/diagnostic.js b/prelude/diagnostic.js index c66d0a453..343048c6d 100644 --- a/prelude/diagnostic.js +++ b/prelude/diagnostic.js @@ -1,34 +1,43 @@ /* eslint-disable global-require */ /* eslint-disable no-console */ +/* global DICT */ 'use strict'; (function installDiagnostic() { const fs = require('fs'); const path = require('path'); - const win32 = process.platform === 'win32'; - function dumpLevel(folderPath, level) { + if (process.env.DEBUG_PKG === '2') { + console.log(Object.entries(DICT)); + } + function dumpLevel(filename, level, tree) { let totalSize = 0; - const d = fs.readdirSync(folderPath); + const d = fs.readdirSync(filename); for (let j = 0; j < d.length; j += 1) { - const f = path.join(folderPath, d[j]); - // const isSymbolicLink = fs.statSync(f).isSymbolicLink(); + const f = path.join(filename, d[j]); const realPath = fs.realpathSync(f); const isSymbolicLink2 = f !== realPath; const s = fs.statSync(f); totalSize += s.size; - console.log( - ' '.padStart(level * 2, ' '), - d[j], - s.size, - isSymbolicLink2 ? `=> ${realPath}` : ' ' - ); if (s.isDirectory() && !isSymbolicLink2) { - totalSize += dumpLevel(f, level + 1); + const tree1 = []; + totalSize += dumpLevel(f, level + 1, tree1); + const str = + (' '.padStart(level * 2, ' ') + d[j]).padEnd(40, ' ') + + (totalSize.toString().padStart(10, ' ') + + (isSymbolicLink2 ? `=> ${realPath}` : ' ')); + tree.push(str); + tree1.forEach((x) => tree.push(x)); + } else { + const str = + (' '.padStart(level * 2, ' ') + d[j]).padEnd(40, ' ') + + (s.size.toString().padStart(10, ' ') + + (isSymbolicLink2 ? `=> ${realPath}` : ' ')); + tree.push(str); } } return totalSize; @@ -47,31 +56,38 @@ if (process.env.DEBUG_PKG) { console.log('------------------------------- virtual file system'); const startFolder = win32 ? 'C:\\snapshot' : '/snapshot'; - const totalSize = dumpLevel(startFolder, 2); + console.log(startFolder); + + const tree = []; + const totalSize = dumpLevel(startFolder, 1, tree); + console.log(tree.join('\n')); + console.log('Total size = ', totalSize); - wrap(fs, 'openSync'); - wrap(fs, 'open'); - wrap(fs, 'readSync'); - wrap(fs, 'read'); - wrap(fs, 'writeSync'); - wrap(fs, 'write'); - wrap(fs, 'closeSync'); - wrap(fs, 'readFileSync'); - wrap(fs, 'close'); - wrap(fs, 'readFile'); - wrap(fs, 'readdirSync'); - wrap(fs, 'readdir'); - wrap(fs, 'realpathSync'); - wrap(fs, 'realpath'); - wrap(fs, 'statSync'); - wrap(fs, 'stat'); - wrap(fs, 'lstatSync'); - wrap(fs, 'lstat'); - wrap(fs, 'fstatSync'); - wrap(fs, 'fstat'); - wrap(fs, 'existsSync'); - wrap(fs, 'exists'); - wrap(fs, 'accessSync'); - wrap(fs, 'access'); + if (process.env.DEBUG_PKG === '2') { + wrap(fs, 'openSync'); + wrap(fs, 'open'); + wrap(fs, 'readSync'); + wrap(fs, 'read'); + wrap(fs, 'writeSync'); + wrap(fs, 'write'); + wrap(fs, 'closeSync'); + wrap(fs, 'readFileSync'); + wrap(fs, 'close'); + wrap(fs, 'readFile'); + wrap(fs, 'readdirSync'); + wrap(fs, 'readdir'); + wrap(fs, 'realpathSync'); + wrap(fs, 'realpath'); + wrap(fs, 'statSync'); + wrap(fs, 'stat'); + wrap(fs, 'lstatSync'); + wrap(fs, 'lstat'); + wrap(fs, 'fstatSync'); + wrap(fs, 'fstat'); + wrap(fs, 'existsSync'); + wrap(fs, 'exists'); + wrap(fs, 'accessSync'); + wrap(fs, 'access'); + } } })(); diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 000000000..c2658d7d1 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/test/test-10-pnpm/main.js b/test/test-10-pnpm/main.js index 07c24b816..248a32008 100644 --- a/test/test-10-pnpm/main.js +++ b/test/test-10-pnpm/main.js @@ -48,6 +48,8 @@ const logPkg = utils.pkg.sync([ '--target', target, '--debug', + '--compress', + 'Brotli', '--output', output, input, diff --git a/test/test-12-compression-node-opcua/.gitignore b/test/test-12-compression-node-opcua/.gitignore new file mode 100644 index 000000000..a56a7ef43 --- /dev/null +++ b/test/test-12-compression-node-opcua/.gitignore @@ -0,0 +1,2 @@ +node_modules + diff --git a/test/test-12-compression-node-opcua/main.js b/test/test-12-compression-node-opcua/main.js new file mode 100644 index 000000000..fb3dd4bf4 --- /dev/null +++ b/test/test-12-compression-node-opcua/main.js @@ -0,0 +1,137 @@ +#!/usr/bin/env node + +'use strict'; + +/* + * A test with a large number of modules with symlinks + * (installed with npm) and compress + * + */ + +const fs = require('fs'); +const path = require('path'); +const assert = require('assert'); +const utils = require('../utils.js'); + +assert(!module.parent); +assert(__dirname === process.cwd()); + +// ignore this test if nodejs <= 10 , as recent version of PNPM do not support nodejs=10 +const MAJOR_VERSION = parseInt(process.version.match(/v([0-9]+)/)[1], 10); +if (MAJOR_VERSION < 12) { + return; +} + +// remove any possible left-over +utils.vacuum.sync('./node_modules'); +utils.vacuum.sync('./pnpm-lock.yaml'); + +// launch `pnpm install` +const pnpmlog = utils.spawn.sync( + path.join( + path.dirname(process.argv[0]), + 'npx' + (process.platform === 'win32' ? '.cmd' : '') + ), + ['pnpm', 'install'], + { cwd: path.dirname(__filename), expect: 0 } +); +console.log('pnpm log :', pnpmlog); + +// verify that we have the .pnpm folder and a symlinks module in node_modules +assert(fs.lstatSync(path.join(__dirname, 'node_modules/.pnpm')).isDirectory()); +assert( + fs + .lstatSync(path.join(__dirname, 'node_modules/node-opcua-address-space')) + .isSymbolicLink() +); + +/* eslint-disable no-unused-vars */ +const input = 'package.json'; +const target = 'host'; +const ext = process.platform === 'win32' ? '.exe' : ''; +const outputRef = 'test-output-empty' + ext; +const outputNone = 'test-output-None' + ext; +const outputGZip = 'test-output-GZip' + ext; +const outputBrotli = 'test-output-Brotli' + ext; +const outputBrotliDebug = 'test-output-Brotli-debug' + ext; + +const inspect = ['ignore', 'ignore', 'pipe']; + +console.log(' compiling empty '); +const logPkg0 = utils.pkg.sync( + [ + '--target', + target, + '--compress', + 'None', + '--output', + outputRef, + './test-empty.js', + ], + { stdio: inspect, expect: 0 } +); +const sizeReference = fs.statSync(outputRef).size; + +function pkgCompress(compressMode, output) { + console.log(` compiling compression ${compressMode} `); + const logPkg1 = utils.pkg.sync( + ['--target', target, '--compress', compressMode, '--output', output, input], + { stdio: inspect, expect: 0 } + ); + // check that produced executable is running and produce the expected output. + const log = utils.spawn.sync(path.join(__dirname, output), [], { + cwd: __dirname, + expect: 0, + }); + assert(log === '42\n'); + return fs.statSync(output).size; +} + +const sizeNoneFull = pkgCompress('None', outputNone); +const sizeGZipFull = pkgCompress('GZip', outputGZip); +const sizeBrotliFull = pkgCompress('Brotli', outputBrotli); + +const sizeNone = sizeNoneFull - sizeReference; +const sizeBrotli = sizeBrotliFull - sizeReference; +const sizeGZip = sizeGZipFull - sizeReference; + +console.log('empty = ', sizeReference); +console.log('no compression = ', sizeNoneFull, sizeNone); +console.log('Brotli = ', sizeBrotliFull, sizeBrotli); +console.log('GZip = ', sizeGZipFull, sizeGZip); + +console.log( + ' Δ GZip = ', + sizeGZip - sizeNone, + '(', + (((sizeGZip - sizeNone) / sizeNone) * 100).toFixed(0), + '%)' +); +console.log( + ' Δ Brotli = ', + sizeBrotli - sizeNone, + '(', + (((sizeBrotli - sizeNone) / sizeNone) * 100).toFixed(0), + '%)' +); + +assert(sizeNone > sizeGZip); +assert(sizeGZip > sizeBrotli); + +const logPkg5 = utils.pkg.sync( + ['--target', target, '--compress', 'Crap', '--output', outputBrotli, input], + { expect: 2 } +); + +// xx console.log(logPkg4); +assert(logPkg5.match(/Invalid compression algorithm/g)); + +utils.vacuum.sync(outputRef); +utils.vacuum.sync(outputNone); +utils.vacuum.sync(outputBrotli); +utils.vacuum.sync(outputGZip); +utils.vacuum.sync(outputBrotliDebug); +utils.vacuum.sync('node_modules'); +utils.vacuum.sync('./pnpm-lock.yaml'); + +console.log('OK'); diff --git a/test/test-12-compression-node-opcua/package.json b/test/test-12-compression-node-opcua/package.json new file mode 100644 index 000000000..fa88fb8f1 --- /dev/null +++ b/test/test-12-compression-node-opcua/package.json @@ -0,0 +1,24 @@ +{ + "name": "test-12-compression", + "version": "1.0.0", + "description": "", + "main": "test-x.js", + "scripts": { + "preinstall": "npx only-allow pnpm", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "node-opcua-address-space": "^2.36.0", + "node-opcua-crypto": "^1.7.1", + "node-opcua-nodesets": "^2.36.0" + }, + "bin": "test-x.js", + "pkg": { + "assets": [ + "./node_modules/node-opcua-nodesets/nodesets/*.xml" + ] + } +} diff --git a/test/test-12-compression-node-opcua/test-empty.js b/test/test-12-compression-node-opcua/test-empty.js new file mode 100644 index 000000000..e56b212ef --- /dev/null +++ b/test/test-12-compression-node-opcua/test-empty.js @@ -0,0 +1 @@ +/***/ diff --git a/test/test-12-compression-node-opcua/test-x.js b/test/test-12-compression-node-opcua/test-x.js new file mode 100644 index 000000000..cbcb95309 --- /dev/null +++ b/test/test-12-compression-node-opcua/test-x.js @@ -0,0 +1,12 @@ +'use strict'; + +const { AddressSpace } = require('node-opcua-address-space'); + +const { generateAddressSpace } = require('node-opcua-address-space/nodeJS'); +const { nodesets } = require('node-opcua-nodesets'); + +(async () => { + const addressSpace = AddressSpace.create({}); + await generateAddressSpace(addressSpace, [nodesets.standard]); + console.log('42'); +})(); diff --git a/test/test-12-compression-node-opcua/test-y.js b/test/test-12-compression-node-opcua/test-y.js new file mode 100644 index 000000000..a55e4000e --- /dev/null +++ b/test/test-12-compression-node-opcua/test-y.js @@ -0,0 +1,6 @@ +'use strict'; + +const fs = require('fs'); +const { nodesets } = require('node-opcua-nodesets'); +const a = fs.readFileSync(nodesets.adi); +console.log(a.length); diff --git a/test/test-12-compression/.gitignore b/test/test-12-compression/.gitignore new file mode 100644 index 000000000..402229549 --- /dev/null +++ b/test/test-12-compression/.gitignore @@ -0,0 +1 @@ +ouput-* diff --git a/test/test-12-compression/main.js b/test/test-12-compression/main.js new file mode 100644 index 000000000..80cdbf799 --- /dev/null +++ b/test/test-12-compression/main.js @@ -0,0 +1,111 @@ +#!/usr/bin/env node + +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const assert = require('assert'); +const utils = require('../utils.js'); + +assert(!module.parent); +assert(__dirname === process.cwd()); + +/* eslint-disable no-unused-vars */ +const input = 'test-x.js'; +const target = 'host'; +const ext = process.platform === 'win32' ? '.exe' : ''; +const outputRef = 'output-empty' + ext; +const outputNone = 'output-None' + ext; +const outputGZip = 'output-Brotli' + ext; +const outputBrotli = 'output-GZip' + ext; +const outputBrotliDebug = 'output-debug' + ext; + +const inspect = ['ignore', 'ignore', 'pipe']; + +console.log(' compiling empty '); +const logPkg0 = utils.pkg.sync( + [ + '--target', + target, + '--compress', + 'None', + '--output', + outputRef, + './test-empty.js', + ], + { expect: 0 } +); +const sizeReference = fs.statSync(outputRef).size; + +function pkgCompress(compressMode, output) { + console.log(` compiling compression ${compressMode} `); + const logPkg1 = utils.pkg.sync( + ['--target', target, '--compress', compressMode, '--output', output, input], + { stdio: inspect, expect: 0 } + ); + // check that produced executable is running and produce the expected output. + const log = utils.spawn.sync(path.join(__dirname, output), [], { + cwd: __dirname, + expect: 0, + }); + assert(log === '42\n'); + return fs.statSync(output).size; +} + +const sizeNoneFull = pkgCompress('None', outputNone); +const sizeGZipFull = pkgCompress('GZip', outputGZip); +const sizeBrotliFull = pkgCompress('Brotli', outputBrotli); + +const sizeNone = sizeNoneFull - sizeReference; +const sizeBrotli = sizeBrotliFull - sizeReference; +const sizeGZip = sizeGZipFull - sizeReference; + +console.log(' compiling compression Brotli + debug'); +const logPkg4 = utils.pkg.sync( + [ + '--target', + target, + '--debug', + '--compress', + 'Brotli', + '--output', + outputBrotliDebug, + input, + ], + { expect: 0 } +); + +console.log('node.exe size =', sizeReference); +console.log('virtual file system'); +console.log('No compression = ', sizeNone - sizeReference); +console.log( + ' Δ GZip = ', + sizeGZip - sizeNone, + '(', + (((sizeGZip - sizeNone) / sizeNone) * 100).toFixed(0), + '%)' +); +console.log( + ' Δ Brotli = ', + sizeBrotli - sizeNone, + '(', + (((sizeBrotli - sizeNone) / sizeNone) * 100).toFixed(0), + '%)' +); + +assert(sizeNone > sizeGZip); +assert(sizeGZip > sizeBrotli); + +const logPkg5 = utils.pkg.sync( + ['--target', target, '--compress', 'Crap', '--output', outputBrotli, input], + { expect: 2 } +); + +// xx console.log(logPkg4); +assert(logPkg5.match(/Invalid compression algorithm/g)); + +utils.vacuum.sync(outputRef); +utils.vacuum.sync(outputNone); +utils.vacuum.sync(outputBrotli); +utils.vacuum.sync(outputGZip); +utils.vacuum.sync(outputBrotliDebug); diff --git a/test/test-12-compression/test-empty.js b/test/test-12-compression/test-empty.js new file mode 100644 index 000000000..10df28383 --- /dev/null +++ b/test/test-12-compression/test-empty.js @@ -0,0 +1 @@ +/** empty */ diff --git a/test/test-12-compression/test-x.js b/test/test-12-compression/test-x.js new file mode 100644 index 000000000..1d4708028 --- /dev/null +++ b/test/test-12-compression/test-x.js @@ -0,0 +1,13 @@ +'use strict'; + +/* eslint-disable no-unused-vars */ + +const m = require('minimist'); +const c = require('chalk'); + +const loremIpsum = + 'Unus audio pluribus sibi quibusdam amicitias habere qua satis molestum sapientes molestum est vel frui non pluribus nimias possit quam esse sollicitum adducas persequantur esse audio nihil sollicitum laxissimas enim rerum vel non ad tamquam habitos implicari placuisse quibusdam nihil.'; +const loremIpsum2 = + 'Semper praetorio satisfaceret semper sit militem ut ipse ordinarias ad atque sit ire in ad sit ut more trusus dignitates more compellebatur ultimum praefectus discrimen et in ut tempestate et dignitates impedita convectio in est inopia ad alioqui et ob.'; + +console.log(42 || loremIpsum2 || loremIpsum2); diff --git a/test/test-50-ast-parsing-2/test-output.exe b/test/test-50-ast-parsing-2/test-output.exe deleted file mode 100755 index bdaf46f5d..000000000 Binary files a/test/test-50-ast-parsing-2/test-output.exe and /dev/null differ diff --git a/test/test-50-fs-runtime-layer-3/test-x-index.js b/test/test-50-fs-runtime-layer-3/test-x-index.js index 3f27bae32..f8a7687c1 100644 --- a/test/test-50-fs-runtime-layer-3/test-x-index.js +++ b/test/test-50-fs-runtime-layer-3/test-x-index.js @@ -12,7 +12,7 @@ test01(); function test01() { fs.open(theFile, 'w+', function (error, fd) { console.log(error === null); - var buffer = new Buffer('FOO'); + var buffer = Buffer.from('FOO'); fs.write(fd, buffer, 0, buffer.length, null, function (error2) { console.log(error2 === null); if (error2) console.log(error2.message); @@ -31,7 +31,7 @@ function test01() { function test02() { var fd = fs.openSync(theFile, 'w+'); - var buffer = new Buffer('QUX BARABAZ'); + var buffer = Buffer.from('QUX BARABAZ'); var bytesWritten; try { bytesWritten = fs.writeSync(fd, buffer, 0, buffer.length); diff --git a/test/test-50-mountpoints/main.js b/test/test-50-mountpoints/main.js index 2c51e92af..b33ba9c95 100644 --- a/test/test-50-mountpoints/main.js +++ b/test/test-50-mountpoints/main.js @@ -23,7 +23,7 @@ fs.writeFileSync( fs.readFileSync('./plugins-D-ext/test-y-require-D.js') ); -utils.pkg.sync(['--target', target, '--output', output, input]); +utils.pkg.sync(['--debug', '--target', target, '--output', output, input]); right = utils.spawn.sync('./' + path.basename(output), [], { cwd: path.dirname(output), diff --git a/test/test-79-npm/homebridge/homebridge.js b/test/test-79-npm/homebridge/homebridge.js index 140eb1d9f..5b1247231 100644 --- a/test/test-79-npm/homebridge/homebridge.js +++ b/test/test-79-npm/homebridge/homebridge.js @@ -4,7 +4,6 @@ process.argv.push('--help'); require('homebridge/lib/cli.js'); // dont run. only load var Server = require('homebridge/lib/server.js').Server; var server = new Server(); - // eslint-disable-next-line no-underscore-dangle if (server._api) { console.log('ok'); diff --git a/test/utils.js b/test/utils.js index 467c04aae..f53edb805 100644 --- a/test/utils.js +++ b/test/utils.js @@ -5,9 +5,9 @@ const path = require('path'); const mkdirp = require('mkdirp'); const rimraf = require('rimraf'); const globby = require('globby'); -const execSync = require('child_process').execSync; -const spawnSync = require('child_process').spawnSync; -const existsSync = require('fs').existsSync; +const { execSync } = require('child_process'); +const { spawnSync } = require('child_process'); +const { existsSync } = require('fs'); const stableStringify = require('json-stable-stringify'); module.exports.mkdirp = mkdirp;