compress files in virtual file system (vercel#885)

README.md

@@ -178,6 +178,16 @@ option to `pkg`. First ensure your computer meets the
 requirements to compile original Node.js:
 [BUILDING.md](https:/nodejs/node/blob/master/BUILDING.md)
 
+### Compression
+
+Pass `--compress Brolti` or `--compress GZip` to `pkg` to compress further the content of the files store in the exectable.
+
+This option can reduce the size of the embedded file system by up to 60%.
+
+The startup time of the application might be reduced slightly.
+
+`-C` can be used as a shortcut for `--compress `.
+
 ### Environment
 
 | Var | Description |

lib/help.js

@@ -17,6 +17,7 @@ export default function help() {
  -d, --debug show more information during packaging process [off]
  -b, --build don't download prebuilt base binaries, build them
  --public speed up and disclose the sources of top-level project
+ -C, --compress [default=None] compression algorithm = Brotli or GZip
 
  ${chalk.dim('Examples:')}
 
@@ -30,6 +31,9 @@ export default function help() {
  ${chalk.cyan('$ pkg -t node4-linux,node6-linux,node6-win index.js')}
  ${chalk.gray('–')} Bakes '--expose-gc' into executable
  ${chalk.cyan('$ pkg --options expose-gc index.js')}
-
+ ${chalk.gray(
+ '–'
+ )} reduce size of the data packed inside the executable with GZip
+ ${chalk.cyan('$ pkg --compress GZip index.js')}
 `);
 }

lib/index.js

@@ -1,6 +1,6 @@
 /* eslint-disable require-atomic-updates */
 
-import { exists, mkdirp, readFile, remove, stat } from 'fs-extra';
+import { existsSync, mkdirp, readFile, remove, stat } from 'fs-extra';
 import { need, system } from 'pkg-fetch';
 import assert from 'assert';
 import minimist from 'minimist';
@@ -157,6 +157,8 @@ export async function exec(argv2) {
  't',
  'target',
  'targets',
+ 'C',
+ 'compress',
  ],
  default: { bytecode: true },
  });
@@ -184,7 +186,31 @@ export async function exec(argv2) {
 
  const forceBuild = argv.b || argv.build;
 
- // _
+ // doCompress
+ const algo = argv.C || argv.compress || 'None';
+ let doCompress = 'None';
+ switch (algo.toLowerCase()) {
+ case 'brotli':
+ case 'br':
+ doCompress = 'BR';
+ break;
+ case 'gzip':
+ case 'gz':
+ doCompress = 'GZ';
+ break;
+ case 'none':
+ break;
+ default:
+ // eslint-disable-next-line no-console
+ console.log(
+ `Invalid compression algorithm ${algo} ( should be None, Brolti or Gzip)`
+ );
+ process.exit(-1);
+ }
+ if (doCompress !== 'None') {
+ // eslint-disable-next-line no-console
+ console.log('compression: ', doCompress);
+ }
 
  if (!argv._.length) {
  throw wasReported('Entry file/directory is expected', [
@@ -199,12 +225,12 @@ export async function exec(argv2) {
 
  let input = path.resolve(argv._[0]);
 
- if (!(await exists(input))) {
+ if (!existsSync(input)) {
  throw wasReported('Input file does not exist', [input]);
  }
  if ((await stat(input)).isDirectory()) {
  input = path.join(input, 'package.json');
- if (!(await exists(input))) {
+ if (!existsSync(input)) {
  throw wasReported('Input file does not exist', [input]);
  }
  }
@@ -239,7 +265,7 @@ export async function exec(argv2) {
  }
  }
  inputBin = path.resolve(path.dirname(input), bin);
- if (!(await exists(inputBin))) {
+ if (!existsSync(inputBin)) {
  throw wasReported(
  'Bin file does not exist (taken from package.json ' +
  "'bin' property)",
@@ -271,7 +297,7 @@ export async function exec(argv2) {
 
  if (config) {
  config = path.resolve(config);
- if (!(await exists(config))) {
+ if (!existsSync(config)) {
  throw wasReported('Config file does not exist', [config]);
  }
  // eslint-disable-next-line import/no-dynamic-require, global-require
@@ -485,7 +511,7 @@ export async function exec(argv2) {
  log.debug('Targets:', JSON.stringify(targets, null, 2));
 
  for (const target of targets) {
- if (await exists(target.output)) {
+ if (existsSync(target.output)) {
  if ((await stat(target.output)).isFile()) {
  await remove(target.output);
  } else {
@@ -498,7 +524,7 @@ export async function exec(argv2) {
  }
 
  const slash = target.platform === 'win' ? '\\' : '/';
- await producer({ backpack, bakes, slash, target, symLinks });
+ await producer({ backpack, bakes, slash, target, symLinks, doCompress });
  if (target.platform !== 'win') {
  await plusx(target.output);
  }

lib/packer.js

@@ -132,7 +132,7 @@ export default function packer({ records, entrypoint, bytecode }) {
  }
  }
  const prelude =
- `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS) { 
+ `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS, DICT, DOCOMPRESS) { 
  ${bootstrapText}${
  log.debugMode ? diagnosticText : ''
  }\n})(function (exports) {\n${commonText}\n},\n` +
@@ -141,6 +141,10 @@ export default function packer({ records, entrypoint, bytecode }) {
  `%DEFAULT_ENTRYPOINT%` +
  `\n,\n` +
  `%SYMLINKS%` +
+ '\n,\n' +
+ '%DICT%' +
+ '\n,\n' +
+ '%DOCOMPRESS%' +
  `\n);`;
 
  return { prelude, entrypoint, stripes };

lib/producer.js

@@ -1,3 +1,4 @@
+import { createBrotliCompress, createGzip } from 'zlib';
 import Multistream from 'multistream';
 import assert from 'assert';
 import { execSync } from 'child_process';
@@ -166,7 +167,30 @@ function nativePrebuildInstall(target, nodeFile) {
  return nativeFile;
 }
 
-export default function producer({ backpack, bakes, slash, target, symLinks }) {
+const fileDictionary = {};
+let counter = 0;
+function replace(k) {
+ let existingKey = fileDictionary[k];
+ if (!existingKey) {
+ const newkey = counter;
+ counter += 1;
+ existingKey = newkey.toString(36);
+ fileDictionary[k] = existingKey;
+ }
+ return existingKey;
+}
+function makeKey(filename, slash) {
+ const a = filename.split(slash).map(replace).join('$');
+ return a;
+}
+export default function producer({
+ backpack,
+ bakes,
+ slash,
+ target,
+ symLinks,
+ doCompress,
+}) {
  return new Promise((resolve, reject) => {
  if (!Buffer.alloc) {
  throw wasReported(
@@ -183,14 +207,16 @@ export default function producer({ backpack, bakes, slash, target, symLinks }) {
  for (const stripe of stripes) {
  let { snap } = stripe;
  snap = snapshotify(snap, slash);
- if (!vfs[snap]) vfs[snap] = {};
+ const vfsKey = makeKey(snap, slash);
+ if (!vfs[vfsKey]) vfs[vfsKey] = {};
  }
 
  const snapshotSymLinks = {};
  for (const [key, value] of Object.entries(symLinks)) {
  const k = snapshotify(key, slash);
  const v = snapshotify(value, slash);
- snapshotSymLinks[k] = v;
+ const vfsKey = makeKey(k, slash);
+ snapshotSymLinks[vfsKey] = makeKey(v, slash);
  }
  let meter;
  let count = 0;
@@ -199,6 +225,15 @@ export default function producer({ backpack, bakes, slash, target, symLinks }) {
  meter = streamMeter();
  return s.pipe(meter);
  }
+ function pipeMayCompressToNewMeter(s) {
+ if (doCompress === 'GZ') {
+ return pipeToNewMeter(s.pipe(createGzip()));
+ }
+ if (doCompress === 'BR') {
+ return pipeToNewMeter(s.pipe(createBrotliCompress()));
+ }
+ return pipeToNewMeter(s);
+ }
 
  function next(s) {
  count += 1;
@@ -231,7 +266,8 @@ export default function producer({ backpack, bakes, slash, target, symLinks }) {
  const { store } = prevStripe;
  let { snap } = prevStripe;
  snap = snapshotify(snap, slash);
- vfs[snap][store] = [track, meter.bytes];
+ const vfsKey = makeKey(snap, slash);
+ vfs[vfsKey][store] = [track, meter.bytes];
  track += meter.bytes;
  }
 
@@ -257,12 +293,14 @@ export default function producer({ backpack, bakes, slash, target, symLinks }) {
  return cb(undefined, intoStream(Buffer.alloc(0)));
  }
 
- cb(undefined, pipeToNewMeter(intoStream(buffer)));
+ cb(undefined, pipeMayCompressToNewMeter(intoStream(buffer)));
  }
  );
  }
-
- return cb(undefined, pipeToNewMeter(intoStream(stripe.buffer)));
+ return cb(
+ undefined,
+ pipeMayCompressToNewMeter(intoStream(stripe.buffer))
+ );
  }
 
  if (stripe.file) {
@@ -282,7 +320,7 @@ export default function producer({ backpack, bakes, slash, target, symLinks }) {
  if (fs.existsSync(platformFile)) {
  return cb(
  undefined,
- pipeToNewMeter(fs.createReadStream(platformFile))
+ pipeMayCompressToNewMeter(fs.createReadStream(platformFile))
  );
  }
  } catch (err) {
@@ -291,7 +329,7 @@ export default function producer({ backpack, bakes, slash, target, symLinks }) {
  }
  return cb(
  undefined,
- pipeToNewMeter(fs.createReadStream(stripe.file))
+ pipeMayCompressToNewMeter(fs.createReadStream(stripe.file))
  );
  }
 
@@ -307,15 +345,23 @@ export default function producer({ backpack, bakes, slash, target, symLinks }) {
  replaceDollarWise(
  replaceDollarWise(
  replaceDollarWise(
- prelude,
- '%VIRTUAL_FILESYSTEM%',
- JSON.stringify(vfs)
+ replaceDollarWise(
+ replaceDollarWise(
+ prelude,
+ '%VIRTUAL_FILESYSTEM%',
+ JSON.stringify(vfs)
+ ),
+ '%DEFAULT_ENTRYPOINT%',
+ JSON.stringify(entrypoint)
+ ),
+ '%SYMLINKS%',
+ JSON.stringify(snapshotSymLinks)
  ),
- '%DEFAULT_ENTRYPOINT%',
- JSON.stringify(entrypoint)
+ '%DICT%',
+ JSON.stringify(fileDictionary)
  ),
- '%SYMLINKS%',
- JSON.stringify(snapshotSymLinks)
+ '%DOCOMPRESS%',
+ JSON.stringify(doCompress)
  )
  )
  )

lib/walker.js

@@ -294,6 +294,8 @@ class Walker {
  if (assets) {
  assets = expandFiles(assets, base);
  for (const asset of assets) {
+ console.log(' Adding asset : .... ', asset);
+
  const stat = await fs.stat(asset);
  if (stat.isFile()) {
  this.appendBlobOrContent({

package.json

@@ -77,7 +77,7 @@
  }
  },
  "scripts": {
- "babel": "node test/rimraf-es5.js && babel lib --out-dir lib-es5",
+ "babel": "node test/rimraf-es5.js && babel lib  -s --out-dir lib-es5",
  "lint": "eslint lib prelude test || true",
  "lint:fix": "eslint lib prelude test --fix",
  "prepare": "npm run babel",