compress files in virtual file system (vercel#885)

.eslintrc

@@ -1,4 +1,5 @@
 {
+ // note you must disable the base rule as it can report incorrect errors
  "extends": ["airbnb-base", "prettier"],
  "rules": {
  "no-bitwise": "off",
@@ -10,8 +11,10 @@
  "consistent-return": "off",
  "no-restricted-syntax": "off",
  "import/prefer-default-export": "off",
- "camelcase": "off"
- },
+ "camelcase": "off",
+ "no-shadow": "off",
+ "@typescript-eslint/no-shadow": ["error"]
+ },
  "overrides": [
  {
  "files": ["*.ts"],

README.md

@@ -178,6 +178,16 @@ option to `pkg`. First ensure your computer meets the
 requirements to compile original Node.js:
 [BUILDING.md](https:/nodejs/node/blob/master/BUILDING.md)
 
+### Compression
+
+Pass `--compress Brolti` or `--compress GZip` to `pkg` to compress further the content of the files store in the exectable.
+
+This option can reduce the size of the embedded file system by up to 60%.
+
+The startup time of the application might be reduced slightly.
+
+`-C` can be used as a shortcut for `--compress `.
+
 ### Environment
 
 | Var | Description |

lib/compress_type.ts

@@ -0,0 +1,6 @@
+
+export enum CompressType {
+ None=0,
+ GZip= 1,
+ Brolti= 2
+};

lib/help.ts

@@ -17,6 +17,7 @@ export default function help() {
  -d, --debug show more information during packaging process [off]
  -b, --build don't download prebuilt base binaries, build them
  --public speed up and disclose the sources of top-level project
+ -C, --compress [default=None] compression algorithm = Brotli or GZip
 
  ${chalk.dim('Examples:')}
 
@@ -30,5 +31,9 @@ export default function help() {
  ${chalk.cyan('$ pkg -t node12-linux,node14-linux,node14-win index.js')}
  ${chalk.gray('–')} Bakes '--expose-gc' into executable
  ${chalk.cyan('$ pkg --options expose-gc index.js')}
+ ${chalk.gray(
+ '–'
+ )} reduce size of the data packed inside the executable with GZip
+ ${chalk.cyan('$ pkg --compress GZip index.js')}
 `);
 }

lib/index.ts

@@ -1,6 +1,6 @@
 /* eslint-disable require-atomic-updates */
 
-import { mkdirp, readFile, remove, stat, readFileSync } from 'fs-extra';
+import { existsSync, mkdirp, readFile, remove, stat, readFileSync } from 'fs-extra';
 import { need, system } from 'pkg-fetch';
 import assert from 'assert';
 import minimist from 'minimist';
@@ -16,6 +16,7 @@ import refine from './refiner';
 import { shutdown } from './fabricator';
 import walk, { Marker, WalkerParams } from './walker';
 import { Target, NodeTarget, SymLinks } from './types';
+import { CompressType } from './compress_type';
 
 const { version } = JSON.parse(
  readFileSync(path.join(__dirname, '../package.json'), 'utf-8')
@@ -25,15 +26,6 @@ function isConfiguration(file: string) {
  return isPackageJson(file) || file.endsWith('.config.json');
 }
 
-async function exists(file: string) {
- try {
- await stat(file);
- return true;
- } catch (error) {
- return false;
- }
-}
-
 // http://www.openwall.com/lists/musl/2012/12/08/4
 
 const {
@@ -231,6 +223,8 @@ export async function exec(argv2: string[]) {
  't',
  'target',
  'targets',
+ 'C',
+ 'compress',
  ],
  default: { bytecode: true },
  });
@@ -258,7 +252,31 @@ export async function exec(argv2: string[]) {
 
  const forceBuild = argv.b || argv.build;
 
- // _
+ // doCompress
+ const algo = argv.C || argv.compress || 'None';
+
+
+ let doCompress: CompressType = CompressType.None;
+ switch (algo.toLowerCase()) {
+ case 'brotli':
+ case 'br':
+ doCompress = CompressType.Brolti;
+ break;
+ case 'gzip':
+ case 'gz':
+ doCompress = CompressType.GZip;
+ break;
+ case 'none':
+ break;
+ default:
+ // eslint-disable-next-line no-console
+ throw wasReported(`Invalid compression algorithm ${algo} ( should be None, Brolti or Gzip)`);
+
+ }
+ if (doCompress !== CompressType.None) {
+ // eslint-disable-next-line no-console
+ console.log('compression: ', CompressType[doCompress]);
+ }
 
  if (!argv._.length) {
  throw wasReported('Entry file/directory is expected', [
@@ -274,14 +292,13 @@ export async function exec(argv2: string[]) {
 
  let input = path.resolve(argv._[0]);
 
- if (!(await exists(input))) {
+ if (!existsSync(input)) {
  throw wasReported('Input file does not exist', [input]);
  }
 
  if ((await stat(input)).isDirectory()) {
  input = path.join(input, 'package.json');
-
- if (!(await exists(input))) {
+ if (!existsSync(input)) {
  throw wasReported('Input file does not exist', [input]);
  }
  }
@@ -316,7 +333,7 @@ export async function exec(argv2: string[]) {
  }
  }
  inputBin = path.resolve(path.dirname(input), bin);
- if (!(await exists(inputBin))) {
+ if (!existsSync(inputBin)) {
  throw wasReported(
  'Bin file does not exist (taken from package.json ' +
  "'bin' property)",
@@ -348,8 +365,7 @@ export async function exec(argv2: string[]) {
 
  if (config) {
  config = path.resolve(config);
-
- if (!(await exists(config))) {
+ if (!existsSync(config)) {
  throw wasReported('Config file does not exist', [config]);
  }
 
@@ -589,7 +605,7 @@ export async function exec(argv2: string[]) {
  log.debug('Targets:', JSON.stringify(targets, null, 2));
 
  for (const target of targets) {
- if (target.output && (await exists(target.output))) {
+ if (target.output &&  existsSync(target.output)) {
  if ((await stat(target.output)).isFile()) {
  await remove(target.output);
  } else {
@@ -601,14 +617,8 @@ export async function exec(argv2: string[]) {
  await mkdirp(path.dirname(target.output));
  }
 
- await producer({
- backpack,
- bakes,
- slash: target.platform === 'win' ? '\\' : '/',
- target: target as Target,
- symLinks,
- });
-
+ const slash = target.platform === 'win' ? '\\' : '/';
+ await producer({ backpack, bakes, slash, target: target as Target, symLinks, doCompress });
  if (target.platform !== 'win' && target.output) {
  await plusx(target.output);
  }

lib/packer.ts

@@ -1,8 +1,8 @@
 /* eslint-disable complexity */
 
 import assert from 'assert';
-import fs from 'fs-extra';
-import path from 'path';
+import * as fs from 'fs-extra';
+import * as path from 'path';
 
 import {
  STORE_BLOB,
@@ -157,7 +157,7 @@ export default function packer({
  }
  }
  const prelude =
- `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS) { 
+ `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS, DICT, DOCOMPRESS) { 
  ${bootstrapText}${
  log.debugMode ? diagnosticText : ''
  }\n})(function (exports) {\n${commonText}\n},\n` +
@@ -166,6 +166,10 @@ export default function packer({
  `%DEFAULT_ENTRYPOINT%` +
  `\n,\n` +
  `%SYMLINKS%` +
+ '\n,\n' +
+ '%DICT%' +
+ '\n,\n' +
+ '%DOCOMPRESS%' +
  `\n);`;
 
  return { prelude, entrypoint, stripes };

lib/producer.ts

@@ -1,3 +1,4 @@
+import { createBrotliCompress, createGzip } from 'zlib';
 import Multistream from 'multistream';
 import assert from 'assert';
 import { execSync } from 'child_process';
@@ -12,6 +13,7 @@ import { log, wasReported } from './log';
 import { fabricateTwice } from './fabricator';
 import { platform, SymLinks, Target } from './types';
 import { Stripe } from './packer';
+import { CompressType } from './compress_type';
 
 interface NotFound {
  notFound: true;
@@ -242,14 +244,34 @@ interface ProducerOptions {
  slash: string;
  target: Target;
  symLinks: SymLinks;
+ doCompress: CompressType;
 }
 
+const fileDictionary: {[key:string]: string} = {};
+let counter = 0;
+function replace(k: string) {
+ let existingKey = fileDictionary[k];
+ if (!existingKey) {
+ const newkey = counter;
+ counter += 1;
+ existingKey = newkey.toString(36);
+ fileDictionary[k] = existingKey;
+ }
+ return existingKey;
+}
+const separator = '$';
+
+function makeKey(filename: string, slash: string): string {
+ const a = filename.split(slash).map(replace).join(separator);
+ return a;
+}
 export default function producer({
  backpack,
  bakes,
  slash,
  target,
  symLinks,
+ doCompress
 }: ProducerOptions) {
  return new Promise<void>((resolve, reject) => {
  if (!Buffer.alloc) {
@@ -268,18 +290,17 @@ export default function producer({
  for (const stripe of stripes) {
  let { snap } = stripe;
  snap = snapshotify(snap, slash);
-
- if (!vfs[snap]) {
- vfs[snap] = {};
- }
+ const vfsKey = makeKey(snap, slash);
+ if (!vfs[vfsKey]) vfs[vfsKey] = {};
  }
 
  const snapshotSymLinks: SymLinks = {};
 
  for (const [key, value] of Object.entries(symLinks)) {
  const k = snapshotify(key, slash);
  const v = snapshotify(value, slash);
- snapshotSymLinks[k] = v;
+ const vfsKey = makeKey(k, slash);
+ snapshotSymLinks[vfsKey] = makeKey(v, slash);
  }
 
  let meter: streamMeter.StreamMeter;
@@ -289,6 +310,15 @@ export default function producer({
  meter = streamMeter();
  return s.pipe(meter);
  }
+ function pipeMayCompressToNewMeter(s: Readable): streamMeter.StreamMeter {
+ if (doCompress === CompressType.GZip) {
+ return pipeToNewMeter(s.pipe(createGzip()));
+ }
+ if (doCompress === CompressType.Brolti) {
+ return pipeToNewMeter(s.pipe(createBrotliCompress()));
+ }
+ return pipeToNewMeter(s);
+ }
 
  function next(s: Readable) {
  count += 1;
@@ -321,7 +351,8 @@ export default function producer({
  const { store } = prevStripe;
  let { snap } = prevStripe;
  snap = snapshotify(snap, slash);
- vfs[snap][store] = [track, meter.bytes];
+ const vfsKey = makeKey(snap, slash);
+ vfs[vfsKey][store] = [track, meter.bytes];
  track += meter.bytes;
  }
 
@@ -347,15 +378,14 @@ export default function producer({
  return cb(null, intoStream(Buffer.alloc(0)));
  }
 
- cb(
- null,
- pipeToNewMeter(intoStream(buffer || Buffer.from('')))
- );
+ cb(null, pipeMayCompressToNewMeter(intoStream(buffer || Buffer.from(''))));
  }
  );
  }
-
- return cb(null, pipeToNewMeter(intoStream(stripe.buffer)));
+ return cb(
+ null,
+ pipeMayCompressToNewMeter(intoStream(stripe.buffer))
+ );
  }
 
  if (stripe.file) {
@@ -378,15 +408,17 @@ export default function producer({
  if (fs.existsSync(platformFile)) {
  return cb(
  null,
- pipeToNewMeter(fs.createReadStream(platformFile))
+ pipeMayCompressToNewMeter(fs.createReadStream(platformFile))
  );
  }
  } catch (err) {
  log.debug(`prebuild-install failed[${stripe.file}]:`, err);
  }
  }
-
- return cb(null, pipeToNewMeter(fs.createReadStream(stripe.file)));
+ return cb(
+ null,
+ pipeMayCompressToNewMeter(fs.createReadStream(stripe.file))
+ );
  }
 
  assert(false, 'producer: bad stripe');
@@ -401,15 +433,23 @@ export default function producer({
  replaceDollarWise(
  replaceDollarWise(
  replaceDollarWise(
- prelude,
- '%VIRTUAL_FILESYSTEM%',
- JSON.stringify(vfs)
+ replaceDollarWise(
+ replaceDollarWise(
+ prelude,
+ '%VIRTUAL_FILESYSTEM%',
+ JSON.stringify(vfs)
+ ),
+ '%DEFAULT_ENTRYPOINT%',
+ JSON.stringify(entrypoint)
+ ),
+ '%SYMLINKS%',
+ JSON.stringify(snapshotSymLinks)
  ),
- '%DEFAULT_ENTRYPOINT%',
- JSON.stringify(entrypoint)
+ '%DICT%',
+ JSON.stringify(fileDictionary)
  ),
- '%SYMLINKS%',
- JSON.stringify(snapshotSymLinks)
+ '%DOCOMPRESS%',
+ JSON.stringify(doCompress)
  )
  )
  )