compress files in virtual file system (vercel#885)

.eslintrc

@@ -1,4 +1,5 @@
 {
+ // note you must disable the base rule as it can report incorrect errors
  "extends": ["airbnb-base", "prettier"],
  "rules": {
  "no-bitwise": "off",
@@ -10,8 +11,10 @@
  "consistent-return": "off",
  "no-restricted-syntax": "off",
  "import/prefer-default-export": "off",
- "camelcase": "off"
- },
+ "camelcase": "off",
+ "no-shadow": "off",
+ "@typescript-eslint/no-shadow": ["error"]
+ },
  "overrides": [
  {
  "files": ["*.ts"],

README.md

@@ -178,6 +178,16 @@ option to `pkg`. First ensure your computer meets the
 requirements to compile original Node.js:
 [BUILDING.md](https:/nodejs/node/blob/master/BUILDING.md)
 
+### Compression
+
+Pass `--compress Brolti` or `--compress GZip` to `pkg` to compress further the content of the files store in the exectable.
+
+This option can reduce the size of the embedded file system by up to 60%.
+
+The startup time of the application might be reduced slightly.
+
+`-C` can be used as a shortcut for `--compress `.
+
 ### Environment
 
 | Var | Description |

lib/compress_type.ts

@@ -0,0 +1,6 @@
+
+export enum CompressType {
+ None=0,
+ GZip= 1,
+ Brolti= 2
+};

lib/help.ts

@@ -17,6 +17,7 @@ export default function help() {
  -d, --debug show more information during packaging process [off]
  -b, --build don't download prebuilt base binaries, build them
  --public speed up and disclose the sources of top-level project
+ -C, --compress [default=None] compression algorithm = Brotli or GZip
 
  ${chalk.dim('Examples:')}
 
@@ -30,5 +31,9 @@ export default function help() {
  ${chalk.cyan('$ pkg -t node4-linux,node6-linux,node6-win index.js')}
  ${chalk.gray('–')} Bakes '--expose-gc' into executable
  ${chalk.cyan('$ pkg --options expose-gc index.js')}
+ ${chalk.gray(
+ '–'
+ )} reduce size of the data packed inside the executable with GZip
+ ${chalk.cyan('$ pkg --compress GZip index.js')}
 `);
 }

lib/index.ts

@@ -1,6 +1,6 @@
 /* eslint-disable require-atomic-updates */
 
-import { mkdirp, readFile, remove, stat, readFileSync } from 'fs-extra';
+import { existsSync, mkdirp, readFile, remove, stat, readFileSync } from 'fs-extra';
 import { need, system } from 'pkg-fetch';
 import assert from 'assert';
 import minimist from 'minimist';
@@ -16,6 +16,7 @@ import refine from './refiner';
 import { shutdown } from './fabricator';
 import walk, { Marker, WalkerParams } from './walker';
 import { Target, NodeTarget, SymLinks } from './types';
+import { CompressType } from './compress_type';
 
 const { version } = JSON.parse(
  readFileSync(path.join(__dirname, '../package.json'), 'utf-8')
@@ -25,15 +26,6 @@ function isConfiguration(file: string) {
  return isPackageJson(file) || file.endsWith('.config.json');
 }
 
-async function exists(file: string) {
- try {
- await stat(file);
- return true;
- } catch (error) {
- return false;
- }
-}
-
 // http://www.openwall.com/lists/musl/2012/12/08/4
 
 const {
@@ -213,6 +205,8 @@ export async function exec(argv2: string[]) {
  't',
  'target',
  'targets',
+ 'C',
+ 'compress',
  ],
  default: { bytecode: true },
  });
@@ -240,7 +234,33 @@ export async function exec(argv2: string[]) {
 
  const forceBuild = argv.b || argv.build;
 
- // _
+ // doCompress
+ const algo = argv.C || argv.compress || 'None';
+
+
+ let doCompress: CompressType = CompressType.None;
+ switch (algo.toLowerCase()) {
+ case 'brotli':
+ case 'br':
+ doCompress = CompressType.Brolti;
+ break;
+ case 'gzip':
+ case 'gz':
+ doCompress = CompressType.GZip;
+ break;
+ case 'none':
+ break;
+ default:
+ // eslint-disable-next-line no-console
+ console.log(
+ `Invalid compression algorithm ${algo} ( should be None, Brolti or Gzip)`
+ );
+ process.exit(-1);
+ }
+ if (doCompress !== CompressType.None) {
+ // eslint-disable-next-line no-console
+ console.log('compression: ', CompressType[doCompress]);
+ }
 
  if (!argv._.length) {
  throw wasReported('Entry file/directory is expected', [
@@ -256,14 +276,13 @@ export async function exec(argv2: string[]) {
 
  let input = path.resolve(argv._[0]);
 
- if (!(await exists(input))) {
+ if (!existsSync(input)) {
  throw wasReported('Input file does not exist', [input]);
  }
 
  if ((await stat(input)).isDirectory()) {
  input = path.join(input, 'package.json');
-
- if (!(await exists(input))) {
+ if (!existsSync(input)) {
  throw wasReported('Input file does not exist', [input]);
  }
  }
@@ -298,7 +317,7 @@ export async function exec(argv2: string[]) {
  }
  }
  inputBin = path.resolve(path.dirname(input), bin);
- if (!(await exists(inputBin))) {
+ if (!existsSync(inputBin)) {
  throw wasReported(
  'Bin file does not exist (taken from package.json ' +
  "'bin' property)",
@@ -330,8 +349,7 @@ export async function exec(argv2: string[]) {
 
  if (config) {
  config = path.resolve(config);
-
- if (!(await exists(config))) {
+ if (!existsSync(config)) {
  throw wasReported('Config file does not exist', [config]);
  }
 
@@ -568,7 +586,7 @@ export async function exec(argv2: string[]) {
  log.debug('Targets:', JSON.stringify(targets, null, 2));
 
  for (const target of targets) {
- if (target.output && (await exists(target.output))) {
+ if (target.output &&  existsSync(target.output)) {
  if ((await stat(target.output)).isFile()) {
  await remove(target.output);
  } else {
@@ -580,14 +598,8 @@ export async function exec(argv2: string[]) {
  await mkdirp(path.dirname(target.output));
  }
 
- await producer({
- backpack,
- bakes,
- slash: target.platform === 'win' ? '\\' : '/',
- target: target as Target,
- symLinks,
- });
-
+ const slash = target.platform === 'win' ? '\\' : '/';
+ await producer({ backpack, bakes, slash, target: target as Target, symLinks, doCompress });
  if (target.platform !== 'win' && target.output) {
  await plusx(target.output);
  }

lib/packer.ts

@@ -1,8 +1,8 @@
 /* eslint-disable complexity */
 
 import assert from 'assert';
-import fs from 'fs-extra';
-import path from 'path';
+import * as fs from 'fs-extra';
+import * as path from 'path';
 
 import {
  STORE_BLOB,
@@ -157,7 +157,7 @@ export default function packer({
  }
  }
  const prelude =
- `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS) { 
+ `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS, DICT, DOCOMPRESS) { 
  ${bootstrapText}${
  log.debugMode ? diagnosticText : ''
  }\n})(function (exports) {\n${commonText}\n},\n` +
@@ -166,6 +166,10 @@ export default function packer({
  `%DEFAULT_ENTRYPOINT%` +
  `\n,\n` +
  `%SYMLINKS%` +
+ '\n,\n' +
+ '%DICT%' +
+ '\n,\n' +
+ '%DOCOMPRESS%' +
  `\n);`;
 
  return { prelude, entrypoint, stripes };