Revert^2 "allow to compress files in virtual file system (#1115)" (#1200

)

v2:

- fix issue #1191 
- only compress VFS if --compress is set (#1192)
- improve random access with the compressed virtual file system

README.md

@@ -209,6 +209,16 @@ requirements to compile original Node.js:
 
 See [pkg-fetch](https:/vercel/pkg-fetch) for more info.
 
+### Compression
+
+Pass `--compress Brotli` or `--compress GZip` to `pkg` to compress further the content of the files store in the exectable.
+
+This option can reduce the size of the embedded file system by up to 60%.
+
+The startup time of the application might be reduced slightly.
+
+`-C` can be used as a shortcut for `--compress `.
+
 ### Environment
 
 | Var | Description |
@@ -358,7 +368,7 @@ and check that all the required files for your application are properly
 incorporated to the final executable.
 
  $ pkg --debug app.js -o output
- $ DEBUG_PKG output
+ $ DEBUG_PKG=1 output
 
 or
 

dictionary/better-sqlite3.js

@@ -0,0 +1,3 @@
+'use strict';
+
+module.exports = {};

lib/compress_type.ts

@@ -0,0 +1,6 @@
+
+export enum CompressType {
+ None = 0,
+ GZip = 1,
+ Brotli = 2
+};

lib/help.ts

@@ -19,6 +19,7 @@ export default function help() {
  --public speed up and disclose the sources of top-level project
  --public-packages force specified packages to be considered public
  --no-bytecode skip bytecode generation and include source files as plain js
+ -C, --compress [default=None] compression algorithm = Brotli or GZip
 
  ${chalk.dim('Examples:')}
 
@@ -36,5 +37,11 @@ export default function help() {
  ${chalk.cyan('$ pkg --public-packages "packageA,packageB" index.js')}
  ${chalk.gray('–')} Consider all packages to be public
  ${chalk.cyan('$ pkg --public-packages "*" index.js')}
+ ${chalk.gray('–')} Bakes '--expose-gc' into executable
+ ${chalk.cyan('$ pkg --options expose-gc index.js')}
+ ${chalk.gray(
+ '–'
+ )} reduce size of the data packed inside the executable with GZip
+ ${chalk.cyan('$ pkg --compress GZip index.js')}
 `);
 }

lib/index.ts

@@ -26,6 +26,7 @@ import refine from './refiner';
 import { shutdown } from './fabricator';
 import walk, { Marker, WalkerParams } from './walker';
 import { Target, NodeTarget, SymLinks } from './types';
+import { CompressType } from './compress_type';
 import { patchMachOExecutable } from './mach-o';
 
 const { version } = JSON.parse(
@@ -247,6 +248,8 @@ export async function exec(argv2: string[]) {
  't',
  'target',
  'targets',
+ 'C',
+ 'compress',
  ],
  default: { bytecode: true },
  });
@@ -274,6 +277,32 @@ export async function exec(argv2: string[]) {
 
  const forceBuild = argv.b || argv.build;
 
+ // doCompress
+ const algo = argv.C || argv.compress || 'None';
+
+ let doCompress: CompressType = CompressType.None;
+ switch (algo.toLowerCase()) {
+ case 'brotli':
+ case 'br':
+ doCompress = CompressType.Brotli;
+ break;
+ case 'gzip':
+ case 'gz':
+ doCompress = CompressType.GZip;
+ break;
+ case 'none':
+ break;
+ default:
+ // eslint-disable-next-line no-console
+ throw wasReported(
+ `Invalid compression algorithm ${algo} ( should be None, Brotli or Gzip)`
+ );
+ }
+ if (doCompress !== CompressType.None) {
+ // eslint-disable-next-line no-console
+ console.log('compression: ', CompressType[doCompress]);
+ }
+
  // _
 
  if (!argv._.length) {
@@ -631,6 +660,7 @@ export async function exec(argv2: string[]) {
  slash: target.platform === 'win' ? '\\' : '/',
  target: target as Target,
  symLinks,
+ doCompress,
  });
 
  if (target.platform !== 'win' && target.output) {

lib/packer.ts

@@ -157,7 +157,7 @@ export default function packer({
  }
  }
  const prelude =
- `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS) { 
+ `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS, DICT, DOCOMPRESS) { 
  ${bootstrapText}${
  log.debugMode ? diagnosticText : ''
  }\n})(function (exports) {\n${commonText}\n},\n` +
@@ -166,6 +166,10 @@ export default function packer({
  `%DEFAULT_ENTRYPOINT%` +
  `\n,\n` +
  `%SYMLINKS%` +
+ '\n,\n' +
+ '%DICT%' +
+ '\n,\n' +
+ '%DOCOMPRESS%' +
  `\n);`;
 
  return { prelude, entrypoint, stripes };

lib/producer.ts

@@ -1,3 +1,4 @@
+import { createBrotliCompress, createGzip } from 'zlib';
 import Multistream from 'multistream';
 import assert from 'assert';
 import { execFileSync } from 'child_process';
@@ -12,6 +13,7 @@ import { log, wasReported } from './log';
 import { fabricateTwice } from './fabricator';
 import { platform, SymLinks, Target } from './types';
 import { Stripe } from './packer';
+import { CompressType } from './compress_type';
 
 interface NotFound {
  notFound: true;
@@ -248,6 +250,61 @@ interface ProducerOptions {
  slash: string;
  target: Target;
  symLinks: SymLinks;
+ doCompress: CompressType;
+}
+
+/**
+ * instead of creating a vfs dicionnary with actual path as key
+ * we use a compression mechanism that can reduce significantly
+ * the memory footprint of the vfs in the code.
+ *
+ * without vfs compression:
+ *
+ * vfs = {
+ * "/folder1/folder2/file1.js": {};
+ * "/folder1/folder2/folder3/file2.js": {};
+ * "/folder1/folder2/folder3/file3.js": {};
+ * }
+ *
+ * with compression :
+ *
+ * fileDictionary = {
+ * "folder1": "1",
+ * "folder2": "2",
+ * "file1": "3",
+ * "folder3": "4",
+ * "file2": "5",
+ * "file3": "6",
+ * }
+ * vfs = {
+ * "/1/2/3": {};
+ * "/1/2/4/5": {};
+ * "/1/2/4/6": {};
+ * }
+ *
+ * note: the key is computed in base36 for further compression.
+ */
+const fileDictionary: { [key: string]: string } = {};
+let counter = 0;
+function getOrCreateHash(fileOrFolderName: string) {
+ let existingKey = fileDictionary[fileOrFolderName];
+ if (!existingKey) {
+ const newkey = counter;
+ counter += 1;
+ existingKey = newkey.toString(36);
+ fileDictionary[fileOrFolderName] = existingKey;
+ }
+ return existingKey;
+}
+const separator = '/';
+
+function makeKey(
+ doCompression: CompressType,
+ fullpath: string,
+ slash: string
+): string {
+ if (doCompression === CompressType.None) return fullpath;
+ return fullpath.split(slash).map(getOrCreateHash).join(separator);
 }
 
 export default function producer({
@@ -256,6 +313,7 @@ export default function producer({
  slash,
  target,
  symLinks,
+ doCompress,
 }: ProducerOptions) {
  return new Promise<void>((resolve, reject) => {
  if (!Buffer.alloc) {
@@ -274,18 +332,17 @@ export default function producer({
  for (const stripe of stripes) {
  let { snap } = stripe;
  snap = snapshotify(snap, slash);
-
- if (!vfs[snap]) {
- vfs[snap] = {};
- }
+ const vfsKey = makeKey(doCompress, snap, slash);
+ if (!vfs[vfsKey]) vfs[vfsKey] = {};
  }
 
  const snapshotSymLinks: SymLinks = {};
 
  for (const [key, value] of Object.entries(symLinks)) {
  const k = snapshotify(key, slash);
  const v = snapshotify(value, slash);
- snapshotSymLinks[k] = v;
+ const vfsKey = makeKey(doCompress, k, slash);
+ snapshotSymLinks[vfsKey] = makeKey(doCompress, v, slash);
  }
 
  let meter: streamMeter.StreamMeter;
@@ -295,6 +352,15 @@ export default function producer({
  meter = streamMeter();
  return s.pipe(meter);
  }
+ function pipeMayCompressToNewMeter(s: Readable): streamMeter.StreamMeter {
+ if (doCompress === CompressType.GZip) {
+ return pipeToNewMeter(s.pipe(createGzip()));
+ }
+ if (doCompress === CompressType.Brotli) {
+ return pipeToNewMeter(s.pipe(createBrotliCompress()));
+ }
+ return pipeToNewMeter(s);
+ }
 
  function next(s: Readable) {
  count += 1;
@@ -327,7 +393,8 @@ export default function producer({
  const { store } = prevStripe;
  let { snap } = prevStripe;
  snap = snapshotify(snap, slash);
- vfs[snap][store] = [track, meter.bytes];
+ const vfsKey = makeKey(doCompress, snap, slash);
+ vfs[vfsKey][store] = [track, meter.bytes];
  track += meter.bytes;
  }
 
@@ -355,13 +422,17 @@ export default function producer({
 
  cb(
  null,
- pipeToNewMeter(intoStream(buffer || Buffer.from('')))
+ pipeMayCompressToNewMeter(
+ intoStream(buffer || Buffer.from(''))
+ )
  );
  }
  );
  }
-
- return cb(null, pipeToNewMeter(intoStream(stripe.buffer)));
+ return cb(
+ null,
+ pipeMayCompressToNewMeter(intoStream(stripe.buffer))
+ );
  }
 
  if (stripe.file) {
@@ -384,15 +455,17 @@ export default function producer({
  if (fs.existsSync(platformFile)) {
  return cb(
  null,
- pipeToNewMeter(fs.createReadStream(platformFile))
+ pipeMayCompressToNewMeter(fs.createReadStream(platformFile))
  );
  }
  } catch (err) {
  log.debug(`prebuild-install failed[${stripe.file}]:`, err);
  }
  }
-
- return cb(null, pipeToNewMeter(fs.createReadStream(stripe.file)));
+ return cb(
+ null,
+ pipeMayCompressToNewMeter(fs.createReadStream(stripe.file))
+ );
  }
 
  assert(false, 'producer: bad stripe');
@@ -407,15 +480,23 @@ export default function producer({
  replaceDollarWise(
  replaceDollarWise(
  replaceDollarWise(
- prelude,
- '%VIRTUAL_FILESYSTEM%',
- JSON.stringify(vfs)
+ replaceDollarWise(
+ replaceDollarWise(
+ prelude,
+ '%VIRTUAL_FILESYSTEM%',
+ JSON.stringify(vfs)
+ ),
+ '%DEFAULT_ENTRYPOINT%',
+ JSON.stringify(entrypoint)
+ ),
+ '%SYMLINKS%',
+ JSON.stringify(snapshotSymLinks)
  ),
- '%DEFAULT_ENTRYPOINT%',
- JSON.stringify(entrypoint)
+ '%DICT%',
+ JSON.stringify(fileDictionary)
  ),
- '%SYMLINKS%',
- JSON.stringify(snapshotSymLinks)
+ '%DOCOMPRESS%',
+ JSON.stringify(doCompress)
  )
  )
  )

lib/walker.ts

@@ -497,6 +497,7 @@ class Walker {
  assets = expandFiles(assets, base);
 
  for (const asset of assets) {
+ log.debug(' Adding asset : .... ', asset);
  const stat = await fs.stat(asset);
 
  if (stat.isFile()) {

package.json

@@ -72,6 +72,7 @@
  "lint": "eslint lib prelude test",
  "lint:fix": "npm run lint -- --fix",
  "prepare": "npm run build",
+ "prepublishOnly": "npm run lint",
  "test": "npm run build && node test/test.js node14 no-npm && node test/test.js node12 no-npm && node test/test.js node10 no-npm && node test/test.js host only-npm"
  },
  "greenkeeper": {