No longer package X-Pack as a node module

package.json

@@ -247,7 +247,6 @@
  "webpack-merge": "4.1.4",
  "whatwg-fetch": "^3.0.0",
  "wreck": "^14.0.2",
- "x-pack": "8.0.0",
  "yauzl": "2.7.0"
  },
  "devDependencies": {

packages/kbn-pm/src/commands/bootstrap.test.ts

@@ -27,12 +27,13 @@ import { linkProjectExecutables } from '../utils/link_project_executables';
 import { IPackageJson } from '../utils/package_json';
 import { Project } from '../utils/project';
 import { buildProjectGraph } from '../utils/projects';
-import { installInDir, runScriptInPackageStreaming } from '../utils/scripts';
+import { installInDir, runScriptInPackageStreaming, yarnWorkspacesInfo } from '../utils/scripts';
 import { BootstrapCommand } from './bootstrap';
 
 const mockInstallInDir = installInDir as jest.Mock;
 const mockRunScriptInPackageStreaming = runScriptInPackageStreaming as jest.Mock;
 const mockLinkProjectExecutables = linkProjectExecutables as jest.Mock;
+const mockYarnWorkspacesInfo = yarnWorkspacesInfo as jest.Mock;
 
 const createProject = (packageJson: IPackageJson, path = '.') => {
  const project = new Project(
@@ -57,6 +58,10 @@ const noop = () => {
  // noop
 };
 
+beforeEach(() => {
+ mockYarnWorkspacesInfo.mockResolvedValue({});
+});
+
 afterEach(() => {
  jest.resetAllMocks();
  jest.restoreAllMocks();

packages/kbn-pm/src/config.ts

@@ -27,7 +27,7 @@ export interface IProjectPathOptions {
 /**
  * Returns all the paths where plugins are located
  */
-export function getProjectPaths(rootPath: string, options: IProjectPathOptions) {
+export function getProjectPaths(rootPath: string, options: IProjectPathOptions = {}) {
  const skipKibanaPlugins = Boolean(options['skip-kibana-plugins']);
  const ossOnly = Boolean(options.oss);
 

packages/kbn-pm/src/index.ts

@@ -19,4 +19,7 @@
 
 export { run } from './cli';
 export { buildProductionProjects, prepareExternalProjectDependencies } from './production';
+export { getProjects } from './utils/projects';
+export { Project } from './utils/project';
 export { copyWorkspacePackages } from './utils/workspaces';
+export { getProjectPaths } from './config';

packages/kbn-pm/src/production/build_production_projects.ts

@@ -35,12 +35,14 @@ import {
 
 export async function buildProductionProjects({
  kibanaRoot,
- buildRoots,
+ buildRoot,
+ onlyOSS,
 }: {
  kibanaRoot: string;
- buildRoots: string[];
+ buildRoot: string;
+ onlyOSS?: boolean;
 }) {
- const projects = await getProductionProjects(kibanaRoot);
+ const projects = await getProductionProjects(kibanaRoot, onlyOSS);
  const projectGraph = buildProjectGraph(projects);
  const batchedProjects = topologicallyBatchProjects(projects, projectGraph);
 
@@ -51,9 +53,7 @@ export async function buildProductionProjects({
  for (const project of batch) {
  await deleteTarget(project);
  await buildProject(project);
- for (const buildRoot of buildRoots) {
- await copyToBuild(project, kibanaRoot, buildRoot);
- }
+ await copyToBuild(project, kibanaRoot, buildRoot);
  }
  }
 }
@@ -62,19 +62,33 @@ export async function buildProductionProjects({
  * Returns the subset of projects that should be built into the production
  * bundle. As we copy these into Kibana's `node_modules` during the build step,
  * and let Kibana's build process be responsible for installing dependencies,
- * we only include Kibana's transitive _production_ dependencies.
+ * we only include Kibana's transitive _production_ dependencies. If onlyOSS
+ * is supplied, we omit projects with build.oss in their package.json set to false.
  */
-async function getProductionProjects(rootPath: string) {
+async function getProductionProjects(rootPath: string, onlyOSS?: boolean) {
  const projectPaths = getProjectPaths(rootPath, {});
  const projects = await getProjects(rootPath, projectPaths);
+ const projectsSubset = [projects.get('kibana')!];
+
+ if (projects.has('x-pack')) {
+ projectsSubset.push(projects.get('x-pack')!);
+ }
 
- const productionProjects = includeTransitiveProjects([projects.get('kibana')!], projects, {
+ const productionProjects = includeTransitiveProjects(projectsSubset, projects, {
  onlyProductionDependencies: true,
  });
 
  // We remove Kibana, as we're already building Kibana
  productionProjects.delete('kibana');
 
+ if (onlyOSS) {
+ productionProjects.forEach(project => {
+ if (project.getBuildConfig().oss === false) {
+ productionProjects.delete(project.json.name);
+ }
+ });
+ }
+
  return productionProjects;
 }
 

packages/kbn-pm/src/production/integration_tests/__fixtures__/packages/foo/package.json

@@ -3,6 +3,11 @@
  "version": "1.0.0",
  "private": true,
  "main": "./target/index.js",
+ "kibana": {
+ "build": {
+ "oss": false
+ }
+ },
  "dependencies": {
  "@elastic/bar": "link:../bar"
  },

packages/kbn-pm/src/production/integration_tests/build_production_projects.test.ts

@@ -27,31 +27,38 @@ import { getProjects } from '../../utils/projects';
 import { buildProductionProjects } from '../build_production_projects';
 
 describe('kbn-pm production', () => {
- test(
- 'builds and copies projects for production',
- async () => {
- const tmpDir = tempy.directory();
- const buildRoot = tempy.directory();
- const fixturesPath = resolve(__dirname, '__fixtures__');
+ let tmpDir: string;
+ let buildRoot: string;
 
- // Copy all the test fixtures into a tmp dir, as we will be mutating them
- await copy(['**/*'], tmpDir, {
- cwd: fixturesPath,
- dot: true,
- nodir: true,
- parents: true,
- });
+ const timeout = 1 * 60 * 1000;
 
- const projects = await getProjects(tmpDir, ['.', './packages/*']);
+ beforeEach(async () => {
+ tmpDir = tempy.directory();
+ buildRoot = tempy.directory();
+ const fixturesPath = resolve(__dirname, '__fixtures__');
 
- for (const project of projects.values()) {
- // This will both install dependencies and generate `yarn.lock` files
- await project.installDependencies({
- extraArgs: ['--silent', '--no-progress'],
- });
- }
+ // Copy all the test fixtures into a tmp dir, as we will be mutating them
+ await copy(['**/*'], tmpDir, {
+ cwd: fixturesPath,
+ dot: true,
+ nodir: true,
+ parents: true,
+ });
+
+ const projects = await getProjects(tmpDir, ['.', './packages/*']);
 
- await buildProductionProjects({ kibanaRoot: tmpDir, buildRoots: [buildRoot] });
+ for (const project of projects.values()) {
+ // This will both install dependencies and generate `yarn.lock` files
+ await project.installDependencies({
+ extraArgs: ['--silent', '--no-progress'],
+ });
+ }
+ }, timeout);
+
+ test(
+ 'builds and copies projects for production',
+ async () => {
+ await buildProductionProjects({ kibanaRoot: tmpDir, buildRoot });
 
  const files = await globby(['**/*', '!**/node_modules/**'], {
  cwd: buildRoot,
@@ -65,6 +72,20 @@ describe('kbn-pm production', () => {
  }
  }
  },
- 2 * 60 * 1000
+ timeout
+ );
+
+ test(
+ 'builds and copies only OSS projects for production',
+ async () => {
+ await buildProductionProjects({ kibanaRoot: tmpDir, buildRoot, onlyOSS: true });
+
+ const files = await globby(['**/*', '!**/node_modules/**'], {
+ cwd: buildRoot,
+ });
+
+ expect(files.sort()).toMatchSnapshot();
+ },
+ timeout
  );
 });

packages/kbn-pm/src/utils/child_process.ts

@@ -33,8 +33,8 @@ function generateColors() {
 
 export function spawn(command: string, args: string[], opts: execa.Options) {
  return execa(command, args, {
- ...opts,
  stdio: 'inherit',
+ ...opts,
  });
 }
 
@@ -47,8 +47,8 @@ export function spawnStreaming(
  { prefix }: { prefix: string }
 ) {
  const spawned = execa(command, args, {
- ...opts,
  stdio: ['ignore', 'pipe', 'pipe'],
+ ...opts,
  });
 
  const color = nextColor();

packages/kbn-pm/src/utils/fs.ts

@@ -24,7 +24,7 @@ import { ncp } from 'ncp';
 import { dirname, relative } from 'path';
 import { promisify } from 'util';
 
-const stat = promisify(fs.stat);
+const lstat = promisify(fs.lstat);
 const readFile = promisify(fs.readFile);
 const symlink = promisify(fs.symlink);
 const chmod = promisify(fs.chmod);
@@ -37,7 +37,7 @@ export { chmod, readFile, mkdirp };
 
 async function statTest(path: string, block: (stats: fs.Stats) => boolean) {
  try {
- return block(await stat(path));
+ return block(await lstat(path));
  } catch (e) {
  if (e.code === 'ENOENT') {
  return false;
@@ -46,6 +46,14 @@ async function statTest(path: string, block: (stats: fs.Stats) => boolean) {
  }
 }
 
+/**
+ * Test if a path points to a symlink.
+ * @param path
+ */
+export async function isSymlink(path: string) {
+ return await statTest(path, stats => stats.isSymbolicLink());
+}
+
 /**
  * Test if a path points to a directory.
  * @param path

packages/kbn-pm/src/utils/project.ts

@@ -18,6 +18,7 @@
  */
 
 import chalk from 'chalk';
+import fs from 'fs';
 import { relative, resolve as resolvePath } from 'path';
 import { inspect } from 'util';
 
@@ -30,11 +31,17 @@ import {
  isLinkDependency,
  readPackageJson,
 } from './package_json';
-import { installInDir, runScriptInPackage, runScriptInPackageStreaming } from './scripts';
+import {
+ installInDir,
+ runScriptInPackage,
+ runScriptInPackageStreaming,
+ yarnWorkspacesInfo,
+} from './scripts';
 
 interface BuildConfig {
  skip?: boolean;
  intermediateBuildDirectory?: string;
+ oss?: boolean;
 }
 
 interface CleanConfig {
@@ -190,7 +197,41 @@ export class Project {
 
  public async installDependencies({ extraArgs }: { extraArgs: string[] }) {
  log.write(chalk.bold(`\n\nInstalling dependencies in [${chalk.green(this.name)}]:\n`));
- return installInDir(this.path, extraArgs);
+ await installInDir(this.path, extraArgs);
+ await this.removeExtraneousNodeModules();
+ }
+
+ /**
+ * Yarn workspaces symlinks workspace projects to the root node_modules, even
+ * when there is no depenency on the project. This results in unnecicary, and
+ * often duplicated code in the build archives.
+ */
+ public async removeExtraneousNodeModules() {
+ // this is only relevant for the root workspace
+ if (!this.isWorkspaceRoot) {
+ return;
+ }
+
+ const workspacesInfo = await yarnWorkspacesInfo(this.path);
+ const unusedWorkspaces = new Set(Object.keys(workspacesInfo));
+
+ // check for any cross-project dependency
+ for (const name of Object.keys(workspacesInfo)) {
+ const workspace = workspacesInfo[name];
+ workspace.workspaceDependencies.forEach(w => unusedWorkspaces.delete(w));
+ }
+
+ unusedWorkspaces.forEach(name => {
+ const { dependencies, devDependencies } = this.json;
+ const nodeModulesPath = resolvePath(this.nodeModulesLocation, name);
+ const isDependency = dependencies && dependencies.hasOwnProperty(name);
+ const isDevDependency = devDependencies && devDependencies.hasOwnProperty(name);
+
+ if (!isDependency && !isDevDependency && fs.existsSync(nodeModulesPath)) {
+ log.write(`No dependency on ${name}, removing link in node_modules`);
+ fs.unlinkSync(nodeModulesPath);
+ }
+ });
  }
 }