fix: re-install lmdb when detecting the absence of @LMDB prebuilt packages

.circleci/config.yml

@@ -313,6 +313,13 @@ jobs:
  test_path: integration-tests/esm-in-gatsby-files
  test_command: yarn test
 
+ integration_tests_lmdb_regeneration:
+ executor: node
+ steps:
+ - e2e-test:
+ test_path: integration-tests/lmdb-regeneration
+ test_command: yarn test
+
  e2e_tests_path-prefix:
  <<: *e2e-executor
  steps:
@@ -590,6 +597,8 @@ workflows:
  <<: *e2e-test-workflow
  - integration_tests_esm_in_gatsby_files:
  <<: *e2e-test-workflow
+ - integration_tests_lmdb_regeneration:
+ <<: *e2e-test-workflow
  - integration_tests_gatsby_cli:
  requires:
  - bootstrap

integration-tests/lmdb-regeneration/.gitignore

@@ -0,0 +1,3 @@
+__tests__/__debug__
+node_modules
+yarn.lock

integration-tests/lmdb-regeneration/.npmrc

@@ -0,0 +1 @@
+build_from_source=true

integration-tests/lmdb-regeneration/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 gatsbyjs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

integration-tests/lmdb-regeneration/README.md

@@ -0,0 +1,3 @@
+## Artifacts test suite
+
+This integration test suite helps us assert our mechanism to heal back from a broken lmdb installation is working by validating we install and use the correct prebundled binary for our platform

integration-tests/lmdb-regeneration/__tests__/index.js

@@ -0,0 +1,59 @@
+const path = require(`path`)
+const execa = require(`execa`)
+const mod = require("module")
+const fs = require(`fs-extra`)
+
+jest.setTimeout(100000)
+
+const rootPath = path.resolve(__dirname, "../")
+
+describe(`Lmdb regeneration`, () => {
+ test(`gatbsy build detects lmdb setup built from source and installs pre-buit package`, async () => {
+ const lmdbNodeModulesPath = path.resolve(rootPath, "node_modules", "lmdb")
+ // Make sure we clear out the current `@lmdb` optional dependencies
+ const pathsToRemove = [
+ path.resolve(rootPath, "node_modules", "@lmdb"),
+ path.resolve(rootPath, "node_modules", "gatsby", "node_modules", "@lmdb"),
+ ]
+ for (let path of pathsToRemove) {
+ fs.rmSync(path, { force: true, recursive: true })
+ }
+ // Check the lmdb instance we have installed does have a binary built from source since we need it to reproduce the fix we're trying to test
+ // If this check fails then it means our fixture is wrong and we're relying on an lmdb instance with prebuilt binaries
+ const builtFromSource = fs.existsSync(
+ path.resolve(lmdbNodeModulesPath, "build", "Release", "lmdb.node")
+ )
+ expect(builtFromSource).toEqual(true)
+
+ const options = {
+ stderr: `inherit`,
+ stdout: `inherit`,
+ cwd: rootPath,
+ }
+ const gatsbyBin = path.resolve(rootPath, `node_modules`, `gatsby`, `cli.js`)
+ await execa(gatsbyBin, [`build`], options)
+
+ // lmdb module with prebuilt binaries for our platform
+ const lmdbPackage = `@lmdb/lmdb-${process.platform}-${process.arch}`
+
+ // If the fix worked correctly we should have installed the prebuilt binary for our platform under our `.cache` directory
+ const lmdbRequire = mod.createRequire(
+ path.resolve(rootPath, ".cache", "internal-packages", "package.json")
+ )
+ expect(() => {
+ lmdbRequire.resolve(lmdbPackage)
+ }).not.toThrow()
+
+ // The resulting query-engine bundle should not contain the binary built from source
+ const binaryBuiltFromSource = path.resolve(
+ rootPath,
+ ".cache",
+ "query-engine",
+ "assets",
+ "build",
+ "Release",
+ "lmdb.node"
+ )
+ expect(fs.existsSync(binaryBuiltFromSource)).toEqual(false)
+ })
+})

integration-tests/lmdb-regeneration/gatsby-config.js

@@ -0,0 +1,9 @@
+module.exports = {
+ siteMetadata: {
+ siteUrl: `https://www.yourdomain.tld`,
+ },
+ plugins: [],
+ flags: {
+ DEV_SSR: true,
+ },
+}

integration-tests/lmdb-regeneration/jest.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+ testPathIgnorePatterns: [`/node_modules/`, `__tests__/fixtures`, `.cache`],
+}

integration-tests/lmdb-regeneration/package.json

@@ -0,0 +1,22 @@
+{
+ "name": "lmdb-regeneration",
+ "private": true,
+ "author": "Sid Chatterjee",
+ "description": "A simplified bare-bones starter for Gatsby with DSG",
+ "version": "0.1.0",
+ "license": "MIT",
+ "scripts": {
+ "build": "gatsby build",
+ "clean": "gatsby clean",
+ "test": "jest --runInBand"
+ },
+ "dependencies": {
+ "gatsby": "next",
+ "react": "^18.2.0",
+ "react-dom": "^18.2.0"
+ },
+ "devDependencies": {
+ "fs-extra": "^11.1.0",
+ "jest": "^29.3.1"
+ }
+}

integration-tests/lmdb-regeneration/src/pages/404.jsx

@@ -0,0 +1,49 @@
+import * as React from "react"
+import { Link } from "gatsby"
+
+const pageStyles = {
+ color: "#232129",
+ padding: "96px",
+ fontFamily: "-apple-system, Roboto, sans-serif, serif",
+}
+const headingStyles = {
+ marginTop: 0,
+ marginBottom: 64,
+ maxWidth: 320,
+}
+
+const paragraphStyles = {
+ marginBottom: 48,
+}
+const codeStyles = {
+ color: "#8A6534",
+ padding: 4,
+ backgroundColor: "#FFF4DB",
+ fontSize: "1.25rem",
+ borderRadius: 4,
+}
+
+const NotFoundPage = () => {
+ return (
+ <main style={pageStyles}>
+ <h1 style={headingStyles}>Page not found</h1>
+ <p style={paragraphStyles}>
+ Sorry 😔, we couldn’t find what you were looking for.
+ <br />
+ {process.env.NODE_ENV === "development" ? (
+ <>
+ <br />
+ Try creating a page in <code style={codeStyles}>src/pages/</code>.
+ <br />
+ </>
+ ) : null}
+ <br />
+ <Link to="/">Go home</Link>.
+ </p>
+ </main>
+ )
+}
+
+export default NotFoundPage
+
+export const Head = () => <title>Not found</title>

integration-tests/lmdb-regeneration/src/pages/test-dsg.jsx

@@ -0,0 +1,17 @@
+import * as React from "react";
+
+const DSG = () => {
+ return <h1>DSG</h1>;
+};
+
+export default DSG;
+
+export const Head = () => <title>DSG</title>;
+
+export async function config() {
+ return () => {
+ return {
+ defer: true,
+ };
+ };
+}

packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts

@@ -2,8 +2,10 @@
 
 import * as path from "path"
 import * as fs from "fs-extra"
+import execa, { Options as ExecaOptions } from "execa"
 import webpack, { Module, NormalModule, Compilation } from "webpack"
 import ConcatenatedModule from "webpack/lib/optimize/ConcatenatedModule"
+import { dependencies } from "gatsby/package.json"
 import { printQueryEnginePlugins } from "./print-plugins"
 import mod from "module"
 import { WebpackLoggingPlugin } from "../../utils/webpack/plugins/webpack-logging"
@@ -12,6 +14,7 @@ import { schemaCustomizationAPIs } from "./print-plugins"
 import type { GatsbyNodeAPI } from "../../redux/types"
 import * as nodeApis from "../../utils/api-node-docs"
 import { store } from "../../redux"
+import { PackageJson } from "../../.."
 
 type Reporter = typeof reporter
 
@@ -35,6 +38,92 @@ function getApisToRemoveForQueryEngine(): Array<GatsbyNodeAPI> {
  return apisToRemove
 }
 
+const getInternalPackagesCacheDir = (): string =>
+ path.join(process.cwd(), `.cache/internal-packages`)
+
+// Create a directory and JS module where we install internally used packages
+const createInternalPackagesCacheDir = async (): Promise<void> => {
+ const cacheDir = getInternalPackagesCacheDir()
+ await fs.ensureDir(cacheDir)
+ await fs.emptyDir(cacheDir)
+
+ const packageJsonPath = path.join(cacheDir, `package.json`)
+
+ await fs.outputJson(packageJsonPath, {
+ name: `gatsby-internal-packages`,
+ description: `This directory contains internal packages installed by Gatsby used to comply with the current platform requirements`,
+ version: `1.0.0`,
+ private: true,
+ author: `Gatsby`,
+ license: `MIT`,
+ })
+}
+
+// lmdb module with prebuilt binaries for our platform
+const lmdbPackage = `@lmdb/lmdb-${process.platform}-${process.arch}`
+
+// Detect if the prebuilt binaries for lmdb have been installed. These are installed under @lmdb and are tied to each platform/arch. We've seen instances where regular installations lack these modules because of a broken lockfile or skipping optional dependencies installs
+function installPrebuiltLmdb(): boolean {
+ // Read lmdb's package.json, go through its optional depedencies and validate if there's a prebuilt lmdb module with a compatible binary to our platform and arch
+ let packageJson: PackageJson
+ try {
+ const modulePath = path
+ .dirname(require.resolve(`lmdb`))
+ .replace(`/dist`, ``)
+ const packageJsonPath = path.join(modulePath, `package.json`)
+ packageJson = JSON.parse(fs.readFileSync(packageJsonPath, `utf-8`))
+ } catch (e) {
+ // If we fail to read lmdb's package.json there's bigger problems here so just skip installation
+ return false
+ }
+ // If there's no lmdb prebuilt package for our arch/platform listed as optional dep no point in trying to install it
+ const { optionalDependencies } = packageJson
+ if (!optionalDependencies) return false
+ if (!Object.keys(optionalDependencies).find(p => p === lmdbPackage))
+ return false
+ try {
+ const lmdbRequire = mod.createRequire(require.resolve(`lmdb`))
+ lmdbRequire.resolve(lmdbPackage)
+ return false
+ } catch (e) {
+ return true
+ }
+}
+
+// Install lmdb's native system module under our internal cache if we detect the current installation
+// isn't using the pre-build binaries
+async function installIfMissingLmdb(): Promise<string | undefined> {
+ if (!installPrebuiltLmdb()) return undefined
+
+ await createInternalPackagesCacheDir()
+
+ const cacheDir = getInternalPackagesCacheDir()
+ const options: ExecaOptions = {
+ stderr: `inherit`,
+ cwd: cacheDir,
+ }
+
+ const npmAdditionalCliArgs = [
+ `--no-progress`,
+ `--no-audit`,
+ `--no-fund`,
+ `--loglevel`,
+ `error`,
+ `--color`,
+ `always`,
+ `--legacy-peer-deps`,
+ `--save-exact`,
+ ]
+
+ await execa(
+ `npm`,
+ [`install`, ...npmAdditionalCliArgs, `${lmdbPackage}@${dependencies.lmdb}`],
+ options
+ )
+
+ return path.join(cacheDir, `node_modules`, lmdbPackage)
+}
+
 export async function createGraphqlEngineBundle(
  rootDir: string,
  reporter: Reporter,
@@ -57,6 +146,19 @@ export async function createGraphqlEngineBundle(
  require.resolve(`gatsby-plugin-typescript`)
  )
 
+ // Alternative lmdb path we've created to self heal from a "broken" lmdb installation
+ const alternativeLmdbPath = await installIfMissingLmdb()
+
+ // We force a specific lmdb binary module if we detected a broken lmdb installation or if we detect the presence of an adapter
+ let forcedLmdbBinaryModule: string | undefined = undefined
+ if (store.getState().adapter.instance) {
+ forcedLmdbBinaryModule = `${lmdbPackage}/node.abi83.glibc.node`
+ }
+ // We always force the binary if we've installed an alternative path
+ if (alternativeLmdbPath) {
+ forcedLmdbBinaryModule = `${alternativeLmdbPath}/node.abi83.glibc.node`
+ }
+
  const compiler = webpack({
  name: `Query Engine`,
  // mode: `production`,
@@ -121,9 +223,7 @@ export async function createGraphqlEngineBundle(
  {
  loader: require.resolve(`./lmdb-bundling-patch`),
  options: {
- forcedBinaryModule: store.getState().adapter.instance
- ? `@lmdb/lmdb-${process.platform}-${process.arch}/node.abi83.glibc.node`
- : undefined,
+ forcedBinaryModule: forcedLmdbBinaryModule,
  },
  },
  ],