chore(benchmarks): Add ability to fix broken images for contentful si…

…tes (#22882)

* chore(benchmarks): Add ability to fix broken images for contentful sites

* fix: added missing await

* fix: removed accidental console.log

benchmarks/source-contentful/README.md

@@ -7,16 +7,28 @@ Those individual article pages and the homepage share a common "Layout" componen
 
 ## Setup Contentful benchmark site
 
-1. TODO <Setup data source>
+1. Setup will-it-build data source
 2. Copy `.env.example` to `.env.development` and make sure all variables are set
 3. Run `yarn setup`
 
 Note that the script is idempotent, so you can re-run it on failures.
 
 Also use `yarn setup --skip [N:number]` to skip first `N` articles
-(for example articles created during a previous run)
+(for example articles created during a previous run which failed)
+
+### Fixing broken images
+
+Sometimes Contentful silently fails to process images which causes builds to fail.
+Use following approach to fix those:
+
+1. Run `yarn site find-broken-images`
+2. Change image URLs in will-it-build dataset for this site to some other images
+ (or just use one of the larger sites and set `BENCHMARK_SITE_ID` appropriately)
+3. Run `yarn site fix-broken-images imageid1 imageid2 imageid3`
+ This command updates broken images with images from the `BENCHMARK_SITE_ID` dataset
 
 ## Build a site
 
-1. Copy `.env.example` to `.env.production` and make sure all variables are set
-2. Run `yarn build`
+1. Copy `.env.example` to `.env.production`
+2. Set `BENCHMARK_CONTENTFUL_SPACE_ID` and `BENCHMARK_CONTENTFUL_ACCESS_TOKEN` variables
+3. Run `yarn build`

benchmarks/source-contentful/bin/setup.js → benchmarks/source-contentful/bin/site.js

@@ -2,6 +2,7 @@ const fs = require(`fs`)
 const path = require(`path`)
 const contentful = require(`contentful-management`)
 const chalk = require("chalk")
+const yargs = require("yargs")
 
 require("dotenv").config({
  path: `.env.${process.env.NODE_ENV}`,
@@ -29,12 +30,43 @@ const contentfulConfig = {
 const { spaceId, managementToken } = contentfulConfig
 
 if (!spaceId || !managementToken) {
- console.error(
- `Contentful space id and management API token are required.`
- )
+ console.error(`Contentful space id and management API token are required.`)
  process.exit(1)
 }
 
+yargs
+ .scriptName("site")
+ .usage("$0 <command> [arguments]")
+ .command({
+ command: `setup [--skip=number]`,
+ desc: `Setup new Contentful Benchmark Site from the dataset`,
+ builder: yargs =>
+ yargs.option(`skip`, {
+ type: `number`,
+ default: 0,
+ description: `Skip this number of entries`,
+ }),
+ handler: ({ skip = 0 }) => {
+ runSetup({ skip }).catch(console.error)
+ },
+ })
+ .command({
+ command: "find-broken-images",
+ desc: `Find broken images in current contentful site`,
+ handler: () => {
+ runFindBrokenImages().catch(console.error)
+ },
+ })
+ .command({
+ command: "fix-broken-images <ids...>",
+ desc: `Fix images found by find-broken-images`,
+ handler: ({ ids }) => {
+ runFixBrokenImages(ids).catch(console.error)
+ },
+ })
+ .demandCommand(1)
+ .help().argv
+
 /**
  * Transforms an article from source dataset to contentful data model
  */
@@ -201,16 +233,7 @@ async function createArticle(env, articleData) {
  }
 }
 
-const resolveSkip = () => {
- const index = process.argv.findIndex(param => param === `--skip`)
- if (index >= 0) {
- const skipValue = process.argv[index + 1]
- return Number(skipValue) || 0
- }
- return 0
-}
-
-async function createEntries(env) {
+async function createEntries({ env, skip = 0 }) {
  const processBatch = sourceArticles =>
  Promise.all(
  sourceArticles.map(async sourceArticle => {
@@ -219,14 +242,12 @@ async function createEntries(env) {
  const articleCreated = await createArticle(env, article)
  console.log(
  `Processed ${chalk.green(article.sys.id)} (` +
- `asset ${assetCreated ? `created` : chalk.yellow(`exists`)}, ` +
- `article ${articleCreated ? `created` : chalk.yellow(`exists`)})`
+  `asset ${assetCreated ? `created` : chalk.yellow(`exists`)}, ` +
+  `article ${articleCreated ? `created` : chalk.yellow(`exists`)})`
  )
  })
  )
 
- const skip = resolveSkip()
-
  if (skip) {
  console.log(`Skipping first ${chalk.yellow(skip)} articles`)
  }
@@ -248,28 +269,87 @@ async function createEntries(env) {
  }
 }
 
-async function run() {
+async function updateAssets({ env, assetIds }) {
+ for await (const sourceArticle of readSourceArticles(inputDir)) {
+ const { asset: assetData } = extractEntities(sourceArticle)
+ if (assetIds.has(assetData.sys.id)) {
+ try {
+ let asset = await env.getAsset(assetData.sys.id)
+ try {
+ asset = await asset.unpublish()
+ } catch (e) {}
+ asset = await asset.delete()
+ asset = await createAsset(env, assetData)
+ console.log(`Updated asset ${chalk.yellow(assetData.sys.id)}`)
+ } catch (e) {
+ console.warn(`Could not update asset ${chalk.yellow(assetData.sys.id)}`)
+ console.log(e)
+ }
+ }
+ }
+}
+
+async function findBrokenImages(env) {
+ let assets
+ let skip = 0
+ let ids = []
+
+ do {
+ assets = await env.getAssets({ skip })
+ for (let asset of assets.items) {
+ const details = asset.fields.file[`en-US`].details
+ if (!details || !details.image || !details.image.width) {
+ ids.push(asset.sys.id)
+ }
+ }
+ skip = assets.skip + assets.limit
+ } while (assets && assets.items.length > 0)
+ return ids
+}
+
+async function createClient() {
  const client = contentful.createClient({
  accessToken: contentfulConfig.managementToken,
  })
 
  const space = await client.getSpace(contentfulConfig.spaceId)
  const env = await space.getEnvironment(`master`)
 
- // Create content model only:
- createContentModel(env)
- .then(() => {
- console.log(`Content model ${chalk.green(`created`)}`)
- })
- .then(() => createEntries(env))
- .then(() => {
- console.log(
- `All set! You can now run ${chalk.yellow(
- "gatsby develop"
- )} to see the site in action.`
- )
- })
- .catch(error => console.error(error))
+ return { client, space, env }
+}
+
+async function runSetup({ skip }) {
+ const { env } = await createClient()
+
+ await createContentModel(env)
+ console.log(`Content model ${chalk.green(`created`)}`)
+ await createEntries({ env, skip })
+
+ console.log(
+ `All set! You can now run ${chalk.yellow(
+ "gatsby develop"
+ )} to see the site in action.`
+ )
+}
+
+async function runFindBrokenImages() {
+ const { env } = await createClient()
+ const ids = await findBrokenImages(env)
+ if (ids.length) {
+ console.log(chalk.yellow(`Broken images:`))
+ console.log(ids.join(` `))
+ console.log(``)
+ } else {
+ console.log(chalk.green(`No broken images!`))
+ }
 }
 
-run().catch(console.error)
+async function runFixBrokenImages(ids) {
+ if (!ids.length) {
+ console.log(`Nothing to do: no broken images!`)
+ return
+ }
+ const { env } = await createClient()
+ console.log(`Fixing ${chalk.yellow(ids.length)} broken images`)
+ await updateAssets({ env, assetIds: new Set(ids) })
+}

benchmarks/source-contentful/package.json

@@ -10,7 +10,8 @@
  "develop": "gatsby develop",
  "format": "prettier --write \"**/*.{js,jsx,json,md}\"",
  "serve": "gatsby serve",
- "setup": "cross-env NODE_ENV=development node bin/setup.js",
+ "site": "cross-env NODE_ENV=development node bin/site.js",
+ "setup": "cross-env NODE_ENV=development node bin/site.js setup",
  "start": "npm run develop",
  "test": "echo \"Write tests! -> https://gatsby.dev/unit-testing\" && exit 1"
  },
@@ -31,7 +32,8 @@
  "chalk": "^2.4.2",
  "cross-env": "^7.0.0",
  "gatsby-plugin-benchmark-reporting": "*",
- "prettier": "^1.19.1"
+ "prettier": "^1.19.1",
+ "yargs": "^15.3.1"
  },
  "repository": {
  "type": "git",