Merge branch 'GoogleChrome:main' into main

Author: Z-byte-prog

.cloudbuild/cleanup.yaml

@@ -1,4 +1,4 @@
 steps:
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  args: ['-c', './.cloudbuild/cleanup.sh']
+  - name: 'gcr.io/cloud-builders/gcloud'
+    entrypoint: 'bash'
+    args: ['-c', './.cloudbuild/cleanup.sh']

.cloudbuild/deploy.yaml

@@ -1,30 +1,36 @@
 timeout: 2700s # set build timeout to 45 mins
 steps:
-- name: node
-  entrypoint: npm
-  args: ['ci']
-- name: node
-  entrypoint: npm
-  args: ['run', 'cloud-secrets']
-  env:
-  - 'PROJECT_ID=$PROJECT_ID'
-- name: node
-  entrypoint: npm
-  args: ['run', 'production']
-- name: node
-  entrypoint: npm
-  args: ['run', 'algolia']
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  args:
-  - '-c'
-  - |
-    # This snippet lets us pass additional args via the Cloud Build console
-    # in the _EXTRA_GCLOUD_ARGS var.
-    # nb. We don't have to specify --project; it's part of the environment.
-    gcloud app deploy ${_EXTRA_GCLOUD_ARGS}
+  - name: node
+    entrypoint: npm
+    args: ['ci']
+
+  - name: node
+    entrypoint: npm
+    args: ['run', 'cloud-secrets']
+
+  - name: node
+    entrypoint: npm
+    args: ['run', 'production']
+
+  - name: node
+    entrypoint: npm
+    args: ['run', 'algolia']
+
+  - name: 'gcr.io/cloud-builders/gcloud'
+    entrypoint: 'bash'
+    args:
+    - '-c'
+    - |
+      # This snippet lets us pass additional args via the Cloud Build console
+      # in the _EXTRA_GCLOUD_ARGS var.
+      # nb. We don't have to specify --project; it's part of the environment.
+      gcloud app deploy ${_EXTRA_GCLOUD_ARGS}
+
 substitutions:
   _EXTRA_GCLOUD_ARGS:  # default empty
 
 options:
   machineType: 'E2_HIGHCPU_8'  # yolo
+  env:
+  - 'NODE_OPTIONS=--unhandled-rejections=strict'
+  - 'PROJECT_ID=$PROJECT_ID'

.cloudbuild/external.yaml

@@ -0,0 +1,51 @@
+# This builds external data, and if the site passes a basic build, writes the data to
+# Cloud Storage.
+
+timeout: 900s # set build timeout to 15 mins
+
+steps:
+  - name: node:14
+    id: 'Install dependencies'
+    entrypoint: npm
+    args: ['ci']
+
+  - name: node:14
+    id: 'Configure secrets'
+    entrypoint: npm
+    args: ['run', 'cloud-secrets']
+
+  - name: node:14
+    id: 'Build external data'
+    entrypoint: npm
+    args: ['run', 'build-external']
+    env:
+    - 'NODE_ENV=production'
+
+  - name: node:14
+    id: 'Create .eleventyignore file'
+    entrypoint: npm
+    args: ['run', 'ignore']
+    env:
+    # TODO: We should try to remove more stuff here, because it's supposed to
+    # be a fast build to sanity-check output.
+    # It needs to include extensions and API generation.
+    - 'ELEVENTY_IGNORE_NACL=true'
+
+  - name: node:14
+    id: 'Build eleventy in dev mode to confirm'
+    entrypoint: npm
+    args: ['run', 'eleventy']
+    # This does NOT set `NODE_ENV=production`, as we don't need the full build.
+
+  - name: 'gcr.io/cloud-builders/gcloud'
+    id: 'Synchronize content to external-dcc-data bucket'
+    entrypoint: bash
+    args:
+    - '-c'
+    - |
+      gsutil rsync external/data/ gs://external-dcc-data
+
+options:
+  env:
+  - 'PROJECT_ID=$PROJECT_ID'
+  - 'NODE_OPTIONS=--unhandled-rejections=strict'

.eleventy.js

@@ -61,6 +61,9 @@ module.exports = eleventyConfig => {
   // to use it for its build.
   eleventyConfig.setUseGitIgnore(false);
 
+  // Watch our external data in case it is synchronized or rebuilt.
+  eleventyConfig.addWatchTarget('./external/data/');
+
   // Merge eleventy's data cascade. This means directory data files will
   // cascade down to any child directories.
   eleventyConfig.setDataDeepMerge(true);

.github/workflows/check.yml

@@ -7,6 +7,9 @@ jobs:
   lint_and_test:
     name: Lint, test and build
     runs-on: ubuntu-latest
+    env:
+      # Increase RAM limit, and make uncaught exceptions crash (default in 16+).
+      NODE_OPTIONS: --max_old_space_size=4096 --unhandled-rejections=strict
     steps:
       - name: Checkout
         uses: actions/[email protected]
@@ -46,6 +49,8 @@ jobs:
               - '**/*.scss'
             njk:
               - '**/*.njk'
+            _external:
+              - 'external/**/*'
       
       # Use the filter to check if files with a specific file type were changed
       # in the PR. If they were, run the relevant linters. Otherwise, skip.
@@ -66,9 +71,13 @@ jobs:
         if: ${{ steps.filter.outputs.scss == 'true' }}
         run: npm run lint:scss
 
+      - name: Build External
+        if: ${{ steps.filter.outputs._external == 'true' }}
+        run: npm run build-external
+
       - name: Build Eleventy
-        if: ${{ steps.filter.outputs.md == 'true' || steps.filter.outputs.js == 'true' || steps.filter.outputs.yml == 'true' || steps.filter.outputs.njk == 'true'}}
-        run: npm run ignore && npm run eleventy
+        if: ${{ steps.filter.outputs.md == 'true' || steps.filter.outputs.js == 'true' || steps.filter.outputs.yml == 'true' || steps.filter.outputs.njk == 'true' }}
+        run: npm run ignore && npm run maybe-sync-external && npm run eleventy
 
       # Only run tests if the PR touches behavior related files.
       - name: Test

.github/workflows/lighthouse-ci.yml

@@ -15,6 +15,9 @@ jobs:
   lhci:
     if: ${{ github.repository == 'GoogleChrome/developer.chrome.com' }}
     runs-on: ubuntu-latest
+    env:
+      # Increase RAM limit, and make uncaught exceptions crash (default in 16+).
+      NODE_OPTIONS: --max_old_space_size=4096 --unhandled-rejections=strict
     steps:
       - name: Checkout
         uses: actions/[email protected]

.github/workflows/percy.yml

@@ -23,6 +23,9 @@ jobs:
   percy:
     if: ${{ github.repository == 'GoogleChrome/developer.chrome.com' }}
     runs-on: ubuntu-latest
+    env:
+      # Increase RAM limit, and make uncaught exceptions crash (default in 16+).
+      NODE_OPTIONS: --max_old_space_size=4096 --unhandled-rejections=strict
     steps:
       - name: Checkout
         uses: actions/[email protected]

.gitignore

@@ -25,3 +25,7 @@ dist
 # Eleventy
 # We generate our own .eleventyignore file dynamically during builds
 .eleventyignore
+
+# External source files
+external/data/
+external/local-build-flag

external/README.md

@@ -0,0 +1,23 @@
+This folder contains the system that deals with external data for developer.chrome.com, that is, data that regularly changes yet is required to publish the site.
+
+- the "data/" folder is used as a working dir (and isn't checked in)
+
+- all scripts inside "build/" are run when `npm run build-external` is run, and a local flag is set
+
+- otherwise, run `npm run sync-external` to retrieve the last known good files stored in Cloud Storage
+
+- running `npm run dev` will automatically pull external data
+
+## Changes
+
+If you make changes to the build script(s), you can manually kick off a Cloud Build task to confirm that the output builds and to write it to storage for other users.
+
+```bash
+$ gcloud builds submit --config .cloudbuild/external.yaml .
+```
+
+You can confirm the contents of the bucket by:
+
+```bash
+$ gsutil ls -l gs://external-dcc-data
+```

external/build-external.js

@@ -0,0 +1,84 @@
+/**
+ * @fileoverview Builds all external sources. This clears the data/ folder here
+ * and runs all scripts in build/, which can write new files there.
+ *
+ * This is intended for use by Cloud Build, or by site devs doing local work.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const glob = require('glob');
+const childProcess = require('child_process');
+const crypto = require('crypto');
+const syncTestdata = require('./lib/sync-testdata');
+
+async function run() {
+  let errors = 0;
+
+  const scripts = glob.sync('build/*.js', {cwd: __dirname});
+  scripts.sort(); // run in alphabetical order
+
+  const projectRoot = path.join(__dirname, '..');
+
+  const dataTarget = path.join(__dirname, 'data');
+  fs.rmSync(dataTarget, {recursive: true, force: true});
+  fs.mkdirSync(dataTarget, {recursive: true});
+
+  // If this is a CI build, we start with everything found in "fallback/". It won't win, but it
+  // will be used in cases where credentials and such aren't available.
+  if (process.env.CI) {
+    const all = await syncTestdata();
+    console.info('! Using fallback before build in CI, copied:', all);
+  }
+
+  /** @type {childProcess.CommonExecOptions} */
+  const options = {cwd: projectRoot, stdio: 'inherit'};
+
+  for (const script of scripts) {
+    const r = path.join(__dirname, script);
+    console.info('> Running', r);
+    try {
+      childProcess.execFileSync('node', [r], options);
+    } catch (e) {
+      // We don't log the error here, as we're already getting STDERR piped above.
+      console.warn(`! Failed to execute "${script}" (${e.status})`);
+      ++errors;
+    }
+  }
+
+  // Determine the hash for everything in data/.
+  const hash = crypto.createHash('sha256');
+  const allFiles = glob.sync('data/**/*', {cwd: __dirname});
+  if (!allFiles.length) {
+    throw new Error('no files generated, cowardly refusing to hash');
+  }
+
+  // Sort allFiles, in case glob.sync is inconsistent.
+  allFiles.sort();
+
+  for (const f of allFiles) {
+    const p = path.join(__dirname, f);
+    const bytes = fs.readFileSync(p);
+    hash.update(bytes);
+  }
+  const digest = hash.digest('hex');
+  console.info(
+    `@ Generated digest=${digest} for ${allFiles.length} files:`,
+    allFiles
+  );
+  fs.writeFileSync(path.join(__dirname, 'data/.hash'), digest);
+
+  // If there were any errors, return with a non-zero status code anyway.
+  if (errors) {
+    // eslint-disable-next-line no-process-exit
+    process.exit(1);
+  }
+
+  // Mark this local environment as being build-only, so it won't automatically sync.
+  const payload =
+    '// This file blocks synchronizing local data, because you ran `npm run build-external`.\n' +
+    '// Delete it to bring back automatic sync when you run `npm run dev`.';
+  fs.writeFileSync(path.join(__dirname, 'local-build-flag'), payload);
+}
+
+run();

external/build/chrome-release.js

@@ -0,0 +1,16 @@
+/**
+ * @fileoverview Fetches Chrome release data and writes to storage.
+ */
+
+const buildVersionInformation = require('./lib/chrome-release.js');
+const fs = require('fs');
+const path = require('path');
+
+async function run() {
+  const versionInformation = await buildVersionInformation();
+
+  const targetFile = path.join(__dirname, '../data/chrome-release.json');
+  fs.writeFileSync(targetFile, JSON.stringify(versionInformation));
+}
+
+run();