cleanup external task (GoogleChrome#1561)

Author: Sam Thorogood

.cloudbuild/cleanup.yaml

@@ -1,4 +1,4 @@
 steps:
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  args: ['-c', './.cloudbuild/cleanup.sh']
+  - name: 'gcr.io/cloud-builders/gcloud'
+    entrypoint: 'bash'
+    args: ['-c', './.cloudbuild/cleanup.sh']

.cloudbuild/deploy.yaml

@@ -1,32 +1,36 @@
 timeout: 2700s # set build timeout to 45 mins
 steps:
-- name: node
-  entrypoint: npm
-  args: ['ci']
-- name: node
-  entrypoint: npm
-  args: ['run', 'cloud-secrets']
-  env:
-  - 'PROJECT_ID=$PROJECT_ID'
-- name: node
-  entrypoint: npm
-  args: ['run', 'production']
-- name: node
-  entrypoint: npm
-  args: ['run', 'algolia']
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  args:
-  - '-c'
-  - |
-    # This snippet lets us pass additional args via the Cloud Build console
-    # in the _EXTRA_GCLOUD_ARGS var.
-    # nb. We don't have to specify --project; it's part of the environment.
-    gcloud app deploy ${_EXTRA_GCLOUD_ARGS}
+  - name: node
+    entrypoint: npm
+    args: ['ci']
+
+  - name: node
+    entrypoint: npm
+    args: ['run', 'cloud-secrets']
+
+  - name: node
+    entrypoint: npm
+    args: ['run', 'production']
+
+  - name: node
+    entrypoint: npm
+    args: ['run', 'algolia']
+
+  - name: 'gcr.io/cloud-builders/gcloud'
+    entrypoint: 'bash'
+    args:
+    - '-c'
+    - |
+      # This snippet lets us pass additional args via the Cloud Build console
+      # in the _EXTRA_GCLOUD_ARGS var.
+      # nb. We don't have to specify --project; it's part of the environment.
+      gcloud app deploy ${_EXTRA_GCLOUD_ARGS}
+
 substitutions:
   _EXTRA_GCLOUD_ARGS:  # default empty
 
 options:
   machineType: 'E2_HIGHCPU_8'  # yolo
   env:
-  - 'NODE_OPTIONS=--max_old_space_size=4096 --unhandled-rejections=strict'
+  - 'NODE_OPTIONS=--unhandled-rejections=strict'
+  - 'PROJECT_ID=$PROJECT_ID'

.cloudbuild/external.yaml

@@ -9,12 +9,16 @@ steps:
     entrypoint: npm
     args: ['ci']
 
+  - name: node:14
+    id: 'Configure secrets'
+    entrypoint: npm
+    args: ['run', 'cloud-secrets']
+
   - name: node:14
     id: 'Build external data'
     entrypoint: npm
     args: ['run', 'build-external']
     env:
-    - 'PROJECT_ID=$PROJECT_ID'
     - 'NODE_ENV=production'
 
   - name: node:14
@@ -43,4 +47,5 @@ steps:
 
 options:
   env:
-  - 'NODE_OPTIONS=--max_old_space_size=4096 --unhandled-rejections=strict'
+  - 'PROJECT_ID=$PROJECT_ID'
+  - 'NODE_OPTIONS=--unhandled-rejections=strict'

external/README.md

@@ -7,3 +7,17 @@ This folder contains the system that deals with external data for developer.chro
 - otherwise, run `npm run sync-external` to retrieve the last known good files stored in Cloud Storage
 
 - running `npm run dev` will automatically pull external data
+
+## Changes
+
+If you make changes to the build script(s), you can manually kick off a Cloud Build task to confirm that the output builds and to write it to storage for other users.
+
+```bash
+$ gcloud builds submit --config .cloudbuild/external.yaml .
+```
+
+You can confirm the contents of the bucket by:
+
+```bash
+$ gsutil ls -l gs://external-dcc-data
+```