|
| 1 | +workflow: |
| 2 | + rules: |
| 3 | + # Disable merge request pipelines |
| 4 | + - if: $CI_MERGE_REQUEST_ID |
| 5 | + when: never |
| 6 | + - when: always |
| 7 | + |
| 8 | +variables: |
| 9 | + GIT_SUBMODULE_STRATEGY: recursive |
| 10 | + GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}" |
| 11 | + GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git" |
| 12 | + # Cache .npm |
| 13 | + npm_config_cache: "${CI_PROJECT_DIR}/tmp/npm" |
| 14 | + # Prefer offline node module installation |
| 15 | + npm_config_prefer_offline: "true" |
| 16 | + # Homebrew cache only used by macos runner |
| 17 | + HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew" |
| 18 | + |
| 19 | +default: |
| 20 | + interruptible: true |
| 21 | + before_script: |
| 22 | + # Replace this in windows runners that use powershell |
| 23 | + # with `mkdir -Force "$CI_PROJECT_DIR/tmp"` |
| 24 | + - mkdir -p "$CI_PROJECT_DIR/tmp" |
| 25 | + |
| 26 | +# Cached directories shared between jobs & pipelines per-branch per-runner |
| 27 | +cache: |
| 28 | + key: $CI_COMMIT_REF_SLUG |
| 29 | + # Preserve cache even if job fails |
| 30 | + when: 'always' |
| 31 | + paths: |
| 32 | + - ./tmp/npm/ |
| 33 | + # Homebrew cache is only used by the macos runner |
| 34 | + - ./tmp/Homebrew |
| 35 | + # Chocolatey cache is only used by the windows runner |
| 36 | + - ./tmp/chocolatey/ |
| 37 | + # `jest` cache is configured in jest.config.js |
| 38 | + - ./tmp/jest/ |
| 39 | + |
| 40 | +stages: |
| 41 | + - check # Linting, unit tests |
| 42 | + - build # Cross-platform library compilation, unit tests |
| 43 | + - integration # Cross-platform application bundling, integration tests, and pre-release |
| 44 | + - release # Cross-platform distribution and deployment |
| 45 | + |
| 46 | +image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner |
| 47 | + |
| 48 | +check:lint: |
| 49 | + stage: check |
| 50 | + needs: [] |
| 51 | + script: |
| 52 | + - > |
| 53 | + nix-shell --arg ci true --run $' |
| 54 | + npm run lint; |
| 55 | + npm run lint-shell; |
| 56 | + ' |
| 57 | + rules: |
| 58 | + # Runs on feature and staging commits and ignores version commits |
| 59 | + - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 60 | + # Runs on tag pipeline where the tag is a prerelease or release version |
| 61 | + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 62 | + # Manually run on commits other than master and ignore version commits |
| 63 | + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 64 | + when: manual |
| 65 | + |
| 66 | +check:test: |
| 67 | + stage: check |
| 68 | + needs: [] |
| 69 | + script: |
| 70 | + - > |
| 71 | + nix-shell --arg ci true --run $' |
| 72 | + npm run prebuild --verbose; |
| 73 | + npm test -- --ci --coverage; |
| 74 | + ' |
| 75 | + artifacts: |
| 76 | + when: always |
| 77 | + reports: |
| 78 | + junit: |
| 79 | + - ./tmp/junit/junit.xml |
| 80 | + coverage_report: |
| 81 | + coverage_format: cobertura |
| 82 | + path: ./tmp/coverage/cobertura-coverage.xml |
| 83 | + coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' |
| 84 | + rules: |
| 85 | + # Runs on feature commits and ignores version commits |
| 86 | + - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 87 | + # Manually run on commits other than master and staging and ignore version commits |
| 88 | + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 89 | + when: manual |
| 90 | + |
| 91 | +build:merge: |
| 92 | + stage: build |
| 93 | + needs: [] |
| 94 | + allow_failure: true |
| 95 | + script: |
| 96 | + # Required for `gh pr create` |
| 97 | + - git remote add upstream "$GH_PROJECT_URL" |
| 98 | + - > |
| 99 | + nix-shell --arg ci true --run $' |
| 100 | + gh pr create \ |
| 101 | + --head staging \ |
| 102 | + --base master \ |
| 103 | + --title "ci: merge staging to master" \ |
| 104 | + --body "This is an automatic PR generated by the pipeline CI/CD. This will be automatically fast-forward merged if successful." \ |
| 105 | + --assignee "@me" \ |
| 106 | + --no-maintainer-edit \ |
| 107 | + --repo "$GH_PROJECT_PATH" || true; |
| 108 | + printf "Pipeline Attempt on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \ |
| 109 | + | gh pr comment staging \ |
| 110 | + --body-file - \ |
| 111 | + --repo "$GH_PROJECT_PATH"; |
| 112 | + ' |
| 113 | + rules: |
| 114 | + # Runs on staging commits and ignores version commits |
| 115 | + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 116 | + # Runs on tag pipeline where the tag is a prerelease or release version |
| 117 | + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 118 | + |
| 119 | +build:dist: |
| 120 | + stage: build |
| 121 | + needs: [] |
| 122 | + script: |
| 123 | + - > |
| 124 | + nix-shell --arg ci true --run $' |
| 125 | + npm run build --ignore-scripts --verbose; |
| 126 | + ' |
| 127 | + artifacts: |
| 128 | + when: always |
| 129 | + paths: |
| 130 | + - ./dist |
| 131 | + rules: |
| 132 | + # Runs on staging commits and ignores version commits |
| 133 | + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 134 | + # Runs on tag pipeline where the tag is a prerelease or release version |
| 135 | + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 136 | + |
| 137 | +build:linux: |
| 138 | + stage: build |
| 139 | + needs: [] |
| 140 | + variables: |
| 141 | + # Only x64 architecture is needed |
| 142 | + npm_config_arch: "x64" |
| 143 | + script: |
| 144 | + - > |
| 145 | + nix-shell --arg ci true --run $' |
| 146 | + npm run prebuild --verbose -- --production; |
| 147 | + npm test -- --ci --coverage; |
| 148 | + npm run bench; |
| 149 | + ' |
| 150 | + artifacts: |
| 151 | + when: always |
| 152 | + reports: |
| 153 | + junit: |
| 154 | + - ./tmp/junit/junit.xml |
| 155 | + coverage_report: |
| 156 | + coverage_format: cobertura |
| 157 | + path: ./tmp/coverage/cobertura-coverage.xml |
| 158 | + metrics: ./benches/results/metrics.txt |
| 159 | + paths: |
| 160 | + - ./prebuild/ |
| 161 | + coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' |
| 162 | + rules: |
| 163 | + # Runs on staging commits and ignores version commits |
| 164 | + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 165 | + # Runs on tag pipeline where the tag is a prerelease or release version |
| 166 | + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 167 | + |
| 168 | +build:windows: |
| 169 | + stage: build |
| 170 | + needs: [] |
| 171 | + tags: |
| 172 | + - windows |
| 173 | + variables: |
| 174 | + # Only x64 architecture is needed |
| 175 | + npm_config_arch: "x64" |
| 176 | + before_script: |
| 177 | + - mkdir -Force "$CI_PROJECT_DIR/tmp" |
| 178 | + script: |
| 179 | + - ./scripts/choco-install.ps1 |
| 180 | + - refreshenv |
| 181 | + - npm install --ignore-scripts |
| 182 | + - $env:Path = "$(npm bin);" + $env:Path |
| 183 | + - npm run prebuild --verbose -- --production |
| 184 | + - npm test -- --ci --coverage |
| 185 | + - npm run bench |
| 186 | + artifacts: |
| 187 | + when: always |
| 188 | + reports: |
| 189 | + junit: |
| 190 | + - ./tmp/junit/junit.xml |
| 191 | + coverage_report: |
| 192 | + coverage_format: cobertura |
| 193 | + path: ./tmp/coverage/cobertura-coverage.xml |
| 194 | + metrics: ./benches/results/metrics.txt |
| 195 | + paths: |
| 196 | + - ./prebuild/ |
| 197 | + rules: |
| 198 | + # Runs on staging commits and ignores version commits |
| 199 | + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 200 | + # Runs on tag pipeline where the tag is a prerelease or release version |
| 201 | + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 202 | + |
| 203 | +build:macos: |
| 204 | + stage: build |
| 205 | + needs: [] |
| 206 | + tags: |
| 207 | + - shared-macos-amd64 |
| 208 | + image: macos-11-xcode-12 |
| 209 | + variables: |
| 210 | + # Produce universal binary |
| 211 | + npm_config_arch: 'x64+arm64' |
| 212 | + script: |
| 213 | + - eval "$(brew shellenv)" |
| 214 | + - ./scripts/brew-install.sh |
| 215 | + - hash -r |
| 216 | + - npm install --ignore-scripts |
| 217 | + - export PATH="$(npm bin):$PATH" |
| 218 | + - npm run prebuild --verbose -- --production |
| 219 | + - npm test -- --ci --coverage |
| 220 | + - npm run bench |
| 221 | + artifacts: |
| 222 | + when: always |
| 223 | + reports: |
| 224 | + junit: |
| 225 | + - ./tmp/junit/junit.xml |
| 226 | + coverage_report: |
| 227 | + coverage_format: cobertura |
| 228 | + path: ./tmp/coverage/cobertura-coverage.xml |
| 229 | + metrics: ./benches/results/metrics.txt |
| 230 | + paths: |
| 231 | + - ./prebuild/ |
| 232 | + rules: |
| 233 | + # Runs on staging commits and ignores version commits |
| 234 | + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 235 | + # Runs on tag pipeline where the tag is a prerelease or release version |
| 236 | + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 237 | + |
| 238 | +build:prerelease: |
| 239 | + stage: build |
| 240 | + needs: |
| 241 | + - build:dist |
| 242 | + - build:linux |
| 243 | + - build:windows |
| 244 | + - build:macos |
| 245 | + # Don't interrupt publishing job |
| 246 | + interruptible: false |
| 247 | + variables: |
| 248 | + # Set the prerelease tag for prepublishOnly script |
| 249 | + npm_config_tag: 'prerelease' |
| 250 | + script: |
| 251 | + - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc |
| 252 | + - echo 'Publishing library prerelease' |
| 253 | + - > |
| 254 | + nix-shell --arg ci true --run $' |
| 255 | + npm publish --tag prerelease --access public; |
| 256 | + ' |
| 257 | + - > |
| 258 | + for d in prebuild/*; do |
| 259 | + tar \ |
| 260 | + --create \ |
| 261 | + --verbose \ |
| 262 | + --file="prebuild/$(basename $d).tar" \ |
| 263 | + --directory=prebuild \ |
| 264 | + "$(basename $d)"; |
| 265 | + done |
| 266 | + - > |
| 267 | + nix-shell --arg ci true --run $' |
| 268 | + gh release \ |
| 269 | + create "$CI_COMMIT_TAG" \ |
| 270 | + prebuild/*.tar \ |
| 271 | + --title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ |
| 272 | + --notes "" \ |
| 273 | + --prerelease \ |
| 274 | + --target staging \ |
| 275 | + --repo "$GH_PROJECT_PATH"; |
| 276 | + ' |
| 277 | + after_script: |
| 278 | + - rm -f ./.npmrc |
| 279 | + rules: |
| 280 | + # Only runs on tag pipeline where the tag is a prerelease version |
| 281 | + # This requires dependencies to also run on tag pipeline |
| 282 | + # However version tag comes with a version commit |
| 283 | + # Dependencies must not run on the version commit |
| 284 | + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+-.*[0-9]+$/ |
| 285 | + |
| 286 | +integration:merge: |
| 287 | + stage: integration |
| 288 | + needs: |
| 289 | + - build:merge |
| 290 | + - job: build:linux |
| 291 | + optional: true |
| 292 | + - job: build:windows |
| 293 | + optional: true |
| 294 | + - job: build:macos |
| 295 | + optional: true |
| 296 | + # Requires mutual exclusion |
| 297 | + resource_group: integration:merge |
| 298 | + allow_failure: true |
| 299 | + variables: |
| 300 | + # Ensure that CI/CD is fetching all commits |
| 301 | + # this is necessary to checkout origin/master |
| 302 | + # and to also merge origin/staging |
| 303 | + GIT_DEPTH: 0 |
| 304 | + script: |
| 305 | + - > |
| 306 | + nix-shell --arg ci true --run $' |
| 307 | + printf "Pipeline Succeeded on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \ |
| 308 | + | gh pr comment staging \ |
| 309 | + --body-file - \ |
| 310 | + --repo "$GH_PROJECT_PATH"; |
| 311 | + ' |
| 312 | + - git remote add upstream "$GH_PROJECT_URL" |
| 313 | + - git checkout origin/master |
| 314 | + # Merge up to the current commit (not the latest commit) |
| 315 | + - git merge --ff-only "$CI_COMMIT_SHA" |
| 316 | + - git push upstream HEAD:master |
| 317 | + rules: |
| 318 | + # Runs on staging commits and ignores version commits |
| 319 | + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 320 | + # Runs on tag pipeline where the tag is a prerelease or release version |
| 321 | + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ |
| 322 | + |
| 323 | +release:distribution: |
| 324 | + stage: release |
| 325 | + needs: |
| 326 | + - build:dist |
| 327 | + - build:linux |
| 328 | + - build:windows |
| 329 | + - build:macos |
| 330 | + - integration:merge |
| 331 | + # Don't interrupt publishing job |
| 332 | + interruptible: false |
| 333 | + script: |
| 334 | + - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc |
| 335 | + - echo 'Publishing library' |
| 336 | + - > |
| 337 | + nix-shell --arg ci true --run $' |
| 338 | + npm publish --access public; |
| 339 | + ' |
| 340 | + - > |
| 341 | + for d in prebuild/*; do |
| 342 | + tar \ |
| 343 | + --create \ |
| 344 | + --verbose \ |
| 345 | + --file="prebuild/$(basename $d).tar" \ |
| 346 | + --directory=prebuild \ |
| 347 | + "$(basename $d)"; |
| 348 | + done |
| 349 | + - > |
| 350 | + nix-shell --arg ci true --run $' |
| 351 | + gh release \ |
| 352 | + create "$CI_COMMIT_TAG" \ |
| 353 | + prebuild/*.tar \ |
| 354 | + --title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ |
| 355 | + --notes "" \ |
| 356 | + --target master \ |
| 357 | + --repo "$GH_PROJECT_PATH"; |
| 358 | + ' |
| 359 | + after_script: |
| 360 | + - rm -f ./.npmrc |
| 361 | + rules: |
| 362 | + # Only runs on tag pipeline where the tag is a release version |
| 363 | + # This requires dependencies to also run on tag pipeline |
| 364 | + # However version tag comes with a version commit |
| 365 | + # Dependencies must not run on the version commit |
| 366 | + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ |
0 commit comments