Merge remote-tracking branch 'mozilla/master' into syncstorage-sqlite

Author: Paula Van Zuijlen

.circleci/config.yml

@@ -7,7 +7,7 @@
 # To avoid collision with other GCP connections we create specific vars for
 # the ETE Test Pipeline.
 # ETE_GOOGLE_PROJECT_ID     - GCP Project ID for ecosystem-test-eng
-# ETE_GCLOUD_SERVICE_KEY    - GCP syncstorage specific Service Account JSON Key 
+# ETE_GCLOUD_SERVICE_KEY    - GCP syncstorage specific Service Account JSON Key
 # ETE_GOOGLE_PROJECT_NUMBER - GCP Project Number for ecosystem-test-eng
 #
 version: 2.1
@@ -63,20 +63,17 @@ commands:
     steps:
       - run:
           name: Rust Clippy MySQL
-          command: |
-            cargo clippy --workspace --all-targets --no-default-features --features=mysql,py_verifier -- -D warnings
+          command: make clippy_mysql
   rust-clippy-sqlite:
     steps:
       - run:
           name: Rust Clippy SQLite
-          command: |
-            cargo clippy --workspace --all-targets --no-default-features --features=sqlite,py_verifier -- -D warnings
+          command: make clippy_sqlite
   rust-clippy-spanner:
     steps:
       - run:
           name: Rust Clippy Spanner
-          command: |
-            cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/spanner --features=tokenserver-db/mysql --features=py_verifier -- -D warnings
+          command: make clippy_spanner
   setup-mysql:
     steps:
       - run:
@@ -105,9 +102,9 @@ commands:
 
   install-test-deps:
     steps:
-      - run: 
+      - run:
           name: Install test dependencies
-          command: cargo install cargo-nextest cargo-llvm-cov
+          command: cargo install --locked cargo-nextest cargo-llvm-cov
 
   make-test-dir:
     steps:
@@ -124,7 +121,7 @@ commands:
           name: nextest with code coverage (quota enforced)
           command: make test_with_coverage
           environment:
-              SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1
+            SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1
 
   merge-unit-test-coverage:
     steps:
@@ -146,54 +143,25 @@ commands:
           when: always
           name: Tokenserver integration tests
           command: |
-              # NOTE: Python3.12 requires `--break-system-packages`.
-              # This command is run on the circleci/rust image, which is running python 3.10
-               make run_token_server_integration_tests
-          environment:
-              SYNCSTORAGE_RS_IMAGE: app:build
-  run-e2e-mysql-tests:
-    steps:
-      - run:
-          name: e2e tests (syncstorage mysql)
-          command: >
-               /usr/local/bin/docker-compose
-               -f docker-compose.mysql.yaml
-               -f docker-compose.e2e.mysql.yaml
-               up
-               --exit-code-from mysql-e2e-tests
-               --abort-on-container-exit
-          environment:
-              SYNCSTORAGE_RS_IMAGE: app:build
-
-  run-e2e-sqlite-tests:
-    steps:
-      - run:
-          name: e2e tests (syncstorage sqlite)
-          command: >
-               /usr/local/bin/docker-compose
-               -f docker-compose.sqlite.yaml
-               -f docker-compose.e2e.sqlite.yaml
-               up
-               --exit-code-from sqlite-e2e-tests
-               --abort-on-container-exit
+            # NOTE: Python3.12 requires `--break-system-packages`.
+            # This command is run on the circleci/rust image, which is running python 3.10
+             make run_token_server_integration_tests
           environment:
-              SYNCSTORAGE_RS_IMAGE: app:build
-
-  run-e2e-spanner-tests:
+            SYNCSTORAGE_RS_IMAGE: app:build
+  run-e2e-tests:
+    parameters:
+      db:
+        type: enum
+        enum: ["mysql", "spanner", "sqlite"]
     steps:
       - run:
-          name: e2e tests (syncstorage spanner)
+          name: e2e tests (syncstorage << parameters.db >>)
           command: >
-               /usr/local/bin/docker-compose
-               -f docker-compose.spanner.yaml
-               -f docker-compose.e2e.spanner.yaml
-               up
-               --exit-code-from spanner-e2e-tests
-               --abort-on-container-exit
+            make docker_run_<< parameters.db >>_e2e_tests
           environment:
-              SYNCSTORAGE_RS_IMAGE: app:build
+            SYNCSTORAGE_RS_IMAGE: app:build
 
-  upload-to-gcs:
+  gcs-configure-and-upload:
     parameters:
       source:
         type: string
@@ -203,9 +171,13 @@ commands:
         type: enum
         enum: ["xml", "json"]
     steps:
+      - gcp-cli/setup:
+          google_project_id: ETE_GOOGLE_PROJECT_ID
+          gcloud_service_key: ETE_GCLOUD_SERVICE_KEY
+          google_project_number: ETE_GOOGLE_PROJECT_NUMBER
       - run:
           name: Upload << parameters.source >> << parameters.extension >> Files to GCS
-          when: always  # Ensure the step runs even if previous steps, like test runs, fail
+          when: always # Ensure the step runs even if previous steps, like test runs, fail
           command: |
             if [ "$CIRCLE_BRANCH" = "master" ]; then
               FILES=$(ls -1 << parameters.source>>/*.<< parameters.extension>> )
@@ -244,7 +216,7 @@ commands:
 jobs:
   checks:
     docker:
-      - image: cimg/rust:1.81.0  # RUST_VER
+      - image: cimg/rust:1.86 # RUST_VER
         auth:
           username: $DOCKER_USER
           password: $DOCKER_PASS
@@ -261,31 +233,27 @@ jobs:
 
   build-and-test:
     docker:
-      - image: cimg/rust:1.81.0  # RUST_VER
+      - image: cimg/rust:1.86 # RUST_VER
         auth:
           username: $DOCKER_USER
           password: $DOCKER_PASS
         environment:
-            SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:[email protected]/syncstorage
-            SYNC_TOKENSERVER__DATABASE_URL: mysql://test:[email protected]/tokenserver
-            RUST_BACKTRACE: 1
-            # XXX: begin_test_transaction doesn't play nice over threaded tests
-            RUST_TEST_THREADS: 1
+          SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:[email protected]/syncstorage
+          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:[email protected]/tokenserver
+          RUST_BACKTRACE: 1
+          # XXX: begin_test_transaction doesn't play nice over threaded tests
+          RUST_TEST_THREADS: 1
       - image: cimg/mysql:5.7
         auth:
           username: $DOCKER_USER
           password: $DOCKER_PASS
         environment:
-            MYSQL_ROOT_PASSWORD: password
-            MYSQL_USER: test
-            MYSQL_PASSWORD: test
-            MYSQL_DATABASE: syncstorage
+          MYSQL_ROOT_PASSWORD: password
+          MYSQL_USER: test
+          MYSQL_PASSWORD: test
+          MYSQL_DATABASE: syncstorage
     resource_class: large
     steps:
-      - gcp-cli/setup:
-          google_project_id: ETE_GOOGLE_PROJECT_ID
-          gcloud_service_key: ETE_GCLOUD_SERVICE_KEY
-          google_project_number: ETE_GOOGLE_PROJECT_NUMBER
       - checkout
       - display-versions
       - setup-python
@@ -302,20 +270,19 @@ jobs:
       # if the above tests don't run tokenserver-db tests (i.e. using --workspace)
       # then run-tokenserver-scripts-tests will fail. These tests expect the db to be
       # configured already, and it appears unit-tests modify the db to the expected state
-      - run-tokenserver-integration-tests
       - store-test-results
-      - upload-to-gcs:
+      - gcs-configure-and-upload:
           source: workflow/test-results
           destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/junit
           extension: xml
-      - upload-to-gcs:
+      - gcs-configure-and-upload:
           source: workflow/test-results
           destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/coverage
           extension: json
       #- save-sccache-cache
   build-mysql-image:
     docker:
-      - image: cimg/rust:1.81.0  # RUST_VER
+      - image: cimg/rust:1.86 # RUST_VER
         auth:
           username: $DOCKER_USER
           password: $DOCKER_PASS
@@ -328,7 +295,10 @@ jobs:
       - write-version
       - run:
           name: Build MySQL Docker image
-          command: docker build -t app:build --build-arg DATABASE_BACKEND=mysql .
+          command: >
+            docker build -t app:build
+            --build-arg DATABASE_BACKEND=mysql
+            .
           no_output_timeout: 30m
       # save the built docker container into CircleCI's cache. This is
       # required since Workflows do not have the same remote docker instance.
@@ -340,6 +310,9 @@ jobs:
       - run:
           name: Save docker-compose config
           command: cp docker-compose*mysql.yaml /home/circleci/cache
+      - run:
+          name: Save Makefile to cache
+          command: cp Makefile /home/circleci/cache
       - save_cache:
           key: mysql-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}-{{ epoch }}
           paths:
@@ -379,7 +352,7 @@ jobs:
 
   build-spanner-image:
     docker:
-      - image: cimg/rust:1.81.0  # RUST_VER
+      - image: cimg/rust:1.86 # RUST_VER
         auth:
           username: $DOCKER_USER
           password: $DOCKER_PASS
@@ -392,7 +365,12 @@ jobs:
       - write-version
       - run:
           name: Build Spanner Docker image
-          command: docker build -t app:build --build-arg DATABASE_BACKEND=spanner .
+          # Build w/ the Oracle libmysqlclient-dev for TLS support
+          command: >
+            docker build -t app:build
+            --build-arg DATABASE_BACKEND=spanner
+            --build-arg MYSQLCLIENT_PKG=libmysqlclient-dev
+            .
           no_output_timeout: 30m
       # save the built docker container into CircleCI's cache. This is
       # required since Workflows do not have the same remote docker instance.
@@ -404,6 +382,9 @@ jobs:
       - run:
           name: Save docker-compose config
           command: cp docker-compose*spanner.yaml /home/circleci/cache
+      - run:
+          name: Save Makefile to cache
+          command: cp Makefile /home/circleci/cache
       - save_cache:
           key: spanner-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}-{{ epoch }}
           paths:
@@ -465,7 +446,7 @@ jobs:
 
   mysql-e2e-tests:
     docker:
-      - image: docker/compose:1.24.0
+      - image: cimg/base:2025.04
         auth:
           username: $DOCKER_USER
           password: $DOCKER_PASS
@@ -477,14 +458,24 @@ jobs:
       - run:
           name: Restore Docker image cache
           command: docker load -i /home/circleci/cache/docker.tar
+      - run:
+          name: Restore Makefile from save_cache
+          command: cp /home/circleci/cache/Makefile .
       - run:
           name: Restore docker-compose config
           command: cp /home/circleci/cache/docker-compose*.yaml .
-      - run-e2e-mysql-tests
+      - make-test-dir
+      - run-e2e-tests:
+          db: mysql
+      - store-test-results
+      - gcs-configure-and-upload:
+          source: workflow/test-results
+          destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/junit
+          extension: xml
 
   spanner-e2e-tests:
     docker:
-      - image: docker/compose:1.24.0
+      - image: cimg/base:2025.04
         auth:
           username: $DOCKER_USER
           password: $DOCKER_PASS
@@ -496,10 +487,20 @@ jobs:
       - run:
           name: Restore Docker image cache
           command: docker load -i /home/circleci/cache/docker.tar
+      - run:
+          name: Restore Makefile from save_cache
+          command: cp /home/circleci/cache/Makefile .
       - run:
           name: Restore docker-compose config
           command: cp /home/circleci/cache/docker-compose*.yaml .
-      - run-e2e-spanner-tests
+      - make-test-dir
+      - run-e2e-tests:
+          db: spanner
+      - store-test-results
+      - gcs-configure-and-upload:
+          source: workflow/test-results
+          destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/junit
+          extension: xml
 
   sqlite-e2e-tests:
     docker:

CHANGELOG.md

@@ -1,3 +1,47 @@
+<a name="0.18.3"></a>
+## 0.18.3 (2025-05-14)
+
+
+#### Chore
+
+*   bump to latest rust ([0148e04d](https://github.com/mozilla-services/syncstorage-rs/commit/0148e04dd2881869ffe52b6ebb93be6929f31a25))
+*   update python cryptography (#1690) ([e93bb882](https://github.com/mozilla-services/syncstorage-rs/commit/e93bb8821ccdf94e34c184f51ad86f0388333f3d))
+*   added build-and-push to GAR (#1654) ([cb37e2aa](https://github.com/mozilla-services/syncstorage-rs/commit/cb37e2aa4134d5e8e0c11178e267d3e7565da05d))
+*   upload test artifacts to gcs ([aeedcf1e](https://github.com/mozilla-services/syncstorage-rs/commit/aeedcf1e19e622b4f0d0e9c813ba4da3c712f125))
+*   switch back to libmariadb-dev (#1665) ([e0093a88](https://github.com/mozilla-services/syncstorage-rs/commit/e0093a88bfc059a891c1a5d3f74cef068b720861))
+*   migrate tokenserver tests to pytest with junit output ([15840c5e](https://github.com/mozilla-services/syncstorage-rs/commit/15840c5ecfd1e6fbcd239bed0f50cf3537631775))
+*   migrate unit tests to nextest and llvm-cov ([8c56cae8](https://github.com/mozilla-services/syncstorage-rs/commit/8c56cae8905325345972a4abe99c12c1fc1b012c))
+
+#### Features
+
+*   build docker w/ Oracle's libmysqlclient (#1695) ([569e5100](https://github.com/mozilla-services/syncstorage-rs/commit/569e5100839245cd5869bb12b655b7fe571fbbcf))
+*   emit oauth verification timeouts as metrics (not sentry) (#1694) ([624eced1](https://github.com/mozilla-services/syncstorage-rs/commit/624eced1e9cad6492a38397c9440b558d263cca0))
+
+#### Bug Fixes
+
+*   re-enable tokensever e2e tests ([d0336c88](https://github.com/mozilla-services/syncstorage-rs/commit/d0336c8869e52a48e49fed989b5ac9573a3b1e55))
+*   avoid underflow of the queued_tasks metric ([10daab06](https://github.com/mozilla-services/syncstorage-rs/commit/10daab06cf35cf5696aa6ed6b790d8115bfeb432))
+*   Revert "fix: avoid underflow of the queued_tasks metric (#1628)" ([31dda136](https://github.com/mozilla-services/syncstorage-rs/commit/31dda136809879b8e7f91f095bc378bb41b9f304))
+*   resolve pyo3 vuln deprecations (#1682) ([0675930a](https://github.com/mozilla-services/syncstorage-rs/commit/0675930a155d27bbf2eca2c0abf81d262a9cfb28))
+* **infra:**  configure gcp utils before upload (#1698) ([5dcfefe2](https://github.com/mozilla-services/syncstorage-rs/commit/5dcfefe2b6a8946f02c7bfac2fd641b0a6a3356b))
+
+#### Test
+
+* **e2e:**  run integration and e2e tests with pytest (#1697) ([6f15ad54](https://github.com/mozilla-services/syncstorage-rs/commit/6f15ad546d3c5234986db09fec485fb911624e5f))
+
+#### Doc
+
+*   add tokenserver documentation to sync (#1681) ([dadbcea3](https://github.com/mozilla-services/syncstorage-rs/commit/dadbcea3f7428ad7f0a5ae6f0c2ad966c331660a))
+*   update purge script's status now that it's running (#1679) ([6f0c7b28](https://github.com/mozilla-services/syncstorage-rs/commit/6f0c7b28db3f8a2701c4af4dfe7a2d691fc079ef))
+*   document pruning scripts (#1645) ([7c9bc008](https://github.com/mozilla-services/syncstorage-rs/commit/7c9bc0089dd73a9ecaba8b33e26634b2a69b5ff0))
+
+#### Refactor
+
+*   kill tokenserver's TokenType now that it's solely oauth ([a26ff490](https://github.com/mozilla-services/syncstorage-rs/commit/a26ff490b8086ce3c12b837ca00cc757caa54169))
+*   simplify metric_label to return a &str ([0ca435fb](https://github.com/mozilla-services/syncstorage-rs/commit/0ca435fb1a05f073d1e78ed420d953a00c8d0d53))
+
+
+
 <a name="0.18.2"></a>
 ## 0.18.2 (2024-12-05)