feat: Syncstorage inital sqlite support

.cargo/audit.toml

@@ -3,4 +3,5 @@ ignore = [
        "RUSTSEC-2024-0365", # Bound by diesel 1.4 (4GB limit n/a to tokenserver)
        "RUSTSEC-2024-0421", # Bound by diesel 1.4, `idna`  < 0.1.5, Upgrade to >=1.0.0
        "RUSTSEC-2024-0437", # Bound by grpcio 0.13,
+       "RUSTSEC-2022-0090", # Bound by diesel 1.4.8, diesel_migrations 1.4.0, diesel_logger 0.1.1
 ]

.circleci/config.yml

@@ -64,6 +64,11 @@ commands:
       - run:
           name: Rust Clippy MySQL
           command: make clippy_mysql
+  rust-clippy-sqlite:
+    steps:
+      - run:
+          name: Rust Clippy SQLite
+          command: make clippy_sqlite
   rust-clippy-spanner:
     steps:
       - run:
@@ -156,6 +161,20 @@ commands:
           environment:
             SYNCSTORAGE_RS_IMAGE: app:build
 
+  run-sqlite-tests:
+    steps:
+      - run:
+          name: e2e tests (syncstorage sqlite)
+          command: >
+               /usr/local/bin/docker-compose
+               -f docker-compose.sqlite.yaml
+               -f docker-compose.e2e.sqlite.yaml
+               up
+               --exit-code-from sqlite-e2e-tests
+               --abort-on-container-exit
+          environment:
+              SYNCSTORAGE_RS_IMAGE: app:build
+
   run-spanner-tests:
     steps:
       - run:
@@ -347,6 +366,42 @@ jobs:
           key: mysql-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}-{{ epoch }}
           paths:
             - /home/circleci/cache
+
+  build-sqlite-image:
+    docker:
+      - image: cimg/rust:1.86  # RUST_VER
+        auth:
+          username: $DOCKER_USER
+          password: $DOCKER_PASS
+    resource_class: large
+    steps:
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - checkout
+      - display-versions
+      - write-version
+      - run:
+          name: Build SQLite Docker image
+          command: docker build -t app:build --build-arg DATABASE_BACKEND=sqlite .
+          no_output_timeout: 30m
+      # save the built docker container into CircleCI's cache. This is
+      # required since Workflows do not have the same remote docker instance.
+      - run:
+          name: docker save app:build
+          command: |
+            mkdir -p /home/circleci/cache
+            docker save -o /home/circleci/cache/docker.tar "app:build"
+      - run:
+          name: Save docker-compose config
+          command: cp docker-compose*sqlite.yaml /home/circleci/cache
+      - run:
+          name: Save Makefile to cache
+          command: cp Makefile /home/circleci/cache
+      - save_cache:
+          key: sqlite-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}-{{ epoch }}
+          paths:
+            - /home/circleci/cache
+
   build-spanner-image:
     docker:
       - image: cimg/rust:1.86 # RUST_VER
@@ -499,6 +554,36 @@ jobs:
           destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/junit
           extension: xml
 
+  sqlite-e2e-tests:
+    docker:
+      - image: cimg/base:2025.04
+        auth:
+          username: $DOCKER_USER
+          password: $DOCKER_PASS
+    steps:
+      - setup_remote_docker
+      - display-versions
+      - restore_cache:
+          key: sqlite-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}
+      - run:
+          name: Restore Docker image cache
+          command: docker load -i /home/circleci/cache/docker.tar
+      - run:
+          name: Restore Makefile from save_cache
+          command: cp /home/circleci/cache/Makefile .
+      - run:
+          name: Restore docker-compose config
+          command: cp /home/circleci/cache/docker-compose*.yaml .
+      - make-test-dir
+      - run-e2e-tests:
+          db: sqlite
+      - store-test-results
+      - gcs-configure-and-upload:
+          source: workflow/test-results
+          destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/junit
+          extension: xml
+      - run-e2e-sqlite-tests
+
   deploy:
     docker:
       - image: docker:18.02.0-ce
@@ -600,6 +685,12 @@ workflows:
           filters:
             tags:
               only: /.*/
+      - build-sqlite-image:
+          requires:
+            - build-and-test
+          filters:
+            tags:
+              only: /.*/
       - mysql-e2e-tests:
           requires:
             - build-mysql-image
@@ -612,10 +703,17 @@ workflows:
           filters:
             tags:
               only: /.*/
+      - sqlite-e2e-tests:
+          requires:
+            - build-sqlite-image
+          filters:
+            tags:
+              only: /.*/
       - deploy:
           requires:
             - mysql-e2e-tests
             - spanner-e2e-tests
+            - sqlite-e2e-tests
           filters:
             tags:
               only: /.*/
@@ -629,6 +727,7 @@ workflows:
           requires:
             - mysql-e2e-tests
             - spanner-e2e-tests
+            - sqlite-e2e-tests
           filters:
             tags:
               only: /.*/
@@ -638,6 +737,7 @@ workflows:
           requires:
             - mysql-e2e-tests
             - spanner-e2e-tests
+            - sqlite-e2e-tests
           filters:
             tags:
               only: /.*/

.gitignore

@@ -15,6 +15,8 @@ html_coverage
 .hgignore
 .idea
 *.iml
+Pipfile
+Pipfile.lock
 site-packages/*
 lib-python/*
 bin/*
@@ -31,11 +33,12 @@ target
 service-account.json
 .sentryclirc
 .envrc
+.env
 
 config/local.toml
 tools/tokenserver/loadtests/*.pem
 tools/tokenserver/loadtests/*.pub
-venv
+venv/
 .vscode/settings.json
 
 # circleci