feat: Add sqlite as a storage backend

First throw at adding sqlite as storage backend for the syncserver and
the tokenserver.

There is probably some duplicated code between:
- syncstorage-mysql and syncstorage-sqlite
- tokenserver-db-mysql and tokenserver-db-sqlite

tokenserver-db-sqlite probably contains Mysql-specific SQL that might
break Sqlite when ran.

Squashed commit of the following:

commit 1047197
Author: Eragon <[email protected]>
Date:   Wed Mar 6 00:24:07 2024 +0100

    fix: Fix default for tokenserver-db

commit 5e2a745
Author: Eragon <[email protected]>
Date:   Sun Feb 11 22:00:46 2024 +0100

    tokenserver-db defaults to use mysql as db backend

commit a587787
Author: Eragon <[email protected]>
Date:   Sun Feb 11 21:47:55 2024 +0100

    Run cargo fmt

commit c8c1458
Author: Eragon <[email protected]>
Date:   Sun Feb 11 20:32:19 2024 +0100

    Better logging of migrations

commit e2b8563
Author: Eragon <[email protected]>
Date:   Tue Jan 30 00:36:11 2024 +0100

    wip: At least it runs now

commit bd24d7c
Author: Eragon <[email protected]>
Date:   Mon Jan 22 11:20:38 2024 +0100

    lll

commit 07ba38f
Author: Eragon <[email protected]>
Date:   Fri Jan 19 14:35:20 2024 +0100

    wip: First throw at adding sqlite as a storage backend

Author: Eragonfr

Cargo.lock

@@ -9,9 +9,13 @@ members = [
   "syncstorage-mysql",
   "syncstorage-settings",
   "syncstorage-spanner",
+  "syncstorage-sqlite",
   "tokenserver-auth",
   "tokenserver-common",
   "tokenserver-db",
+  "tokenserver-db-common",
+  "tokenserver-db-mysql",
+  "tokenserver-db-sqlite",
   "tokenserver-settings",
   "syncserver",
 ]
@@ -23,6 +27,7 @@ authors = [
   "Ben Bangert <[email protected]>",
   "Phil Jenvey <[email protected]>",
   "Mozilla Services Engineering <[email protected]>",
+  "Eragon <[email protected]>",
 ]
 edition = "2021"
 license = "MPL-2.0"

Cargo.toml

@@ -13,13 +13,17 @@ PATH_TO_GRPC_CERT = ../server-syncstorage/local/lib/python2.7/site-packages/grpc
 SRC_ROOT = $(shell pwd)
 PYTHON_SITE_PACKGES = $(shell $(SRC_ROOT)/venv/bin/python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
 
+clippy_sqlite:
+	# Matches what's run in circleci
+	cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/sqlite,tokenserver-db/sqlite --features=py_verifier -- -D warnings
+
 clippy_mysql:
 	# Matches what's run in circleci
-	cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/mysql --features=py_verifier -- -D warnings
+	cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/mysql,tokenserver-db/mysql --features=py_verifier -- -D warnings
 
 clippy_spanner:
 	# Matches what's run in circleci
-	cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/spanner --features=py_verifier -- -D warnings
+	cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/spanner,tokenserver-db/mysql --features=py_verifier  -- -D warnings
 
 clean:
 	cargo clean
@@ -53,23 +57,38 @@ run_mysql: python
 		# See https://github.com/PyO3/pyo3/issues/1741 for discussion re: why we need to set the
 		# below env var
 		PYTHONPATH=$(PYTHON_SITE_PACKGES) \
-	        RUST_LOG=debug \
+		RUST_LOG=debug \
 		RUST_BACKTRACE=full \
-		cargo run --no-default-features --features=syncstorage-db/mysql --features=py_verifier -- --config config/local.toml
+		cargo run --no-default-features --features=syncstorage-db/mysql,tokenserver-db/mysql --features=py_verifier -- --config config/local.toml
+
+run_sqlite: python
+	PATH="./venv/bin:$(PATH)" \
+		# See https://github.com/PyO3/pyo3/issues/1741 for discussion re: why we need to set the
+		# below env var
+		PYTHONPATH=$(PYTHON_SITE_PACKGES) \
+		RUST_LOG=debug \
+		RUST_BACKTRACE=full \
+		cargo run --no-default-features --features=syncstorage-db/sqlite,tokenserver-db/sqlite --features=py_verifier -- --config config/local.toml
 
 run_spanner: python
 	GOOGLE_APPLICATION_CREDENTIALS=$(PATH_TO_SYNC_SPANNER_KEYS) \
 		GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=$(PATH_TO_GRPC_CERT) \
 		# See https://github.com/PyO3/pyo3/issues/1741 for discussion re: why we need to set the
 		# below env var
 		PYTHONPATH=$(PYTHON_SITE_PACKGES) \
-	    PATH="./venv/bin:$(PATH)" \
+		PATH="./venv/bin:$(PATH)" \
 		RUST_LOG=debug \
 		RUST_BACKTRACE=full \
-		cargo run --no-default-features --features=syncstorage-db/spanner --features=py_verifier -- --config config/local.toml
+		cargo run --no-default-features --features=syncstorage-db/spanner,tokenserver-db/mysql --features=py_verifier -- --config config/local.toml
 
-test:
+test_mysql:
 	SYNC_SYNCSTORAGE__DATABASE_URL=mysql://sample_user:sample_password@localhost/syncstorage_rs \
 		SYNC_TOKENSERVER__DATABASE_URL=mysql://sample_user:sample_password@localhost/tokenserver_rs \
 		RUST_TEST_THREADS=1 \
 		cargo test --workspace
+
+test_sqlite:
+	SYNC_SYNCSTORAGE__DATABASE_URL=:memory: \
+		SYNC_TOKENSERVER__DATABASE_URL=:memory: \
+		RUST_TEST_THREADS=1 \
+		cargo test --workspace

Makefile

@@ -12,6 +12,7 @@ Mozilla Sync Storage built with [Rust](https://rust-lang.org).
 - [Local Setup](#local-setup)
   - [MySQL](#mysql)
   - [Spanner](#spanner)
+  - [Sqlite](#sqlite)
   - [Running via Docker](#running-via-docker)
   - [Connecting to Firefox](#connecting-to-firefox)
 - [Logging](#logging)
@@ -179,6 +180,15 @@ To run an application server that points to the local Spanner emulator:
 SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST=localhost:9010 make run_spanner
 ```
 
+### Sqlite
+
+Setting up the server with sqlite only requires a path to the database file,
+which will be created automatically:
+
+`sqlite:path/syncdb.sqlite`
+
+This requires at least sqlite v3.24.0 to be installed on the host system.
+
 ### Running via Docker
 
 This requires access to [Google Cloud Rust (raw)](https://crates.io/crates/google-cloud-rust-raw/) crate. Please note that due to interdependencies, you will need to ensure that `grpcio` and `protobuf` match the version used by `google-cloud-rust-raw`.

README.md

@@ -5,9 +5,13 @@ human_logs = 1
 
 # Example Syncstorage settings:
 # Example MySQL DSN:
-syncstorage.database_url = "mysql://sample_user:sample_password@localhost/syncstorage_rs"
+#syncstorage.database_url = "mysql://sample_user:sample_password@localhost/syncstorage_rs"
 # Example Spanner DSN:
 # database_url="spanner://projects/SAMPLE_GCP_PROJECT/instances/SAMPLE_SPANNER_INSTANCE/databases/SAMPLE_SPANNER_DB"
+# Example SQLite DSN:
+# database_url="sqlite://PATH_TO_FILE/FILE.sqlite"
+syncstorage.database_url = ":memory:"
+
 # enable quota limits
 syncstorage.enable_quota = 0
 # set the quota limit to 2GB.
@@ -16,7 +20,8 @@ syncstorage.enabled = true
 syncstorage.limits.max_total_records = 1666 # See issues #298/#333
 
 # Example Tokenserver settings:
-tokenserver.database_url = "mysql://sample_user:sample_password@localhost/tokenserver_rs"
+#tokenserver.database_url = "mysql://sample_user:sample_password@localhost/tokenserver_rs"
+tokenserver.database_url = ":memory:"
 tokenserver.enabled = true
 tokenserver.fxa_email_domain = "api-accounts.stage.mozaws.net"
 tokenserver.fxa_metrics_hash_secret = "INSERT_SECRET_KEY_HERE"
@@ -26,5 +31,5 @@ tokenserver.fxa_browserid_issuer = "https://api-accounts.stage.mozaws.net"
 tokenserver.fxa_browserid_server_url = "https://verifier.stage.mozaws.net/v2"
 
 # cors settings
-# cors_allowed_origin = "localhost"
+cors_allowed_origin = "localhost"
 # cors_max_age = 86400

config/local.example.toml

@@ -12,6 +12,6 @@ http.workspace=true
 thiserror.workspace=true
 
 deadpool = { git = "https://github.com/mozilla-services/deadpool", tag = "deadpool-v0.7.0" }
-diesel = { version = "1.4", features = ["mysql", "r2d2"] }
-diesel_migrations = { version = "1.4.0", features = ["mysql"] }
+diesel = { version = "1.4", features = ["mysql", "sqlite","r2d2"] }
+diesel_migrations = { version = "1.4.0", features = ["mysql", "sqlite"] }
 syncserver-common = { path = "../syncserver-common" }

syncserver-db-common/Cargo.toml

@@ -8,14 +8,14 @@ use thiserror::Error;
 /// Error specific to any MySQL database backend. These errors are not related to the syncstorage
 /// or tokenserver application logic; rather, they are lower-level errors arising from diesel.
 #[derive(Debug)]
-pub struct MysqlError {
-    kind: MysqlErrorKind,
+pub struct SqlError {
+    kind: SqlErrorKind,
     pub status: StatusCode,
     pub backtrace: Backtrace,
 }
 
 #[derive(Debug, Error)]
-enum MysqlErrorKind {
+enum SqlErrorKind {
     #[error("A database error occurred: {}", _0)]
     DieselQuery(#[from] diesel::result::Error),
 
@@ -29,8 +29,8 @@ enum MysqlErrorKind {
     Migration(diesel_migrations::RunMigrationsError),
 }
 
-impl From<MysqlErrorKind> for MysqlError {
-    fn from(kind: MysqlErrorKind) -> Self {
+impl From<SqlErrorKind> for SqlError {
+    fn from(kind: SqlErrorKind) -> Self {
         Self {
             kind,
             status: StatusCode::INTERNAL_SERVER_ERROR,
@@ -39,21 +39,17 @@ impl From<MysqlErrorKind> for MysqlError {
     }
 }
 
-impl_fmt_display!(MysqlError, MysqlErrorKind);
+impl_fmt_display!(SqlError, SqlErrorKind);
 
-from_error!(
-    diesel::result::Error,
-    MysqlError,
-    MysqlErrorKind::DieselQuery
-);
+from_error!(diesel::result::Error, SqlError, SqlErrorKind::DieselQuery);
 from_error!(
     diesel::result::ConnectionError,
-    MysqlError,
-    MysqlErrorKind::DieselConnection
+    SqlError,
+    SqlErrorKind::DieselConnection
 );
-from_error!(diesel::r2d2::PoolError, MysqlError, MysqlErrorKind::Pool);
+from_error!(diesel::r2d2::PoolError, SqlError, SqlErrorKind::Pool);
 from_error!(
     diesel_migrations::RunMigrationsError,
-    MysqlError,
-    MysqlErrorKind::Migration
+    SqlError,
+    SqlErrorKind::Migration
 );

syncserver-db-common/src/error.rs

@@ -1,6 +1,7 @@
 use diesel::{
     mysql::MysqlConnection,
     r2d2::{CustomizeConnection, Error as PoolError},
+    sqlite::SqliteConnection,
     Connection,
 };
 
@@ -12,3 +13,9 @@ impl CustomizeConnection<MysqlConnection, PoolError> for TestTransactionCustomiz
         conn.begin_test_transaction().map_err(PoolError::QueryError)
     }
 }
+
+impl CustomizeConnection<SqliteConnection, PoolError> for TestTransactionCustomizer {
+    fn on_acquire(&self, conn: &mut SqliteConnection) -> Result<(), PoolError> {
+        conn.begin_test_transaction().map_err(PoolError::QueryError)
+    }
+}

syncserver-db-common/src/test.rs

@@ -65,3 +65,4 @@ no_auth = []
 py_verifier = ["tokenserver-auth/py"]
 mysql = ["syncstorage-db/mysql"]
 spanner = ["syncstorage-db/spanner"]
+sqlite = ["syncstorage-db/sqlite"]

syncserver/Cargo.toml

@@ -3,6 +3,7 @@
 pub mod mock;
 pub mod mysql;
 pub mod spanner;
+pub mod sqlite;
 #[cfg(test)]
 mod tests;
 pub mod transaction;
@@ -35,6 +36,11 @@ pub async fn pool_from_settings(
         "spanner" => Box::new(
             spanner::pool::SpannerDbPool::new(settings, metrics, blocking_threadpool).await?,
         ),
+        "sqlite" => Box::new(sqlite::pool::SqliteDbPool::new(
+            settings,
+            metrics,
+            blocking_threadpool,
+        )?),
         _ => Err(DbErrorKind::InvalidUrl(settings.database_url.to_owned()))?,
     })
 }