Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
5087f57
fix(clippy): replace map_or_else with unwrap_or_else for identity clo…
tobyhede Dec 14, 2025
d30cc4a
test: add jsonb_containment_index module skeleton
tobyhede Dec 11, 2025
01b1748
test: add GIN index helper functions for containment tests
tobyhede Dec 11, 2025
13562ae
test: add baseline test verifying seq scan without GIN index
tobyhede Dec 11, 2025
101c503
test: add @> operator GIN index usage test
tobyhede Dec 11, 2025
e253142
test: optimize bulk insert and fix SQL syntax in GIN index tests
tobyhede Dec 11, 2025
45fb72c
feat(eql-mapper): add jsonb_array, jsonb_contains, jsonb_contained_by…
tobyhede Dec 11, 2025
868f52d
test(eql-mapper): add tests for jsonb_array, jsonb_contains, jsonb_co…
tobyhede Dec 11, 2025
c113ca5
feat(eql-mapper): add RewriteContainmentOps transformation rule
tobyhede Dec 11, 2025
6182ca8
feat(eql-mapper): wire RewriteContainmentOps into transformation pipe…
tobyhede Dec 11, 2025
8045668
test(eql-mapper): add unit tests for containment operator transformat…
tobyhede Dec 11, 2025
33c774a
test(integration): update tests to use direct @> operator
tobyhede Dec 11, 2025
ebddfe2
fix(eql-mapper): preserve NodeKey identity and fix transformation order
tobyhede Dec 12, 2025
4d375eb
feat(eql-mapper): add EXPLAIN statement support for type checking
tobyhede Dec 12, 2025
9ed4707
test(eql-mapper): add test for EXPLAIN with containment operator
tobyhede Dec 12, 2025
52f1f1b
test(integration): WIP - jsonb containment index tests with direct @>…
tobyhede Dec 12, 2025
9be7681
fix(eql-mapper): check EQL types before rewriting containment operators
tobyhede Dec 12, 2025
e7ce483
feat(integration): add OperandType and QueryProtocol enums for contai…
tobyhede Dec 12, 2025
b532646
feat(integration): add ContainmentTestCase struct for containment tests
tobyhede Dec 12, 2025
2d2229d
feat(integration): add build_sql method to ContainmentTestCase
tobyhede Dec 12, 2025
3161dc8
feat(integration): add run method to ContainmentTestCase
tobyhede Dec 12, 2025
f2dcdc9
feat(integration): add containment_test! macro
tobyhede Dec 12, 2025
1304db0
refactor(integration): replace manual test with containment_test! macro
tobyhede Dec 12, 2025
b7b7bc7
test(integration): add encrypted_contains_literal containment test
tobyhede Dec 12, 2025
e3f90fe
test(integration): add param_contains_encrypted containment test
tobyhede Dec 12, 2025
000ec50
test(integration): fix parallel test isolation for JSONB containment …
tobyhede Dec 12, 2025
41cecc8
test(integration): use exact match for param/literal @> encrypted tests
tobyhede Dec 12, 2025
c30ed7d
refactor(eql-mapper): apply code review suggestions
tobyhede Dec 12, 2025
a246ee7
style: apply cargo fmt formatting
tobyhede Dec 12, 2025
40816fb
refactor(integration): add FIXTURE_COUNT constant and document variance
tobyhede Dec 12, 2025
f9bee70
style: fix clippy lints for Rust 1.92
tobyhede Dec 12, 2025
f8cb73e
style: fix manual range contains clippy lint
tobyhede Dec 12, 2025
9502e04
style(eql-mapper): simplify unused variable binding in EXPLAIN handler
tobyhede Dec 12, 2025
ec6fee8
deps: eql-2.2.1
tobyhede Dec 13, 2025
4453ea5
test(integration): remove jsonb type casting from literal tests
tobyhede Dec 14, 2025
63631d0
test(integration): add JSONB encryption sanity check
tobyhede Dec 14, 2025
0a33b74
docs: document JSONB literals work without explicit type casts
tobyhede Dec 14, 2025
65efaad
chore: rename port vars to reflect new matrix testing
tobyhede Dec 15, 2025
4fbd72e
???
tobyhede Dec 15, 2025
6d5d45c
ci: connect to pg directly.
tobyhede Dec 15, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/reference/searchable-json.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ SELECT eql_v2.add_search_config(
);
```

> **Note:** JSONB literals in INSERT and UPDATE statements work directly without explicit `::jsonb` type casts. The proxy infers the JSONB type from the target column and handles encryption transparently.

### JSON document structure

Examples assume an encrypted JSON document with the following structure:
Expand Down Expand Up @@ -591,6 +593,8 @@ SELECT jsonb_array_length(jsonb_path_query(encrypted_jsonb, '$.unknown')) FROM c

## Containment Operators

> **Note:** Containment operators work directly with JSONB literals without requiring explicit `::jsonb` type casts. The examples below use the simplified syntax intentionally.

### `@>` (Contains Operator)

Tests whether the left JSONB value contains the right JSONB value.
Expand Down
20 changes: 15 additions & 5 deletions mise.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ CS_PROXY__HOST = "host.docker.internal"
# Misc
DOCKER_CLI_HINTS = "false" # Please don't show us What's Next.

CS_EQL_VERSION = "eql-2.1.8"
CS_EQL_VERSION = "eql-2.2.1"


[tools]
Expand Down Expand Up @@ -174,6 +174,18 @@ run = """
cargo nextest run --no-fail-fast --nocapture -p cipherstash-proxy-integration
"""

[tasks."test:integration:without_multitenant"]
description = "Runs integration tests excluding multitenant"
run = """
cargo nextest run --no-fail-fast --nocapture -E 'package(cipherstash-proxy-integration) and not test(multitenant)'
"""

[tasks."test:integration:multitenant"]
description = "Runs multitenant integration tests only"
run = """
cargo nextest run --no-fail-fast --nocapture -E 'package(cipherstash-proxy-integration) and test(multitenant)'
"""

[tasks."test:local:mapper"]
alias = 'lm'
description = "Runs test/s"
Expand Down Expand Up @@ -311,8 +323,6 @@ echo
mise --env tcp run postgres:setup
mise --env tls run postgres:setup

mise run test:integration:showcase

echo
echo '###############################################'
echo '# Test: Prometheus'
Expand Down Expand Up @@ -354,7 +364,7 @@ echo

mise --env tls run proxy:up proxy-tls --extra-args "--detach --wait"
mise --env tls run test:wait_for_postgres_to_quack --port 6432 --max-retries 20 --tls
cargo nextest run --no-fail-fast --nocapture -E 'package(cipherstash-proxy-integration) and not test(multitenant)'
mise --env tls run test:integration:without_multitenant
mise --env tls run proxy:down

echo
Expand All @@ -369,7 +379,7 @@ unset CS_DEFAULT_KEYSET_ID

mise --env tls run proxy:up proxy-tls --extra-args "--detach --wait"
mise --env tls run test:wait_for_postgres_to_quack --port 6432 --max-retries 20 --tls
cargo nextest run --no-fail-fast --nocapture -E 'package(cipherstash-proxy-integration) and test(multitenant)'
mise --env tls run test:integration:multitenant

echo "'set CS_DEFAULT_KEYSET_ID = {{default_keyset_id}}'"
export CS_DEFAULT_KEYSET_ID="{{default_keyset_id}}"
Expand Down
31 changes: 26 additions & 5 deletions packages/cipherstash-proxy-integration/src/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@ use rustls::{
use serde_json::Value;
use std::sync::{Arc, Once};
use tokio_postgres::{types::ToSql, Client, NoTls, Row, SimpleQueryMessage};
use tracing::info;
use tracing_subscriber::{filter::Directive, EnvFilter, FmtSubscriber};

pub const PROXY: u16 = 6432;
pub const PG_LATEST: u16 = 5532;
pub const PG_V17_TLS: u16 = 5617;
pub const PG_PORT: u16 = 5532;
pub const PG_TLS_PORT: u16 = 5617;

pub const TEST_SCHEMA_SQL: &str = include_str!(concat!("../../../tests/sql/schema.sql"));

Expand Down Expand Up @@ -52,7 +53,7 @@ pub async fn clear() {
pub async fn reset_schema() {
let port = std::env::var("CS_DATABASE__PORT")
.map(|s| s.parse().unwrap())
.unwrap_or(PG_LATEST);
.unwrap_or(PG_PORT);

let client = connect_with_tls(port).await;
client.simple_query(TEST_SCHEMA_SQL).await.unwrap();
Expand All @@ -61,7 +62,7 @@ pub async fn reset_schema() {
pub async fn reset_schema_to(schema: &'static str) {
let port = std::env::var("CS_DATABASE__PORT")
.map(|s| s.parse().unwrap())
.unwrap_or(PG_LATEST);
.unwrap_or(PG_PORT);

let client = connect_with_tls(port).await;
client.simple_query(schema).await.unwrap();
Expand All @@ -81,7 +82,7 @@ pub async fn table_exists(table: &str) -> bool {

let port = std::env::var("CS_DATABASE__PORT")
.map(|s| s.parse().unwrap())
.unwrap_or(PG_LATEST);
.unwrap_or(PG_PORT);

let client = connect_with_tls(port).await;
let messages = client.simple_query(&query).await.unwrap();
Expand Down Expand Up @@ -209,6 +210,26 @@ where
rows.iter().map(|row| row.get(0)).collect::<Vec<T>>()
}

/// Get database port from environment or use default.
fn get_database_port() -> u16 {
std::env::var("CS_DATABASE__PORT")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(PG_PORT)
}

pub async fn query_direct_by<T>(sql: &str, param: &(dyn ToSql + Sync)) -> Vec<T>
where
T: for<'a> tokio_postgres::types::FromSql<'a>,
{
let port = get_database_port();
info!(port);

let client = connect_with_tls(port).await;
let rows = client.query(sql, &[param]).await.unwrap();
rows.iter().map(|row| row.get(0)).collect()
}

pub async fn simple_query<T: std::str::FromStr>(sql: &str) -> Vec<T>
where
<T as std::str::FromStr>::Err: std::fmt::Debug,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,6 @@ mod tests {

macro_rules! test_insert_with_literal {
($name: ident, $type: ident, $pg_type: ident) => {
test_insert_with_literal!($name, $type, $pg_type, false);
};

($name: ident, $type: ident, $pg_type: ident, $cast: expr) => {
#[tokio::test]
pub async fn $name() {
trace();
Expand All @@ -26,14 +22,8 @@ mod tests {

let expected = vec![encrypted_val.clone()];

let cast_to_type: &str = if $cast {
&format!("::{}", stringify!($pg_type))
} else {
""
};

let insert_sql = format!("INSERT INTO encrypted (id, {encrypted_col}) VALUES ($1, '{encrypted_val}'{cast_to_type})");
let select_sql = format!("SELECT {encrypted_col}{cast_to_type} FROM encrypted WHERE id = $1");
let insert_sql = format!("INSERT INTO encrypted (id, {encrypted_col}) VALUES ($1, '{encrypted_val}')");
let select_sql = format!("SELECT {encrypted_col} FROM encrypted WHERE id = $1");

execute_query(&insert_sql, &[&id]).await;
let actual = query_by::<$type>(&select_sql, &id).await;
Expand All @@ -46,10 +36,6 @@ mod tests {

macro_rules! test_insert_simple_query_with_literal {
($name: ident, $type: ident, $pg_type: ident) => {
test_insert_simple_query_with_literal!($name, $type, $pg_type, false);
};

($name: ident, $type: ident, $pg_type: ident, $cast: expr) => {
#[tokio::test]
pub async fn $name() {
trace();
Expand All @@ -62,15 +48,8 @@ mod tests {
let encrypted_col = format!("encrypted_{}", stringify!($pg_type));
let encrypted_val = crate::value_for_type!($type, random_limited());

let cast_to_type: &str = if $cast {
&format!("::{}", stringify!($pg_type))
} else {
""
};

let insert_sql = format!("INSERT INTO encrypted (id, {encrypted_col}) VALUES ({id}, '{encrypted_val}'{cast_to_type})");
let select_sql = format!("SELECT {encrypted_col}{cast_to_type} FROM encrypted WHERE id = {id}");

let insert_sql = format!("INSERT INTO encrypted (id, {encrypted_col}) VALUES ({id}, '{encrypted_val}')");
let select_sql = format!("SELECT {encrypted_col} FROM encrypted WHERE id = {id}");

let expected = vec![encrypted_val];

Expand All @@ -89,7 +68,7 @@ mod tests {
test_insert_with_literal!(insert_with_literal_bool, bool, bool);
test_insert_with_literal!(insert_with_literal_text, String, text);
test_insert_with_literal!(insert_with_literal_date, NaiveDate, date);
test_insert_with_literal!(insert_with_literal_jsonb, Value, jsonb, true);
test_insert_with_literal!(insert_with_literal_jsonb, Value, jsonb);

test_insert_simple_query_with_literal!(insert_simple_query_with_literal_int2, i16, int2);
test_insert_simple_query_with_literal!(insert_simple_query_with_literal_int4, i32, int4);
Expand All @@ -98,12 +77,7 @@ mod tests {
test_insert_simple_query_with_literal!(insert_simple_query_with_literal_bool, bool, bool);
test_insert_simple_query_with_literal!(insert_simple_query_with_literal_text, String, text);
test_insert_simple_query_with_literal!(insert_simple_query_with_literal_date, NaiveDate, date);
test_insert_simple_query_with_literal!(
insert_simple_query_with_literal_jsonb,
Value,
jsonb,
true
);
test_insert_simple_query_with_literal!(insert_simple_query_with_literal_jsonb, Value, jsonb);

// -----------------------------------------------------------------

Expand Down
64 changes: 61 additions & 3 deletions packages/cipherstash-proxy-integration/src/map_literals.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#[cfg(test)]
mod tests {
use crate::common::{clear, connect_with_tls, random_id, PROXY};
use crate::common::{clear, connect_with_tls, query_direct_by, random_id, trace, PROXY};

#[tokio::test]
async fn map_literal() {
Expand Down Expand Up @@ -45,8 +45,14 @@ mod tests {
println!("encrypted: {:?}", rows[0])
}

/// Verify JSONB literal insertion and retrieval without explicit type casts.
///
/// JSONB literals in INSERT and SELECT statements work directly with the proxy
/// without requiring `::jsonb` type annotations. The proxy infers the JSONB type
/// from the target column and handles encryption/decryption transparently.
#[tokio::test]
async fn map_jsonb() {
trace();
clear().await;

let client = connect_with_tls(PROXY).await;
Expand All @@ -55,12 +61,12 @@ mod tests {
let encrypted_jsonb = serde_json::json!({"key": "value"});

let sql = format!(
"INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, '{encrypted_jsonb}'::jsonb)",
"INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, '{encrypted_jsonb}')",
);

client.query(&sql, &[&id]).await.unwrap();

let sql = "SELECT id, encrypted_jsonb::jsonb FROM encrypted WHERE id = $1";
let sql = "SELECT id, encrypted_jsonb FROM encrypted WHERE id = $1";
let rows = client.query(sql, &[&id]).await.unwrap();

assert_eq!(rows.len(), 1);
Expand All @@ -74,6 +80,58 @@ mod tests {
}
}

/// Sanity check: verify JSONB is actually encrypted in database
///
/// This test catches silent encryption failures where plaintext is stored.
/// Insert via proxy, query DIRECT from database to verify encryption,
/// then query via proxy to verify decryption round-trip.
#[tokio::test]
async fn jsonb_encryption_sanity_check() {
trace();
clear().await;

let id = random_id();
let plaintext_json = serde_json::json!({"key": "value"});

// Insert through proxy (should encrypt)
let client = connect_with_tls(PROXY).await;
let sql = "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)";
client.query(sql, &[&id, &plaintext_json]).await.unwrap();

// Query DIRECT from database (bypassing proxy, no decryption)
// The stored value should NOT be readable as the original JSON
let sql = "SELECT encrypted_jsonb::text FROM encrypted WHERE id = $1";
let stored: Vec<String> = query_direct_by(sql, &id).await;

assert_eq!(stored.len(), 1, "Expected exactly one row");
let stored_text = &stored[0];

// Verify it's NOT the plaintext JSON (encryption actually happened)
let plaintext_str = plaintext_json.to_string();
assert_ne!(
stored_text, &plaintext_str,
"ENCRYPTION FAILED: Stored value matches plaintext! Data was not encrypted."
);

// Additional verification: the encrypted format should be different structure
if let Ok(stored_json) = serde_json::from_str::<serde_json::Value>(stored_text) {
assert_ne!(
stored_json, plaintext_json,
"ENCRYPTION FAILED: Stored JSON structure matches plaintext!"
);
}

// Round-trip: query through proxy should decrypt back to original
let sql = "SELECT encrypted_jsonb FROM encrypted WHERE id = $1";
let rows = client.query(sql, &[&id]).await.unwrap();
assert_eq!(rows.len(), 1, "Expected exactly one row for round-trip");
let decrypted: serde_json::Value = rows[0].get(0);
assert_eq!(
decrypted, plaintext_json,
"DECRYPTION FAILED: Round-trip value doesn't match original!"
);
}

#[tokio::test]
async fn map_repeated_literals_different_columns_regression() {
clear().await;
Expand Down
Loading