From 192cd9a0f95c16ee5b0aae9636fce123afd045cd Mon Sep 17 00:00:00 2001 From: Kuan-Po Tseng Date: Mon, 2 Jun 2025 20:55:53 +0800 Subject: [PATCH 1/7] KAFKA-18462: Upgrade RocksDB dependency from 9.7.3 to 10.2.1 --- LICENSE-binary | 2 +- gradle/dependencies.gradle | 2 +- ...ToDbOptionsColumnFamilyOptionsAdapter.java | 22 ------------------- 3 files changed, 2 insertions(+), 24 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 7456e7fc60e4e..38b91555a24b5 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -246,7 +246,7 @@ License Version 2.0: - metrics-core-2.2.0 - opentelemetry-proto-1.0.0-alpha - plexus-utils-3.5.1 -- rocksdbjni-9.7.3 +- rocksdbjni-10.2.1 - scala-library-2.13.16 - scala-logging_2.13-3.9.5 - scala-reflect-2.13.16 diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index edde8a4e136bd..b5b9af6940177 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -113,7 +113,7 @@ versions += [ protobuf: "3.25.5", // a dependency of opentelemetryProto pcollections: "4.0.1", re2j: "1.7", - rocksDB: "9.7.3", + rocksDB: "10.2.1", // When updating the scalafmt version please also update the version field in checkstyle/.scalafmt.conf. scalafmt now // has the version field as mandatory in its configuration, see // https://github.com/scalameta/scalafmt/releases/tag/v3.1.0. diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java index 5b2f1e06b344f..85a87bbc9dc9a 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java @@ -574,17 +574,6 @@ public long compactionReadaheadSize() { return dbOptions.compactionReadaheadSize(); } - @Override - public Options setRandomAccessMaxBufferSize(final long randomAccessMaxBufferSize) { - dbOptions.setRandomAccessMaxBufferSize(randomAccessMaxBufferSize); - return this; - } - - @Override - public long randomAccessMaxBufferSize() { - return dbOptions.randomAccessMaxBufferSize(); - } - @Override public Options setWritableFileMaxBufferSize(final long writableFileMaxBufferSize) { dbOptions.setWritableFileMaxBufferSize(writableFileMaxBufferSize); @@ -1230,17 +1219,6 @@ public boolean paranoidFileChecks() { return columnFamilyOptions.paranoidFileChecks(); } - @Override - public Options setMaxWriteBufferNumberToMaintain(final int maxWriteBufferNumberToMaintain) { - columnFamilyOptions.setMaxWriteBufferNumberToMaintain(maxWriteBufferNumberToMaintain); - return this; - } - - @Override - public int maxWriteBufferNumberToMaintain() { - return columnFamilyOptions.maxWriteBufferNumberToMaintain(); - } - @Override public Options setCompactionPriority(final CompactionPriority compactionPriority) { columnFamilyOptions.setCompactionPriority(compactionPriority); From a9f8104cf0949b0f3eeb895e406e381e0eaedf33 Mon Sep 17 00:00:00 2001 From: Kuan-Po Tseng Date: Tue, 3 Jun 2025 00:32:38 +0800 Subject: [PATCH 2/7] Add new db option dailyOffpeakTimeUTC --- ...cOptionsToDbOptionsColumnFamilyOptionsAdapter.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java index 85a87bbc9dc9a..8036ed35dbb97 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java @@ -1741,6 +1741,17 @@ public int blobFileStartingLevel() { return columnFamilyOptions.blobFileStartingLevel(); } + @Override + public Options setDailyOffpeakTimeUTC(final String offpeakTimeUTC) { + dbOptions.setDailyOffpeakTimeUTC(offpeakTimeUTC); + return this; + } + + @Override + public String dailyOffpeakTimeUTC() { + return dbOptions.dailyOffpeakTimeUTC(); + } + // // END options for blobs (integrated BlobDB) // From 13f8c94195540dcf940f4432aa3a4f4dbebd7797 Mon Sep 17 00:00:00 2001 From: Kuan-Po Tseng Date: Thu, 12 Jun 2025 20:04:03 +0800 Subject: [PATCH 3/7] Revert "KAFKA-18462: Upgrade RocksDB dependency from 9.7.3 to 10.2.1" This reverts commit 192cd9a0f95c16ee5b0aae9636fce123afd045cd. --- LICENSE-binary | 2 +- gradle/dependencies.gradle | 2 +- ...ToDbOptionsColumnFamilyOptionsAdapter.java | 22 +++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 38b91555a24b5..7456e7fc60e4e 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -246,7 +246,7 @@ License Version 2.0: - metrics-core-2.2.0 - opentelemetry-proto-1.0.0-alpha - plexus-utils-3.5.1 -- rocksdbjni-10.2.1 +- rocksdbjni-9.7.3 - scala-library-2.13.16 - scala-logging_2.13-3.9.5 - scala-reflect-2.13.16 diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index b5b9af6940177..edde8a4e136bd 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -113,7 +113,7 @@ versions += [ protobuf: "3.25.5", // a dependency of opentelemetryProto pcollections: "4.0.1", re2j: "1.7", - rocksDB: "10.2.1", + rocksDB: "9.7.3", // When updating the scalafmt version please also update the version field in checkstyle/.scalafmt.conf. scalafmt now // has the version field as mandatory in its configuration, see // https://github.com/scalameta/scalafmt/releases/tag/v3.1.0. diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java index 8036ed35dbb97..104a8f5d19746 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java @@ -574,6 +574,17 @@ public long compactionReadaheadSize() { return dbOptions.compactionReadaheadSize(); } + @Override + public Options setRandomAccessMaxBufferSize(final long randomAccessMaxBufferSize) { + dbOptions.setRandomAccessMaxBufferSize(randomAccessMaxBufferSize); + return this; + } + + @Override + public long randomAccessMaxBufferSize() { + return dbOptions.randomAccessMaxBufferSize(); + } + @Override public Options setWritableFileMaxBufferSize(final long writableFileMaxBufferSize) { dbOptions.setWritableFileMaxBufferSize(writableFileMaxBufferSize); @@ -1219,6 +1230,17 @@ public boolean paranoidFileChecks() { return columnFamilyOptions.paranoidFileChecks(); } + @Override + public Options setMaxWriteBufferNumberToMaintain(final int maxWriteBufferNumberToMaintain) { + columnFamilyOptions.setMaxWriteBufferNumberToMaintain(maxWriteBufferNumberToMaintain); + return this; + } + + @Override + public int maxWriteBufferNumberToMaintain() { + return columnFamilyOptions.maxWriteBufferNumberToMaintain(); + } + @Override public Options setCompactionPriority(final CompactionPriority compactionPriority) { columnFamilyOptions.setCompactionPriority(compactionPriority); From f5a382e40afed2faefe4c9059a8f5b082a913643 Mon Sep 17 00:00:00 2001 From: Kuan-Po Tseng Date: Thu, 12 Jun 2025 20:14:39 +0800 Subject: [PATCH 4/7] KAFKA-18462: Upgrade RocksDB dependency from 9.7.3 to 10.1.3 --- LICENSE-binary | 2 +- gradle/dependencies.gradle | 2 +- ...tionsToDbOptionsColumnFamilyOptionsAdapter.java | 14 +++++++++----- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 7456e7fc60e4e..440c08c2c7873 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -246,7 +246,7 @@ License Version 2.0: - metrics-core-2.2.0 - opentelemetry-proto-1.0.0-alpha - plexus-utils-3.5.1 -- rocksdbjni-9.7.3 +- rocksdbjni-10.1.3 - scala-library-2.13.16 - scala-logging_2.13-3.9.5 - scala-reflect-2.13.16 diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index edde8a4e136bd..e90187654dd32 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -113,7 +113,7 @@ versions += [ protobuf: "3.25.5", // a dependency of opentelemetryProto pcollections: "4.0.1", re2j: "1.7", - rocksDB: "9.7.3", + rocksDB: "10.1.3", // When updating the scalafmt version please also update the version field in checkstyle/.scalafmt.conf. scalafmt now // has the version field as mandatory in its configuration, see // https://github.com/scalameta/scalafmt/releases/tag/v3.1.0. diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java index 104a8f5d19746..4b78cb07fe5f1 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java @@ -574,15 +574,19 @@ public long compactionReadaheadSize() { return dbOptions.compactionReadaheadSize(); } - @Override - public Options setRandomAccessMaxBufferSize(final long randomAccessMaxBufferSize) { - dbOptions.setRandomAccessMaxBufferSize(randomAccessMaxBufferSize); + @Deprecated(since = "4.2.0") + public Options setRandomAccessMaxBufferSize(final long ignored) { + log.warn("random_access_max_buffer_size has been removed in RocksDB v9.11.1." + + " See https://github.com/facebook/rocksdb/pull/13288"); + return this; } - @Override + @Deprecated(since = "4.2.0") public long randomAccessMaxBufferSize() { - return dbOptions.randomAccessMaxBufferSize(); + log.warn("random_access_max_buffer_size has been removed in RocksDB v9.11.1." + + " See https://github.com/facebook/rocksdb/pull/13288"); + return 0; } @Override From e6422431567d583079e5148931a30fd8a48b9774 Mon Sep 17 00:00:00 2001 From: Kuan-Po Tseng Date: Thu, 12 Jun 2025 23:48:38 +0800 Subject: [PATCH 5/7] refine the deprecated annotations --- ...DBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java index 4b78cb07fe5f1..5d4ec4eb4b964 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java @@ -574,7 +574,7 @@ public long compactionReadaheadSize() { return dbOptions.compactionReadaheadSize(); } - @Deprecated(since = "4.2.0") + @Deprecated(since = "4.2", forRemoval = true) public Options setRandomAccessMaxBufferSize(final long ignored) { log.warn("random_access_max_buffer_size has been removed in RocksDB v9.11.1." + " See https://github.com/facebook/rocksdb/pull/13288"); @@ -582,7 +582,7 @@ public Options setRandomAccessMaxBufferSize(final long ignored) { return this; } - @Deprecated(since = "4.2.0") + @Deprecated(since = "4.2", forRemoval = true) public long randomAccessMaxBufferSize() { log.warn("random_access_max_buffer_size has been removed in RocksDB v9.11.1." + " See https://github.com/facebook/rocksdb/pull/13288"); From ccfa842aad69da327453548811b473416cea363c Mon Sep 17 00:00:00 2001 From: Kuan-Po Tseng Date: Thu, 12 Jun 2025 23:50:32 +0800 Subject: [PATCH 6/7] remove blank line --- ...cksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java index 5d4ec4eb4b964..810ae8f0dd79e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java @@ -578,7 +578,6 @@ public long compactionReadaheadSize() { public Options setRandomAccessMaxBufferSize(final long ignored) { log.warn("random_access_max_buffer_size has been removed in RocksDB v9.11.1." + " See https://github.com/facebook/rocksdb/pull/13288"); - return this; } From 68753199fb2ea322f14adbb4f89e1a21fb5ce15c Mon Sep 17 00:00:00 2001 From: Kuan-Po Tseng Date: Fri, 13 Jun 2025 00:35:05 +0800 Subject: [PATCH 7/7] Mark rate_limiter option deprecated --- ...sDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java index 810ae8f0dd79e..99bfd6ba75184 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java @@ -797,8 +797,11 @@ public Options setMemTableConfig(final MemTableConfig config) { return this; } + @Deprecated(since = "4.2", forRemoval = true) @Override public Options setRateLimiter(final RateLimiter rateLimiter) { + log.warn("rate_limiter has been deprecated in RocksDB v7.6.0." + + " See https://github.com/facebook/rocksdb/pull/10378"); dbOptions.setRateLimiter(rateLimiter); return this; }