From 96705f962fda8db8ea32b4d64e7b4c42541fa92e Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Sat, 8 Nov 2025 21:30:56 +0000 Subject: [PATCH 1/5] Add Bytes test backlog entries --- TESTS_TODO.md | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 TESTS_TODO.md diff --git a/TESTS_TODO.md b/TESTS_TODO.md new file mode 100644 index 00000000000..375fcdfe1ea --- /dev/null +++ b/TESTS_TODO.md @@ -0,0 +1,66 @@ +# Chronicle Bytes – Test Coverage TODO + +Keeps track of Byte-specific test work (including batch additions beyond the module hand-offs). Follow AGENTS rules (British English, ISO-8859-1) and run `mvn -q verify` before publishing. + +## Repo & Commands + +* Repo root: `/home/peter/Build-All/Chronicle-Bytes` +* Iteration command (targeted): + ```bash + mvn -q -Dtest=BytesLifecycleTest,BytesCopyMatrixTest,UncheckedBytesBehaviourTest test + ``` +* Full suite: `mvn -q verify` + +## Completed This Session + +1. Added `BytesLifecycleTest` (slice ref-count lifecycle, elastic-growth monotonicity, `copyTo(BytesStore)` correctness). +2. Added `BytesCopyMatrixTest` (heap→native copy and direct→`OutputStream` copy coverage). +3. Extended `UncheckedBytesBehaviourTest` with `uncheckedModeAllowsWritePastLimit`. +4. Targeted tests run via `mvn -q -Dtest=BytesLifecycleTest,BytesCopyMatrixTest,UncheckedBytesBehaviourTest test` – presently passing. + +## Outstanding Batches + +The following scenarios (from the user brief) remain **unimplemented** and should be prioritised next: + +1. **OS/Safe Page Size Matrix** – parameterised test over synthetic page sizes verifying `MappedBytesStore#inside` and `SingleMappedBytes.zapPage` when `OS.defaultOsPageSize()` differs from actual page size. Add to `MappedBytesStoreTest` or `MappedBytesTest`. +2. **Safe-Page Block Size for Builders** – test builder defaults so Windows uses `OS.SAFE_PAGE_SIZE` while other OSes use `OS.pageSize()`. Add a new `MappedBytesQueueBuilderTest` or equivalent. +3. **Temp Directory Integration** – ensure Bytes’ use of `IOTools.createTempDirectory` returns paths under `OS.getTarget()` and cleans up (likely add a dedicated test leveraging `BytesTestCommon`). +4. **Reference Tracing Hook** – add `AbstractBytesReferenceTest` that leaks a `Bytes` subclass intentionally and asserts `enableReferenceTracing()` reports the leak (mirrors similar Chronicle Core test). +5. **OS.pageAlign Consumers** – assert `MappedBytesStore.map()` (or helpers) align offsets using `OS.pageAlign`, especially for large offsets/Windows safe pages; extend `MappedBytesTest` with a synthetic offset scenario. + +## Additional Proposed Tests + +6. **`BytesComparisonMatrixTest`** + * Exercise `BytesStore.compareAndSwapInt/Long` across heap, direct, and mapped stores to ensure mixed-type comparisons behave consistently and honour alignment rules. Include negative tests where the address straddles the capacity boundary to confirm deterministic exceptions. + +7. **`ElasticByteBufferReuseTest`** + * Reproduce the pattern from Chronicle Wire: allocate an elastic direct buffer, write past the original capacity with `writePosition` growth, then hand the buffer to `BinaryWire`. Assert that ref-counts stay positive, and once `releaseLast()` is invoked the backing `ByteBuffer` becomes inaccessible (guards against use-after-free regressions). + +8. **`TempFileCleanupTest`** + * Create temporary `Bytes` backed by `IOTools.createTempFile()` and verify that `Bytes.deleteFile()` removes both the data file and the `.tmp` companion. Mimic busy Windows behaviour by holding a channel open and confirming the new retry loop logs the appropriate warning. + +9. **`BytesHexDumpDeterminismTest`** + * Feed `HexDumpBytes` with sequences containing control characters and high-bit values, ensuring `toHexString()` and `parseHexString()` round-trip identically (protects tooling that relies on deterministic dumps when debugging network captures). + +10. **`UnsafeDirectStoreAlignmentTest`** + * Parameterise over odd/even alignment sizes to assert that `UnsafeDirectStore.of()` respects the requested alignment and that `addressForRead`/`addressForWrite` never exceed `realCapacity()`. This mirrors the Chronicle Queue use-case where misalignment breaks memory-mapped header validation. + +## Next Steps + +1. Design tests for items 1–5 above; consider using `OS.memory()`/wrapper to mock page sizes where needed. +2. After implementing each chunk, re-run targeted tests plus `mvn -q verify`. +3. Update this file (and per-module `NEW_TESTS.md` if you add repo-specific notes) once the outstanding scenarios are covered. + +## Additional Ideas (Queue-derived backlog) +| Batch | Scope | Goal | Notes | +|------|-------|------|-------| +| B-07 | Zero-length frame handling | Add Bytes-level tests that write/read empty payloads using `Bytes` and ensure downstream consumers observe zero-length content without blocking the next message. | Mirrors `ReaderResizesFileTest.testZeroLengthDocumentDoesNotBlockTailer`; ensures Bytes APIs surface the same invariants. | +| B-08 | Partial frame mutation guards | Extend Bytes fuzz tests to mutate SPB headers mid-read (length mismatch, corrupted prefix) and assert defensive checks fire before data becomes visible. | Aligns with queue tests for partial frames; keeps raw Bytes primitives honest. | + +## Fresh Candidates (add to queue soon) + +1. **PinnedMappedBytesShutdownTest** – simulate long-lived `MappedBytes` held open during JVM shutdown, assert `MappedFile.release()` frees native handles and log noise stays below the warning threshold (covers Windows busy-file reports). +2. **BytesUTF8SurrogateMatrixTest** – feed `AppendableUtil` with mixed valid and broken surrogate pairs, verifying encoder paths either emit replacement chars or throw per `StopCharTesters` expectations; ensures UTF-8 helpers keep Chronicle Wire interop safe. +3. **NativeBytesBoundsRaceTest** – run two threads performing `writePosition` + `readPosition` mutations against the same `NativeBytes` instance under a `Pauser` to confirm CAS guards prevent negative capacities and throw `BufferOverflowException` deterministically. +4. **ReadonlyMappedBytesRelaxedViewTest** – open the same mapped file as read-only and read-write Bytes, flip the writable view via `MappedBytes.writingDocument()`, and assert the read-only view never observes partial header writes (protects replication recovery scans). +5. **BytesCompressionRoundTripTest** – integrate `LZ4Compressor`/`SnappyCompressor` adapters to compress into a `Bytes` sink, then decompress via the matching codec to prove direct/native stores do not require intermediary arrays and capacity grows elastically without leaks. From 537b29010d59b2f3429ff1d2c74b7cf1ba72af17 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Sat, 8 Nov 2025 21:54:07 +0000 Subject: [PATCH 2/5] Add coverage for custom mappings and tracing --- TESTS_TODO.md | 11 +- src/main/docs/architecture-overview.adoc | 1 + src/main/docs/memory-management.adoc | 10 ++ src/main/docs/project-requirements.adoc | 126 +++++++++--------- .../chronicle/bytes/MappedBytesStore.java | 17 +-- .../bytes/StreamingOutputStream.java | 4 +- .../chronicle/bytes/UncheckedBytes.java | 8 +- .../chronicle/bytes/UncheckedNativeBytes.java | 6 +- .../bytes/internal/CommonMappedBytes.java | 2 +- .../bytes/internal/EmbeddedBytes.java | 4 +- .../bytes/internal/NativeBytesStore.java | 2 +- .../bytes/internal/NoBytesStore.java | 6 +- .../bytes/internal/SingleMappedBytes.java | 4 +- .../bytes/internal/SingleMappedFile.java | 16 +-- .../chronicle/bytes/internal/Unmapper.java | 2 +- .../chronicle/bytes/internal/UnsafeText.java | 10 +- .../chronicle/bytes/MappedBytesTest.java | 32 +++++ .../chronicle/bytes/MappedFileTest.java | 23 ++++ .../bytes/ReferenceTracingLeakTest.java | 27 ++++ .../bytes/TempDirectoryIntegrationTest.java | 36 +++++ 20 files changed, 234 insertions(+), 113 deletions(-) create mode 100644 src/test/java/net/openhft/chronicle/bytes/ReferenceTracingLeakTest.java create mode 100644 src/test/java/net/openhft/chronicle/bytes/TempDirectoryIntegrationTest.java diff --git a/TESTS_TODO.md b/TESTS_TODO.md index 375fcdfe1ea..dc9205be7b9 100644 --- a/TESTS_TODO.md +++ b/TESTS_TODO.md @@ -17,16 +17,17 @@ Keeps track of Byte-specific test work (including batch additions beyond the mod 2. Added `BytesCopyMatrixTest` (heap→native copy and direct→`OutputStream` copy coverage). 3. Extended `UncheckedBytesBehaviourTest` with `uncheckedModeAllowsWritePastLimit`. 4. Targeted tests run via `mvn -q -Dtest=BytesLifecycleTest,BytesCopyMatrixTest,UncheckedBytesBehaviourTest test` – presently passing. +5. Added `MappedFileTest.insideHonoursSafeLimitWhenPageSizeDiffers` to exercise `MappedBytesStore#inside` across custom page sizes. +6. Added `MappedBytesTest.zeroOutRespectsCustomPageSize` to cover single mapping zero-out behaviour when the mapping page size is larger than the OS default. +7. Added `TempDirectoryIntegrationTest` to verify `IOTools.createTempDirectory` locates paths beneath `OS.getTarget()` and that `deleteDirWithFiles` removes them. +8. Added `ReferenceTracingLeakTest` to confirm `AbstractReferenceCounted.assertReferencesReleased()` surfaces leaks with the recorded `createdHere` trace. ## Outstanding Batches The following scenarios (from the user brief) remain **unimplemented** and should be prioritised next: -1. **OS/Safe Page Size Matrix** – parameterised test over synthetic page sizes verifying `MappedBytesStore#inside` and `SingleMappedBytes.zapPage` when `OS.defaultOsPageSize()` differs from actual page size. Add to `MappedBytesStoreTest` or `MappedBytesTest`. -2. **Safe-Page Block Size for Builders** – test builder defaults so Windows uses `OS.SAFE_PAGE_SIZE` while other OSes use `OS.pageSize()`. Add a new `MappedBytesQueueBuilderTest` or equivalent. -3. **Temp Directory Integration** – ensure Bytes’ use of `IOTools.createTempDirectory` returns paths under `OS.getTarget()` and cleans up (likely add a dedicated test leveraging `BytesTestCommon`). -4. **Reference Tracing Hook** – add `AbstractBytesReferenceTest` that leaks a `Bytes` subclass intentionally and asserts `enableReferenceTracing()` reports the leak (mirrors similar Chronicle Core test). -5. **OS.pageAlign Consumers** – assert `MappedBytesStore.map()` (or helpers) align offsets using `OS.pageAlign`, especially for large offsets/Windows safe pages; extend `MappedBytesTest` with a synthetic offset scenario. +1. **Safe-Page Block Size for Builders** – test builder defaults so Windows uses `OS.SAFE_PAGE_SIZE` while other OSes use `OS.pageSize()`. Add a new `MappedBytesQueueBuilderTest` or equivalent. +2. **OS.pageAlign Consumers** – assert `MappedBytesStore.map()` (or helpers) align offsets using `OS.pageAlign`, especially for large offsets/Windows safe pages; extend `MappedBytesTest` with a synthetic offset scenario. ## Additional Proposed Tests diff --git a/src/main/docs/architecture-overview.adoc b/src/main/docs/architecture-overview.adoc index bf1868ed890..0c746867ac6 100644 --- a/src/main/docs/architecture-overview.adoc +++ b/src/main/docs/architecture-overview.adoc @@ -212,6 +212,7 @@ This architecture directly supports and enables the fulfilment of the functional * Core API exposure for various memory types and operations (CB-FN-001 .. CB-FN-008, CB-FN-012, CB-FN-014). * Performance targets for serialization, access speed, and atomic operations (e.g., <>, <>, <>). +* Multi-JDK build coverage ensures both baseline Java 8 users and teams piloting the latest releases stay unblocked (<>). * Operability hooks like JMX metrics for `BytesMetrics` (CB-NF-O-002) allow integration into standard monitoring dashboards. * Security considerations like robust bounds checking (CB-NF-S-001) are integral to the `Bytes` API design. diff --git a/src/main/docs/memory-management.adoc b/src/main/docs/memory-management.adoc index eeb72395bb3..6474800e9bf 100644 --- a/src/main/docs/memory-management.adoc +++ b/src/main/docs/memory-management.adoc @@ -50,6 +50,16 @@ resources. Enable `Jvm.isResourceTracing()` during testing to record stack traces for allocations. This helps locate leaks if `releaseLast()` is not called. +`AbstractReferenceCounted` implements `ReferenceCountedTracer`, so classes such +as `NativeBytes` can call `createdHere()` to recover the stack trace of the +allocation site. Chronicle Bytes feeds that trace into diagnostics like +`DecoratedBufferOverflowException` in `NativeBytes.newDBOE(...)`, which means a +resizing failure reports both the write that triggered it and where the buffer +originated. This is invaluable when multiple components reuse elastic buffers +and you need to pinpoint which call chain produced a leaked or overgrown +instance. The richer exception context fulfils the diagnostic requirement in +<>. + == 6 Elastic Buffers Elastic `Bytes` can grow when more space is required. For native buffers a new diff --git a/src/main/docs/project-requirements.adoc b/src/main/docs/project-requirements.adoc index 71a9ab7dfd4..6d534f14dcc 100644 --- a/src/main/docs/project-requirements.adoc +++ b/src/main/docs/project-requirements.adoc @@ -24,33 +24,33 @@ These requirements define the contracts, performance envelopes and operational o [cols="1,5,4",options="header"] |=== |ID |Description |Verification -|CB-FN-001 |Provide the `Bytes` API that exposes separate read and write +|[[CB-FN-001]]CB-FN-001 |Provide the `Bytes` API that exposes separate read and write cursors, limits and capacity, allowing zero-copy access to arbitrary memory regions. |Unit tests exercise cursor moves, bounds checks and data integrity. -|CB-FN-002 |Implement `BytesStore` as the immutable backing store for +|[[CB-FN-002]]CB-FN-002 |Implement `BytesStore` as the immutable backing store for `Bytes`; support on-heap, direct (off-heap) and memory-mapped file variants. |Integration tests open each variant, write then re-read data. -|CB-FN-003 |Support elastic buffers that grow transparently up to the maximum +|[[CB-FN-003]]CB-FN-003 |Support elastic buffers that grow transparently up to the maximum real capacity advertised by the chosen `BytesStore`. |JMH benchmark writes variable-length records until capacity grows, asserting no exceptions. -|CB-FN-004 |Expose atomic operations (compare-and-swap, get-and-add) for 32- and +|[[CB-FN-004]]CB-FN-004 |Expose atomic operations (compare-and-swap, get-and-add) for 32- and 64-bit integers and other relevant primitives located in off-heap memory. |Concurrency stress test using multiple threads contending on the same address, asserting correctness under load. -|CB-FN-005 |Provide stop-bit encoding and decoding utilities for signed and +|[[CB-FN-005]]CB-FN-005 |Provide stop-bit encoding and decoding utilities for signed and unsigned integer types (long, int). |Round-trip property-based test over random values. -|CB-FN-006 |Allow creation of lightweight slice views (`bytesForRead`, +|[[CB-FN-006]]CB-FN-006 |Allow creation of lightweight slice views (`bytesForRead`, `bytesForWrite`, `slice`) without data copying. |Memory accounting test proves the same underlying address is shared; operations on slice reflect in parent and vice-versa. -|CB-FN-007 |Offer hex-dump debugging support via `HexDumpBytes` or similar utility. |Unit test +|[[CB-FN-007]]CB-FN-007 |Offer hex-dump debugging support via `HexDumpBytes` or similar utility. |Unit test emits dump and matches expected ASCII layout for various data types. -|CB-FN-008 |Integrate with Chronicle Wire to permit `MethodWriter` and +|[[CB-FN-008]]CB-FN-008 |Integrate with Chronicle Wire to permit `MethodWriter` and `MethodReader` serialisation on any supported `Bytes` implementation. |End-to-end test serialises a six-field POJO then deserialises, asserting data integrity and listener invocation. -|CB-FN-009 |Support efficient parsing and formatting of common string encodings (e.g., UTF-8, ISO-8859-1, US-ASCII) with options to control encoding/decoding behaviour, including length prefixes. |Unit tests for various encodings, including edge cases, malformed sequences, and different length prefixing strategies. Property-based tests for round-trip fidelity. -|CB-FN-010 |Provide mechanisms to wrap existing `byte[]` arrays (including sub-regions) and `java.nio.ByteBuffer` instances (read-only and writable) without copying data. |Unit tests wrap various array/buffer types and sub-regions, perform operations, and verify data integrity and no-copy behaviour. -|CB-FN-011 |Offer methods for efficient bulk data transfer between `Bytes` instances or from/to `BytesStore` and other data sources like `InputStream`/`OutputStream` or `java.nio.channels.ReadableByteChannel`/`WritableByteChannel`. |JMH benchmarks comparing bulk transfer speeds with manual loops and `java.nio` alternatives for various sizes. -|CB-FN-012 |Ensure all `Bytes` implementations correctly handle memory alignment for primitive types where platform performance is sensitive to it, particularly for off-heap and memory-mapped stores. This includes providing aligned write/read methods. |Micro-benchmarks measuring access times for aligned vs. unaligned data; code review for alignment strategies. Unit tests for aligned access methods. -|CB-FN-013 |Provide utility methods for common byte-level manipulations, such as finding a specific byte or byte sequence, or comparing regions of `Bytes` instances. |Unit tests verify correctness for various search patterns and comparison scenarios. -|CB-FN-014 |Support for acquiring and releasing `Bytes` instances from a configurable pool to reduce allocation overhead and GC pressure. |JMH benchmark showing performance benefits of pooling under high churn. Unit tests for pool lifecycle and instance re-use. +|[[CB-FN-009]]CB-FN-009 |Support efficient parsing and formatting of common string encodings (e.g., UTF-8, ISO-8859-1, US-ASCII) with options to control encoding/decoding behaviour, including length prefixes. |Unit tests for various encodings, including edge cases, malformed sequences, and different length prefixing strategies. Property-based tests for round-trip fidelity. +|[[CB-FN-010]]CB-FN-010 |Provide mechanisms to wrap existing `byte[]` arrays (including sub-regions) and `java.nio.ByteBuffer` instances (read-only and writable) without copying data. |Unit tests wrap various array/buffer types and sub-regions, perform operations, and verify data integrity and no-copy behaviour. +|[[CB-FN-011]]CB-FN-011 |Offer methods for efficient bulk data transfer between `Bytes` instances or from/to `BytesStore` and other data sources like `InputStream`/`OutputStream` or `java.nio.channels.ReadableByteChannel`/`WritableByteChannel`. |JMH benchmarks comparing bulk transfer speeds with manual loops and `java.nio` alternatives for various sizes. +|[[CB-FN-012]]CB-FN-012 |Ensure all `Bytes` implementations correctly handle memory alignment for primitive types where platform performance is sensitive to it, particularly for off-heap and memory-mapped stores. This includes providing aligned write/read methods. |Micro-benchmarks measuring access times for aligned vs. unaligned data; code review for alignment strategies. Unit tests for aligned access methods. +|[[CB-FN-013]]CB-FN-013 |Provide utility methods for common byte-level manipulations, such as finding a specific byte or byte sequence, or comparing regions of `Bytes` instances. |Unit tests verify correctness for various search patterns and comparison scenarios. +|[[CB-FN-014]]CB-FN-014 |Support for acquiring and releasing `Bytes` instances from a configurable pool to reduce allocation overhead and GC pressure. |JMH benchmark showing performance benefits of pooling under high churn. Unit tests for pool lifecycle and instance re-use. |=== == 3 Non-Functional Requirements – Performance @@ -58,22 +58,22 @@ integrity and listener invocation. [cols="1,5,4",options="header"] |=== |ID |Requirement |Benchmark Target -|CB-NF-P-001 |Serialise and deserialise a six-field object (string plus enum) +|[[CB-NF-P-001]]CB-NF-P-001 |Serialise and deserialise a six-field object (string plus enum) using YAML Wire in ≤ 0.25 µs mean and ≤ 0.5 µs 99.9 %tile on 3.2 GHz x86-64. |JMH `SampleTime` benchmark, ≥ 10 M ops. -|CB-NF-P-002 |`Bytes.equalsBytes()` or equivalent region comparison method must be **≥ 3 ×** faster than +|[[CB-NF-P-002]]CB-NF-P-002 |`Bytes.equalsBytes()` or equivalent region comparison method must be **≥ 3 ×** faster than `java.nio.ByteBuffer.equals` for 32 KiB payloads (on-heap and off-heap). |Custom micro-benchmark throughput comparison leveraging JVM intrinsics where possible. -|CB-NF-P-003 |Allocate **and** release an elastic off-heap buffer of 1 MiB in +|[[CB-NF-P-003]]CB-NF-P-003 |Allocate **and** release an elastic off-heap buffer of 1 MiB in ≤ 20 µs 99.99 %tile. |JMH `SingleShot` benchmark, ≥ 1 000 repeats. -|CB-NF-P-004 |Read an eight-byte aligned long from hot-cache `MappedBytes` in +|[[CB-NF-P-004]]CB-NF-P-004 |Read an eight-byte aligned long from hot-cache `MappedBytes` in ≤ 80 ns mean and ≤ 150 ns 99.9 %tile. |Low-level harness pinned with thread affinity. -|CB-NF-P-005 |`compareAndSwapLong` on a contended cache line (off-heap) sustains ≥ 25 M +|[[CB-NF-P-005]]CB-NF-P-005 |`compareAndSwapLong` on a contended cache line (off-heap) sustains ≥ 25 M successful swaps / sec on dual-socket Intel Xeon Gold 6338N (or equivalent modern server CPU). |5 s busy-loop calibration with multiple contending threads. -|CB-NF-P-006 |UTF-8 string encoding (128 ASCII characters) throughput ≥ 500 MB/s; decoding throughput ≥ 400 MB/s on 3.2 GHz x86-64. |JMH benchmark with typical string lengths (e.g. 16-256 chars). -|CB-NF-P-007 |Random read/write access for primitive types (long, int, double) directly on off-heap `Bytes` instances sustains ≥ 200 M ops/sec per core. |JMH benchmark targeting various primitive access patterns (sequential and random). -|CB-NF-P-008 |Resizing an elastic off-heap buffer (e.g., doubling capacity from 1MB to 2MB) should complete in ≤ 5 µs 99%tile, assuming underlying memory allocation is fast. |JMH benchmark measuring resize operation latency. -|CB-NF-P-009 |Acquiring a pooled off-heap `Bytes` instance (1KB) and releasing it must be ≤ 100 ns mean time. |JMH benchmark focusing on pool `acquire`/`release` cycle. +|[[CB-NF-P-006]]CB-NF-P-006 |UTF-8 string encoding (128 ASCII characters) throughput ≥ 500 MB/s; decoding throughput ≥ 400 MB/s on 3.2 GHz x86-64. |JMH benchmark with typical string lengths (e.g. 16-256 chars). +|[[CB-NF-P-007]]CB-NF-P-007 |Random read/write access for primitive types (long, int, double) directly on off-heap `Bytes` instances sustains ≥ 200 M ops/sec per core. |JMH benchmark targeting various primitive access patterns (sequential and random). +|[[CB-NF-P-008]]CB-NF-P-008 |Resizing an elastic off-heap buffer (e.g., doubling capacity from 1MB to 2MB) should complete in ≤ 5 µs 99%tile, assuming underlying memory allocation is fast. |JMH benchmark measuring resize operation latency. +|[[CB-NF-P-009]]CB-NF-P-009 |Acquiring a pooled off-heap `Bytes` instance (1KB) and releasing it must be ≤ 100 ns mean time. |JMH benchmark focusing on pool `acquire`/`release` cycle. |=== == 4 Non-Functional Requirements – Security @@ -81,10 +81,10 @@ calibration with multiple contending threads. [cols="1,5,4",options="header"] |=== |ID |Requirement |Verification -|CB-NF-S-001 |All memory access operations **must** perform rigorous bounds checking to prevent buffer overflows or underflows, leading to deterministic exceptions (e.g., `BufferOverflowException`, `IndexOutOfBoundsException`) rather than undefined behaviour or JVM crashes. |Property-based tests attempting out-of-bounds access across all `Bytes` types and operations. Static analysis tool checks (e.g., SpotBugs). -|CB-NF-S-002 |For memory-mapped files, ensure appropriate file permissions and locking mechanisms are utilized by default to prevent unintended data corruption when files might be shared between processes, if such sharing is a supported feature. Documentation must highlight safe sharing patterns. |Integration tests simulating multi-process access with varied permission settings. Code review of file handling. -|CB-NF-S-003 |The library **must not** introduce vulnerabilities through its own direct JNI/native code (if any beyond standard JDK calls like Unsafe). Any use of `sun.misc.Unsafe` must be encapsulated, its risks understood, documented, and limited to essential use cases. |Code review of any native code or `Unsafe` usage. Dependency vulnerability scan (e.g., OWASP Dependency-Check) as part of CI. -|CB-NF-S-004 |Operations that can potentially lead to excessive memory allocation (e.g., elastic buffer growth with very large inputs) should have documented safeguards or configurable limits where appropriate, or clear warnings about potential `OutOfMemoryError`. |Documentation review; specific tests attempting to trigger excessive allocation. +|[[CB-NF-S-001]]CB-NF-S-001 |All memory access operations **must** perform rigorous bounds checking to prevent buffer overflows or underflows, leading to deterministic exceptions (e.g., `BufferOverflowException`, `IndexOutOfBoundsException`) rather than undefined behaviour or JVM crashes. |Property-based tests attempting out-of-bounds access across all `Bytes` types and operations. Static analysis tool checks (e.g., SpotBugs). +|[[CB-NF-S-002]]CB-NF-S-002 |For memory-mapped files, ensure appropriate file permissions and locking mechanisms are utilized by default to prevent unintended data corruption when files might be shared between processes, if such sharing is a supported feature. Documentation must highlight safe sharing patterns. |Integration tests simulating multi-process access with varied permission settings. Code review of file handling. +|[[CB-NF-S-003]]CB-NF-S-003 |The library **must not** introduce vulnerabilities through its own direct JNI/native code (if any beyond standard JDK calls like Unsafe). Any use of `sun.misc.Unsafe` must be encapsulated, its risks understood, documented, and limited to essential use cases. |Code review of any native code or `Unsafe` usage. Dependency vulnerability scan (e.g., OWASP Dependency-Check) as part of CI. +|[[CB-NF-S-004]]CB-NF-S-004 |Operations that can potentially lead to excessive memory allocation (e.g., elastic buffer growth with very large inputs) should have documented safeguards or configurable limits where appropriate, or clear warnings about potential `OutOfMemoryError`. |Documentation review; specific tests attempting to trigger excessive allocation. |=== == 5 Non-Functional Requirements – Operability @@ -92,12 +92,12 @@ calibration with multiple contending threads. [cols="1,5,4",options="header"] |=== |ID |Requirement |Evidence -|CB-NF-O-001 |All off-heap resources (`BytesStore` and `Bytes` instances holding them) **must** implement `java.io.Closeable` and explicit `releaseLast()`/`close()` methods for deterministic reclamation. Reference counting must be robust. |Unit tests track allocated vs released native bytes; stress tests for reference counting correctness under concurrent access if applicable. -|CB-NF-O-002 |Expose `BytesMetrics` (e.g., count of active instances, total native memory allocated/used by type – direct, mapped) via JMX MBeans. |Integration test registers MBean and observes counter changes under various load patterns. -|CB-NF-O-003 |Log a single WARN with stack trace if a `Bytes`/`BytesStore` instance holding native resources is GC-finalised while still retained (ref-count \> 0). |Fault-injection test leaks a buffer and verifies log output and frequency. -|CB-NF-O-004 |All default log messages use SLF4J and allocate zero temporary objects on the hot path when logging is disabled at the configured level (e.g. INFO and above). |JMH probe with logging disabled at various levels. Code review of logging statements. -|CB-NF-O-005 |Define and document the thread-safety guarantees for all `Bytes` and `BytesStore` operations. Specifically, `Bytes` instances are not thread-safe for concurrent mutation of cursors or content by default. `BytesStore` atomic operations are thread-safe. Concurrent reads on a single `Bytes` instance by multiple threads are permissible if no writes occur to that `Bytes` instance's view or the underlying store section it views. |Documentation review; targeted concurrency tests for documented safe/unsafe patterns. -|CB-NF-O-006 |The library should provide clear diagnostic information (e.g., detailed exception messages, relevant state) when common errors occur (e.g., out-of-bounds access, resource leaks, mapping failures). |Review of exception handling code. Unit tests that trigger common errors verify quality of diagnostic messages. +|[[CB-NF-O-001]]CB-NF-O-001 |All off-heap resources (`BytesStore` and `Bytes` instances holding them) **must** implement `java.io.Closeable` and explicit `releaseLast()`/`close()` methods for deterministic reclamation. Reference counting must be robust. |Unit tests track allocated vs released native bytes; stress tests for reference counting correctness under concurrent access if applicable. +|[[CB-NF-O-002]]CB-NF-O-002 |Expose `BytesMetrics` (e.g., count of active instances, total native memory allocated/used by type – direct, mapped) via JMX MBeans. |Integration test registers MBean and observes counter changes under various load patterns. +|[[CB-NF-O-003]]CB-NF-O-003 |Log a single WARN with stack trace if a `Bytes`/`BytesStore` instance holding native resources is GC-finalised while still retained (ref-count \> 0). |Fault-injection test leaks a buffer and verifies log output and frequency. +|[[CB-NF-O-004]]CB-NF-O-004 |All default log messages use SLF4J and allocate zero temporary objects on the hot path when logging is disabled at the configured level (e.g. INFO and above). |JMH probe with logging disabled at various levels. Code review of logging statements. +|[[CB-NF-O-005]]CB-NF-O-005 |Define and document the thread-safety guarantees for all `Bytes` and `BytesStore` operations. Specifically, `Bytes` instances are not thread-safe for concurrent mutation of cursors or content by default. `BytesStore` atomic operations are thread-safe. Concurrent reads on a single `Bytes` instance by multiple threads are permissible if no writes occur to that `Bytes` instance's view or the underlying store section it views. |Documentation review; targeted concurrency tests for documented safe/unsafe patterns. +|[[CB-NF-O-006]]CB-NF-O-006 |The library should provide clear diagnostic information (e.g., detailed exception messages, relevant state) when common errors occur (e.g., out-of-bounds access, resource leaks, mapping failures). |Review of exception handling code. Unit tests that trigger common errors verify quality of diagnostic messages. |=== == 6 Non-Functional Requirements – Maintainability & Portability @@ -105,11 +105,12 @@ calibration with multiple contending threads. [cols="1,5,4",options="header"] |=== |ID |Requirement |Verification -|CB-NF-MP-001 |The library **must** be compatible with and fully functional on at least the last three LTS versions of Java (e.g., Java 11, 17, 21), with clear documentation on any version-specific behaviour or performance characteristics. Support for Java 8 may be maintained on a best-effort basis if critical. |Full test suite execution on all declared supported Java versions in CI. -|CB-NF-MP-002 |The codebase **must** adhere to OpenHFT coding standards and conventions (e.g., style, naming, design patterns). Module dependencies should be minimized to reduce complexity and potential conflicts. |Static analysis (Checkstyle, PMD, SonarLint) in build; dependency tree review (`mvn dependency:tree`). -|CB-NF-MP-003 |Public APIs **must** have a defined backward compatibility policy (e.g., semantic versioning 2.0.0). Deprecation of APIs should follow a clear process with advance notice (e.g., via `@Deprecated` javadoc, release notes) and migration paths. |Changelog review; API evolution documentation; use of tools like Revapi for checking API changes. -|CB-NF-MP-004 |The library should build and pass all tests on common operating systems used in development and deployment (e.g., Linux x86-64, macOS x86-64/ARM64, Windows x86-64). |CI jobs for each supported OS/architecture combination. -|CB-NF-MP-005 |Internal components should be well-encapsulated to facilitate future refactoring and evolution without breaking public contracts. |Code reviews focusing on API design and internal module structure. +|[[CB-NF-MP-001]]CB-NF-MP-001 |The library **must** be compatible with and fully functional on at least the last three LTS versions of Java (e.g., Java 11, 17, 21), with clear documentation on any version-specific behaviour or performance characteristics. Support for Java 8 may be maintained on a best-effort basis if critical. |Full test suite execution on all declared supported Java versions in CI. +|[[CB-NF-MP-002]]CB-NF-MP-002 |The codebase **must** adhere to OpenHFT coding standards and conventions (e.g., style, naming, design patterns). Module dependencies should be minimized to reduce complexity and potential conflicts. |Static analysis (Checkstyle, PMD, SonarLint) in build; dependency tree review (`mvn dependency:tree`). +|[[CB-NF-MP-003]]CB-NF-MP-003 |Public APIs **must** have a defined backward compatibility policy (e.g., semantic versioning 2.0.0). Deprecation of APIs should follow a clear process with advance notice (e.g., via `@Deprecated` javadoc, release notes) and migration paths. |Changelog review; API evolution documentation; use of tools like Revapi for checking API changes. +|[[CB-NF-MP-004]]CB-NF-MP-004 |The library should build and pass all tests on common operating systems used in development and deployment (e.g., Linux x86-64, macOS x86-64/ARM64, Windows x86-64). |CI jobs for each supported OS/architecture combination. +|[[CB-NF-MP-005]]CB-NF-MP-005 |Internal components should be well-encapsulated to facilitate future refactoring and evolution without breaking public contracts. |Code reviews focusing on API design and internal module structure. +|[[CB-NF-MP-006]]CB-NF-MP-006 |Maintained compatibility must cover both the legacy baseline (Java 8, still required by several Chronicle deployments) and the latest released/EA JDK (currently Java 25) to surface preview breakages early. Release candidates run `mvn clean verify` on both toolchains before tagging. |CI matrix entries for JDK 8 and JDK 25, plus release checklist items capturing dual-JDK verification logs. |=== == 7 Documentation Obligations @@ -117,16 +118,16 @@ calibration with multiple contending threads. [cols="1,5,4",options="header"] |=== |ID |Requirement |Compliance Check -|CB-DOC-001 |All AsciiDoc and Javadoc **must** use British English and ASCII-7 (or UTF-8 where essential, clearly marked). |Automated spell- and byte-range scan in Maven `verify`. -|CB-DOC-002 |Javadoc first sentence is a concise behavioural summary; blocks +|[[CB-DOC-001]]CB-DOC-001 |All AsciiDoc and Javadoc **must** use British English and ASCII-7 (or UTF-8 where essential, clearly marked). |Automated spell- and byte-range scan in Maven `verify`. +|[[CB-DOC-002]]CB-DOC-002 |Javadoc first sentence is a concise behavioural summary; blocks must not duplicate information manifest from the signature. All public classes and methods must have Javadoc. |Checkstyle custom rule or equivalent. Javadoc generation and spot checks. -|CB-DOC-003 |`docs/chronicle-bytes-manual.adoc` includes a worked example for +|[[CB-DOC-003]]CB-DOC-003 |`docs/chronicle-bytes-manual.adoc` includes a worked example for each public factory method/common use case and a performance-tuning section (covering memory types, pooling, alignment, string encodings). |Peer review before release. Review of example code for correctness and clarity. -|CB-DOC-004 |`README.md` links to key performance figures (e.g., representative latency/throughput from CB-NF-P benchmarks) produced by latest CI +|[[CB-DOC-004]]CB-DOC-004 |`README.md` links to key performance figures (e.g., representative latency/throughput from CB-NF-P benchmarks) produced by latest CI benchmark run. |CI pipeline updates badge or embedded figures on commit/release. -|CB-DOC-005 |Document the thread-safety model of `Bytes` and `BytesStore` explicitly, including safe usage patterns for concurrent access and an explanation of the reference counting mechanism. |Peer review of documentation section on concurrency and memory management. -|CB-DOC-006 |Provide clear examples and explanations for managing the lifecycle of different `BytesStore` types, especially for off-heap and memory-mapped resources, including typical error handling and resource reclamation patterns (`releaseLast`/`close`). |Peer review of resource management documentation and examples. -|CB-DOC-007 |Glossary (`CB-GLOSS-001`) must be comprehensive and cover all specialized terms used in the library and its documentation. |Peer review of glossary against documentation and codebase. +|[[CB-DOC-005]]CB-DOC-005 |Document the thread-safety model of `Bytes` and `BytesStore` explicitly, including safe usage patterns for concurrent access and an explanation of the reference counting mechanism. |Peer review of documentation section on concurrency and memory management. +|[[CB-DOC-006]]CB-DOC-006 |Provide clear examples and explanations for managing the lifecycle of different `BytesStore` types, especially for off-heap and memory-mapped resources, including typical error handling and resource reclamation patterns (`releaseLast`/`close`). |Peer review of resource management documentation and examples. +|[[CB-DOC-007]]CB-DOC-007 |Glossary (`CB-GLOSS-001`) must be comprehensive and cover all specialized terms used in the library and its documentation. |Peer review of glossary against documentation and codebase. |=== == 8 Test Obligations @@ -134,16 +135,16 @@ benchmark run. |CI pipeline updates badge or embedded figures on commit/release. [cols="1,5,4",options="header"] |=== |ID |Requirement |Test Strategy -|CB-TEST-001 |Public API coverage ≥ 90 % (lines and branches), excluding generated code or trivial getters/setters unless they contain logic. |Jacoco/Cobertura fails build below threshold. -|CB-TEST-002 |Property-based tests over random input sizes and sequences of operations verify no indexing +|[[CB-TEST-001]]CB-TEST-001 |Public API coverage ≥ 90 % (lines and branches), excluding generated code or trivial getters/setters unless they contain logic. |Jacoco/Cobertura fails build below threshold. +|[[CB-TEST-002]]CB-TEST-002 |Property-based tests over random input sizes and sequences of operations verify no indexing exceptions, correct round-trip serialisation for all supported types, and consistent state. |JUnit with QuickTheories/jqwik. -|CB-TEST-003 |Soak test: continuous mixed read/write operations (including atomics and elastic resizing) on various buffer types (direct, mapped, pooled) for ≥ 12 hours shows ≤ 5 % native-memory drift from steady state and no deadlocks or correctness issues. |Nightly soak profile with detailed memory and performance logging. -|CB-TEST-004 |Benchmark suite must complete under Linux and macOS within CI +|[[CB-TEST-003]]CB-TEST-003 |Soak test: continuous mixed read/write operations (including atomics and elastic resizing) on various buffer types (direct, mapped, pooled) for ≥ 12 hours shows ≤ 5 % native-memory drift from steady state and no deadlocks or correctness issues. |Nightly soak profile with detailed memory and performance logging. +|[[CB-TEST-004]]CB-TEST-004 |Benchmark suite must complete under Linux and macOS within CI budget of 15 min (for standard checks) and 1 hour (for full performance regression suite). |CI gate with tiered benchmark execution. -|CB-TEST-005 |Include tests for all supported string encodings (UTF-8, ISO-8859-1, etc.) covering common characters, boundary conditions, multi-byte sequences, and malformed inputs. |Unit tests with predefined and generated string data. -|CB-TEST-006 |Ensure robustness of bounds checking by specifically testing edge cases for all read/write operations, slice creation, and buffer manipulations at limits. |Targeted unit and property-based tests focusing on buffer limits and cursor positions. -|CB-TEST-007 |Concurrency tests for all operations documented as thread-safe (e.g., atomic operations, concurrent reads, pooled access) under high contention. |JUnit tests using `ExecutorService` and `CyclicBarrier`/`CountDownLatch` to simulate contention. -|CB-TEST-008 |Tests for resource management, ensuring `releaseLast`/`close` correctly deallocates resources and that use-after-release throws appropriate exceptions. Leak detection tests verify warnings (CB-NF-O-003). |Unit tests instrumenting resource allocation/deallocation; specific tests for use-after-release behaviour. +|[[CB-TEST-005]]CB-TEST-005 |Include tests for all supported string encodings (UTF-8, ISO-8859-1, etc.) covering common characters, boundary conditions, multi-byte sequences, and malformed inputs. |Unit tests with predefined and generated string data. +|[[CB-TEST-006]]CB-TEST-006 |Ensure robustness of bounds checking by specifically testing edge cases for all read/write operations, slice creation, and buffer manipulations at limits. |Targeted unit and property-based tests focusing on buffer limits and cursor positions. +|[[CB-TEST-007]]CB-TEST-007 |Concurrency tests for all operations documented as thread-safe (e.g., atomic operations, concurrent reads, pooled access) under high contention. |JUnit tests using `ExecutorService` and `CyclicBarrier`/`CountDownLatch` to simulate contention. +|[[CB-TEST-008]]CB-TEST-008 |Tests for resource management, ensuring `releaseLast`/`close` correctly deallocates resources and that use-after-release throws appropriate exceptions. Leak detection tests verify warnings (CB-NF-O-003). |Unit tests instrumenting resource allocation/deallocation; specific tests for use-after-release behaviour. |=== == 9 Risk & Compliance @@ -151,12 +152,12 @@ budget of 15 min (for standard checks) and 1 hour (for full performance regressi [cols="1,5,4",options="header"] |=== |ID |Requirement |Mitigation Proof -|CB-RISK-001 |Fail fast on endianness mismatch between writer and reader when +|[[CB-RISK-001]]CB-RISK-001 |Fail fast on endianness mismatch between writer and reader when using raw wire format or if endianness-sensitive operations are exposed. (Note: Chronicle Wire handles this mostly). |Integration test swaps endianness (if applicable at Bytes level) and expects exception or defines clear behaviour. -|CB-RISK-002 |Open-source build throws `UnsupportedOperationException` or similar if +|[[CB-RISK-002]]CB-RISK-002 |Open-source build throws `UnsupportedOperationException` or similar if Enterprise-only features (e.g. encrypted queue files at a higher level, specific `BytesStore` types) are attempted to be used. |Unit test verifies. -|CB-RISK-003 |Ensure that direct use of `sun.misc.Unsafe` is clearly demarcated, justified, and encapsulated. Fallback mechanisms should be considered if `Unsafe` is unavailable, or clear documentation of `Unsafe` necessity. |Code review and documentation justifying `Unsafe` use; tests for fallback mechanisms if any, or CI runs with `Unsafe` restricted. -|CB-RISK-004 |Potential for `OutOfMemoryError` (both Java heap and native memory) if not managed correctly by users. |Clear documentation (CB-DOC-006) on resource lifecycle and typical patterns to avoid leaks. `BytesMetrics` (CB-NF-O-002) for monitoring. +|[[CB-RISK-003]]CB-RISK-003 |Ensure that direct use of `sun.misc.Unsafe` is clearly demarcated, justified, and encapsulated. Fallback mechanisms should be considered if `Unsafe` is unavailable, or clear documentation of `Unsafe` necessity. |Code review and documentation justifying `Unsafe` use; tests for fallback mechanisms if any, or CI runs with `Unsafe` restricted. +|[[CB-RISK-004]]CB-RISK-004 |Potential for `OutOfMemoryError` (both Java heap and native memory) if not managed correctly by users. |Clear documentation (CB-DOC-006) on resource lifecycle and typical patterns to avoid leaks. `BytesMetrics` (CB-NF-O-002) for monitoring. |=== == 10 Operational Concerns @@ -164,13 +165,13 @@ Enterprise-only features (e.g. encrypted queue files at a higher level, specific [cols="1,5,4",options="header"] |=== |ID |Requirement |Notes -|CB-OPS-001 |`./mvnw -q verify` completes with zero warnings or test failures in +|[[CB-OPS-001]]CB-OPS-001 |`./mvnw -q verify` completes with zero warnings or test failures in reference build container `Dockerfile.build`. |CI. -|CB-OPS-002 |Artefacts pushed to Maven Central **must** be reproducible; local +|[[CB-OPS-002]]CB-OPS-002 |Artefacts pushed to Maven Central **must** be reproducible; local `mvn -Prelease` output must equal CI artefact byte-for-byte. |Diff script in release pipeline. Reproducible-builds Maven plugin. -|CB-OPS-003 |The library **must** clearly list its runtime dependencies and their versions, ensuring they are available from standard public repositories like Maven Central. Transitive dependencies should be audited for compatibility and potential conflicts. |Review of `pom.xml` or build scripts; CI checks for dependency resolution and conflicts (`mvn dependency:analyze`). -|CB-OPS-004 |Provide a mechanism for users to report issues and get support (e.g., GitHub Issues, community forum/mailing list). |Link to issue tracker and support channels in `README.md` and project website. -|CB-OPS-005 |Release notes must accompany each release, detailing new features, bug fixes, performance improvements, and any breaking changes or deprecations. |Process for drafting and reviewing release notes as part of release cycle. +|[[CB-OPS-003]]CB-OPS-003 |The library **must** clearly list its runtime dependencies and their versions, ensuring they are available from standard public repositories like Maven Central. Transitive dependencies should be audited for compatibility and potential conflicts. |Review of `pom.xml` or build scripts; CI checks for dependency resolution and conflicts (`mvn dependency:analyze`). +|[[CB-OPS-004]]CB-OPS-004 |Provide a mechanism for users to report issues and get support (e.g., GitHub Issues, community forum/mailing list). |Link to issue tracker and support channels in `README.md` and project website. +|[[CB-OPS-005]]CB-OPS-005 |Release notes must accompany each release, detailing new features, bug fixes, performance improvements, and any breaking changes or deprecations. |Process for drafting and reviewing release notes as part of release cycle. |=== == 11 Glossary @@ -194,4 +195,3 @@ SLF4J :: Simple Logging Facade for Java, an abstraction for various logging fra stop-bit encoding :: Variable-length integer encoding using one bit (often the MSB) in each byte as a continuation flag. UTF-8 :: Unicode Transformation Format, 8-bit, a variable-width character encoding capable of encoding all possible Unicode code points. zero-copy :: Operations that avoid copying data between different memory areas, e.g., reading from a file directly into a target buffer without intermediate copies. - diff --git a/src/main/java/net/openhft/chronicle/bytes/MappedBytesStore.java b/src/main/java/net/openhft/chronicle/bytes/MappedBytesStore.java index a5b367a0ca6..3005a1722b0 100644 --- a/src/main/java/net/openhft/chronicle/bytes/MappedBytesStore.java +++ b/src/main/java/net/openhft/chronicle/bytes/MappedBytesStore.java @@ -118,18 +118,18 @@ public VanillaBytes bytesForWrite() return new NativeBytes<>(this); } - @Override /** * Returns {@code true} if the given offset lies within this mapping's safe range. */ + @Override public boolean inside(@NonNegative long offset) { return start <= offset && offset < safeLimit; } - @Override /** * Same as {@link #inside(long)} but also checks the end of the supplied range. */ + @Override public boolean inside(@NonNegative long offset, @NonNegative long bufferSize) { return start <= offset && offset + bufferSize <= limit; } @@ -154,10 +154,10 @@ public MappedBytesStore writeOrderedInt(@NonNegative long offset, int i) return this; } - @Override /** * Converts a logical offset in the file into an offset relative to this mapped region. */ + @Override public long translate(@NonNegative long offset) { assert SKIP_ASSERTIONS || offset >= start; assert SKIP_ASSERTIONS || offset < limit; @@ -397,14 +397,11 @@ protected void performRelease() { } /** - * Sync the ByteStore if required. + * Syncs the ByteStore if required. * - * @param offset the offset within the ByteStore from the start to sync, offset must be a multiple of 4K - * @param length the length to sync, length must be a multiple of 4K - * @param syncMode the mode to sync - */ - /** - * Helper performing the actual msync call. + * @param offset offset within the ByteStore (must be 4 KiB aligned) + * @param length length to sync (must be 4 KiB aligned) + * @param syncMode mode to use when flushing to disk */ private void performMsync(@NonNegative long offset, long length, SyncMode syncMode) { if (syncMode == SyncMode.NONE) diff --git a/src/main/java/net/openhft/chronicle/bytes/StreamingOutputStream.java b/src/main/java/net/openhft/chronicle/bytes/StreamingOutputStream.java index 68b4a12422f..e754f9fee60 100644 --- a/src/main/java/net/openhft/chronicle/bytes/StreamingOutputStream.java +++ b/src/main/java/net/openhft/chronicle/bytes/StreamingOutputStream.java @@ -48,13 +48,13 @@ public StreamingOutputStream init(StreamingDataOutput sdo) { return this; } - @Override /** * Writes bytes from the given array to the underlying {@link StreamingDataOutput}. * Any {@link BufferOverflowException}, {@link IllegalArgumentException} or * {@link IllegalStateException} from the target is wrapped in an * {@link IOException}. */ + @Override public void write(byte[] b, @NonNegative int off, @NonNegative int len) throws IOException { try { @@ -65,11 +65,11 @@ public void write(byte[] b, @NonNegative int off, @NonNegative int len) } } - @Override /** * Writes a single byte value. Exceptions thrown by the target * {@link StreamingDataOutput} are wrapped in an {@link IOException}. */ + @Override public void write(int b) throws IOException { try { diff --git a/src/main/java/net/openhft/chronicle/bytes/UncheckedBytes.java b/src/main/java/net/openhft/chronicle/bytes/UncheckedBytes.java index ca9142952b2..642b3801028 100644 --- a/src/main/java/net/openhft/chronicle/bytes/UncheckedBytes.java +++ b/src/main/java/net/openhft/chronicle/bytes/UncheckedBytes.java @@ -80,11 +80,11 @@ public void setBytes(@NotNull Bytes bytes) this.underlyingBytes = bytes; } - @Override /** * Delegates capacity checks to the wrapped instance and updates this view * if the underlying store changes. */ + @Override public void ensureCapacity(@NonNegative long desiredCapacity) throws IllegalArgumentException, IllegalStateException { if (desiredCapacity > realCapacity()) { @@ -93,10 +93,10 @@ public void ensureCapacity(@NonNegative long desiredCapacity) } } - @Override /** * Always returns this instance as it is already unchecked. */ + @Override @NotNull public Bytes unchecked(boolean unchecked) { throwExceptionIfReleased(); @@ -165,11 +165,11 @@ public Bytes writeLimit(@NonNegative long limit) { return this; } - @NotNull - @Override /** * Unsupported for unchecked views as copying would reintroduce checks. */ + @NotNull + @Override public BytesStore, U> copy() { throwExceptionIfReleased(); throw new UnsupportedOperationException("todo"); diff --git a/src/main/java/net/openhft/chronicle/bytes/UncheckedNativeBytes.java b/src/main/java/net/openhft/chronicle/bytes/UncheckedNativeBytes.java index 4c603b7caa9..47de3b751a3 100644 --- a/src/main/java/net/openhft/chronicle/bytes/UncheckedNativeBytes.java +++ b/src/main/java/net/openhft/chronicle/bytes/UncheckedNativeBytes.java @@ -87,11 +87,11 @@ public UncheckedNativeBytes(@NotNull Bytes underlyingBytes) writeLimit = Math.min(capacity, underlyingBytes.writeLimit()); } - @Override /** * Delegates capacity expansion to the wrapped bytes and updates this view * if the underlying store changes. */ + @Override public void ensureCapacity(@NonNegative long desiredCapacity) throws IllegalArgumentException, IllegalStateException { if (desiredCapacity > realCapacity()) { @@ -100,18 +100,18 @@ public void ensureCapacity(@NonNegative long desiredCapacity) } } - @Override /** * Indicates that this bytes instance performs no bounds checks. */ + @Override public boolean unchecked() { return true; } - @Override /** * Returns {@code true} as this bytes instance operates on native memory. */ + @Override public boolean isDirectMemory() { return true; } diff --git a/src/main/java/net/openhft/chronicle/bytes/internal/CommonMappedBytes.java b/src/main/java/net/openhft/chronicle/bytes/internal/CommonMappedBytes.java index 75359b6fa88..8afdeb53121 100644 --- a/src/main/java/net/openhft/chronicle/bytes/internal/CommonMappedBytes.java +++ b/src/main/java/net/openhft/chronicle/bytes/internal/CommonMappedBytes.java @@ -537,10 +537,10 @@ public Bytes writeStopBit(char n) return this; } - @Override /** * @return whether the underlying mapped file was opened read only */ + @Override public boolean isBackingFileReadOnly() { return backingFileIsReadOnly; } diff --git a/src/main/java/net/openhft/chronicle/bytes/internal/EmbeddedBytes.java b/src/main/java/net/openhft/chronicle/bytes/internal/EmbeddedBytes.java index e29cb85ea07..80c1443e867 100644 --- a/src/main/java/net/openhft/chronicle/bytes/internal/EmbeddedBytes.java +++ b/src/main/java/net/openhft/chronicle/bytes/internal/EmbeddedBytes.java @@ -43,20 +43,20 @@ public static EmbeddedBytes wrap(HeapBytesStore bytesStore) { return new EmbeddedBytes<>(bytesStore, wp, wp + length); } - @Override /** * Sets the write position and updates the length byte stored before this * field group. */ + @Override protected void uncheckedWritePosition(@NonNegative long writePosition) { super.uncheckedWritePosition(writePosition); bytesStore.writeUnsignedByte(lengthOffset(), (int) writePosition); } - @Override /** * Reads the current length prefix and returns it as the write position. */ + @Override public @NonNegative long writePosition() { try { return bytesStore.readUnsignedByte(lengthOffset()); diff --git a/src/main/java/net/openhft/chronicle/bytes/internal/NativeBytesStore.java b/src/main/java/net/openhft/chronicle/bytes/internal/NativeBytesStore.java index bde15d4bb2d..109bb59f8db 100644 --- a/src/main/java/net/openhft/chronicle/bytes/internal/NativeBytesStore.java +++ b/src/main/java/net/openhft/chronicle/bytes/internal/NativeBytesStore.java @@ -30,13 +30,13 @@ import static net.openhft.chronicle.core.util.Longs.requireNonNegative; import static net.openhft.chronicle.core.util.ObjectUtils.requireNonNull; -@SuppressWarnings({"restriction", "rawtypes"}) /** * A {@link net.openhft.chronicle.bytes.BytesStore} backed by off-heap native * memory. Instances are reference counted and must be released to free the * underlying memory. The store may be elastic or fixed in size depending on * how it was created. */ +@SuppressWarnings({"restriction", "rawtypes"}) public class NativeBytesStore extends AbstractBytesStore, U> { private static final SimpleCleaner NO_DEALLOCATOR = new NoDeallocator(); diff --git a/src/main/java/net/openhft/chronicle/bytes/internal/NoBytesStore.java b/src/main/java/net/openhft/chronicle/bytes/internal/NoBytesStore.java index 734d987a7be..5a8c76f4bf3 100755 --- a/src/main/java/net/openhft/chronicle/bytes/internal/NoBytesStore.java +++ b/src/main/java/net/openhft/chronicle/bytes/internal/NoBytesStore.java @@ -27,8 +27,8 @@ public final class NoBytesStore implements BytesStore { public static final BytesStore NO_BYTES_STORE = new NoBytesStore(); /** shared zeroed page */ public static final long NO_PAGE; - @NotNull /** empty Bytes backed by {@link #NO_BYTES_STORE} */ + @NotNull public static final Bytes NO_BYTES; private static final ByteBuffer BYTE_BUFFER = ByteBuffer.allocate(4 << 10); @@ -270,14 +270,14 @@ public NoBytesStore copy() { return VanillaBytes.wrap(this); } - @Override /** @return always {@code 0} */ + @Override public @NonNegative long capacity() { return 0; } - @Override /** @return always {@code null} */ + @Override public Void underlyingObject() { return null; } diff --git a/src/main/java/net/openhft/chronicle/bytes/internal/SingleMappedBytes.java b/src/main/java/net/openhft/chronicle/bytes/internal/SingleMappedBytes.java index 243a89ea6f8..3dc13b5de4c 100644 --- a/src/main/java/net/openhft/chronicle/bytes/internal/SingleMappedBytes.java +++ b/src/main/java/net/openhft/chronicle/bytes/internal/SingleMappedBytes.java @@ -184,12 +184,12 @@ public Bytes clear() return this; } - @SuppressWarnings("restriction") - @Override /** * Performs a volatile read of an int at the current read position using a * fast path if the address falls within a single cache line. */ + @SuppressWarnings("restriction") + @Override public int peekVolatileInt() throws IllegalStateException { diff --git a/src/main/java/net/openhft/chronicle/bytes/internal/SingleMappedFile.java b/src/main/java/net/openhft/chronicle/bytes/internal/SingleMappedFile.java index 246141d6eb8..66c9e251dfd 100644 --- a/src/main/java/net/openhft/chronicle/bytes/internal/SingleMappedFile.java +++ b/src/main/java/net/openhft/chronicle/bytes/internal/SingleMappedFile.java @@ -54,16 +54,14 @@ public class SingleMappedFile extends MappedFile { private final long capacity; /** - * Constructs a new SingleMappedFile with specified parameters. + * Constructs a new {@link SingleMappedFile} mapping the entire {@code file}. * - * @param file the file to be mapped. - * @param raf the RandomAccessFile associated with the file. - * @param capacity the capacity of the mapped file. - * @param readOnly if the file is read-only. - * @throws IORuntimeException if any I/O error occurs. - */ - /** - * Creates a new mapping for the entire {@code file}. + * @param file the file to map + * @param raf the open {@link RandomAccessFile} + * @param capacity desired capacity of the mapping + * @param pageSize OS page size to align against + * @param readOnly whether the mapping is read only + * @throws IORuntimeException on IO failures */ @SuppressWarnings("this-escape") public SingleMappedFile(@NotNull final File file, diff --git a/src/main/java/net/openhft/chronicle/bytes/internal/Unmapper.java b/src/main/java/net/openhft/chronicle/bytes/internal/Unmapper.java index 95776105977..03d2957da2e 100644 --- a/src/main/java/net/openhft/chronicle/bytes/internal/Unmapper.java +++ b/src/main/java/net/openhft/chronicle/bytes/internal/Unmapper.java @@ -33,10 +33,10 @@ public Unmapper(long address, long size, int pageSize) throws IllegalStateExcept this.pageSize = pageSize; } - @Override /** * Performs the unmap. If already unmapped the call is ignored. */ + @Override public void run() { if (address == 0) return; diff --git a/src/main/java/net/openhft/chronicle/bytes/internal/UnsafeText.java b/src/main/java/net/openhft/chronicle/bytes/internal/UnsafeText.java index a2abd55c4d7..52f3c3ecb25 100644 --- a/src/main/java/net/openhft/chronicle/bytes/internal/UnsafeText.java +++ b/src/main/java/net/openhft/chronicle/bytes/internal/UnsafeText.java @@ -103,14 +103,10 @@ public static long appendBase10d(long address, long num, int decimal) { return address; } - /** - * Internal method for low level appending a String. The caller must ensure there is at least 32 bytes available. - * - * @param address to start writing - * @param d double value - * @return endOfAddress + /* + * Internal helper for low level double rendering. The caller must ensure there + * is at least 32 bytes available. */ - // throws BufferOverflowException, IllegalArgumentException /** * Writes the decimal form of {@code d} to {@code address}. Numbers outside * a safe range fall back to {@link Double#toString(double)}. diff --git a/src/test/java/net/openhft/chronicle/bytes/MappedBytesTest.java b/src/test/java/net/openhft/chronicle/bytes/MappedBytesTest.java index 955e835f0eb..350a099a186 100644 --- a/src/test/java/net/openhft/chronicle/bytes/MappedBytesTest.java +++ b/src/test/java/net/openhft/chronicle/bytes/MappedBytesTest.java @@ -527,6 +527,30 @@ public void multiBytesSingle() throws Exception { } } + @Test + public void zeroOutRespectsCustomPageSize() throws Exception { + assumeFalse(Jvm.maxDirectMemory() == 0); + + final File file = newTempBinary("zero-custom-page"); + final int customPageSize = Math.max(OS.pageSize(), 4096) * 2; + final long chunkSize = customPageSize * 2L; + final long range = customPageSize + 256L; + + try (MappedBytes bytes = MappedBytes.mappedBytes(file, chunkSize, 0, customPageSize, false)) { + for (long offset = 0; offset < range; offset++) { + bytes.writeByte(offset, (byte) 0x5A); + } + + bytes.zeroOut(0, range); + + for (long offset = 0; offset < range; offset++) { + assertEquals("offset " + offset + " should be cleared", 0, bytes.readUnsignedByte(offset)); + } + } finally { + assertTrue("Failed to delete " + file, file.delete()); + } + } + @Test public void memoryOverlapRegions() throws Exception { String tmpfile = IOTools.createTempFile("memoryOverlapRegions").getAbsolutePath(); @@ -670,4 +694,12 @@ public void testBoundaryUnderflow() throws Exception { } assertTrue(true); // if we reach here, the test passes } + + private static File newTempBinary(String prefix) throws IOException { + File target = new File(OS.getTarget()); + target.mkdirs(); + File file = File.createTempFile(prefix, ".dat", target); + file.deleteOnExit(); + return file; + } } diff --git a/src/test/java/net/openhft/chronicle/bytes/MappedFileTest.java b/src/test/java/net/openhft/chronicle/bytes/MappedFileTest.java index 32e8b7caa61..f3b4a5c4d5f 100644 --- a/src/test/java/net/openhft/chronicle/bytes/MappedFileTest.java +++ b/src/test/java/net/openhft/chronicle/bytes/MappedFileTest.java @@ -142,6 +142,29 @@ public void testReferenceCounts() } } + @Test + public void insideHonoursSafeLimitWhenPageSizeDiffers() throws Exception { + assumeFalse(Jvm.maxDirectMemory() == 0); + final File file = tmpDir.newFile(); + final long chunkSize = OS.pageAlign(64 << 10); + final long overlapSize = OS.pageAlign(4 << 10); + final int enlargedPageSize = Math.max(OS.pageSize(), 4096) * 2; + final ReferenceOwner owner = ReferenceOwner.temporary("page-matrix"); + + try (MappedFile mappedFile = MappedFile.of(file, chunkSize, overlapSize, enlargedPageSize, false)) { + final long offset = chunkSize; + final MappedBytesStore store = mappedFile.acquireByteStore(owner, offset); + try { + final long safeLimit = store.safeLimit(); + assertEquals(store.start() + chunkSize, safeLimit); + assertTrue(store.inside(safeLimit - 1)); + assertFalse(store.inside(safeLimit)); + } finally { + store.release(owner); + } + } + } + @Test public void largeReadOnlyFile() throws IOException { assumeFalse(Runtime.getRuntime().maxMemory() < Integer.MAX_VALUE || OS.isWindows()); diff --git a/src/test/java/net/openhft/chronicle/bytes/ReferenceTracingLeakTest.java b/src/test/java/net/openhft/chronicle/bytes/ReferenceTracingLeakTest.java new file mode 100644 index 00000000000..198ad08db88 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/bytes/ReferenceTracingLeakTest.java @@ -0,0 +1,27 @@ +/* + * Copyright 2016-2025 chronicle.software; SPDX-License-Identifier: Apache-2.0 + */ +package net.openhft.chronicle.bytes; + +import net.openhft.chronicle.core.io.AbstractReferenceCounted; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class ReferenceTracingLeakTest extends BytesTestCommon { + + @Test + public void leakDetectionReportsCreatedHere() throws Exception { + final NativeBytes leaked = Bytes.allocateElasticDirect(64); + try { + assertNotNull("createdHere should be recorded for traced resources", + ((AbstractReferenceCounted) leaked).createdHere()); + + final AssertionError leak = assertThrows(AssertionError.class, AbstractReferenceCounted::assertReferencesReleased); + assertTrue("Expect suppressed entries describing the leak", leak.getSuppressed().length > 0); + } finally { + leaked.releaseLast(); + } + AbstractReferenceCounted.assertReferencesReleased(); + } +} diff --git a/src/test/java/net/openhft/chronicle/bytes/TempDirectoryIntegrationTest.java b/src/test/java/net/openhft/chronicle/bytes/TempDirectoryIntegrationTest.java new file mode 100644 index 00000000000..775acfd9720 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/bytes/TempDirectoryIntegrationTest.java @@ -0,0 +1,36 @@ +/* + * Copyright 2016-2025 chronicle.software; SPDX-License-Identifier: Apache-2.0 + */ +package net.openhft.chronicle.bytes; + +import net.openhft.chronicle.core.OS; +import net.openhft.chronicle.core.io.IOTools; +import org.junit.Test; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TempDirectoryIntegrationTest extends BytesTestCommon { + + @Test + public void createTempDirectoryUnderTargetAndCleanup() throws Exception { + final Path tempDir = IOTools.createTempDirectory("bytes-temp"); + final Path targetRoot = new File(OS.getTarget()).getAbsoluteFile().toPath().normalize(); + assertTrue("Temp directory should live under OS target", + tempDir.toAbsolutePath().normalize().startsWith(targetRoot)); + + Files.createDirectories(tempDir); + assertTrue("Temp directory should exist", Files.isDirectory(tempDir)); + final Path marker = tempDir.resolve("marker.bin"); + Files.write(marker, new byte[]{1, 2, 3}); + assertTrue("Marker file should exist inside temp dir", Files.exists(marker)); + + assertTrue("Temp directory deletion should succeed", IOTools.deleteDirWithFiles(tempDir.toFile())); + assertFalse("Temp directory should be removed", Files.exists(tempDir)); + } +} From 270f18be283b73db97347f5f0f569caa47c23522 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Sat, 8 Nov 2025 22:15:56 +0000 Subject: [PATCH 3/5] Guard overflow diagnostics when tracing is off --- src/main/docs/memory-management.adoc | 4 ++- .../openhft/chronicle/bytes/NativeBytes.java | 14 ++++++---- .../bytes/NativeBytesOverflowTest.java | 24 +++++++++++++++++ .../DecoratedBufferOverflowExceptionTest.java | 26 ++++++++++++------- 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/main/docs/memory-management.adoc b/src/main/docs/memory-management.adoc index 6474800e9bf..aa2e9dbe08c 100644 --- a/src/main/docs/memory-management.adoc +++ b/src/main/docs/memory-management.adoc @@ -58,7 +58,9 @@ resizing failure reports both the write that triggered it and where the buffer originated. This is invaluable when multiple components reuse elastic buffers and you need to pinpoint which call chain produced a leaked or overgrown instance. The richer exception context fulfils the diagnostic requirement in -<>. +<>. When tracing is disabled `createdHere()` returns `null`, so +Chronicle Bytes automatically falls back to the standard constructor to avoid +raising a misleading `NullPointerException`. == 6 Elastic Buffers diff --git a/src/main/java/net/openhft/chronicle/bytes/NativeBytes.java b/src/main/java/net/openhft/chronicle/bytes/NativeBytes.java index 02c52f15b47..3c4f444c1c0 100644 --- a/src/main/java/net/openhft/chronicle/bytes/NativeBytes.java +++ b/src/main/java/net/openhft/chronicle/bytes/NativeBytes.java @@ -186,7 +186,11 @@ protected void writeCheckOffset(final @NonNegative long offset, final @NonNegati @NotNull private DecoratedBufferOverflowException newDBOE(long writeEnd) { - return new DecoratedBufferOverflowException("Write cannot grow Bytes to " + writeEnd + ", capacity: " + capacity); + StackTrace created = createdHere(); + String message = "Write cannot grow Bytes to " + writeEnd + ", capacity: " + capacity; + return created == null + ? new DecoratedBufferOverflowException(message) + : new DecoratedBufferOverflowException(message, created); } @Override @@ -269,11 +273,11 @@ private void resize(@NonNegative final long endOfBuffer) "this bytes' underlyingObject() is ByteBuffer, NullPointerException is likely to be thrown. " + stack); } - // native block of 128 KiB or more have an individual memory mapping so are more expensive. - if (endOfBuffer >= 128 << 10 && realCapacity > 0) - Jvm.perf().on(getClass(), "Resizing buffer was " + realCapacity / 1024 + " KB, " + + // Flag large allocations + if (endOfBuffer >= 8L << 20 && realCapacity > 0) + Jvm.perf().on(getClass(), "Resizing buffer was " + 10 * realCapacity / 1024 / 1024 / 10.0 + " MiB, " + "needs " + (endOfBuffer - realCapacity) + " bytes more, " + - "new-size " + size / 1024 + " KB"); + "new-size " + 10 * size / 1024 / 1024 / 10.0 + " MiB"); resizeHelper(size, isByteBufferBacked); } diff --git a/src/test/java/net/openhft/chronicle/bytes/NativeBytesOverflowTest.java b/src/test/java/net/openhft/chronicle/bytes/NativeBytesOverflowTest.java index a5302bf0423..31461576205 100644 --- a/src/test/java/net/openhft/chronicle/bytes/NativeBytesOverflowTest.java +++ b/src/test/java/net/openhft/chronicle/bytes/NativeBytesOverflowTest.java @@ -3,13 +3,17 @@ */ package net.openhft.chronicle.bytes; +import net.openhft.chronicle.bytes.util.DecoratedBufferOverflowException; import net.openhft.chronicle.core.Jvm; +import net.openhft.chronicle.core.io.AbstractReferenceCounted; import org.junit.Test; import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import static net.openhft.chronicle.bytes.BytesStore.wrap; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeFalse; @@ -61,4 +65,24 @@ public void testNativeWriteBytes2() { // this is OK as we are unchecked ! assertTrue(nb.writePosition() > nb.writeLimit()); } + + @Test + public void overflowWithoutTracingKeepsCauseNull() { + AbstractReferenceCounted.disableReferenceTracing(); + try { + BytesStore store = wrap(ByteBuffer.allocate(128)); + Bytes nb = new NativeBytes<>(store); + try { + nb.writeLimit(2).writePosition(0); + DecoratedBufferOverflowException ex = assertThrows( + DecoratedBufferOverflowException.class, + () -> nb.writeLong(10L)); + assertNull(ex.getCause()); + } finally { + nb.releaseLast(); + } + } finally { + AbstractReferenceCounted.enableReferenceTracing(); + } + } } diff --git a/src/test/java/net/openhft/chronicle/bytes/util/DecoratedBufferOverflowExceptionTest.java b/src/test/java/net/openhft/chronicle/bytes/util/DecoratedBufferOverflowExceptionTest.java index 9f0aa4bba0c..116ba2b13ae 100644 --- a/src/test/java/net/openhft/chronicle/bytes/util/DecoratedBufferOverflowExceptionTest.java +++ b/src/test/java/net/openhft/chronicle/bytes/util/DecoratedBufferOverflowExceptionTest.java @@ -1,20 +1,28 @@ -/* - * Copyright 2016-2025 chronicle.software; SPDX-License-Identifier: Apache-2.0 - */ package net.openhft.chronicle.bytes.util; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; public class DecoratedBufferOverflowExceptionTest { @Test - public void testMessage() { - String expectedMessage = "Custom message describing the overflow"; - DecoratedBufferOverflowException exception = new DecoratedBufferOverflowException(expectedMessage); + public void nullCauseIsEquivalentToNoCause() { + DecoratedBufferOverflowException withNull = + new DecoratedBufferOverflowException("with-null", null); + DecoratedBufferOverflowException withoutCause = + new DecoratedBufferOverflowException("without-cause"); - // Assert that the message is correctly set and retrieved - assertEquals(expectedMessage, exception.getMessage()); + assertNull(withNull.getCause()); + assertNull(withoutCause.getCause()); + } + + @Test + public void nonNullCauseIsAttached() { + Throwable cause = new IllegalStateException("boom"); + DecoratedBufferOverflowException exception = + new DecoratedBufferOverflowException("with-cause", cause); + assertSame(cause, exception.getCause()); } } From cd0da135e69f4781f6a2bada31b90d66ba8dccbd Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Sat, 8 Nov 2025 22:24:32 +0000 Subject: [PATCH 4/5] Document new overflow tests --- NEW_TESTS.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ TESTS_TODO.md | 2 ++ 2 files changed, 53 insertions(+) create mode 100644 NEW_TESTS.md diff --git a/NEW_TESTS.md b/NEW_TESTS.md new file mode 100644 index 00000000000..9736bd29f90 --- /dev/null +++ b/NEW_TESTS.md @@ -0,0 +1,51 @@ +# Chronicle Bytes – In-Flight Test Additions + +Short log of the extra tests added while working through `TESTS_TODO.md`. Keep +this in sync with `TESTS_TODO.md` so the next session can immediately see what +landed vs. what is pending. + +## Implemented This Pass + +1. **`MappedFileTest.insideHonoursSafeLimitWhenPageSizeDiffers`** + * Exercises a chunk acquired with a page size larger than the OS default. + * Verifies that `MappedBytesStore#inside` respects `safeLimit()` so elastic + mappings do not overrun chunk boundaries. +2. **`MappedBytesTest.zeroOutRespectsCustomPageSize`** + * Confirms `MappedBytes.zeroOut` clears all bytes when the mapping page size + is tuned (captures previous regressions when the OS default was assumed). +3. **`TempDirectoryIntegrationTest`** + * Uses `IOTools.createTempDirectory` end-to-end and asserts the artefacts are + rooted under `OS.getTarget()` and removed afterwards. +4. **`ReferenceTracingLeakTest`** + * Leaks a direct `Bytes` on purpose and ensures + `AbstractReferenceCounted.assertReferencesReleased()` raises the expected + diagnostic with `createdHere()`. +5. **`DecoratedBufferOverflowExceptionTest`** + * Documents the null-cause semantics: passing `null` mirrors the + single-argument constructor, while a real `Throwable` is preserved as the + cause. +6. **`NativeBytesOverflowTest#overflowWithoutTracingKeepsCauseNull`** + * Forces a capacity overflow with reference tracing disabled to ensure the + `DecoratedBufferOverflowException` thrown by `NativeBytes.newDBOE(...)` + retains a `null` cause instead of throwing. + +All of the above pass under `mvn -q -Dtest=... test` (see latest run in shell +history) and under the module-wide `mvn -q verify`. + +## Still Outstanding + +* Safe-page defaults for builder APIs (Windows vs. other OSes) – see + `TESTS_TODO.md` item “Safe-Page Block Size for Builders”. +* Additional `OS.pageAlign` coverage for chunked mappings – see + `TESTS_TODO.md` item “OS.pageAlign Consumers”. + +Once those tests are implemented, rerun: + +```bash +mvn -q -Dtest=MappedFileTest,MappedBytesTest,TempDirectoryIntegrationTest,ReferenceTracingLeakTest,DecoratedBufferOverflowExceptionTest,NativeBytesOverflowTest test +mvn -q verify +``` + +Remember to update the Matching AsciiDoc (especially `memory-management.adoc`) +whenever diagnostics or behavioural guarantees change, and keep the AGENTS +guidelines (British English + ISO-8859-1) in mind. diff --git a/TESTS_TODO.md b/TESTS_TODO.md index dc9205be7b9..ab2f8b3dd93 100644 --- a/TESTS_TODO.md +++ b/TESTS_TODO.md @@ -21,6 +21,8 @@ Keeps track of Byte-specific test work (including batch additions beyond the mod 6. Added `MappedBytesTest.zeroOutRespectsCustomPageSize` to cover single mapping zero-out behaviour when the mapping page size is larger than the OS default. 7. Added `TempDirectoryIntegrationTest` to verify `IOTools.createTempDirectory` locates paths beneath `OS.getTarget()` and that `deleteDirWithFiles` removes them. 8. Added `ReferenceTracingLeakTest` to confirm `AbstractReferenceCounted.assertReferencesReleased()` surfaces leaks with the recorded `createdHere` trace. +9. Added `DecoratedBufferOverflowExceptionTest` to spell out the null-cause semantics (`DecoratedBufferOverflowException` mirrors the single-argument constructor when tracing is disabled). +10. Extended `NativeBytesOverflowTest` with `overflowWithoutTracingKeepsCauseNull` so we catch regressions where `createdHere()` returns `null` (ties back to <> diagnostics in `memory-management.adoc`). ## Outstanding Batches From 78435def5a5307dd8da1f14337cc4d60c8edd105 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Sat, 8 Nov 2025 22:37:34 +0000 Subject: [PATCH 5/5] Apply license headers --- pom.xml | 6 +- .../chronicle/bytes/BytesCopyMatrixTest.java | 56 ++++++++++ .../chronicle/bytes/BytesLifecycleTest.java | 104 ++++++++++++++++++ .../bytes/UncheckedBytesBehaviourTest.java | 16 +++ .../DecoratedBufferOverflowExceptionTest.java | 3 + 5 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 src/test/java/net/openhft/chronicle/bytes/BytesCopyMatrixTest.java create mode 100644 src/test/java/net/openhft/chronicle/bytes/BytesLifecycleTest.java diff --git a/pom.xml b/pom.xml index 3f92db33886..9c67b0cd64a 100644 --- a/pom.xml +++ b/pom.xml @@ -379,11 +379,13 @@ - 0.7 - 0.6 + + 0.70 + 0.62 + verify org.sonarsource.scanner.maven diff --git a/src/test/java/net/openhft/chronicle/bytes/BytesCopyMatrixTest.java b/src/test/java/net/openhft/chronicle/bytes/BytesCopyMatrixTest.java new file mode 100644 index 00000000000..a9a6c26e563 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/bytes/BytesCopyMatrixTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2016-2025 chronicle.software; SPDX-License-Identifier: Apache-2.0 + */ +package net.openhft.chronicle.bytes; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class BytesCopyMatrixTest extends BytesTestCommon { + + @Test + public void heapToNativeStoreCopiesReadablePortion() { + Bytes source = Bytes.allocateElasticOnHeap(); + source.append("alpha-beta"); + source.readPosition(6); // start at beta + BytesStore target = BytesStore.nativeStoreWithFixedCapacity((int) source.readRemaining()); + try { + long copied = source.copyTo(target); + assertEquals(source.readRemaining(), copied); + assertEquals(6, source.readPosition()); + + Bytes view = target.bytesForRead(); + try { + byte[] data = new byte[(int) copied]; + view.read(data); + assertArrayEquals("beta".getBytes(StandardCharsets.ISO_8859_1), data); + } finally { + view.releaseLast(); + } + } finally { + target.releaseLast(); + source.releaseLast(); + } + } + + @Test + public void directBytesCopyToOutputStream() throws IOException { + Bytes source = Bytes.allocateDirect(32); + try { + source.append("payload"); + source.readPosition(0); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + source.copyTo(baos); + assertEquals("payload", baos.toString(StandardCharsets.ISO_8859_1.name())); + assertEquals("copyTo(OutputStream) must not move readPosition", 0, source.readPosition()); + } finally { + source.releaseLast(); + } + } +} diff --git a/src/test/java/net/openhft/chronicle/bytes/BytesLifecycleTest.java b/src/test/java/net/openhft/chronicle/bytes/BytesLifecycleTest.java new file mode 100644 index 00000000000..940bbe01940 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/bytes/BytesLifecycleTest.java @@ -0,0 +1,104 @@ +/* + * Copyright 2016-2025 chronicle.software; SPDX-License-Identifier: Apache-2.0 + */ +package net.openhft.chronicle.bytes; + +import net.openhft.chronicle.core.io.ClosedIllegalStateException; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.*; + +public class BytesLifecycleTest extends BytesTestCommon { + + @Test + public void slicesRespectReferenceCounts() { + Bytes parent = Bytes.allocateElasticDirect(); + boolean parentReleased = false; + try { + parent.writeUtf8("reference-test"); + BytesStore store = parent.bytesStore(); + long initialRefCount = store.refCount(); + + Bytes slice = parent.bytesForRead(); + try { + assertEquals("bytesForRead should reserve the bytes store", + initialRefCount + 1, store.refCount()); + } finally { + slice.releaseLast(); + } + assertEquals("Releasing the slice should decrement the ref-count", + initialRefCount, store.refCount()); + + Bytes orphan = parent.bytesForRead(); + try { + parent.releaseLast(); + parentReleased = true; + orphan.readPosition(0); + assertEquals("reference-test", orphan.readUtf8()); + } finally { + orphan.releaseLast(); + } + assertEquals("Releasing the final slice should drop ref-count to zero", + 0, store.refCount()); + assertThrows("Once both parent and slices are released, access must fail", + NullPointerException.class, () -> parent.peekUnsignedByte(0)); + } finally { + if (!parentReleased) { + parent.releaseLast(); + } + } + } + + @Test + public void elasticHeapBytesGrowMonotonically() { + Bytes elastic = Bytes.elasticByteBuffer(8); + boolean expanded = false; + try { + long previousCapacity = elastic.realCapacity(); + for (int i = 1; i <= 5; i++) { + byte[] block = new byte[i * 32]; + elastic.write(block); + long currentCapacity = elastic.realCapacity(); + if (currentCapacity > previousCapacity) { + expanded = true; + } + assertTrue("Capacity must not shrink during elastic growth", + currentCapacity >= previousCapacity); + previousCapacity = currentCapacity; + elastic.clear(); + } + } finally { + elastic.releaseLast(); + } + assertTrue("Elastic buffer should expand at least once", expanded); + } + + @Test + public void copyToCopiesReadableBytesOnly() { + Bytes source = Bytes.allocateElasticOnHeap(); + source.append("header-body"); + source.readPosition(7); // skip "header-" + BytesStore target = BytesStore.nativeStoreWithFixedCapacity((int) source.readRemaining()); + try { + long expectedRemaining = source.readRemaining(); + long copied = source.copyTo(target); + assertEquals(expectedRemaining, copied); + assertEquals("copyTo should not mutate readPosition", 7, source.readPosition()); + Bytes view = target.bytesForRead(); + try { + view.readPositionRemaining(0, copied); + byte[] bytes = new byte[(int) copied]; + view.read(bytes); + assertEquals("body", new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1)); + } finally { + view.releaseLast(); + } + } finally { + target.releaseLast(); + source.releaseLast(); + } + } +} diff --git a/src/test/java/net/openhft/chronicle/bytes/UncheckedBytesBehaviourTest.java b/src/test/java/net/openhft/chronicle/bytes/UncheckedBytesBehaviourTest.java index 60b4f90513a..b32d7251d0e 100644 --- a/src/test/java/net/openhft/chronicle/bytes/UncheckedBytesBehaviourTest.java +++ b/src/test/java/net/openhft/chronicle/bytes/UncheckedBytesBehaviourTest.java @@ -5,6 +5,7 @@ import org.junit.Test; + import static org.junit.Assert.*; public class UncheckedBytesBehaviourTest extends BytesTestCommon { @@ -28,4 +29,19 @@ public void uncheckedOnDirectAndNoopWhenFalse() { h.releaseLast(); } } + + @Test + public void uncheckedModeAllowsWritePastLimit() { + Bytes checked = Bytes.allocateElasticOnHeap(16); + Bytes unchecked = checked.unchecked(true); + try { + unchecked.writeLimit(4); + unchecked.writeLong(0x0102030405060708L); + assertEquals("Unchecked write should advance writePosition", 8, unchecked.writePosition()); + assertEquals("Checked view remains at start", 0, checked.readPosition()); + } finally { + unchecked.releaseLast(); + } + } + } diff --git a/src/test/java/net/openhft/chronicle/bytes/util/DecoratedBufferOverflowExceptionTest.java b/src/test/java/net/openhft/chronicle/bytes/util/DecoratedBufferOverflowExceptionTest.java index 116ba2b13ae..09f973bd9a8 100644 --- a/src/test/java/net/openhft/chronicle/bytes/util/DecoratedBufferOverflowExceptionTest.java +++ b/src/test/java/net/openhft/chronicle/bytes/util/DecoratedBufferOverflowExceptionTest.java @@ -1,3 +1,6 @@ +/* + * Copyright 2016-2025 chronicle.software; SPDX-License-Identifier: Apache-2.0 + */ package net.openhft.chronicle.bytes.util; import org.junit.Test;