[YTDB-382] Transactional schema operations and per-class schema records

core/src/main/java/com/jetbrains/youtrackdb/api/config/GlobalConfiguration.java

@@ -275,6 +275,14 @@ public enum GlobalConfiguration {
       "Amount storage collections allocated for storing blobs", Integer.class,
       8),
 
+  EXPORT_RECORD_SPILL_THRESHOLD("youtrackdb.export.recordSpillThreshold",
+      "Size in bytes above which a single record's JSON rendering spills from memory to a"
+          + " transient file during database export. Bounds the export's memory use; an"
+          + " oversized-but-healthy record is still exported whole, never shed. Operational"
+          + " tuning, not correctness.",
+      Integer.class,
+      32 * 1024 * 1024),
+
   STORAGE_SNAPSHOT_INDEX_CLEANUP_THRESHOLD(
       "youtrackdb.storage.snapshotIndex.cleanupThreshold",
       "Number of entries in the shared snapshot index that triggers GC of stale entries",

core/src/main/java/com/jetbrains/youtrackdb/internal/common/concur/lock/ScalableRWLock.java

@@ -23,6 +23,8 @@
  */
 package com.jetbrains.youtrackdb.internal.common.concur.lock;
 
+import com.jetbrains.youtrackdb.internal.core.exception.BaseException;
+import com.jetbrains.youtrackdb.internal.core.exception.DatabaseException;
 import java.lang.ref.Cleaner;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.TimeUnit;
@@ -32,6 +34,7 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.StampedLock;
+import java.util.function.BooleanSupplier;
 
 /**
  * <h1>Scalable Read-Write Lock </h1>
@@ -275,6 +278,17 @@ public Lock writeLock() {
     return writerLock;
   }
 
+  /**
+   * Whether the exclusive (write) lock is currently held by some thread. The underlying
+   * {@link StampedLock} tracks no owner, so this cannot distinguish the current thread from
+   * another holder; it exists for assertions that a code path runs inside an exclusive-lock
+   * window (a caller that must itself hold the lock cannot be foiled by another holder anyway,
+   * because that holder would have blocked it).
+   */
+  public boolean isWriteLocked() {
+    return stampedLock.isWriteLocked();
+  }
+
   /**
    * Creates a new ReadersEntry instance for the current thread and its associated AtomicInteger to
    * store the state of the Reader
@@ -605,4 +619,148 @@ public boolean exclusiveTryLockNanos(long nanosTimeout) throws java.lang.Interru
 
     return true;
   }
+
+  /**
+   * Acquires the write lock ONCE with an abort predicate, for a waiter that must give way to an
+   * external condition (an operator freeze engaging) without ever spuriously failing on
+   * contention alone.
+   *
+   * <p>The two-guarantee contract this primitive exists for (both load-bearing for the freezer
+   * gate's third checkpoint; see the correctness comments inline):
+   *
+   * <ul>
+   *   <li><b>Bounded acquisition under sustained readers.</b> The write bit is acquired exactly
+   *       once and then HELD through the reader drain — writer preference: from the moment the
+   *       bit is set, new readers observe {@code isWriteLocked()} and back off exactly as they do
+   *       against {@link #exclusiveLock()}. There is no inter-attempt release window (unlike an
+   *       {@link #exclusiveTryLockNanos} retry loop, which releases the bit on every drain
+   *       timeout and forfeits admission to slip-in readers), so the acquisition completes within
+   *       the maximum residual reader residence — deterministic, no starvation, no retry storm.
+   *   <li><b>Abort within one poll granularity.</b> The predicate is polled between phase-1
+   *       {@code tryWriteLock} attempts (each bounded by {@code pollNanos}) and on every yield
+   *       iteration of the phase-2 reader-drain spin (the tightest granularity available; the
+   *       intended predicate is a single atomic-counter read, so per-iteration polling costs
+   *       less than the yield beside it). On abort the write bit is released fully — no queue
+   *       entry, no held bit, no residual writer-intent state — so no reader or writer is
+   *       stranded (parked readers spin on {@code isWriteLocked()} and proceed; there is no
+   *       wait/notify channel to lose a wakeup on) and the primitive is immediately reusable.
+   * </ul>
+   *
+   * <p>The predicate is additionally re-checked immediately after the drain completes (which is
+   * also immediately after bit acquisition when there are no residual readers to drain), BEFORE
+   * returning {@code true}: a condition arriving exactly at the acquisition-success edge aborts
+   * here rather than being missed and caught only by later downstream gates with the lock held.
+   *
+   * <p>No new deadlock edge: the method blocks only on the same two waits {@link #exclusiveLock()}
+   * already performs (the stamped writer queue and the reader-drain spin), both now bounded by the
+   * abort predicate; the abort path releases everything before returning, and a queued phase-1
+   * candidate holds nothing at all (readers never consult the stamped writer queue — they poll
+   * only {@code isWriteLocked()}).
+   *
+   * <p>Interruption: an interrupt while parked in the phase-1 timed acquire restores the interrupt
+   * flag and throws {@link DatabaseException} naming the lock state. The phase-2 drain is a yield
+   * spin, uninterruptible exactly like {@link #exclusiveLock()}'s.
+   *
+   * <p>All other methods of this class are byte-for-byte unaffected; readers pay nothing new.
+   *
+   * <p>Fairness note: a phase-1 timeout re-enters the stamped writer queue at its tail, so under
+   * sustained contention from plain {@link #exclusiveLock()} writers this waiter can lose its
+   * queue position once per {@code pollNanos} — phase 1 is unbounded in theory under a permanent
+   * writer storm. Acceptable for the intended consumer (storage state locks have rare, short
+   * writers); a fairness-sensitive consumer would need a different phase-1 shape.
+   *
+   * <p>A predicate that THROWS is propagated — but never with the write bit held: a phase-1 throw
+   * happens with nothing acquired, and the phase-2 evaluation sites are guarded so the bit is
+   * released before the failure escapes (an ownerless write bit would wedge every reader and
+   * writer of this lock forever).
+   *
+   * @param abort     polled condition; when it returns {@code true} the acquisition is abandoned
+   *                  and this method returns {@code false} with no lock state held. Must be cheap
+   *                  (it is polled per drain iteration) and must not itself touch this lock.
+   * @param pollNanos the phase-1 per-attempt park bound; also the coarsest abort-detection
+   *                  latency while queued against another writer. Must be positive.
+   * @return {@code true} when the write lock was acquired (caller releases via
+   * {@link #exclusiveUnlock()}); {@code false} when the abort predicate turned true first.
+   */
+  public boolean exclusiveLockWithAbort(final BooleanSupplier abort, final long pollNanos) {
+    java.util.Objects.requireNonNull(abort, "abort predicate must not be null");
+    if (pollNanos <= 0) {
+      throw new IllegalArgumentException("pollNanos must be positive, got " + pollNanos);
+    }
+    // Phase 1: queue against writers only. A parked tryWriteLock candidate blocks no readers:
+    // readers poll isWriteLocked(), which stays false until an acquisition actually succeeds,
+    // so aborting from this phase leaves no trace at all.
+    long stamp = 0;
+    while (stamp == 0) {
+      if (abort.getAsBoolean()) {
+        return false;
+      }
+      try {
+        stamp = stampedLock.tryWriteLock(pollNanos, TimeUnit.NANOSECONDS);
+      } catch (final InterruptedException e) {
+        // Restore the flag and fail loudly naming the state: a swallowed interrupt would turn
+        // into an unbounded uninterruptible wait. The message stays generic (this is a shared
+        // primitive, not a storage-only one) and reports only what is certain: the write bit was
+        // not acquired by this call. The holder snapshot is best-effort — it can name a free lock
+        // when a pre-interrupted thread never actually parked (StampedLock checks the interrupt
+        // flag before attempting) or when the holder released concurrently.
+        Thread.currentThread().interrupt();
+        throw BaseException.wrapException(
+            new DatabaseException(
+                "interrupted while acquiring the write lock with an abort predicate (write bit"
+                    + " not acquired by this call; "
+                    + (stampedLock.isWriteLocked()
+                        ? "another writer currently holds the write bit"
+                        : "no writer currently holds the write bit")
+                    + ")"),
+            e, (String) null);
+      }
+    }
+
+    // Phase 2: the write bit is HELD from here on — writer preference engaged exactly like
+    // exclusiveLock (new readers observe isWriteLocked() and back off). Drain the residual
+    // readers, polling the abort predicate on every yield iteration. The whole phase is guarded:
+    // a THROW from the predicate (or any phase-2 failure) must release the bit before
+    // propagating — an ownerless write bit would wedge every reader (spinning on isWriteLocked)
+    // and writer of this lock forever. No double-unlock is possible: the two abort branches
+    // below unlock and RETURN immediately, so any throw reaching the catch arrives with the bit
+    // still held.
+    try {
+      var localReadersStateArray = readersStateArrayRef.get();
+      if (localReadersStateArray == null) {
+        // Set to dummyArray before scanning the readersStateList to impose
+        // a linearizability condition
+        readersStateArrayRef.set(dummyArray);
+        // Copy readersStateList to an array
+        localReadersStateArray =
+            readersStateList.toArray(new AtomicInteger[readersStateList.size()]);
+        readersStateArrayRef.compareAndSet(dummyArray, localReadersStateArray);
+      }
+
+      for (var readerState : localReadersStateArray) {
+        while (readerState != null && readerState.get() == SRWL_STATE_READING) {
+          if (abort.getAsBoolean()) {
+            // Full release: the bit drops, backed-off readers spinning on isWriteLocked() proceed
+            // (no lost wakeup possible — there is no parking channel, only the polled bit), and
+            // the primitive is immediately reusable.
+            stampedLock.asWriteLock().unlock();
+            return false;
+          }
+          Thread.yield();
+        }
+      }
+
+      // Predicate re-check at the acquisition-success edge, before returning true. This closes
+      // the window where the condition arrives exactly as the drain completes — including the
+      // zero-residual-readers case, where the drain loop body never ran and so never polled.
+      if (abort.getAsBoolean()) {
+        stampedLock.asWriteLock().unlock();
+        return false;
+      }
+      return true;
+    } catch (final Throwable phaseTwoFailure) {
+      stampedLock.asWriteLock().unlock();
+      throw phaseTwoFailure;
+    }
+  }
 }

core/src/main/java/com/jetbrains/youtrackdb/internal/common/io/FileUtils.java

@@ -24,6 +24,7 @@
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.nio.channels.FileChannel;
 import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.CopyOption;
 import java.nio.file.FileSystems;
@@ -33,6 +34,7 @@
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Locale;
 import javax.annotation.Nullable;
@@ -161,8 +163,7 @@ public static void createDirectoryTree(final String iFileName) {
     }
   }
 
-  @Nullable
-  public static String getPath(final String iPath) {
+  @Nullable public static String getPath(final String iPath) {
     if (iPath == null) {
       return null;
     }
@@ -318,4 +319,58 @@ public static void atomicMoveWithFallback(Path source, Path target, Object reque
       Files.move(source, target);
     }
   }
+
+  /**
+   * Durably promotes {@code source} to {@code target} (the CS40 promote recipe of Track 8's
+   * export hardening): (1) the source file's content is fsynced through a freshly opened
+   * channel — the writing stream is already closed, so the sync needs its own channel; (2) the
+   * file is renamed with {@code ATOMIC_MOVE} + {@code REPLACE_EXISTING}, so a pre-existing
+   * target is replaced only by this whole, durable file and a crash can never leave a torn
+   * target; (3) the target's parent directory is fsynced (POSIX rename durability — without it
+   * a crash after the rename can lose the directory entry itself).
+   *
+   * <p>Deliberately FAIL-CLOSED: unlike {@link #atomicMoveWithFallback} there is NO regular-move
+   * fallback — a filesystem that cannot perform the atomic replace fails the promote rather
+   * than silently degrading to a copy that can tear the target. The parent-directory fsync
+   * carve-out is NARROW: only the directory-channel OPEN failure is tolerated (platforms like
+   * Windows cannot open directory channels, and the POSIX directory-entry hazard does not
+   * apply there in the same form); an I/O failure from {@code force(true)} on a successfully
+   * opened directory channel is a GENUINE fsync failure and propagates — reporting success
+   * over it would let a crash revert the rename after the caller already reported the promote
+   * durable.
+   *
+   * <p>Contract note: only the TARGET's parent directory is fsynced. The current callers move
+   * within a single directory (the export temp file sits next to its final name); a future
+   * CROSS-directory caller would additionally need the SOURCE's parent fsynced, or the
+   * source-entry removal may not be durable.
+   */
+  public static void durableAtomicMove(Path source, Path target, Object requester)
+      throws IOException {
+    try (var channel = FileChannel.open(source, StandardOpenOption.WRITE)) {
+      channel.force(true);
+    }
+    Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+    final var parent = target.toAbsolutePath().getParent();
+    if (parent != null) {
+      FileChannel directoryChannel = null;
+      try {
+        directoryChannel = FileChannel.open(parent, StandardOpenOption.READ);
+      } catch (IOException e) {
+        // The documented platform carve-out: the directory cannot be opened as a channel
+        // (e.g. Windows). Only the OPEN failure is tolerated — see the javadoc.
+        LogManager.instance()
+            .warn(requester,
+                "Cannot open the parent directory of '%s' for fsync after the atomic move;"
+                    + " continuing (the platform does not support directory channels)",
+                e, target);
+      }
+      if (directoryChannel != null) {
+        // A failure from force(true) here is a genuine fsync failure and MUST propagate
+        // (fail-closed): the rename's durability cannot be vouched for.
+        try (var openedDirectoryChannel = directoryChannel) {
+          openedDirectoryChannel.force(true);
+        }
+      }
+    }
+  }
 }

core/src/main/java/com/jetbrains/youtrackdb/internal/core/config/IndexEngineData.java

@@ -11,6 +11,17 @@
 public final class IndexEngineData {
 
   private final int indexId;
+
+  /**
+   * The engine's stable file base id: a monotonically allocated, never-reused number that keys
+   * the engine's storage files (stem {@code ie_<fileBaseId>}). Unlike {@link #indexId} — a
+   * first-null-slot registry index that is deliberately reused after a drop or a failed commit —
+   * the file base id is unique for the storage's whole lifetime, so a drop-and-recreate of a
+   * same-named index can never collide on files, and an index rename never has to touch them.
+   * Allocated by the storage's high-water-mark allocator inside the creating atomic operation and
+   * persisted with the engine entry (engine-property binary version 2+).
+   */
+  private final int fileBaseId;
   @Nonnull
   private final String name;
   private final String algorithm;
@@ -39,6 +50,7 @@ public final class IndexEngineData {
 
   public IndexEngineData(
       int indexId,
+      int fileBaseId,
       final IndexMetadata metadata,
       final Boolean durableInNonTxMode,
       final byte valueSerializerId,
@@ -49,6 +61,7 @@ public IndexEngineData(
       final String encryptionOptions,
       final Map<String, String> engineProperties) {
     this.indexId = indexId;
+    this.fileBaseId = fileBaseId;
     var definition = metadata.getIndexDefinition();
     this.name = metadata.getName();
     this.algorithm = metadata.getAlgorithm();
@@ -73,6 +86,7 @@ public IndexEngineData(
 
   public IndexEngineData(
       int indexId,
+      int fileBaseId,
       @Nonnull final String name,
       final String algorithm,
       String indexType,
@@ -90,6 +104,7 @@ public IndexEngineData(
       final String encryptionOptions,
       final Map<String, String> engineProperties) {
     this.indexId = indexId;
+    this.fileBaseId = fileBaseId;
     this.name = name;
     this.algorithm = algorithm;
     this.indexType = indexType;
@@ -116,6 +131,13 @@ public int getIndexId() {
     return indexId;
   }
 
+  /**
+   * The engine's stable, never-reused file base id. See {@link #fileBaseId}.
+   */
+  public int getFileBaseId() {
+    return fileBaseId;
+  }
+
   public int getKeySize() {
     return keySize;
   }
@@ -175,8 +197,7 @@ public boolean isNullValuesSupport() {
     return nullValuesSupport;
   }
 
-  @Nullable
-  public Map<String, String> getEngineProperties() {
+  @Nullable public Map<String, String> getEngineProperties() {
     if (engineProperties == null) {
       return null;
     }