From bfcdf927f812a8b2cbfb01183df57cbc632db92d Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Mon, 6 Jul 2026 10:29:06 +0100 Subject: [PATCH 01/10] Fix SignalLock.incrementAsyncWork (wrong guard, can lead to ISE) --- .../service/accord/AccordExecutor.java | 18 +++---- .../AccordExecutorAbstractLockLoop.java | 4 +- .../accord/AccordExecutorSignalLoop.java | 4 +- .../service/accord/AccordExecutorSimple.java | 2 +- .../cassandra/service/accord/AccordTask.java | 2 +- .../utils/concurrent/SignalLock.java | 8 +++- .../simulator/test/AccordExecutorTest.java | 48 +++++++++++++++---- 7 files changed, 60 insertions(+), 26 deletions(-) diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java index b1aff08f13f7..cff519396a26 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java @@ -1176,7 +1176,7 @@ static Task reverse(Task unqueued) /** * Run the command; the state cache lock may or may not be held depending on the executor implementation */ - protected abstract void runInternal(); + protected abstract void run(); /** * Fail the command; the state cache lock may or may not be held depending on the executor implementation @@ -1235,7 +1235,7 @@ protected void preRunExclusive() } @Override - protected void runInternal() + protected void run() { queue.runTask(); } @@ -1322,7 +1322,7 @@ void runTask() if (stopped && reject(task)) task.fail(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((AccordTask) task).preLoadContext())); else - task.runInternal(); + task.run(); // NOTE: cannot safely release owner here, in case an immediate-execution runs before we can release our references and store their changes to the cache } @@ -1631,7 +1631,7 @@ static class CancelTask extends Task private CancelTask(Task cancel) { this.cancel = cancel; } @Override void submitExclusive(AccordExecutor owner) { cancel.cancelExclusive(owner); } @Override protected void preRunExclusive() { throw new UnsupportedOperationException(); } - @Override protected void runInternal() { throw new UnsupportedOperationException(); } + @Override protected void run() { throw new UnsupportedOperationException(); } @Override protected void fail(Throwable fail) { throw new UnsupportedOperationException(); } @Override protected void addToQueue(TaskQueue queue) { throw new UnsupportedOperationException(); } } @@ -1706,7 +1706,7 @@ class PlainRunnable extends Plain implements Cancellable } @Override - protected void runInternal() + protected void run() { onRunning(); try (Closeable close = resources.get()) @@ -1809,7 +1809,7 @@ void postRunExclusive() } @Override - public void runInternal() + public void run() { onRunning(); try (Closeable close = resources.get()) @@ -1856,7 +1856,7 @@ class WrappedIOTask extends IOTask } @Override - protected void runInternal() + protected void run() { onRunning(); try (Closeable close = resources.get()) @@ -1907,7 +1907,7 @@ void postRunExclusive() } @Override - public void runInternal() + public void run() { onRunning(); try (Closeable close = resources.get()) @@ -1955,7 +1955,7 @@ SequentialExecutor executor() } @Override - protected void runInternal() + protected void run() { onRunning(); try (Closeable close = resources.get()) diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java index 5afeaf8a7b30..d8051f40dc2d 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java @@ -148,7 +148,7 @@ public void run() try { task.preRunExclusive(); - task.runInternal(); + task.run(); } catch (Throwable t) { @@ -282,7 +282,7 @@ public void run() try { ++count; - task.runInternal(); + task.run(); } catch (Throwable t) { diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java index 034544de4ec8..9ec7fa95d2ed 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java @@ -61,7 +61,7 @@ public AccordExecutorSignalLoop(int executorId, Mode mode, int threads, long spi public AccordExecutorSignalLoop(SignalLock lock, int executorId, Mode mode, int threads, IntFunction name, Agent agent) { super(lock, executorId, agent); - Invariants.require(threads < SignalLock.MAX_THREADS); + Invariants.require(threads <= SignalLock.MAX_THREADS); this.lock = lock; this.loops = new AccordExecutorLoops(mode, threads, name, this::task); this.readyToRunLimit = Math.min(threads * 4, SignalLock.MAX_SIGNAL_COUNT); @@ -397,7 +397,7 @@ public void run() try { setRunning(task); - task.runInternal(); + task.run(); } catch (Throwable t) { diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java index 4e89f068ab60..ab6c8c25e736 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java @@ -95,7 +95,7 @@ protected void run() return; } - try { task.preRunExclusive(); task.runInternal(); } + try { task.preRunExclusive(); task.run(); } catch (Throwable t) { task.fail(t); } finally { diff --git a/src/java/org/apache/cassandra/service/accord/AccordTask.java b/src/java/org/apache/cassandra/service/accord/AccordTask.java index e3fe9656e01f..a766cc209790 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTask.java +++ b/src/java/org/apache/cassandra/service/accord/AccordTask.java @@ -672,7 +672,7 @@ protected void preRunExclusive() } @Override - public void runInternal() + public void run() { onRunning(); logger.trace("Running {} with state {}", this, state); diff --git a/src/java/org/apache/cassandra/utils/concurrent/SignalLock.java b/src/java/org/apache/cassandra/utils/concurrent/SignalLock.java index 9542d396bda5..b84cb78a3289 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/SignalLock.java +++ b/src/java/org/apache/cassandra/utils/concurrent/SignalLock.java @@ -424,11 +424,17 @@ private boolean tryAcquireAsyncInAwaitLoop(long cur, int thread, long threadBitI public boolean incrementAsyncWork(boolean signalIfWaiting) { + return incrementAsyncWork(signalIfWaiting, 1); + } + + public boolean incrementAsyncWork(boolean signalIfWaiting, int increment) + { + Invariants.require(increment >= 1 && increment <= MAX_SIGNAL_COUNT); while (true) { long cur = state; int count = asyncSignalCount(cur); - if (count == MAX_THREADS) + if (count == MAX_SIGNAL_COUNT) return false; if (signalIfWaiting) diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java index 44888830cc82..103f45b2db98 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java @@ -18,7 +18,9 @@ package org.apache.cassandra.simulator.test; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; @@ -36,6 +38,7 @@ import accord.local.SequentialAsyncExecutor; import org.apache.cassandra.concurrent.ExecutorFactory; +import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; import org.apache.cassandra.service.accord.AccordExecutor; @@ -43,47 +46,71 @@ import org.apache.cassandra.service.accord.AccordExecutorSignalLoop; import org.apache.cassandra.service.accord.api.AccordAgent; +import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; import static org.apache.cassandra.service.accord.AccordService.toFuture; // TODO (required): have simulator intercept ReentantLock so we can test SyncSubmit and SemiSyncSubmit public class AccordExecutorTest extends SimulationTestBase { - static final int THREAD_COUNT = 8; + static final int EXECUTOR_THREAD_COUNT = 44; @Test public void signalLoopTest() { - executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, THREAD_COUNT, -1, -1, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent())); + executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, EXECUTOR_THREAD_COUNT, -1, -1, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent()), + 16); } @Test public void signalSpinLoopTest() { - executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, THREAD_COUNT, 10, 100, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent())); + executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, EXECUTOR_THREAD_COUNT, 10, 100, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent()), + 16); } @Test public void ayncSubmitTest() { - executorTest(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, THREAD_COUNT, i -> "Loop" + i, new AccordAgent())); + executorTest(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, EXECUTOR_THREAD_COUNT, i -> "Loop" + i, new AccordAgent()), + 16); } - public void executorTest(SerializableSupplier supplier) + public void executorTest(SerializableSupplier supplier, int submissionThreads) { simulate(arr(() -> { try { DatabaseDescriptor.daemonInitialization(); + ExecutorPlus submit = executorFactory().pooled("submit-test", submissionThreads); AccordExecutor executor = supplier.get(); Lock lock = executor.unsafeLock(); SequentialAsyncExecutor sequentialExecutor = executor.newSequentialExecutor(); - Executor lockExecutor = ExecutorFactory.Global.executorFactory().sequential("lock"); - ConcurrentLinkedQueue> await = new ConcurrentLinkedQueue<>(); + Executor lockExecutor = executorFactory().sequential("lock"); for (float sleepChance : new float[] { 0f, 0.01f, 0.1f }) + { for (float lockChance : new float[] { 0f, 0.01f, 0.1f }) - submitLoop(lock, executor, sequentialExecutor, lockExecutor, 200, 10, await, sleepChance, lockChance); + { + List> done = new ArrayList<>(); + for (int i = 0 ; i < submissionThreads ; ++i) + { + int id = i; + done.add(submit.submit(() -> { + try + { + submitLoop(id, lock, executor, sequentialExecutor, lockExecutor, 20, 10, sleepChance, lockChance); + } + catch (ExecutionException | InterruptedException e) + { + throw new RuntimeException(e); + } + })); + } + for (Future f : done) + f.get(); + } + } } catch (Throwable t) { @@ -93,8 +120,9 @@ public void executorTest(SerializableSupplier supplier) () -> {}, 1L); } - private static void submitLoop(Lock lock, AccordExecutor executor, SequentialAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, ConcurrentLinkedQueue> await, float sleepChance, float lockChance) throws ExecutionException, InterruptedException + private static void submitLoop(int id, Lock lock, AccordExecutor executor, SequentialAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance) throws ExecutionException, InterruptedException { + ConcurrentLinkedQueue> await = new ConcurrentLinkedQueue<>(); while (outerLoop-- > 0) { for (int i = 0; i < innerLoop; ++i) @@ -105,7 +133,7 @@ private static void submitLoop(Lock lock, AccordExecutor executor, SequentialAsy while (!await.isEmpty()) await.poll().get(); done.set(true); - System.out.println("Loop " + (1 + outerLoop)); + System.out.println("Loop " + id + '.' + (1 + outerLoop)); } } From 13956ddf214599f5d47fa55e60f5252c395c1d60 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Thu, 9 Jul 2026 11:25:51 +0100 Subject: [PATCH 02/10] rename PreLoadContext to ExecutionContext, and SequentialAsyncExecutor -> ExclusiveAsyncExecutor --- .gitmodules | 4 +- .../architecture/accord-architecture.adoc | 2 +- .../db/virtual/AccordDebugKeyspace.java | 18 ++--- .../service/accord/AccordCommandStore.java | 22 +++--- .../service/accord/AccordCommandStores.java | 4 +- .../service/accord/AccordExecutor.java | 61 ++++++++-------- .../cassandra/service/accord/AccordTask.java | 70 +++++++++---------- .../accord/debug/DebugBlockedTxns.java | 8 +-- .../service/accord/debug/DebugTxnGraph.java | 8 +-- .../accord/interop/AccordInteropAdapter.java | 10 +-- .../interop/AccordInteropExecution.java | 6 +- .../accord/interop/AccordInteropPersist.java | 4 +- .../test/accord/AccordDropTableBase.java | 4 +- .../test/accord/AccordLoadTest.java | 6 +- .../test/accord/AccordRegainRangesTest.java | 4 +- .../simulator/test/AccordExecutorTest.java | 9 ++- .../CompactionAccordIteratorsTest.java | 2 +- .../accord/AccordCommandStoreTest.java | 6 +- .../service/accord/AccordCommandTest.java | 8 +-- .../service/accord/AccordTaskTest.java | 14 ++-- .../service/accord/AccordTestUtils.java | 10 +-- ...SimpleSimulatedAccordCommandStoreTest.java | 4 +- .../accord/SimulatedAccordCommandStore.java | 10 +-- .../accord/SimulatedAccordTaskTest.java | 10 +-- .../CommandsForKeySerializerTest.java | 16 ++--- 25 files changed, 159 insertions(+), 161 deletions(-) diff --git a/.gitmodules b/.gitmodules index 616dacf610a7..4b1eea6020b2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "modules/accord"] path = modules/accord - url = https://github.com/apache/cassandra-accord.git - branch = trunk + url = https://github.com/belliottsmith/cassandra-accord.git + branch = executor-fair diff --git a/doc/modules/cassandra/pages/architecture/accord-architecture.adoc b/doc/modules/cassandra/pages/architecture/accord-architecture.adoc index 4acb2395d6c4..ac7c6b718013 100644 --- a/doc/modules/cassandra/pages/architecture/accord-architecture.adoc +++ b/doc/modules/cassandra/pages/architecture/accord-architecture.adoc @@ -87,7 +87,7 @@ finally { } .... -In other words, `CommandStore` collects the `PreLoadContext`, state +In other words, `CommandStore` collects the `ExecutionContext`, state required to be in memory for command execution (possible dependencies, such as `TxnId`s, and `Key`s of commands, but also `CommandsForKeys` that will be needed during execution). Once the context is collected and diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 8c63b5767e3d..35b11418a663 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -71,9 +71,9 @@ import accord.local.Commands; import accord.local.Commands.NotifyWaitingOnPlus; import accord.local.DurableBefore; +import accord.local.ExecutionContext; import accord.local.MaxConflicts; import accord.local.Node; -import accord.local.PreLoadContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; @@ -337,16 +337,16 @@ public void collect(PartitionsCollector collector) if (prev != null && info.status() == prev.status() && info.position() == prev.position()) ++uniquePos; else uniquePos = 0; prev = info; - PreLoadContext preLoadContext = info.preLoadContext(); + ExecutionContext executionContext = info.preLoadContext(); rows.add(info.status().name(), info.position(), uniquePos) .lazyCollect(columns -> { columns.add("description", info.describe()) .add("command_store_id", info.commandStoreId()) - .add("txn_id", preLoadContext, PreLoadContext::primaryTxnId, TO_STRING) - .add("txn_id_additional", preLoadContext, PreLoadContext::additionalTxnId, TO_STRING) - .add("keys", preLoadContext, PreLoadContext::keys, TO_STRING) - .add("keys_loading", preLoadContext, PreLoadContext::loadKeys, TO_STRING) - .add("keys_loading_for", preLoadContext, PreLoadContext::loadKeysFor, TO_STRING); + .add("txn_id", executionContext, ExecutionContext::primaryTxnId, TO_STRING) + .add("txn_id_additional", executionContext, ExecutionContext::additionalTxnId, TO_STRING) + .add("keys", executionContext, ExecutionContext::keys, TO_STRING) + .add("keys_loading", executionContext, ExecutionContext::loadKeys, TO_STRING) + .add("keys_loading_for", executionContext, ExecutionContext::loadKeysFor, TO_STRING); }); } }); @@ -740,7 +740,7 @@ private void addPartition(CommandStore commandStore, PartitionsCollector collect collector.partition(commandStore.id()) .collect(rows -> { // TODO (desired): support maybe execute immediately with safeStore - Future future = toFuture(commandStore.chain((PreLoadContext.Empty) metadata::toString, safeStore -> { addRows(safeStore, rows); })); + Future future = toFuture(commandStore.chain((ExecutionContext.Empty) metadata::toString, safeStore -> { addRows(safeStore, rows); })); if (!future.awaitUntilThrowUncheckedOnInterrupt(collector.deadlineNanos())) throw new InternalTimeoutException(); }); @@ -1895,7 +1895,7 @@ private void run(TxnId txnId, int commandStoreId, Function i)); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index f4dd59a9c7ae..7bdb69efc03f 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -58,13 +58,13 @@ import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; import accord.local.CommandSummaries; +import accord.local.ExecutionContext; import accord.local.MaxConflicts; import accord.local.MaxDecidedRX; import accord.local.MinimalCommand; import accord.local.MinimalCommand.MinimalWithDeps; import accord.local.NodeCommandStoreService; -import accord.local.PreLoadContext; -import accord.local.PreLoadContext.Empty; +import accord.local.ExecutionContext.Empty; import accord.local.RedundantBefore; import accord.local.RedundantBefore.Bounds; import accord.local.RedundantStatus.Property; @@ -216,7 +216,7 @@ private boolean isReadyToTerminate() public final String loggingId; public final Journal journal; private final AccordExecutor sharedExecutor; - final AccordExecutor.SequentialExecutor exclusiveExecutor; + final AccordExecutor.ExclusiveExecutor exclusiveExecutor; private final ExclusiveCaches caches; private final RangeIndex rangeIndex; private final TableId tableId; @@ -403,27 +403,27 @@ public long nextSystemTimestampMicros() return lastSystemTimestampMicros.accumulateAndGet(node.now(), (a, b) -> Math.max(a + 1, b)); } @Override - public AsyncChain chain(PreLoadContext loadCtx, Function function) + public AsyncChain chain(ExecutionContext loadCtx, Function function) { return AccordTask.create(this, loadCtx, function).chain(); } @Override - public AsyncChain chain(PreLoadContext preLoadContext, Consumer consumer) + public AsyncChain chain(ExecutionContext executionContext, Consumer consumer) { - return AccordTask.create(this, preLoadContext, consumer).chain(); + return AccordTask.create(this, executionContext, consumer).chain(); } @Override - public AsyncChain priorityChain(PreLoadContext preLoadContext, Consumer consumer) + public AsyncChain priorityChain(ExecutionContext executionContext, Consumer consumer) { - return AccordTask.create(this, preLoadContext, consumer).priorityChain(); + return AccordTask.create(this, executionContext, consumer).priorityChain(); } @Override - public AsyncChain priorityChain(PreLoadContext preLoadContext, Function function) + public AsyncChain priorityChain(ExecutionContext executionContext, Function function) { - return AccordTask.create(this, preLoadContext, function).priorityChain(); + return AccordTask.create(this, executionContext, function).priorityChain(); } @Override @@ -729,7 +729,7 @@ public AsyncChain replay(TxnId txnId) if (!maybeShouldReplay(txnId)) return AsyncChains.success(null); - return commandStore.chain(PreLoadContext.contextFor(txnId, "Replay"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(txnId, "Replay"), safeStore -> { Replay replay = shouldReplay(txnId, safeStore.unsafeGet(txnId).current().participants()); if (replay == Replay.NONE) return null; diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index f362c960c489..929d3d58cbd1 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -36,7 +36,7 @@ import accord.api.ProgressLog; import accord.local.CommandStores; import accord.local.NodeCommandStoreService; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.local.ShardDistributor; import accord.utils.RandomSource; import accord.utils.Reduce; @@ -148,7 +148,7 @@ static Factory factory() } @Override - public SequentialAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someSequentialExecutor() { int idx = ((int) Thread.currentThread().getId()) & mask; return executors[idx].newSequentialExecutor(); diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java index cff519396a26..4277ec55c840 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java @@ -44,8 +44,7 @@ import accord.api.RoutingKey; import accord.impl.AbstractAsyncExecutor; import accord.local.Command; -import accord.local.PreLoadContext; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExecutionContext; import accord.local.cfk.CommandsForKey; import accord.messages.Accept; import accord.messages.Commit; @@ -611,25 +610,25 @@ private void waitingToRun(AccordTask task) task.addToQueue(task.commandStore.exclusiveExecutor); } - private void waitingToRun(Task task, @Nullable SequentialExecutor queue) + private void waitingToRun(Task task, @Nullable ExclusiveExecutor queue) { task.onWaitingToRun(); task.addToQueue(queue == null ? waitingToRun : queue); } - public SequentialExecutor executor() + public ExclusiveExecutor executor() { - return new SequentialExecutor(this); + return new ExclusiveExecutor(this); } - public SequentialExecutor executor(int commandStoreId) + public ExclusiveExecutor executor(int commandStoreId) { - return new SequentialExecutor(this, commandStoreId); + return new ExclusiveExecutor(this, commandStoreId); } - public SequentialAsyncExecutor newSequentialExecutor() + public accord.local.ExclusiveAsyncExecutor newSequentialExecutor() { - return new SequentialExecutor(this); + return new ExclusiveExecutor(this); } public void cancel(AccordTask task) @@ -751,7 +750,7 @@ private void assignQueuePosition(AccordTask task) case PHASE_HLC_FIFO: { // TODO (expected): we should process messages for a TxnId together, to avoid processing delayed messages out of order - PreLoadContext context = task.preLoadContext(); + ExecutionContext context = task.preLoadContext(); if (context instanceof Request) { MessageType type = ((Request) context).type(); @@ -1207,7 +1206,7 @@ void cancelExclusive(AccordExecutor owner) {} } // run the task even on a stopped commandStore - public interface Unstoppable extends PreLoadContext.Empty + public interface Unstoppable extends ExecutionContext.Empty { } @@ -1218,9 +1217,9 @@ public interface Unterminatable extends Unstoppable static final class SequentialQueueTask extends Task { - private final SequentialExecutor queue; + private final ExclusiveExecutor queue; - SequentialQueueTask(SequentialExecutor queue) + SequentialQueueTask(ExclusiveExecutor queue) { super(); this.queue = queue; @@ -1265,8 +1264,8 @@ protected boolean isInHeap() } } - private static final AtomicReferenceFieldUpdater ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(SequentialExecutor.class, Thread.class, "owner"); - public class SequentialExecutor extends TaskQueue implements SequentialAsyncExecutor + private static final AtomicReferenceFieldUpdater ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(ExclusiveExecutor.class, Thread.class, "owner"); + public class ExclusiveExecutor extends TaskQueue implements accord.local.ExclusiveAsyncExecutor { final int commandStoreId; final SequentialQueueTask selfTask; @@ -1278,12 +1277,12 @@ public class SequentialExecutor extends TaskQueue implements SequentialAsy final DebugSequentialExecutor debug; - SequentialExecutor(AccordExecutor executor) + ExclusiveExecutor(AccordExecutor executor) { this(executor, -1); } - SequentialExecutor(AccordExecutor executor, int commandStoreId) + ExclusiveExecutor(AccordExecutor executor, int commandStoreId) { super(WAITING_TO_RUN, commandStoreId < 0); this.commandStoreId = commandStoreId; @@ -1331,7 +1330,7 @@ private boolean reject(Task task) if (!(task instanceof AccordTask)) return true; - PreLoadContext context = ((AccordTask) task).preLoadContext(); + ExecutionContext context = ((AccordTask) task).preLoadContext(); return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); } @@ -1522,7 +1521,7 @@ private long inheritQueuePosition() private Cancellable execute(RunOrFail runOrFail, long queuePosition) { - PlainChain submit = new PlainChain(runOrFail, SequentialExecutor.this, queuePosition); + PlainChain submit = new PlainChain(runOrFail, ExclusiveExecutor.this, queuePosition); return AccordExecutor.this.submit(submit); } @@ -1643,7 +1642,7 @@ static IntFunction constant(O out) abstract class Plain extends Task implements Cancellable { - abstract SequentialExecutor executor(); + abstract ExclusiveExecutor executor(); @Override protected void preRunExclusive() {} @@ -1663,7 +1662,7 @@ public void cancel() void cancelExclusive(AccordExecutor owner) { - SequentialExecutor executor = executor(); + ExclusiveExecutor executor = executor(); TaskQueue queue = executor == null ? waitingToRun : executor; if (queue.contains(this)) { @@ -1685,7 +1684,7 @@ class PlainRunnable extends Plain implements Cancellable { final @Nullable AsyncPromise result; final Runnable run; - final @Nullable SequentialExecutor executor; + final @Nullable ExclusiveExecutor executor; PlainRunnable(Runnable run) { @@ -1697,7 +1696,7 @@ class PlainRunnable extends Plain implements Cancellable this(result, run, null, 0); } - PlainRunnable(AsyncPromise result, Runnable run, @Nullable SequentialExecutor executor, long queuePosition) + PlainRunnable(AsyncPromise result, Runnable run, @Nullable ExclusiveExecutor executor, long queuePosition) { this.result = result; this.run = run; @@ -1727,7 +1726,7 @@ protected void fail(Throwable t) } @Override - SequentialExecutor executor() + ExclusiveExecutor executor() { return executor; } @@ -1755,7 +1754,7 @@ protected void cleanupExclusive(AccordExecutor executor) } @Override - SequentialExecutor executor() + ExclusiveExecutor executor() { return null; } @@ -1934,14 +1933,14 @@ public String description() class PlainChain extends Plain { final RunOrFail runOrFail; - final @Nullable SequentialExecutor executor; + final @Nullable ExclusiveExecutor executor; PlainChain(RunOrFail runOrFail) { this(runOrFail, null, 0); } - PlainChain(RunOrFail runOrFail, @Nullable SequentialExecutor executor, long queuePosition) + PlainChain(RunOrFail runOrFail, @Nullable ExclusiveExecutor executor, long queuePosition) { this.runOrFail = runOrFail; this.executor = executor; @@ -1949,7 +1948,7 @@ class PlainChain extends Plain } @Override - SequentialExecutor executor() + ExclusiveExecutor executor() { return executor; } @@ -1991,7 +1990,7 @@ class DebuggableChain extends PlainChain implements DebuggableTask long startedAtNanos; final Object describe; - DebuggableChain(RunOrFail runOrFail, @Nullable SequentialExecutor executor, int queuePosition, Object describe) + DebuggableChain(RunOrFail runOrFail, @Nullable ExclusiveExecutor executor, int queuePosition, Object describe) { super(runOrFail, executor, queuePosition); this.createdAtNanos = MonotonicClock.Global.approxTime.now(); @@ -2073,7 +2072,7 @@ public long position() return null; } - public @Nullable PreLoadContext preLoadContext() + public @Nullable ExecutionContext preLoadContext() { if (task instanceof AccordTask) return ((AccordTask) task).preLoadContext(); @@ -2119,7 +2118,7 @@ private static void addToSnapshot(List snapshot, TaskQueue queue, T Task t = queue.get(i); if (t instanceof SequentialQueueTask) { - SequentialExecutor q = ((SequentialQueueTask) t).queue; + ExclusiveExecutor q = ((SequentialQueueTask) t).queue; snapshot.add(new TaskInfo(ifCurrent, q.commandStoreId, q.task)); for (int j = 0 ; j < q.size() ; ++j) snapshot.add(new TaskInfo(ifQueued, q.commandStoreId, q.get(j))); diff --git a/src/java/org/apache/cassandra/service/accord/AccordTask.java b/src/java/org/apache/cassandra/service/accord/AccordTask.java index a766cc209790..fe2889e2a6e9 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTask.java +++ b/src/java/org/apache/cassandra/service/accord/AccordTask.java @@ -49,8 +49,8 @@ import accord.local.CommandStore; import accord.local.CommandSummaries; import accord.local.CommandSummaries.Summary; +import accord.local.ExecutionContext; import accord.local.LoadKeys; -import accord.local.PreLoadContext; import accord.local.SafeCommandStore; import accord.local.cfk.CommandsForKey; import accord.primitives.AbstractRanges; @@ -111,7 +111,7 @@ static class ForFunction extends AccordTask { private final Function function; - public ForFunction(AccordCommandStore commandStore, PreLoadContext loadCtx, Function function) + public ForFunction(AccordCommandStore commandStore, ExecutionContext loadCtx, Function function) { super(commandStore, loadCtx); this.function = function; @@ -129,7 +129,7 @@ static class ForConsumer extends AccordTask { private final Consumer consumer; - private ForConsumer(AccordCommandStore commandStore, PreLoadContext loadCtx, Consumer consumer) + private ForConsumer(AccordCommandStore commandStore, ExecutionContext loadCtx, Consumer consumer) { super(commandStore, loadCtx); this.consumer = consumer; @@ -143,12 +143,12 @@ public Void apply(SafeCommandStore commandStore) } } - public static AccordTask create(CommandStore commandStore, PreLoadContext ctx, Function function) + public static AccordTask create(CommandStore commandStore, ExecutionContext ctx, Function function) { return new ForFunction<>((AccordCommandStore) commandStore, ctx, function); } - public static AccordTask create(CommandStore commandStore, PreLoadContext ctx, Consumer consumer) + public static AccordTask create(CommandStore commandStore, ExecutionContext ctx, Consumer consumer) { return new ForConsumer((AccordCommandStore) commandStore, ctx, consumer); } @@ -201,7 +201,7 @@ boolean isComplete() private State state = INITIALIZED; final AccordCommandStore commandStore; - private final PreLoadContext preLoadContext; + private final ExecutionContext executionContext; private volatile String loggingId; private static final AtomicLong nextLoggingId = new AtomicLong(Clock.Global.currentTimeMillis()); private static final AtomicReferenceFieldUpdater loggingIdUpdater = AtomicReferenceFieldUpdater.newUpdater(AccordTask.class, String.class, "loggingId"); @@ -220,10 +220,10 @@ boolean isComplete() private BiConsumer callback; - public AccordTask(@Nonnull AccordCommandStore commandStore, PreLoadContext preLoadContext) + public AccordTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext) { this.commandStore = commandStore; - this.preLoadContext = preLoadContext; + this.executionContext = executionContext; this.loggingId = "0x" + Long.toHexString(nextLoggingId.incrementAndGet()); if (logger.isTraceEnabled()) @@ -245,7 +245,7 @@ private String loggingId() @Override public String toString() { - return preLoadContext.describe() + ' ' + toBriefString(); + return executionContext.describe() + ' ' + toBriefString(); } public String toBriefString() @@ -257,7 +257,7 @@ public String toDescription() { return toBriefString() + ": " + (queued == null ? "unqueued" : queued.kind) - + ", primaryTxnId: " + preLoadContext.primaryTxnId() + + ", primaryTxnId: " + executionContext.primaryTxnId() + ", waitingToLoad: " + summarise(waitingToLoad) + ", loading:" + summarise(loading, AccordSafeState::global) + ", cfks:" + summarise(commandsForKey, AccordSafeState::global) @@ -315,7 +315,7 @@ private void state(State state) Unseekables keys() { - return preLoadContext.keys(); + return executionContext.keys(); } // TODO (expected): try to execute immediately BUT consider ordering requirements @@ -363,22 +363,22 @@ public void presetup(AccordTask parent) // so we do not mutate anything, except the atomic counter of references if (parent.commands != null) { - for (TxnId txnId : preLoadContext.txnIds()) + for (TxnId txnId : executionContext.txnIds()) presetupExclusive(txnId, AccordTask::ensureCommands, parent.commands, commandStore.cachesUnsafe().commands()); } if (parent.commandsForKey == null) return; - if (preLoadContext.keys().domain() != Key) return; - switch (preLoadContext.loadKeys()) + if (executionContext.keys().domain() != Key) return; + switch (executionContext.loadKeys()) { - default: throw new UnhandledEnum(preLoadContext.loadKeys()); + default: throw new UnhandledEnum(executionContext.loadKeys()); case NONE: break; case ASYNC: case INCR: case SYNC: - for (RoutingKey key : (AbstractUnseekableKeys)preLoadContext.keys()) + for (RoutingKey key : (AbstractUnseekableKeys) executionContext.keys()) presetupExclusive(key, AccordTask::ensureCommandsForKey, parent.commandsForKey, commandStore.cachesUnsafe().commandsForKeys()); break; } @@ -402,7 +402,7 @@ private void setupInternal(Caches caches) { { boolean hasPreSetup = commands != null; - for (TxnId txnId : preLoadContext.txnIds()) + for (TxnId txnId : executionContext.txnIds()) { if (hasPreSetup && completePresetupExclusive(txnId, commands, caches.commands())) continue; @@ -410,22 +410,22 @@ private void setupInternal(Caches caches) } } - if (preLoadContext.keys().isEmpty()) + if (executionContext.keys().isEmpty()) return; - switch (preLoadContext.keys().domain()) + switch (executionContext.keys().domain()) { - case Key: setupKeyLoadsExclusive(caches, (AbstractUnseekableKeys)preLoadContext.keys(), false); break; + case Key: setupKeyLoadsExclusive(caches, (AbstractUnseekableKeys) executionContext.keys(), false); break; case Range: setupRangeLoadsExclusive(caches); } } private void setupKeyLoadsExclusive(Caches caches, Iterable keys, boolean isToCompleteRangeScan) { - if (preLoadContext.loadKeys() == LoadKeys.NONE) + if (executionContext.loadKeys() == LoadKeys.NONE) return; - if (!isToCompleteRangeScan && preLoadContext.loadKeysFor() == RECOVERY) + if (!isToCompleteRangeScan && executionContext.loadKeysFor() == RECOVERY) { Invariants.require(rangeScanner == null); rangeScanner = new RangeTxnScanner(); @@ -441,12 +441,12 @@ private void setupKeyLoadsExclusive(Caches caches, Iterable commands() @@ -680,7 +680,7 @@ public void run() try (Closeable close = resources.get()) { if (Tracing.isTracing()) - Tracing.trace(preLoadContext.describe()); + Tracing.trace(executionContext.describe()); state(RUNNING); @@ -789,7 +789,7 @@ public void cancel() void cancelExclusive() { - logger.info("Cancelling {}", preLoadContext); + logger.info("Cancelling {}", executionContext); state(CANCELLED); if (rangeScanner != null) rangeScanner.cancelled = true; @@ -968,7 +968,7 @@ public void onUpdate(AccordCacheEntry state) // TODO (expected): produce key summaries to avoid locking all in memory final Set intersectingKeys = new ObjectHashSet<>(); final KeyWatcher keyWatcher = new KeyWatcher(); - final Ranges ranges = ((AbstractRanges) preLoadContext.keys()).toRanges(); + final Ranges ranges = ((AbstractRanges) executionContext.keys()).toRanges(); final AccordCache.Type.Instance commandsForKeyCache; public RangeTxnAndKeyScanner(AccordCache.Type.Instance commandsForKeyCache) @@ -1082,9 +1082,9 @@ protected void runInternal() loader.load(guardedSummaries, () -> cancelled); } - PreLoadContext preLoadContext() + ExecutionContext preLoadContext() { - return preLoadContext; + return executionContext; } @Override @@ -1109,7 +1109,7 @@ public void start(AccordExecutor executor) void startInternal(Caches caches) { - loader = commandStore.rangeIndex().loader(preLoadContext.primaryTxnId(), preLoadContext.executeAt(), preLoadContext.loadKeysFor(), preLoadContext.keys()); + loader = commandStore.rangeIndex().loader(executionContext.primaryTxnId(), executionContext.executeAt(), executionContext.loadKeysFor(), executionContext.keys()); loader.loadExclusive(guardedSummaries, caches); } @@ -1143,7 +1143,7 @@ CommandSummaries finish(Caches caches) @Override public String description() { - return "Scanning range intersections for " + preLoadContext.reason() + ' ' + toBriefString(); + return "Scanning range intersections for " + executionContext.reason() + ' ' + toBriefString(); } @Override @@ -1174,6 +1174,6 @@ public long startTimeNanos() @Override public String description() { - return preLoadContext.describe(); + return executionContext.describe(); } } diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java b/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java index af38a5c389e8..15cbd81ff279 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java @@ -37,7 +37,7 @@ import accord.local.Command; import accord.local.CommandStore; import accord.local.CommandStores; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommandStore; import accord.local.cfk.SafeCommandsForKey; import accord.primitives.RoutingKeys; @@ -178,7 +178,7 @@ private Future> drainToFuture(Queue> drain, List visitRootTxnAsync(CommandStore commandStore, TxnId txnId) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return null; @@ -188,7 +188,7 @@ private AsyncChain visitRootTxnAsync(CommandStore commandStore, TxnId txnId private AsyncChain visitTxnAsync(CommandStore commandStore, TxnId txnId, Timestamp rootExecuteAt, @Nullable TokenKey byKey, int depth, boolean recurse) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return null; @@ -231,7 +231,7 @@ private Txn visitTxnSync(SafeCommandStore safeStore, Command command, Timestamp private AsyncChain visitKeysAsync(CommandStore commandStore, TokenKey key, Timestamp rootExecuteAt, int depth) { - return commandStore.chain(PreLoadContext.contextFor(RoutingKeys.of(key.toUnseekable()), SYNC, READ_WRITE, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(RoutingKeys.of(key.toUnseekable()), SYNC, READ_WRITE, "Populate txn_blocked_by"), safeStore -> { visitKeysSync(safeStore, key, rootExecuteAt, depth); }); } diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java b/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java index 8bc6a6a53a20..a821db634033 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java @@ -40,7 +40,7 @@ import accord.local.Command; import accord.local.CommandStore; import accord.local.CommandStores; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommandStore; import accord.primitives.PartialDeps; import accord.primitives.Participants; @@ -221,7 +221,7 @@ static Future> drainToFuture(Queue> drain, List> submitRoot(CommandStore commandStore, TxnId txnId) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return AsyncChains.>success(null); @@ -232,7 +232,7 @@ private AsyncChain> submitRoot(CommandStore commandStore, TxnId txnI private AsyncChain> submitParent(CommandStore commandStore, TxnId txnId, P param, Map infos, Set visitedParent, int depth) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return AsyncChains.>success(null); @@ -344,7 +344,7 @@ protected void visitLatestCommitted(List sortedInfos, Command parent, private AsyncChain populateTxnAsync(CommandStore commandStore, TxnId txnId, Map visited) { - return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); visited.putIfAbsent(txnId, command == null || command.saveStatus() == SaveStatus.Uninitialised ? SaveInfo.NONE : new SaveInfo(command.saveStatus(), command.executeAtIfKnown())); }); diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java index 61de28c26c1e..b79fc5f841b2 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java @@ -30,7 +30,7 @@ import accord.coordinate.ExecuteFlag.CoordinationFlags; import accord.coordinate.ExecutePath; import accord.local.Node; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.messages.Apply; import accord.primitives.Ballot; import accord.primitives.Deps; @@ -83,14 +83,14 @@ private AccordInteropAdapter(AccordEndpointMapper endpointMapper, Apply.Kind app } @Override - public void execute(Node node, SequentialAsyncExecutor executor, Topologies any, FullRoute route, Ballot ballot, ExecutePath path, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps stableDeps, Deps sendDeps, BiConsumer callback) + public void execute(Node node, ExclusiveAsyncExecutor executor, Topologies any, FullRoute route, Ballot ballot, ExecutePath path, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps stableDeps, Deps sendDeps, BiConsumer callback) { if (!doInteropExecute(node, executor, route, ballot, txnId, txn, executeAt, stableDeps, callback)) super.execute(node, executor, any, route, ballot, path, flags, txnId, txn, executeAt, stableDeps, sendDeps, callback); } @Override - public void persist(Node node, SequentialAsyncExecutor executor, Topologies any, Route require, Route sendTo, FullRoute route, Ballot ballot, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, boolean informDurableOnDone, BiConsumer callback) + public void persist(Node node, ExclusiveAsyncExecutor executor, Topologies any, Route require, Route sendTo, FullRoute route, Ballot ballot, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, boolean informDurableOnDone, BiConsumer callback) { if (applyKind == Minimal && doInteropPersist(node, executor, any, require, sendTo, ballot, txnId, txn, executeAt, deps, writes, result, route, informDurableOnDone, callback)) return; @@ -98,7 +98,7 @@ public void persist(Node node, SequentialAsyncExecutor executor, Topologies any, super.persist(node, executor, any, require, sendTo, route, ballot, flags, txnId, txn, executeAt, deps, writes, result, informDurableOnDone, callback); } - private boolean doInteropExecute(Node node, SequentialAsyncExecutor executor, FullRoute route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer callback) + private boolean doInteropExecute(Node node, ExclusiveAsyncExecutor executor, FullRoute route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer callback) { // Unrecoverable repair always needs to be run by AccordInteropExecution AccordUpdate.Kind updateKind = AccordUpdate.kind(txn.update()); @@ -121,7 +121,7 @@ private boolean doInteropExecute(Node node, SequentialAsyncExecutor executor, Fu return true; } - private boolean doInteropPersist(Node node, SequentialAsyncExecutor executor, Topologies any, Route require, Route sendTo, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute fullRoute, boolean informDurableOnDone, BiConsumer callback) + private boolean doInteropPersist(Node node, ExclusiveAsyncExecutor executor, Topologies any, Route require, Route sendTo, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute fullRoute, boolean informDurableOnDone, BiConsumer callback) { Update update = txn.update(); ConsistencyLevel consistencyLevel = update instanceof AccordUpdate ? ((AccordUpdate) update).cassandraCommitCL() : null; diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java index 49b467cc7f0a..8b215a903ffd 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java @@ -33,7 +33,7 @@ import accord.coordinate.ExecuteFlag.CoordinationFlags; import accord.local.Node; import accord.local.Node.Id; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.messages.Commit; import accord.messages.Commit.Kind; import accord.primitives.AbstractRanges; @@ -125,7 +125,7 @@ public class AccordInteropExecution implements ReadCoordinator private final Timestamp executeAt; private final Deps deps; private final BiConsumer callback; - private final SequentialAsyncExecutor executor; + private final ExclusiveAsyncExecutor executor; private final ConsistencyLevel consistencyLevel; private final AccordEndpointMapper endpointMapper; @@ -141,7 +141,7 @@ public class AccordInteropExecution implements ReadCoordinator private volatile long uniqueHlc; public AccordInteropExecution(Node node, TxnId txnId, Txn txn, AccordUpdate.Kind updateKind, FullRoute route, Ballot ballot, Timestamp executeAt, Deps deps, BiConsumer callback, - SequentialAsyncExecutor executor, ConsistencyLevel consistencyLevel, AccordEndpointMapper endpointMapper) throws TopologyException + ExclusiveAsyncExecutor executor, ConsistencyLevel consistencyLevel, AccordEndpointMapper endpointMapper) throws TopologyException { requireArgument(!txn.read().keys().isEmpty() || updateKind == AccordUpdate.Kind.UNRECOVERABLE_REPAIR); this.node = node; diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java index 7cdb7032a876..95551f709293 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java @@ -26,7 +26,7 @@ import accord.coordinate.tracking.AllTracker; import accord.coordinate.tracking.QuorumTracker; import accord.local.Node; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.messages.Apply; import accord.primitives.Ballot; import accord.primitives.Deps; @@ -53,7 +53,7 @@ */ public class AccordInteropPersist extends Persist { - public AccordInteropPersist(Node node, SequentialAsyncExecutor executor, Topologies topologies, TxnId txnId, Route sendTo, Ballot ballot, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute fullRoute, ConsistencyLevel consistencyLevel, ExecuteFlag.CoordinationFlags flags, boolean informDurableOnDone, Apply.Kind applyKind, BiConsumer callback) + public AccordInteropPersist(Node node, ExclusiveAsyncExecutor executor, Topologies topologies, TxnId txnId, Route sendTo, Ballot ballot, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute fullRoute, ConsistencyLevel consistencyLevel, ExecuteFlag.CoordinationFlags flags, boolean informDurableOnDone, Apply.Kind applyKind, BiConsumer callback) { super(node, executor, topologies, txnId, ballot, sendTo, txn, executeAt, deps, writes, result.toPersistable(), fullRoute, flags, informDurableOnDone, AccordInteropApply.FACTORY, applyKind, consistencyLevel == ALL ? AllTracker::new : QuorumTracker::new, (ignore, fail) -> { if (fail != null) callback.accept(null, fail); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java index 8b96f0d1da78..04db46248330 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java @@ -24,9 +24,9 @@ import accord.api.RoutingKey; import accord.local.CommandStores; +import accord.local.ExecutionContext; import accord.local.LoadKeys; import accord.local.Node; -import accord.local.PreLoadContext; import accord.local.cfk.CommandsForKey; import accord.primitives.Ranges; import accord.primitives.Routable; @@ -135,7 +135,7 @@ protected static void validateAccord(Cluster cluster, TableId id) TableId tableId = TableId.fromString(s); AccordService accord = (AccordService) AccordService.instance(); TxnId syntheticTxnId = new TxnId(TxnId.MAX_EPOCH, 0, Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range, new Node.Id(1)); - PreLoadContext ctx = PreLoadContext.contextFor(syntheticTxnId, Ranges.single(TokenRange.fullRange(tableId, getPartitioner())), LoadKeys.SYNC, READ_WRITE, "Test"); + ExecutionContext ctx = ExecutionContext.contextFor(syntheticTxnId, Ranges.single(TokenRange.fullRange(tableId, getPartitioner())), LoadKeys.SYNC, READ_WRITE, "Test"); CommandStores stores = accord.node().commandStores(); for (int storeId : stores.ids()) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java index cee395977dbf..8270da184e37 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java @@ -56,7 +56,7 @@ import org.slf4j.LoggerFactory; import accord.local.CommandStore; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommand; import accord.primitives.PartialDeps; import accord.primitives.TxnId; @@ -832,7 +832,7 @@ else if (random.nextFloat() < settings.readRatio) if (storeId.get() >= 0) { CommandStore commandStore = service.node().commandStores().forId(storeId.get()); - List> result = AccordService.getBlocking(commandStore.submit(PreLoadContext.contextFor(candidate, "LoadTest"), safeStore -> { + List> result = AccordService.getBlocking(commandStore.submit(ExecutionContext.contextFor(candidate, "LoadTest"), safeStore -> { SafeCommand safeCommand = safeStore.unsafeGet(candidate); PartialDeps deps = safeCommand.current().partialDeps(); if (deps == null) @@ -855,7 +855,7 @@ else if (random.nextFloat() < settings.readRatio) for (List info : result) { TxnId txnId = TxnId.parse(info.get(0)); - AccordService.getBlocking(commandStore.execute(PreLoadContext.contextFor(txnId, "LoadTest"), safeStore -> { + AccordService.getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, "LoadTest"), safeStore -> { SafeCommand safeCommand = safeStore.unsafeGet(txnId); if (safeCommand.current().executeAt != null) info.add(safeCommand.current().executeAt.toString()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRegainRangesTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRegainRangesTest.java index 990c87502446..eeb57dd204ab 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRegainRangesTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRegainRangesTest.java @@ -29,7 +29,7 @@ import accord.local.CommandStore; import accord.local.CommandStores.PreviouslyOwned; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.primitives.AbstractRanges; import accord.primitives.Ranges; import accord.topology.ActiveEpochs; @@ -121,7 +121,7 @@ public void regainRangesTest() throws Throwable Ranges range = Ranges.EMPTY; for (CommandStore commandStore : AccordService.instance().node().commandStores().all()) { - Ranges safeToReadRanges = getBlocking(commandStore.submit((PreLoadContext.Empty) () -> "No overlapping safeToReadRanges", safeCommandStore -> { + Ranges safeToReadRanges = getBlocking(commandStore.submit((ExecutionContext.Empty) () -> "No overlapping safeToReadRanges", safeCommandStore -> { Ranges mergedRanges = Ranges.EMPTY; for (Ranges r : safeCommandStore.safeToReadAt().values()) mergedRanges = mergedRanges.union(AbstractRanges.UnionMode.MERGE_ADJACENT, r); diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java index 103f45b2db98..eadb7a4f5546 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java @@ -35,9 +35,8 @@ import org.junit.Test; import accord.api.AsyncExecutor; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; -import org.apache.cassandra.concurrent.ExecutorFactory; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; @@ -85,7 +84,7 @@ public void executorTest(SerializableSupplier supplier, int subm ExecutorPlus submit = executorFactory().pooled("submit-test", submissionThreads); AccordExecutor executor = supplier.get(); Lock lock = executor.unsafeLock(); - SequentialAsyncExecutor sequentialExecutor = executor.newSequentialExecutor(); + ExclusiveAsyncExecutor sequentialExecutor = executor.newSequentialExecutor(); Executor lockExecutor = executorFactory().sequential("lock"); for (float sleepChance : new float[] { 0f, 0.01f, 0.1f }) @@ -120,7 +119,7 @@ public void executorTest(SerializableSupplier supplier, int subm () -> {}, 1L); } - private static void submitLoop(int id, Lock lock, AccordExecutor executor, SequentialAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance) throws ExecutionException, InterruptedException + private static void submitLoop(int id, Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance) throws ExecutionException, InterruptedException { ConcurrentLinkedQueue> await = new ConcurrentLinkedQueue<>(); while (outerLoop-- > 0) @@ -137,7 +136,7 @@ private static void submitLoop(int id, Lock lock, AccordExecutor executor, Seque } } - private static void submitRecursive(Lock lock, AccordExecutor executor, SequentialAsyncExecutor sequentialExecutor, int count, Collection> await, float sleepChance, float lockChance) + private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Collection> await, float sleepChance, float lockChance) { AsyncExecutor submitTo = ThreadLocalRandom.current().nextBoolean() ? executor : sequentialExecutor; await.add(toFuture(submitTo.chain(() -> { diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java index 4901566aae4d..a9a5c6db580f 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java @@ -92,7 +92,7 @@ import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; -import static accord.local.PreLoadContext.contextFor; +import static accord.local.ExecutionContext.contextFor; import static accord.local.RedundantStatus.SomeStatus.GC_BEFORE_AND_LOCALLY_DURABLE; import static accord.primitives.Routable.Domain.Range; import static accord.primitives.Timestamp.Flag.HLC_BOUND; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java index f3d94e7827f1..fc691f60fce0 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java @@ -32,7 +32,7 @@ import accord.api.Key; import accord.api.Result.PersistableResult; import accord.local.Command; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.StoreParticipants; import accord.local.cfk.CommandsForKey; import accord.primitives.Ballot; @@ -170,8 +170,8 @@ public void commandsForKeyLoadSave() AccordSafeCommandsForKey cfk = new AccordSafeCommandsForKey(loaded(key, null)); cfk.initialize(); - cfk.set(cfk.current().update(new TestSafeCommandStore(PreLoadContext.contextFor(command1.txnId(), "Test")), command1).cfk()); - cfk.set(cfk.current().update(new TestSafeCommandStore(PreLoadContext.contextFor(command1.txnId(), "Test")), command2).cfk()); + cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.contextFor(command1.txnId(), "Test")), command1).cfk()); + cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.contextFor(command1.txnId(), "Test")), command2).cfk()); CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null, commandStore.nextSystemTimestampMicros()).run(); logger.info("E: {}", cfk); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java index 214e17e6d148..dcd348685283 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java @@ -29,7 +29,7 @@ import accord.local.Command; import accord.local.LoadKeys; import accord.local.Node; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; @@ -98,7 +98,7 @@ private static PartitionKey key(int k) public void basicCycleTest() throws Throwable { AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); - getBlocking(commandStore.execute((PreLoadContext.Empty)() -> "Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); + getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); Txn txn = createWriteTxn(1); @@ -174,7 +174,7 @@ public void basicCycleTest() throws Throwable Commit commit = Commit.SerializerSupport.create(txnId, route, 1, 1, Commit.Kind.StableWithTxnAndDeps, Ballot.ZERO, executeAt, partialTxn, deps, fullRoute); getBlocking(commandStore.execute(commit, commit::apply)); - getBlocking(commandStore.execute(PreLoadContext.contextFor(txnId, Keys.of(key).toParticipants(), LoadKeys.SYNC, READ_WRITE, "Test"), safeStore -> { + getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, Keys.of(key).toParticipants(), LoadKeys.SYNC, READ_WRITE, "Test"), safeStore -> { Command before = safeStore.ifInitialised(txnId).current(); Assert.assertEquals(commit.executeAt, before.executeAt()); Assert.assertTrue(before.hasBeen(Status.Committed)); @@ -191,7 +191,7 @@ public void basicCycleTest() throws Throwable public void computeDeps() throws Throwable { AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); - getBlocking(commandStore.execute((PreLoadContext.Empty)()->"Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); + getBlocking(commandStore.execute((ExecutionContext.Empty)()->"Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); TxnId txnId1 = txnId(1, clock.incrementAndGet(), 1); Txn txn = createWriteTxn(2); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java index c5fafc9103be..ede89a4d77fa 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java @@ -46,7 +46,7 @@ import accord.api.RoutingKey; import accord.local.CheckedCommands; import accord.local.Command; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; @@ -87,7 +87,7 @@ import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; -import static accord.local.PreLoadContext.contextFor; +import static accord.local.ExecutionContext.contextFor; import static accord.utils.Property.qt; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; import static org.apache.cassandra.service.accord.AccordService.getBlocking; @@ -127,7 +127,7 @@ public void optionalCommandTest() throws Throwable AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); - getBlocking(commandStore.execute(PreLoadContext.contextFor(txnId, "Test"), instance -> { + getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, "Test"), instance -> { // TODO review: This change to `ifInitialized` was done in a lot of places and it doesn't preserve this property // I fixed this reference to point to `ifLoadedAndInitialised` and but didn't update other places Assert.assertNull(instance.ifInitialised(txnId)); @@ -141,7 +141,7 @@ public void touchUnknownTxn() throws Throwable AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); - getBlocking(commandStore.execute(PreLoadContext.contextFor(txnId, "Test"), safe -> { + getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, "Test"), safe -> { StoreParticipants participants = StoreParticipants.empty(txnId); SafeCommand command = safe.get(txnId, participants); Assert.assertNotNull(command); @@ -155,7 +155,7 @@ public void optionalCommandsForKeyTest() throws Throwable Txn txn = AccordTestUtils.createWriteTxn((int)clock.incrementAndGet()); TokenKey key = ((PartitionKey) Iterables.getOnlyElement(txn.keys())).toUnseekable(); - getBlocking(commandStore.execute((PreLoadContext.Empty)() -> "Test", instance -> { + getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", instance -> { SafeCommandsForKey cfk = instance.ifLoadedAndInitialised(key); Assert.assertNull(cfk); })); @@ -298,7 +298,7 @@ public void loadFail() awaitDone(commandStore, ids, participants); assertNoReferences(commandStore, ids, participants); - PreLoadContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC, READ_WRITE, "Test"); + ExecutionContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC, READ_WRITE, "Test"); Consumer consumer = Mockito.mock(Consumer.class); Map failed = selectFailedTxn(rs, ids); @@ -363,7 +363,7 @@ public void consumerFails() assertNoReferences(commandStore, ids, participants); createCommand(commandStore, rs, ids); - PreLoadContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC, READ_WRITE, "Test"); + ExecutionContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC, READ_WRITE, "Test"); Consumer consumer = Mockito.mock(Consumer.class); String errorMsg = "txn_ids " + ids; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index fd2301046a44..fc4fdd436755 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -51,12 +51,12 @@ import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; import accord.local.DurableBefore; +import accord.local.ExecutionContext; import accord.local.Node; import accord.local.Node.Id; import accord.local.NodeCommandStoreService; -import accord.local.PreLoadContext; import accord.local.SafeCommandStore; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.durability.DurabilityService; @@ -244,7 +244,7 @@ public static Ballot ballot(long epoch, long hlc, int node) public static AsyncChain> processTxnResult(AccordCommandStore commandStore, TxnId txnId, PartialTxn txn, Timestamp executeAt) throws Throwable { AtomicReference>> result = new AtomicReference<>(); - getBlocking(commandStore.execute((PreLoadContext.Empty)() -> "Test", + getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", safeStore -> result.set(processTxnResultDirect(safeStore, txnId, txn, executeAt)))); return result.get(); } @@ -371,7 +371,7 @@ public AsyncExecutor someExecutor() } @Override - public SequentialAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someSequentialExecutor() { return null; } @@ -419,7 +419,7 @@ public static AccordCommandStore createAccordCommandStore( Node.Id node = new Id(1); Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[] { node }), Sets.newHashSet(node))); AccordCommandStore store = createAccordCommandStore(node, now, topology); - store.execute((PreLoadContext.Empty)()->"Test", safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().cacheUnsafe().setCapacity(1 << 20)); + store.execute((ExecutionContext.Empty)()->"Test", safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().cacheUnsafe().setCapacity(1 << 20)); return store; } diff --git a/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java index 0d2b8ccb9365..fae6c912fa4e 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java @@ -21,7 +21,7 @@ import org.assertj.core.api.Assertions; import org.junit.Test; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.StoreParticipants; import accord.primitives.SaveStatus; import accord.primitives.TxnId; @@ -41,7 +41,7 @@ public void emptyTxns() for (int i = 0, examples = 100; i < examples; i++) { TxnId id = AccordGens.txnIds().next(rs); - instance.process(PreLoadContext.contextFor(id, "Test"), (safe) -> { + instance.process(ExecutionContext.contextFor(id, "Test"), (safe) -> { var safeCommand = safe.get(id, StoreParticipants.empty(id)); var command = safeCommand.current(); Assertions.assertThat(command.saveStatus()).isEqualTo(SaveStatus.Uninitialised); diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java index 0fabb6735202..228e2e564a3d 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java @@ -54,12 +54,12 @@ import accord.local.CommandStores; import accord.local.CommandStores.RangesForEpoch; import accord.local.DurableBefore; +import accord.local.ExecutionContext; import accord.local.Node; import accord.local.NodeCommandStoreService; -import accord.local.PreLoadContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.local.TimeService; import accord.local.durability.DurabilityService; import accord.messages.BeginRecovery; @@ -182,7 +182,7 @@ public AsyncExecutor someExecutor() } @Override - public SequentialAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someSequentialExecutor() { return null; } @@ -448,7 +448,7 @@ public T process(RouteRequest request) throws ExecutionExce return process(request, request); } - public T process(PreLoadContext loadCtx, Function function) throws ExecutionException, InterruptedException + public T process(ExecutionContext loadCtx, Function function) throws ExecutionException, InterruptedException { var result = processAsync(loadCtx, function); processAll(); @@ -460,7 +460,7 @@ public AsyncResult processAsync(RouteRequest request) return processAsync(request, request); } - public AsyncResult processAsync(PreLoadContext loadCtx, Function function) + public AsyncResult processAsync(ExecutionContext loadCtx, Function function) { return commandStore.submit(loadCtx, function); } diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java index 805ba040b85b..ce061f40bf30 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java @@ -32,7 +32,7 @@ import accord.api.RoutingKey; import accord.impl.basic.SimulatedFault; -import accord.local.PreLoadContext; +import accord.local.ExecutionContext; import accord.local.SafeCommandStore; import accord.messages.PreAccept; import accord.primitives.FullRoute; @@ -107,7 +107,7 @@ private static void test(RandomSource rs, int numSamples, TableMetadata tbl, Gen { case Task: { - PreLoadContext ctx = (PreLoadContext.Empty)()->"Test"; + ExecutionContext ctx = (ExecutionContext.Empty)()->"Test"; instance.maybeCacheEvict(ctx.keys()); operation(instance, ctx, actionGen.next(rs), rs::nextBoolean).chain().begin(counter); } @@ -178,7 +178,7 @@ private static TokenRange range(TableId tableId, long start, long end) private enum Action { SUCCESS, FAILURE, LOAD_FAILURE } - private static AccordTask operation(SimulatedAccordCommandStore instance, PreLoadContext ctx, Action action, BooleanSupplier delay) + private static AccordTask operation(SimulatedAccordCommandStore instance, ExecutionContext ctx, Action action, BooleanSupplier delay) { return new SimulatedOperation(instance.commandStore, ctx, action == Action.FAILURE ? SimulatedOperation.Action.FAILURE : SimulatedOperation.Action.SUCCESS); } @@ -201,9 +201,9 @@ private static class SimulatedOperation extends AccordTask enum Action { SUCCESS, FAILURE} private final Action action; - public SimulatedOperation(AccordCommandStore commandStore, PreLoadContext preLoadContext, Action action) + public SimulatedOperation(AccordCommandStore commandStore, ExecutionContext executionContext, Action action) { - super(commandStore, preLoadContext); + super(commandStore, executionContext); this.action = action; } diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java index 138dbb4f12c9..5130dce55339 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java @@ -66,13 +66,13 @@ import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; import accord.local.DurableBefore; +import accord.local.ExecutionContext; import accord.local.Node; import accord.local.NodeCommandStoreService; -import accord.local.PreLoadContext; import accord.local.RedundantBefore; import accord.local.SafeCommand; import accord.local.SafeCommandStore; -import accord.local.SequentialAsyncExecutor; +import accord.local.ExclusiveAsyncExecutor; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.cfk.CommandsForKey; @@ -505,8 +505,8 @@ private static void testOne(long seed) { int next = source.nextInt(commands.size()); Command command = commands.get(next); - if (command.txnId.isSyncPoint()) cfk = cfk.registerUnmanaged(new TestSafeCommandStore(PreLoadContext.contextFor(command.txnId(), "Test")), new TestSafeCommand(command), REGISTER).cfk(); - else cfk = cfk.update(new TestSafeCommandStore(PreLoadContext.contextFor(command.txnId(), "Test")), command).cfk(); + if (command.txnId.isSyncPoint()) cfk = cfk.registerUnmanaged(new TestSafeCommandStore(ExecutionContext.contextFor(command.txnId(), "Test")), new TestSafeCommand(command), REGISTER).cfk(); + else cfk = cfk.update(new TestSafeCommandStore(ExecutionContext.contextFor(command.txnId(), "Test")), command).cfk(); commands.set(next, commands.get(commands.size() - 1)); commands.remove(commands.size() - 1); } @@ -667,8 +667,8 @@ protected TestCommandStore() } @Override public boolean inStore() { return true; } - @Override public AsyncChain chain(PreLoadContext context, Consumer consumer) { throw new UnsupportedOperationException();} - @Override public AsyncChain chain(PreLoadContext context, Function apply) { throw new UnsupportedOperationException(); } + @Override public AsyncChain chain(ExecutionContext context, Consumer consumer) { throw new UnsupportedOperationException();} + @Override public AsyncChain chain(ExecutionContext context, Function apply) { throw new UnsupportedOperationException(); } @Override public Journal.Replayer replayer(AbstractReplayer.Mode mode) { throw new UnsupportedOperationException(); } @@ -702,7 +702,7 @@ protected TestCommandStore() public static class TestSafeCommandStore extends AbstractSafeCommandStore { - public TestSafeCommandStore(PreLoadContext context) + public TestSafeCommandStore(ExecutionContext context) { super(context, TestCommandStore.INSTANCE); } @@ -718,7 +718,7 @@ public TestSafeCommandStore(PreLoadContext context) @Override public NodeCommandStoreService node() { return new NodeCommandStoreService() { @Override public AsyncExecutor someExecutor() { throw new UnsupportedOperationException(); } - @Override public SequentialAsyncExecutor someSequentialExecutor() { throw new UnsupportedOperationException(); } + @Override public ExclusiveAsyncExecutor someSequentialExecutor() { throw new UnsupportedOperationException(); } @Override public long epoch() { return 0;} @Override public Node.Id id() { return Node.Id.NONE; } @Override public Timeouts timeouts() { return null; } From 1cffe9a8a032ce37889e821fb46b165bcd942323 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Thu, 9 Jul 2026 11:42:47 +0100 Subject: [PATCH 03/10] Executor Fairness --- .../cassandra/concurrent/CassandraThread.java | 57 +- .../apache/cassandra/config/AccordConfig.java | 80 +- .../db/virtual/AccordDebugKeyspace.java | 2 +- .../cassandra/service/accord/AccordCache.java | 1 - .../service/accord/AccordCommandStore.java | 40 +- .../service/accord/AccordCommandStores.java | 6 +- .../service/accord/AccordExecutor.java | 2709 +++++++++++++---- .../AccordExecutorAbstractLockLoop.java | 110 +- .../accord/AccordExecutorAbstractLoop.java | 22 +- .../accord/AccordExecutorSemiSyncSubmit.java | 12 - .../accord/AccordExecutorSignalLoop.java | 446 +-- .../service/accord/AccordExecutorSimple.java | 12 + .../accord/AccordExecutorSyncSubmit.java | 5 +- .../accord/AccordFetchCoordinator.java | 2 +- .../accord/AccordSafeCommandStore.java | 2 +- .../service/accord/AccordService.java | 2 +- .../cassandra/service/accord/AccordTask.java | 221 +- .../service/accord/debug/DebugExecution.java | 8 +- .../accord/interop/AccordInteropAdapter.java | 2 +- .../interop/AccordInteropExecution.java | 2 +- .../accord/interop/AccordInteropPersist.java | 2 +- .../utils/concurrent/SignalLock.java | 6 - .../cassandra/simulator/ActionSchedule.java | 5 +- .../simulator/test/AccordExecutorTest.java | 217 +- .../CompactionAccordIteratorsTest.java | 2 +- .../service/accord/AccordCommandTest.java | 2 +- .../service/accord/AccordTaskTest.java | 2 +- .../service/accord/AccordTestUtils.java | 6 +- .../accord/SimulatedAccordCommandStore.java | 4 +- .../CommandsForKeySerializerTest.java | 4 +- .../cassandra/utils/AccordGenerators.java | 2 +- 31 files changed, 2822 insertions(+), 1171 deletions(-) diff --git a/src/java/org/apache/cassandra/concurrent/CassandraThread.java b/src/java/org/apache/cassandra/concurrent/CassandraThread.java index 62cbcdb6a4e8..d66adc24423a 100644 --- a/src/java/org/apache/cassandra/concurrent/CassandraThread.java +++ b/src/java/org/apache/cassandra/concurrent/CassandraThread.java @@ -18,14 +18,21 @@ package org.apache.cassandra.concurrent; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; + import org.apache.cassandra.metrics.ThreadLocalMetrics; +import org.apache.cassandra.service.accord.AccordExecutor; import io.netty.util.concurrent.FastThreadLocalThread; -public class CassandraThread extends FastThreadLocalThread +public class CassandraThread extends FastThreadLocalThread implements AccordExecutor.AccordTaskRunner { private ThreadLocalMetrics threadLocalMetrics; private ExecutorLocals executorLocals; + private AccordExecutor accordActiveExecutor; + private AccordExecutor accordLockedExecutor; + private volatile AccordExecutor.Task accordActiveTask; + private static final AtomicReferenceFieldUpdater accordActiveTaskUpdater = AtomicReferenceFieldUpdater.newUpdater(CassandraThread.class, AccordExecutor.Task.class, "accordActiveTask"); private final ImmediateTaskHolder immediateTaskHolder; @@ -89,6 +96,54 @@ public ExecutorLocals replaceExecutorLocals(ExecutorLocals newExecutorLocals) return current != null ? current : ExecutorLocals.none(); } + public final AccordExecutor accordActiveExecutor() + { + return accordActiveExecutor; + } + + public final void setAccordActiveExecutor(AccordExecutor newExecutor) + { + accordActiveExecutor = newExecutor; + } + + @Override + public final AccordExecutor accordLockedExecutor() + { + return accordLockedExecutor; + } + + @Override + public final boolean trySetAccordLockedExecutor(AccordExecutor newLockedExecutor) + { + if (accordLockedExecutor != null) + return false; + accordLockedExecutor = newLockedExecutor; + return true; + } + + @Override + public final void clearAccordLockedExecutor() + { + accordLockedExecutor = null; + } + + public final AccordExecutor.Task accordActiveTask() + { + return accordActiveTask; + } + + // to be called only by the thread itself, so can (eventually) avoid any memory barriers + public final AccordExecutor.Task accordActiveSelfTask() + { + // TODO (expected): with newer JDK use accordActiveTaskUpdater.getPlain + return accordActiveTask; + } + + public final void setAccordActiveTask(AccordExecutor.Task newActiveTask) + { + accordActiveTaskUpdater.lazySet(this, newActiveTask); + } + // final to avoid skipping of the cleanup logic in child classes final public void run() { diff --git a/src/java/org/apache/cassandra/config/AccordConfig.java b/src/java/org/apache/cassandra/config/AccordConfig.java index b9fe93b9a490..fe226fe92f53 100644 --- a/src/java/org/apache/cassandra/config/AccordConfig.java +++ b/src/java/org/apache/cassandra/config/AccordConfig.java @@ -35,7 +35,6 @@ import org.apache.cassandra.service.consensus.TransactionalMode; import static org.apache.cassandra.config.AccordConfig.CatchupMode.NORMAL; -import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.HLC_FIFO; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_POOL_PER_SHARD; import static org.apache.cassandra.config.AccordConfig.QueueSubmissionModel.SYNC; import static org.apache.cassandra.config.AccordConfig.RangeIndexMode.in_memory; @@ -134,25 +133,65 @@ public enum QueuePriorityModel FIFO, /** - * If the work has an associated TxnId, prioritise by its HLC (and FIFO otherwise) + * If the work has an associated TxnId of Ballot, prioritise by the newest HLC (and FIFO otherwise) */ HLC_FIFO, /** - * Prioritise Apply, Stable, Commit, Accept, and PreAccept messages in that order. - * Within a given message type, prioritise by HLC. - * Other messages will be mixed with PreAccept messages, but using the next counter rather than the HLC of the TxnId. - * Note: this can have some performance edge cases for contended keys, as we may process Stable messages for later commands before - * we process earlier Accept/Commit, which may delay execution + * If the work has an associated TxnId, prioritise by its HLC (and FIFO otherwise) + */ + ORIG_HLC_FIFO + } + + public enum QueueBalancingModel + { + /** + * Always pick the task by priority + */ + PRIORITY_ONLY, + + /** + * Pick by phase first, so the highest phase with work ALWAYS runs. + * Within a phase, pick by priority. + */ + PHASE_ONLY, + + /** + * Always pick the task by priority. + */ + PRIORITY_BUDGET, + + /** + * Pick by phase first, so the highest phase with budget ALWAYS runs. + * Within a phase, pick by priority. */ - PHASE_HLC_FIFO, + PHASE_BUDGET, /** - * Prioritise Apply, Stable, Commit, Accept, and PreAccept messages from the original coordinator only, in that order. - * Within a given message type, prioritise by HLC. - * Other messages will be mixed with PreAccept messages, but using the next counter rather than the HLC of the TxnId. + * Pick by phase first, selecting the phase that has processed the least work recently relative to arrivals. + * Within a phase, pick by priority. */ - ORIG_PHASE_HLC_FIFO + PHASE_FAIR, + + /** + * Pick by phase first, selecting the phase that has processed the least work recently relative to arrivals. + * However, each phase has a budget that is consumed on dispatch, and we pick only from those phases with budget. + * If there is no phase with budget, the budget resets. + * Within a phase, pick by priority. + */ + PHASE_BUDGET_FAIR, + + /** + * While phases are within a threshold of imbalance, pick tasks by priority. + * Once the threshold is crossed, over-processed phases have a small penalty applied + * and work is prioritised by phase with the phase with the least recent work processed picked first, + * until the imbalance is resolved. + * + * Within a phase, pick by priority. + */ + BLENDED_PRIORITY_PHASE_FAIR, + + BLENDED_PRIORITY_PHASE_BUDGET_FAIR, } public QueueShardModel queue_shard_model = THREAD_POOL_PER_SHARD; @@ -168,7 +207,13 @@ public enum QueuePriorityModel */ public volatile OptionaldPositiveInt queue_thread_count = OptionaldPositiveInt.UNDEFINED; - public QueuePriorityModel queue_priority_model = HLC_FIFO; + public QueuePriorityModel queue_priority_model; + public QueueBalancingModel queue_balancing_model; + + public Integer queue_flow_imbalance_onset = null; + public Integer queue_flow_imbalance_width_shift = null; + public String queue_active_limits; + public String queue_budgets; /** * If set, the signal loop does not match park/unpark pairs, but instead consumers perform timed-park spin waits @@ -181,15 +226,6 @@ public enum QueuePriorityModel */ public DurationSpec.LongMicrosecondsBound queue_stop_check_interval; - /** - * If set, the signal loop reduces the number of threads it is using when the time spent parked exceeds real-time - * by this interval. - */ - public DurationSpec.LongMicrosecondsBound queue_signal_stop_check_interval_credit; - - // yield to other executor threads after executing this many tasks in a row, if there are waiting threads and tasks - public int queue_yield_interval = 100; - /** * If the HLC is older than this, queue by FIFO instead */ diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 35b11418a663..86238688f5bf 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -1875,7 +1875,7 @@ private void recover(TxnId txnId, @Nullable Route route, AsyncResult.Settable Node node = AccordService.unsafeInstance().node(); if (Route.isFullRoute(route)) { - PrepareRecovery.recover(node, node.someSequentialExecutor(), txnId, NotKnownToBeInvalid, (FullRoute) route, null, LatentStoreSelector.standard(), (success, fail) -> { + PrepareRecovery.recover(node, node.someExclusiveExecutor(), txnId, NotKnownToBeInvalid, (FullRoute) route, null, LatentStoreSelector.standard(), (success, fail) -> { if (fail != null) result.setFailure(fail); else result.setSuccess(null); }); diff --git a/src/java/org/apache/cassandra/service/accord/AccordCache.java b/src/java/org/apache/cassandra/service/accord/AccordCache.java index 6b6c48b2fc4b..0d679b2f519f 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCache.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCache.java @@ -345,7 +345,6 @@ private void evict(AccordCacheEntry node, boolean updateUnreferenced) Collection> load(LoadExecutor loadExecutor, P1 p1, P2 p2, AccordCacheEntry node) { - Type parent = node.owner.parent(); return node.load(loadExecutor, p1, p2).waiters(); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index 7bdb69efc03f..60205bf890c9 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -27,7 +27,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.concurrent.locks.Lock; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.IntFunction; @@ -59,12 +58,12 @@ import accord.local.CommandStores.RangesForEpoch; import accord.local.CommandSummaries; import accord.local.ExecutionContext; +import accord.local.ExecutionContext.Empty; import accord.local.MaxConflicts; import accord.local.MaxDecidedRX; import accord.local.MinimalCommand; import accord.local.MinimalCommand.MinimalWithDeps; import accord.local.NodeCommandStoreService; -import accord.local.ExecutionContext.Empty; import accord.local.RedundantBefore; import accord.local.RedundantBefore.Bounds; import accord.local.RedundantStatus.Property; @@ -100,6 +99,7 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable; +import org.apache.cassandra.service.accord.AccordExecutor.AccordTaskRunner; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.IAccordService.AccordCompactionInfo; import org.apache.cassandra.service.accord.api.TokenKey; @@ -167,12 +167,12 @@ public final AccordCache.Type { - private final Lock lock; + private final AccordExecutor owner; - public ExclusiveCaches(Lock lock, AccordCache global, AccordCache.Type.Instance commands, AccordCache.Type.Instance commandsForKeys) + public ExclusiveCaches(AccordExecutor owner, AccordCache global, AccordCache.Type.Instance commands, AccordCache.Type.Instance commandsForKeys) { super(global, commands, commandsForKeys); - this.lock = lock; + this.owner = owner; } @Override @@ -190,8 +190,8 @@ public AccordSafeCommandsForKey acquireIfLoaded(RoutingKey key) @Override public void close() { - global().tryShrinkOrEvict(lock); - lock.unlock(); + global().tryShrinkOrEvict(owner.unsafeLock()); + owner.unlock(AccordTaskRunner.get()); } } @@ -261,7 +261,7 @@ public AccordCommandStore(int id, { commands = exclusive.commands.newInstance(this); commandsForKey = exclusive.commandsForKey.newInstance(this); - this.caches = new ExclusiveCaches(sharedExecutor.unsafeLock(), exclusive.global, commands, commandsForKey); + this.caches = new ExclusiveCaches(sharedExecutor, exclusive.global, commands, commandsForKey); } this.exclusiveExecutor = sharedExecutor.executor(id); @@ -323,13 +323,13 @@ public AsyncExecutor taskExecutor() public ExclusiveCaches lockCaches() { //noinspection LockAcquiredButNotSafelyReleased - caches.lock.lock(); + caches.owner.lock(AccordTaskRunner.get()); return caches; } public ExclusiveCaches tryLockCaches() { - if (caches.lock.tryLock()) + if (caches.owner.tryLock(AccordTaskRunner.get())) return caches; return null; } @@ -403,27 +403,15 @@ public long nextSystemTimestampMicros() return lastSystemTimestampMicros.accumulateAndGet(node.now(), (a, b) -> Math.max(a + 1, b)); } @Override - public AsyncChain chain(ExecutionContext loadCtx, Function function) - { - return AccordTask.create(this, loadCtx, function).chain(); - } - - @Override - public AsyncChain chain(ExecutionContext executionContext, Consumer consumer) - { - return AccordTask.create(this, executionContext, consumer).chain(); - } - - @Override - public AsyncChain priorityChain(ExecutionContext executionContext, Consumer consumer) + public AsyncChain chain(ExecutionContext context, Function function) { - return AccordTask.create(this, executionContext, consumer).priorityChain(); + return AccordTask.create(this, context, function).chain(); } @Override - public AsyncChain priorityChain(ExecutionContext executionContext, Function function) + public AsyncChain chain(ExecutionContext context, Consumer consumer) { - return AccordTask.create(this, executionContext, function).priorityChain(); + return AccordTask.create(this, context, consumer).chain(); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index 929d3d58cbd1..f82215919e1e 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -31,12 +31,12 @@ import accord.api.Agent; import accord.api.DataStore; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Journal; import accord.api.LocalListeners; import accord.api.ProgressLog; import accord.local.CommandStores; import accord.local.NodeCommandStoreService; -import accord.local.ExclusiveAsyncExecutor; import accord.local.ShardDistributor; import accord.utils.RandomSource; import accord.utils.Reduce; @@ -148,10 +148,10 @@ static Factory factory() } @Override - public ExclusiveAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someExclusiveExecutor() { int idx = ((int) Thread.currentThread().getId()) & mask; - return executors[idx].newSequentialExecutor(); + return executors[idx].newExclusiveExecutor(); } public synchronized void setCapacity(long bytes) diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java index 4277ec55c840..c60017851344 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java @@ -21,7 +21,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; -import java.util.Queue; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; import java.util.concurrent.RejectedExecutionException; @@ -41,6 +41,7 @@ import org.slf4j.LoggerFactory; import accord.api.Agent; +import accord.api.ExclusiveAsyncExecutor; import accord.api.RoutingKey; import accord.impl.AbstractAsyncExecutor; import accord.local.Command; @@ -54,28 +55,31 @@ import accord.primitives.Ballot; import accord.primitives.SaveStatus; import accord.primitives.TxnId; -import accord.utils.ArrayBuffers.BufferList; +import accord.utils.ArrayBuffers; +import accord.utils.IntrusiveHeapNode; import accord.utils.IntrusivePriorityHeap; import accord.utils.Invariants; import accord.utils.QuadConsumer; import accord.utils.QuadFunction; import accord.utils.QuintConsumer; +import accord.utils.TinyEnumSet; import accord.utils.TriConsumer; import accord.utils.TriFunction; import accord.utils.UnhandledEnum; import accord.utils.async.AsyncCallbacks.CallAndCallback; -import accord.utils.async.AsyncCallbacks.FlatCallAndCallback; -import accord.utils.async.AsyncCallbacks.RunAndCallback; import accord.utils.async.AsyncCallbacks.RunOrFail; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; import accord.utils.async.Cancellable; import org.apache.cassandra.cache.CacheSize; +import org.apache.cassandra.concurrent.CassandraThread; import org.apache.cassandra.concurrent.DebuggableTask; import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; import org.apache.cassandra.concurrent.ExecutorLocals; import org.apache.cassandra.concurrent.Shutdownable; +import org.apache.cassandra.config.AccordConfig; +import org.apache.cassandra.config.AccordConfig.QueueBalancingModel; import org.apache.cassandra.config.AccordConfig.QueuePriorityModel; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.metrics.AccordCacheMetrics; @@ -89,29 +93,52 @@ import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; import org.apache.cassandra.service.accord.AccordCacheEntry.SaveExecutor; import org.apache.cassandra.service.accord.AccordCacheEntry.UniqueSave; +import org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup; +import org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup; +import org.apache.cassandra.service.accord.AccordExecutor.Task.GroupKind; +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExclusiveExecutor; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutor; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugSequentialExecutor; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; import org.apache.cassandra.utils.Closeable; -import org.apache.cassandra.utils.MonotonicClock; import org.apache.cassandra.utils.WithResources; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Condition; import org.apache.cassandra.utils.concurrent.Future; -import io.netty.util.concurrent.FastThreadLocal; - +import static accord.primitives.Routable.Domain.Range; import static accord.utils.Invariants.createIllegalState; -import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.PHASE_HLC_FIFO; +import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.ORIG_HLC_FIFO; import static org.apache.cassandra.service.accord.AccordCache.CommandAdapter.COMMAND_ADAPTER; import static org.apache.cassandra.service.accord.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER; import static org.apache.cassandra.service.accord.AccordCache.registerJfrListener; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; -import static org.apache.cassandra.service.accord.AccordTask.State.LOADING; -import static org.apache.cassandra.service.accord.AccordTask.State.RUNNING; -import static org.apache.cassandra.service.accord.AccordTask.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_RUN; +import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.COUNTER_LOWBITS; +import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.COUNTER_MASKS; +import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.minCounterValue; +import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.selectByOverflowBits; +import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.setOverflowWhenLessEqual; +import static org.apache.cassandra.service.accord.AccordExecutor.RunnableTaskQueue.RUNNABLE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.ACCEPT; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.APPLY; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.COMMIT; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.PREACCEPT; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.RANGE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.STABLE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.COMMAND_STORE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.OTHER; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_SCAN; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.SAVE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.MAX_TRANCHE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.ASSIGNED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.INITIALIZED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.SCANNING_RANGES; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_SCAN_RANGES; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; import static org.apache.cassandra.utils.Clock.Global.nanoTime; @@ -123,11 +150,82 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutorflow. + private static final int BLEND_SHIFT = 6, BLEND_TOTAL = 1 << BLEND_SHIFT; + private static final int FLOW_ONSET, FLOW_WIDTH_SHIFT; + private static final boolean BALANCE_BY_POSITION; + private static final long GLOBAL_QUEUE_LIMITS, GLOBAL_QUEUE_BUDGETS; + private static final long EXCLUSIVE_QUEUE_LIMITS, EXCLUSIVE_QUEUE_BUDGETS; + + static + { + AccordConfig config = DatabaseDescriptor.getAccord(); + AGE_TO_FIFO = config.queue_priority_age_to_fifo.to(TimeUnit.MICROSECONDS); + PRIORITY_MODEL = config.queue_priority_model != null ? config.queue_priority_model : QueuePriorityModel.HLC_FIFO; + BALANCING_MODEL = config.queue_balancing_model != null ? config.queue_balancing_model : QueueBalancingModel.BLENDED_PRIORITY_PHASE_BUDGET_FAIR; + FLOW_ONSET = config.queue_flow_imbalance_onset == null ? 4 : config.queue_flow_imbalance_onset; + FLOW_WIDTH_SHIFT = config.queue_flow_imbalance_width_shift == null ? 5 : config.queue_flow_imbalance_width_shift; + Invariants.require(FLOW_ONSET >= 0 && FLOW_WIDTH_SHIFT >= 0); + switch (BALANCING_MODEL) + { + default: throw new UnhandledEnum(BALANCING_MODEL); + case PRIORITY_ONLY: + case BLENDED_PRIORITY_PHASE_FAIR: + case BLENDED_PRIORITY_PHASE_BUDGET_FAIR: + case PRIORITY_BUDGET: + BALANCE_BY_POSITION = true; + break; + case PHASE_ONLY: + case PHASE_FAIR: + case PHASE_BUDGET: + case PHASE_BUDGET_FAIR: + BALANCE_BY_POSITION = false; + } + + { + long global = COUNTER_MASKS, exclusive = COUNTER_MASKS; + global ^= (0x7f ^ 1) << RANGE_SCAN.ordinal(); + if (config.queue_active_limits != null) + { + long[] limits = parseEnumParams(config.queue_active_limits, "queue_active_limits"); + global = selectByOverflowBits(setOverflowWhenLessEqual(limits[0], 0), global, limits[0]); + exclusive = selectByOverflowBits(setOverflowWhenLessEqual(limits[1], 0), global, limits[1]); + } + GLOBAL_QUEUE_LIMITS = global; + EXCLUSIVE_QUEUE_LIMITS = exclusive; + } + + { + long global = encodeCounters(Map.of(COMMAND_STORE, 16L, LOAD, 16L, SAVE, 16L, OTHER, 16L, RANGE_LOAD, 2L, RANGE_SCAN, 1L)); + long exclusive = encodeCounters(Map.of(APPLY, 8L, STABLE, 7L, COMMIT, 6L, ACCEPT, 5L, OTHER, 4L, PREACCEPT, 4L, RANGE, 1L)); + if (config.queue_budgets != null) + { + long[] limits = parseEnumParams(config.queue_budgets, "queue_budgets"); + // any that are unset, set to the lowest value of any other specified + if (limits[0] != 0) + { + long min = minCounterValue(limits[0], 0); + long mins = min * COUNTER_LOWBITS; + if (min > 1) + { + mins ^= (min ^ 1) << (RANGE_SCAN.ordinal() * 8); // set the default RANGE_SCAN budget to 1 + mins ^= (min ^ 2) << (RANGE_LOAD.ordinal() * 8); // set the default RANGE_LOAD budget to 2 + } + global = selectByOverflowBits(setOverflowWhenLessEqual(limits[0], 0), mins, limits[0]); + } + if (limits[1] != 0) + exclusive = selectByOverflowBits(setOverflowWhenLessEqual(limits[1], 0), minCounterValue(limits[1], 0) * COUNTER_LOWBITS, limits[1]); + } + GLOBAL_QUEUE_BUDGETS = global; + EXCLUSIVE_QUEUE_BUDGETS = exclusive; + } + } + public static final ShardedDecayingHistograms HISTOGRAMS = new ShardedDecayingHistograms(); - private static final FastThreadLocal paranoidPriorityInversionCheck = new FastThreadLocal<>(); public interface AccordExecutorFactory { @@ -136,6 +234,92 @@ public interface AccordExecutorFactory public enum Mode { RUN_WITH_LOCK, RUN_WITHOUT_LOCK } + public interface AccordTaskRunner + { + AccordExecutor accordActiveExecutor(); + void setAccordActiveExecutor(AccordExecutor newExecutor); + + AccordExecutor accordLockedExecutor(); + boolean trySetAccordLockedExecutor(AccordExecutor newLockedExecutor); + void clearAccordLockedExecutor(); + + AccordExecutor.Task accordActiveTask(); + // to be called only by the thread itself, so can (eventually) avoid any memory barriers + AccordExecutor.Task accordActiveSelfTask(); + void setAccordActiveTask(AccordExecutor.Task newActiveTask); + + + static AccordTaskRunner get() + { + return get(Thread.currentThread()); + } + + static AccordTaskRunner get(Thread thread) + { + return thread instanceof CassandraThread ? (CassandraThread)thread : ThreadLocalAccordTaskRunner.threadLocal.get(); + } + } + + public static final class ThreadLocalAccordTaskRunner implements AccordTaskRunner + { + private AccordExecutor lockedExecutor; + private AccordExecutor activeExecutor; + volatile Task activeTask; + + private static final ThreadLocal threadLocal = ThreadLocal.withInitial(ThreadLocalAccordTaskRunner::new); + + @Override + public AccordExecutor accordActiveExecutor() + { + return activeExecutor; + } + + @Override + public void setAccordActiveExecutor(AccordExecutor newExecutor) + { + activeExecutor = newExecutor; + } + + @Override + public AccordExecutor accordLockedExecutor() + { + return lockedExecutor; + } + + @Override + public boolean trySetAccordLockedExecutor(AccordExecutor newLockedExecutor) + { + if (lockedExecutor != null) + return false; + lockedExecutor = newLockedExecutor; + return true; + } + + @Override + public void clearAccordLockedExecutor() + { + lockedExecutor = null; + } + + @Override + public void setAccordActiveTask(Task newActiveTask) + { + activeTask = newActiveTask; + } + + @Override + public Task accordActiveTask() + { + return activeTask; + } + + @Override + public Task accordActiveSelfTask() + { + return activeTask; + } + } + // WARNING: this is a shared object, so close is NOT idempotent public static final class ExclusiveGlobalCaches extends GlobalCaches implements AutoCloseable { @@ -152,7 +336,7 @@ public void close() { executor.beforeUnlockExternal(); global.tryShrinkOrEvict(executor.lock); - executor.unlock(); + executor.unlock(AccordTaskRunner.get()); } } @@ -170,42 +354,214 @@ public GlobalCaches(AccordCache global, AccordCache.Type deferred; + + WithDeferred(Runnable run, Runnable deferred) + { + this.run = run; + this.deferred = new ArrayDeque<>(1); + this.deferred.add(deferred); + } + + WithDeferred(Runnable run, ArrayDeque deferred) + { + this.run = run; + this.deferred = deferred; + } + + @Override + public void run() + { + Runnable register = deferred.poll(); + if (!deferred.isEmpty()) + { + Invariants.require(runs[firstIndex] != null); + Invariants.require(!(runs[firstIndex] instanceof WithDeferred)); + runs[firstIndex] = new WithDeferred(runs[firstIndex], deferred); + } + registerWait(register); + try { run.run(); } + catch (Throwable t) { owner.agent.onException(t); } + } + } + + final AccordExecutor owner; + + int firstTranche; + int firstIndex; + long[] mins = new long[8]; + int[] counts = new int[8]; + Runnable[] runs = new Runnable[8]; - private WaitForCompletion(long position, Runnable run) + // the next tranche, that all new work is being collected against + // (and will move to the array accounting once a new wait is registered) + long nextMin; + int nextTranche; + int nextCount; + + private Tranches(AccordExecutor owner) { - this.position = position; - this.maybeNotify = position - 1; - this.run = run; + this.owner = owner; } - public String toString() + private int trancheToIndex(int tranche) { - return run.toString() + " @" + position; + return firstIndex + trancheToIndexOffset(tranche); } - } - private final Lock lock; - final Agent agent; - final int executorId; - private final AccordCache cache; + private int trancheToIndexOffset(int tranche) + { + int offset = tranche - firstTranche; + if (offset < 0) + offset += MAX_TRANCHE + 1; + return offset; + } - private final TaskQueue> scanningRanges = new TaskQueue<>(SCANNING_RANGES); // never queried, just parked here while scanning - private final TaskQueue> loading = new TaskQueue<>(LOADING); // never queried, just parked here while loading - private final TaskQueue running = new TaskQueue<>(RUNNING); + int size() + { + return trancheToIndexOffset(nextTranche); + } - private final TaskQueue> waitingToLoadRangeTxns = new TaskQueue<>(WAITING_TO_LOAD); - private final TaskQueue> waitingToLoad = new TaskQueue<>(WAITING_TO_LOAD); - private final TaskQueue waitingToRun = new TaskQueue<>(WAITING_TO_RUN); + int capacity() + { + return counts.length; + } - private final ExclusiveGlobalCaches caches; + int addNew(long position) + { + Invariants.require(position >= nextMin); + ++nextCount; + return nextTranche; + } - private List waitingForQuiescence; - private Queue waitingForCompletion; + void addInherited(int tranche, long position) + { + if (tranche == nextTranche) + { + Invariants.require(position >= nextMin); + ++nextCount; + } + else + { + int index = trancheToIndex(tranche); + Invariants.require(counts[index] > 0); + Invariants.require(mins[index] <= position); + ++counts[index]; + } + } + + void complete(int tranche) + { + if (tranche == nextTranche) + { + --nextCount; + } + else + { + + if (counts[trancheToIndex(tranche)] == 1) + owner.drainUnqueuedNewWorkExclusive(); // make sure we don't have any pending + + if (--counts[trancheToIndex(tranche)] == 0 && tranche == firstTranche) + { + do advance(); + while (firstTranche != nextTranche && counts[firstIndex] == 0); + } + + if (firstIndex >= counts.length / 2) + compact(); + } + } + + public void finishAll(long nextPosition) + { + while (firstTranche != nextTranche) + { + logger.warn("{} processed all pending tasks (<{}) but found {} waiting for {}", this, + nextPosition, counts[firstIndex], size() == 1 ? nextMin : mins[firstIndex + 1]); + advance(); + } + } + + private void advance() + { + Runnable run = runs[firstIndex]; + runs[firstIndex] = null; + ++firstIndex; + if (firstTranche == MAX_TRANCHE) firstTranche = 0; + else ++firstTranche; + try { run.run(); } + catch (Throwable t) { owner.agent.onException(t); } + } + + public void registerWait(Runnable run) + { + int newNextTranche = (nextTranche + 1) % (MAX_TRANCHE + 1); + if (newNextTranche == firstTranche) + { + Runnable cur = runs[firstIndex]; + if (cur instanceof WithDeferred) ((WithDeferred) cur).deferred.add(run); + else runs[firstIndex] = new WithDeferred(runs[firstIndex], run); + return; + } + + if ((firstIndex + size()) == capacity()) + growOrCompact(); + + int index = firstIndex + size(); + mins[index] = nextMin; + counts[index] = nextCount; + runs[index] = run; + nextMin = owner.minPosition = owner.nextPosition; + nextCount = 0; + nextTranche = newNextTranche; + } + + private void compact() + { + if (size() <= capacity()/4 && capacity() > 8) resize(capacity()/2); + else compact(mins, counts, runs); + } + + private void growOrCompact() + { + if (size() > capacity()/2) resize(capacity() * 2); + else compact(); + } + + private void resize(int newSize) + { + Invariants.require(newSize > 0); + long[] newMins = new long[newSize]; + int[] newCounts = new int[newSize]; + Runnable[] newRuns = new Runnable[newSize]; + compact(newMins, newCounts, newRuns); + mins = newMins; + counts = newCounts; + runs = newRuns; + } + + private void compact(long[] newMins, int[] newCounts, Runnable[] newRuns) + { + int size = size(); + System.arraycopy(mins, firstIndex, newMins, 0, size); + System.arraycopy(counts, firstIndex, newCounts, 0, size); + System.arraycopy(runs, firstIndex, newRuns, 0, size); + firstIndex = 0; + } + } final LogLinearDecayingHistograms histograms; final LogLinearDecayingHistogram elapsedPreparingToRun; @@ -215,11 +571,20 @@ public String toString() final LogLinearDecayingHistogram keys; public final AccordReplicaMetrics.Shard replicaMetrics; + private final Lock lock; + final Agent agent; + final int executorId; + private final AccordCache cache; + private final ExclusiveGlobalCaches caches; + final DebugExecutor debug = DebugExecutor.maybeDebug(); + + // TODO (expected): remove this /** * The maximum total number of loads we can queue at once - this includes loads for range transactions, * which are subject to this limit as well as that imposed by {@link #maxQueuedRangeLoads} */ private int maxQueuedLoads = 64; + /** * The maximum number of loads exclusively for range transactions we can queue at once; the {@link #maxQueuedLoads} limit also applies. */ @@ -227,11 +592,28 @@ public String toString() private long maxWorkingSetSizeInBytes; private long maxWorkingCapacityInBytes; - private long minPosition, nextPosition; + + private final StandaloneTaskQueue> scanningRanges = new StandaloneTaskQueue<>(TinyEnumSet.encode(SCANNING_RANGES)); // never queried, just parked here while scanning + private final StandaloneTaskQueue> waitingToLoadRangeTxns = new StandaloneTaskQueue<>(TinyEnumSet.encode(WAITING_TO_LOAD)); + private final StandaloneTaskQueue> waitingToLoad = new StandaloneTaskQueue<>(TinyEnumSet.encode(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD)); + private final StandaloneTaskQueue> loading = new StandaloneTaskQueue<>(LOADING); + private final RunnableTaskQueue runnable = new RunnableTaskQueue<>(); + + private final Tranches tranches = new Tranches(this); + + /** + * Newly submitted work must take a position >= minPosition, but this condition does not apply to consequences of + * previously submitted work; this inherits the originating operation's position and tranche. + * This is to ensure afterSubmittedAndConsequences functions correctly. + */ + private long minPosition = 1; + private long nextPosition = 1; + int tasks; + private int activeLoads, activeRangeLoads; private boolean hasPausedLoading; - int tasks; - final DebugExecutor debug = DebugExecutor.maybeDebug(); + + private List waitingForQuiescence; AccordExecutor(Lock lock, int executorId, Agent agent) { @@ -267,7 +649,7 @@ public int executorId() public ExclusiveGlobalCaches lockCaches() { - lock(); + lock(AccordTaskRunner.get(Thread.currentThread())); return caches; } @@ -278,58 +660,38 @@ public final Lock unsafeLock() return lock; } - final void lock() + final void lock(AccordTaskRunner self) { - if (Invariants.isParanoid()) paranoidLockExclusive(); + if (!self.trySetAccordLockedExecutor(this)) + throw new UnsupportedOperationException("To ensure system performance, it is not permitted to lock multiple AccordExecutor simultaneously with the same thread"); //noinspection LockAcquiredButNotSafelyReleased lock.lock(); if (DEBUG_EXECUTION) debug.onEnterLock(); } - final void unlock() + final void unlock(AccordTaskRunner self) { - if (Invariants.isParanoid()) paranoidUnlockExclusive(); + self.clearAccordLockedExecutor(); if (DEBUG_EXECUTION) debug.onExitLock(); lock.unlock(); } - final void paranoidLockExclusive() - { - Lock locked = paranoidPriorityInversionCheck.getAndSet(lock); - Invariants.require(locked == null || locked == lock, "Tried to take multiple AccordExecutor locks on same thread - this is dangerous for progress"); - } - - final void paranoidUnlockExclusive() + final boolean tryLock(AccordTaskRunner self) { - paranoidPriorityInversionCheck.set(null); - } + AccordExecutor locked = self.accordLockedExecutor(); + if (locked != null) + return false; - final boolean tryLock() - { - return onTryLock(lock.tryLock()); + return onTryLock(self, lock.tryLock()); } - final boolean onTryLock(boolean result) + final boolean onTryLock(AccordTaskRunner self, boolean result) { - if (DEBUG_EXECUTION && result) debug.onEnterLock(); - if (Invariants.isParanoid()) + if (result && !self.trySetAccordLockedExecutor(this)) { - if (result) - { - Lock locked = paranoidPriorityInversionCheck.getAndSet(lock); - if (locked != null && locked != lock) - { - lock.unlock(); - paranoidPriorityInversionCheck.set(locked); - Invariants.require(false, "Tried to take multiple AccordExecutor locks on same thread - this is dangerous for progress"); - return false; - } - } - else - { - Lock locked = paranoidPriorityInversionCheck.get(); - Invariants.require(locked == null || locked == lock, "Tried to take multiple AccordExecutor locks on same thread - this is dangerous for progress"); - } + // shouldn't be possible, we should have checked this already + lock.unlock(); + throw new IllegalStateException(); } return result; } @@ -345,15 +707,9 @@ public AccordCache cacheUnsafe() return cache; } - final boolean hasWaitingToRun() - { - updateWaitingToRunExclusive(); - return hasAlreadyWaitingToRun(); - } - final boolean hasAlreadyWaitingToRun() { - return !waitingToRun.isEmpty(); + return runnable.hasWaiting(); } void updateWaitingToRunExclusive() @@ -362,20 +718,18 @@ void updateWaitingToRunExclusive() maybeUnpauseLoading(); } - final Task pollWaitingToRunExclusive() + // drain only new work; specifically leave anything that would call completeTask queued. + // this is to maintain invariants in Tranches.complete, where we may have some consequence of some earlier task + // pending but unqueued, so that we have not incremented its tranche count, and in the interim we set the tranche + // count to zero + void drainUnqueuedNewWorkExclusive() { - updateWaitingToRunExclusive(); - return pollAlreadyWaitingToRunExclusive(); } final Task pollAlreadyWaitingToRunExclusive() { - Task next = waitingToRun.poll(); - if (next != null) - { - if (DEBUG_EXECUTION) DebugTask.get(next).onPolled(); - next.addToQueue(running); - } + Task next = runnable.poll(); + if (DEBUG_EXECUTION && next != null) DebugTask.get(next).onPolled(); return next; } @@ -386,8 +740,9 @@ public Stream active() public void waitForQuiescence() { + AccordTaskRunner self = AccordTaskRunner.get(); Condition condition; - lock(); + lock(self); try { if (tasks == 0) @@ -400,7 +755,7 @@ public void waitForQuiescence() } finally { - unlock(); + unlock(self); } condition.awaitThrowUncheckedOnInterrupt(); } @@ -412,46 +767,37 @@ protected void notifyQuiescentExclusive() waitingForQuiescence.forEach(Condition::signalAll); waitingForQuiescence = null; } - if (waitingForCompletion != null) - { - logger.warn("{} processed all pending tasks (<{}) but found waiting: {}", this, nextPosition, waitingForCompletion); - waitingForCompletion.forEach(w -> w.run.run()); - waitingForCompletion = null; - } + tranches.finishAll(nextPosition); } public void afterSubmittedAndConsequences(Runnable run) { - lock(); + AccordTaskRunner self = AccordTaskRunner.get(); + + lock(self); try { + drainUnqueuedNewWorkExclusive(); if (tasks == 0) { run.run(); return; } - if (waitingForCompletion != null) // escape hatch for some bug that means we lose a notification for a given task's queue position - maybeNotifyWaitingForCompletion(); - if (waitingForCompletion == null) - waitingForCompletion = new ArrayDeque<>(); - - long position = nextPosition; - minPosition = position; - waitingForCompletion.add(new WaitForCompletion(position, run)); + tranches.registerWait(run); } finally { - unlock(); + unlock(self); } } - void maybeUnpauseLoading() + final void maybeUnpauseLoading() { if (!hasPausedLoading) return; - if (cache.weightedSize() < maxWorkingCapacityInBytes || (loading.isEmpty() && waitingToRun.isEmpty())) + if (cache.weightedSize() < maxWorkingCapacityInBytes || (loading.isEmpty() || runnable.hasWaiting())) { hasPausedLoading = false; enqueueLoadsExclusive(); @@ -466,7 +812,7 @@ private void enqueueLoadsExclusive() { outer: while (true) { - TaskQueue> queue = waitingToLoadRangeTxns.isEmpty() || activeRangeLoads >= maxQueuedRangeLoads ? waitingToLoad : waitingToLoadRangeTxns; + StandaloneTaskQueue> queue = waitingToLoadRangeTxns.isEmpty() || activeRangeLoads >= maxQueuedRangeLoads ? waitingToLoad : waitingToLoadRangeTxns; AccordTask next = queue.peek(); if (next == null) return; @@ -474,7 +820,7 @@ private void enqueueLoadsExclusive() if (hasPausedLoading || cache.weightedSize() >= maxWorkingCapacityInBytes) { // we have too much in memory already, and we have work waiting to run, so let that complete before queueing more - if (!loading.isEmpty() || !waitingToRun.isEmpty()) + if (!loading.isEmpty() || runnable.hasWaiting()) { AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); hasPausedLoading = true; @@ -542,43 +888,24 @@ private void enqueueLoadsExclusive() private boolean isForRange(AccordTask task, AccordCacheEntry load) { - boolean isForRangeTxn = task.hasRanges(); + boolean isForRangeTxn = task.isRange(); if (!isForRangeTxn) return false; for (AccordTask t : load.loadingOrWaiting().waiters()) { - if (!t.hasRanges()) + if (!t.isRange()) return false; } return true; } - @Override - public Cancellable execute(RunOrFail runOrFail) - { - PlainChain submit = new PlainChain(runOrFail); - return submit(submit); - } - - public AsyncChain buildDebuggable(Callable task, Object describe) - { - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - return submit(new DebuggableChain(new CallAndCallback<>(task, callback), null, 0, describe)); - } - }; - } - private void parkRangeLoad(AccordTask task) { if (task.queued() != waitingToLoadRangeTxns) { - task.unqueueIfQueued(); - task.addToQueue(waitingToLoadRangeTxns); + task.unqueue(); + waitingToLoadRangeTxns.enqueue(task); } } @@ -590,13 +917,13 @@ private void updateQueue(AccordTask task) default: throw new AssertionError("Unexpected state: " + task.toDescription()); case WAITING_TO_SCAN_RANGES: case WAITING_TO_LOAD: - task.addToQueue(waitingToLoad); + waitingToLoad.enqueue(task); break; case SCANNING_RANGES: - task.addToQueue(scanningRanges); + scanningRanges.enqueue(task); break; case LOADING: - task.addToQueue(loading); + loading.enqueue(task); break; case WAITING_TO_RUN: waitingToRun(task); @@ -607,13 +934,14 @@ private void updateQueue(AccordTask task) private void waitingToRun(AccordTask task) { task.onWaitingToRun(); - task.addToQueue(task.commandStore.exclusiveExecutor); + task.commandStore.exclusiveExecutor.enqueue(task); } private void waitingToRun(Task task, @Nullable ExclusiveExecutor queue) { task.onWaitingToRun(); - task.addToQueue(queue == null ? waitingToRun : queue); + if (queue == null) runnable.enqueue(task); + else queue.enqueue(task); } public ExclusiveExecutor executor() @@ -626,7 +954,7 @@ public ExclusiveExecutor executor(int commandStoreId) return new ExclusiveExecutor(this, commandStoreId); } - public accord.local.ExclusiveAsyncExecutor newSequentialExecutor() + public ExclusiveAsyncExecutor newExclusiveExecutor() { return new ExclusiveExecutor(this); } @@ -678,227 +1006,98 @@ void submit(AccordTask operation) submit(AccordExecutor::submitExclusive, i -> i, operation); } - void submitPriority(AccordTask operation) - { - submit(AccordExecutor::submitPriorityExclusive, i -> i, operation); - } - void submitExclusive(AccordTask task) { - assignQueuePosition(task); - submitInternalExclusive(task); - } - - void submitPriorityExclusive(AccordTask task) - { - assignMinQueuePosition(task); - submitInternalExclusive(task); - } - - private void submitInternalExclusive(AccordTask task) - { + registerExclusive(task); task.setupExclusive(); - ++tasks; updateQueue(task); enqueueLoadsExclusive(); } public void submitExclusive(Runnable runnable) { - submitPlainExclusive(new PlainRunnable(null, runnable)); + submitPlainExclusive(new PlainRunnable(null, runnable, OTHER)); + } + + private Cancellable submitPlain(Plain task) + { + submit(AccordExecutor::submitPlainExclusive, i -> i, task); + return task; } private void submitPlainExclusive(Plain task) { - ++tasks; - assignQueuePosition(task); + registerExclusive(task); waitingToRun(task, task.executor()); } - Cancellable submitPlainExclusive(Task parent, AbstractIOTask task) + Cancellable submitPlainExclusive(Task parent, GlobalGroup group, AbstractIOTask task) { - return submitPlainExclusive(parent, new WrappedIOTask(task)); + return submitPlainExclusive(parent, new WrappedIOTask(task, group, parent.position, parent.tranche())); } - T submitPlainExclusive(Task parent, T task) + final T submitPlainExclusive(Task parent, T task) { Invariants.require(isOwningThread()); - ++tasks; - if (parent != null) inheritQueuePosition(parent, task); - else assignFifoQueuePosition(task); + if (parent == null) registerExclusive(task); + else registerConsequenceExclusive(parent, task); task.onWaitingToRun(); - waitingToRun.append(task); + runnable.enqueue(task); return task; } - private void assignQueuePosition(Task task) + private void registerConsequenceExclusive(Task parent, Task task) { - if (task.queuePosition != 0) updateNextPosition(task); - else assignFifoQueuePosition(task); + ++tasks; + int tranche = parent.tranche(); + tranches.addInherited(tranche, parent.position); + task.position = parent.position; + task.setInheritedTranche(tranche); } - private void assignQueuePosition(AccordTask task) + private void registerExclusive(Task task) { - if (task.queuePosition != 0) updateNextPosition(task); + ++tasks; + if (task.hasInherited()) + { + tranches.addInherited(task.tranche(), task.position); + } else { - long priority_bits = PRIORITY_BITS; - TxnId txnId = null; - switch (PRIORITY_MODEL) + long position = task.position; + if (position == 0) { - case ORIG_PHASE_HLC_FIFO: - case PHASE_HLC_FIFO: - { - // TODO (expected): we should process messages for a TxnId together, to avoid processing delayed messages out of order - ExecutionContext context = task.preLoadContext(); - if (context instanceof Request) - { - MessageType type = ((Request) context).type(); - if (type instanceof StandardMessage) - { - TxnId txnId0 = context.primaryTxnId(); - switch ((StandardMessage)type) - { - case APPLY_REQ: - { - priority_bits = 0L; - txnId = txnId0; - break; - } - case READ_EPHEMERAL_REQ: - case READ_REQ: - case STABLE_THEN_READ_REQ: - { - priority_bits = 1000000000000000L; - txnId = txnId0; - break; - } - case COMMIT_REQ: - { - Commit commit = (Commit) context; - if (PRIORITY_MODEL == PHASE_HLC_FIFO || commit.ballot.equals(Ballot.ZERO)) - txnId = commit.txnId; - if (commit.kind.saveStatus == SaveStatus.Stable) priority_bits = 1000000000000L; - else priority_bits = 2000000000000L; - break; - } - case ACCEPT_REQ: - { - Accept accept = (Accept) context; - if (PRIORITY_MODEL == PHASE_HLC_FIFO || accept.ballot.equals(Ballot.ZERO)) - txnId = accept.txnId; - priority_bits = 3000000000000L; - break; - } - case GET_EPHEMERAL_READ_DEPS_REQ: - case PRE_ACCEPT_REQ: - { - txnId = txnId0; - break; - } - } - } - } - break; - } - case HLC_FIFO: - { - txnId = task.preLoadContext().primaryTxnId(); - break; - } - case FIFO: - { - break; - } + task.position = position = nextPosition++; } - - if (txnId != null) + else { - long hlc = txnId.hlc(); - long delta = nextPosition - hlc; - if (delta < AGE_TO_FIFO) - { - long position = hlc; - if (delta <= 0) nextPosition = position + 1; - else if (position < minPosition) position = minPosition; - position |= priority_bits; - task.queuePosition = position; - return; - } + long delta = nextPosition - position; + if (delta >= AGE_TO_FIFO) + task.position = position = nextPosition++; + else if (delta <= 0) + nextPosition = position + 1; + else if (position < minPosition) + task.position = position = minPosition; } - - assignFifoQueuePosition(task); - } - - } - - private void assignMinQueuePosition(Task task) - { - task.queuePosition = minPosition | PRIORITY_BITS; - } - - private void assignFifoQueuePosition(Task task) - { - task.queuePosition = nextPosition++ | PRIORITY_BITS; - } - - private void updateNextPosition(Task task) - { - nextPosition = Math.max(nextPosition, (task.queuePosition & ~PRIORITY_BITS) + 1); - } - - private void inheritQueuePosition(Task parent, Task task) - { - task.queuePosition = parent.queuePosition; - } - - void completeTaskExclusive(Task task) - { - // for integration with SequentialExecutor, we must : - // - first take the position so that represents the just-executed task - // - call cleanup to submit any following task on the relevant sub-queue - // - remove the previous task from the running collection only if still present (SequentialExecutor will have removed it) - long position = task.queuePosition; - try - { - task.cleanupExclusive(this); - } - finally - { - --tasks; - if (running.contains(task)) - running.remove(task); - - if (waitingForCompletion != null && waitingForCompletion.peek().maybeNotify <= position) - maybeNotifyWaitingForCompletion(); - - cache.tryShrinkOrEvict(lock); + task.setTranche(tranches.addNew(position)); } } - private void maybeNotifyWaitingForCompletion() + final void completeTaskExclusive(Task task) { - long min = minPosition(waitingToRun.peek(), - minPosition(waitingToLoad.peek(), - minPosition(waitingToLoadRangeTxns.peek(), - minPosition(running.peek(), - minPosition(loading.peek(), - minPosition(scanningRanges.peek(), Long.MAX_VALUE)))))); - - while (!waitingForCompletion.isEmpty() && waitingForCompletion.peek().position - min <= 0) - waitingForCompletion.poll().run.run(); - if (waitingForCompletion.isEmpty()) - waitingForCompletion = null; - else - waitingForCompletion.peek().maybeNotify = min; + runnable.complete(task); + try { task.cleanupExclusive(this); } + finally { cache.tryShrinkOrEvict(lock); } } - private static long minPosition(@Nullable Task task, long min) + final void unregisterExclusive(Task task) { - return task == null ? min : Long.min(task.queuePosition, min); + int tranch = task.tranche(); + --tasks; + tranches.complete(tranch); } - void cancelExclusive(AccordTask task) + final void cancelExclusive(AccordTask task) { AccordTask.State state = task.state(); switch (state) @@ -972,7 +1171,7 @@ private void onLoadedExclusive(AccordCacheEntry loaded, V value, Th if (loaded.status() != EVICTED) { - try (BufferList> tasks = loaded.loading().copyWaiters()) + try (ArrayBuffers.BufferList> tasks = loaded.loading().copyWaiters()) { if (fail != null) { @@ -999,35 +1198,74 @@ private void onLoadedExclusive(AccordCacheEntry loaded, V value, Th enqueueLoadsExclusive(); } + private Task inherit() + { + Thread thread = Thread.currentThread(); + AccordTaskRunner self = AccordTaskRunner.get(thread); + + if (self.accordActiveExecutor() != this) + return null; + + Task task = self.accordActiveSelfTask(); + if (task instanceof ExclusiveExecutorTask) + task = ((ExclusiveExecutorTask)task).queue.task; + return task; + } + public Future submit(Runnable run) { - PlainRunnable task = new PlainRunnable(new AsyncPromise<>(), run); - submit(task); + Task inherit = inherit(); + PlainRunnable task = inherit == null ? new PlainRunnable(new AsyncPromise<>(), run, OTHER) + : new PlainRunnable(new AsyncPromise<>(), run, OTHER, inherit.position, inherit.tranche()); + submitPlain(task); return task.result; } - public void execute(Runnable command) + public void execute(Runnable run) { - submit(new PlainRunnable(null, command)); + Task inherit = inherit(); + PlainRunnable task = inherit == null ? new PlainRunnable(null, run, OTHER) + : new PlainRunnable(null, run, OTHER, inherit.position, inherit.tranche()); + submitPlain(task); } - private Cancellable submit(Plain task) + @Override + public Cancellable execute(RunOrFail runOrFail) { - submit(AccordExecutor::submitPlainExclusive, i -> i, task); - return task; + Task inherit = inherit(); + PlainChain submit = inherit == null ? new PlainChain(runOrFail, null, ExclusiveGroup.OTHER) + : new PlainChain(runOrFail, null, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); + return submitPlain(submit); + } + + public AsyncChain buildDebuggable(Callable call, Object describe) + { + return new AsyncChains.Head<>() + { + @Override + protected Cancellable start(BiConsumer callback) + { + Task inherit = inherit(); + return submitPlain(inherit == null ? new DebuggableChain(new CallAndCallback<>(call, callback), null, describe) + : new DebuggableChain(new CallAndCallback<>(call, callback), null, inherit.position, inherit.tranche(), describe)); + } + }; } public void executeDirectlyWithLock(Runnable command) { - lock(); + AccordTaskRunner self = AccordTaskRunner.get(); + lock(self); try { + self.setAccordActiveExecutor(this); command.run(); } finally { beforeUnlockExternal(); - unlock(); + self.clearAccordLockedExecutor(); + unlock(self); } } @@ -1077,60 +1315,287 @@ public long weightedSize() protected static abstract class TaskRunner implements DebuggableTaskRunner { - // TODO (desired): this probably doesn't need to be volatile - private volatile Task running; - private static final AtomicReferenceFieldUpdater runningUpdater = AtomicReferenceFieldUpdater.newUpdater(TaskRunner.class, Task.class, "running"); + private volatile AccordTaskRunner wrapped; + + protected void setWrapped(AccordTaskRunner wrapped) + { + this.wrapped = wrapped; + } @Override public DebuggableTask running() { - Task running = this.running; + Task running = wrapped.accordActiveTask(); return running == null ? null : running.debuggable(); } + } + + public static abstract class Task extends IntrusiveHeapNode + { + enum State + { + INITIALIZED(), + WAITING_TO_SCAN_RANGES(INITIALIZED), + SCANNING_RANGES(WAITING_TO_SCAN_RANGES), + WAITING_TO_LOAD(INITIALIZED, SCANNING_RANGES), + LOADING(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD), + WAITING_TO_RUN(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD, LOADING), + ASSIGNED(WAITING_TO_RUN), + RUNNING(ASSIGNED), + PERSISTING(RUNNING), + FINISHED(RUNNING, PERSISTING), + CANCELLED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, ASSIGNED), + FAILED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, ASSIGNED, RUNNING, PERSISTING); + + private final int permittedFrom; + static final State[] VALUES = values(); + + State() + { + this.permittedFrom = 0; + } + + State(State ... permittedFroms) + { + int permittedFrom = 0; + for (State state : permittedFroms) + permittedFrom |= 1 << state.ordinal(); + this.permittedFrom = permittedFrom; + } + + boolean isPermittedFrom(int prevOrdinal) + { + return (permittedFrom & (1 << prevOrdinal)) != 0; + } + + boolean isExecuted() + { + return this.compareTo(PERSISTING) >= 0; + } + + boolean isComplete() + { + return this.compareTo(FINISHED) >= 0; + } + + static State forOrdinal(int ordinal) + { + return VALUES[ordinal]; + } + } - Task runningTask() + enum GlobalGroup { - return running; + COMMAND_STORE, + LOAD, + SAVE, + OTHER, + RANGE_LOAD(true), + RANGE_SCAN(true), + ; + + final int bits; + + GlobalGroup() + { + this(false); + } + + GlobalGroup(boolean isRange) + { + this.bits = (ordinal() << GLOBAL_GROUP_SHIFT) | (isRange ? RANGE_BIT : 0); + } } - void setRunning(Task debuggable) + enum ExclusiveGroup { - runningUpdater.lazySet(this, debuggable); + APPLY, + STABLE, + COMMIT, + ACCEPT, + OTHER, + PREACCEPT, + RANGE(true), + ; + + final int bits; + + ExclusiveGroup() + { + this(false); + } + + ExclusiveGroup(boolean isRange) + { + this.bits = (ordinal() << EXCLUSIVE_GROUP_SHIFT) | (isRange ? RANGE_BIT : 0); + } } - void clearRunning() + public enum GroupKind { - runningUpdater.lazySet(this, null); + EXCLUSIVE(ExclusiveGroup.values().length, EXCLUSIVE_GROUP_SHIFT), + GLOBAL(GlobalGroup.values().length, GLOBAL_GROUP_SHIFT), + NONE(0, 0); + + final int count; + final byte shift; + GroupKind(int count, int shift) + { + this.count = count; + this.shift = (byte) shift; + } } - } - public static abstract class Task extends IntrusivePriorityHeap.Node - { + private static final int STATE_MASK = 0xf; + private static final int GROUP_MASK = 0x7; + private static final int GLOBAL_GROUP_SHIFT = 7; + private static final int EXCLUSIVE_GROUP_SHIFT = 4; + private static final int RANGE_BIT = 1 << 10; + private static final int CLEANUP_BIT = 1 << 11; + private static final int HAS_TRANCHE_BIT = 1 << 12; + private static final int HAS_INHERITED_BIT = 1 << 13; + private static final int TRANCHE_SHIFT = 14; + static final int MAX_TRANCHE = 0x3ff; + public final WithResources resources; Task next; - long queuePosition; + + long position; + private int info; + + // TODO (expected): do we need this? we should be able to determine the queue from state() if needed for e.g. cancellation + private TaskQueue queued; + + // TODO (expected): expose via executors vtable + // TODO (expected): use just one long and some flag bits to indicate which point it represents, and report incrementally public long createdAt = nanoTime(), waitingToRunAt, runningAt, cleanupAt; - protected Task() + protected Task(GlobalGroup group, State state) + { + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + info = init(group, ExclusiveGroup.OTHER, state); + } + + protected Task(ExclusiveGroup group, State state) + { + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + info = init(GlobalGroup.OTHER, group, state); + } + + protected Task(GlobalGroup group, State state, long position, int tranche) + { + this(group, state); + this.position = position; + setInheritedTranche(tranche); + } + + protected Task(ExclusiveGroup group, State state, long position, int tranche) + { + this(group, state); + this.position = position; + setInheritedTranche(tranche); + } + + protected Task(ExecutionContext context) { resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + ExclusiveGroup group = ExclusiveGroup.OTHER; + TxnId txnId = context.primaryTxnId(); + if (txnId != null) + { + if (txnId.is(Range)) group = ExclusiveGroup.RANGE; + else + { + switch (PRIORITY_MODEL) + { + case HLC_FIFO: + case ORIG_HLC_FIFO: + { + // TODO (expected): we should process messages for a TxnId together, to avoid processing delayed messages out of order + if (context instanceof Request) + { + MessageType type = ((Request) context).type(); + if (type instanceof StandardMessage) + { + switch ((StandardMessage)type) + { + case APPLY_REQ: + { + group = APPLY; + break; + } + case READ_EPHEMERAL_REQ: + case READ_REQ: + case STABLE_THEN_READ_REQ: + { + group = STABLE; + break; + } + case COMMIT_REQ: + { + Commit commit = (Commit) context; + if (PRIORITY_MODEL == ORIG_HLC_FIFO && !commit.ballot.equals(Ballot.ZERO)) + txnId = null; + if (commit.kind.saveStatus == SaveStatus.Stable) group = STABLE; + else group = COMMIT; + break; + } + case ACCEPT_REQ: + { + Accept accept = (Accept) context; + if (PRIORITY_MODEL == ORIG_HLC_FIFO && !accept.ballot.equals(Ballot.ZERO)) + txnId = null; + group = ExclusiveGroup.ACCEPT; + break; + } + case GET_EPHEMERAL_READ_DEPS_REQ: + case PRE_ACCEPT_REQ: + { + group = ExclusiveGroup.PREACCEPT; + break; + } + default: + { + txnId = null; + } + } + } + } + else + { + txnId = null; + } + break; + } + case FIFO: + { + txnId = null; + break; + } + } + } + } + + this.info = init(GlobalGroup.OTHER, group, INITIALIZED); + if (txnId != null) + this.position = txnId.hlc(); } public final Task unwrap() { - if (this instanceof SequentialQueueTask) - return ((SequentialQueueTask) this).queue.task; + if (this instanceof ExclusiveExecutorTask) + return ((ExclusiveExecutorTask) this).queue.task; return this; } final void setReadyToCleanup() { - queuePosition |= Long.MIN_VALUE; + info |= CLEANUP_BIT; } final boolean isReadyToCleanup() { - return 0 != (queuePosition & Long.MIN_VALUE); + return 0 != (info & CLEANUP_BIT); } final void onRunning() @@ -1165,6 +1630,8 @@ static Task reverse(Task unqueued) public DebuggableTask debuggable() { return null; } + abstract String toDescription(); + abstract void submitExclusive(AccordExecutor owner); /** @@ -1182,11 +1649,14 @@ static Task reverse(Task unqueued) */ abstract protected void fail(Throwable fail); + abstract protected boolean isNewWork(); + /** * Cleanup the command while holding the state cache lock */ protected void cleanupExclusive(AccordExecutor executor) { + executor.unregisterExclusive(this); cleanupAt = nanoTime(); if (runningAt != 0) { @@ -1202,7 +1672,127 @@ protected void cleanupExclusive(AccordExecutor executor) void cancelExclusive(AccordExecutor owner) {} - abstract protected void addToQueue(TaskQueue queue); + public final State state() + { + return State.forOrdinal(stateOrdinal()); + } + + private int stateOrdinal() + { + return info & STATE_MASK; + } + + final boolean is(State state) + { + return stateOrdinal() == state.ordinal(); + } + + final boolean is(GlobalGroup group) + { + return globalGroupOrdinal() == group.ordinal(); + } + + boolean isRange() + { + return 0 != (info & RANGE_BIT); + } + + final int compareTo(State state) + { + return stateOrdinal() - state.ordinal(); + } + + final void setState(State state) + { + Invariants.require(state.isPermittedFrom(stateOrdinal()), "%s forbidden from %s", state, this, Task::reportBadStateTransition); + setStateUnsafe(state); + } + + private static String reportBadStateTransition(Task task) + { + return task.state() + " for " + task.toDescription(); + } + + final void setStateUnsafe(State state) + { + info = (info & ~STATE_MASK) | state.ordinal(); + } + + final int globalGroupOrdinal() + { + return (info >>> GLOBAL_GROUP_SHIFT) & GROUP_MASK; + } + + final int exclusiveGroupOrdinal() + { + return (info >>> EXCLUSIVE_GROUP_SHIFT) & GROUP_MASK; + } + + @Nullable + final TaskQueue queued() + { + return queued; + } + + final void unqueueIfQueued() + { + if (queued != null) + unqueue(); + } + + final void unqueue() + { + Invariants.require(queued != null); + queued.unqueue(this); + queued = null; + } + + final void unsetQueue(TaskQueue queue) + { + Invariants.require(queued == queue); + queued = null; + } + + final void setQueue(TaskQueue queue) + { + Invariants.require(queued == null); + Invariants.require(isCompatible(queue)); + queued = queue; + } + + private boolean isCompatible(TaskQueue queue) + { + int self = stateOrdinal(); + return TinyEnumSet.contains(queue.states, self); + } + + private static int init(GlobalGroup global, ExclusiveGroup exclusive, State state) + { + return global.bits | exclusive.bits | state.ordinal(); + } + + final void setTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; + } + + final void setInheritedTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT | HAS_INHERITED_BIT; + } + + final int tranche() + { + Invariants.require((info & HAS_TRANCHE_BIT) != 0); + return info >>> TRANCHE_SHIFT; + } + + final boolean hasInherited() + { + return (info & HAS_INHERITED_BIT) != 0; + } } // run the task even on a stopped commandStore @@ -1215,16 +1805,22 @@ public interface Unterminatable extends Unstoppable { } - static final class SequentialQueueTask extends Task + static final class ExclusiveExecutorTask extends Task { private final ExclusiveExecutor queue; - SequentialQueueTask(ExclusiveExecutor queue) + ExclusiveExecutorTask(ExclusiveExecutor queue) { - super(); + super(COMMAND_STORE, INITIALIZED); this.queue = queue; } + @Override + String toDescription() + { + return queue.task.toDescription(); + } + @Override void submitExclusive(AccordExecutor owner) { throw new UnsupportedOperationException(); } @Override @@ -1246,16 +1842,15 @@ protected void fail(Throwable t) } @Override - protected void cleanupExclusive(AccordExecutor executor) + protected boolean isNewWork() { - queue.cleanupTask(); + throw new UnsupportedOperationException(); } @Override - protected void addToQueue(TaskQueue queue) + protected void cleanupExclusive(AccordExecutor executor) { - Invariants.require(queue.kind == RUNNING); - queue.append(this); + queue.cleanupTask(); } protected boolean isInHeap() @@ -1265,17 +1860,17 @@ protected boolean isInHeap() } private static final AtomicReferenceFieldUpdater ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(ExclusiveExecutor.class, Thread.class, "owner"); - public class ExclusiveExecutor extends TaskQueue implements accord.local.ExclusiveAsyncExecutor + public final class ExclusiveExecutor extends MultiTaskQueue implements ExclusiveAsyncExecutor { final int commandStoreId; - final SequentialQueueTask selfTask; + final ExclusiveExecutorTask selfTask; private Task task; - private volatile Thread owner, waiting; + volatile Thread owner, waiting; private boolean stopped; private volatile boolean visibleStopped; private boolean terminated; - final DebugSequentialExecutor debug; + final DebugExclusiveExecutor debug; ExclusiveExecutor(AccordExecutor executor) { @@ -1284,15 +1879,14 @@ public class ExclusiveExecutor extends TaskQueue implements accord.local.E ExclusiveExecutor(AccordExecutor executor, int commandStoreId) { - super(WAITING_TO_RUN, commandStoreId < 0); + super(RUNNABLE, commandStoreId < 0 ? GroupKind.NONE : GroupKind.EXCLUSIVE, EXCLUSIVE_QUEUE_BUDGETS, EXCLUSIVE_QUEUE_LIMITS); this.commandStoreId = commandStoreId; - this.selfTask = new SequentialQueueTask(this); - this.debug = DebugSequentialExecutor.maybeDebug(executor.debug, commandStoreId); + this.selfTask = new ExclusiveExecutorTask(this); + this.debug = DebugExclusiveExecutor.maybeDebug(executor.debug, commandStoreId); } void preRunTask() { - Invariants.require(task != null); task.preRunExclusive(); } @@ -1318,321 +1912,1055 @@ void runTask() } waiting = null; - if (stopped && reject(task)) - task.fail(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((AccordTask) task).preLoadContext())); - else - task.run(); - // NOTE: cannot safely release owner here, in case an immediate-execution runs before we can release our references and store their changes to the cache + if (stopped && reject(task)) + task.fail(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((AccordTask) task).executionContext())); + else + task.run(); + // NOTE: cannot safely release owner here, in case an immediate-execution runs before we can release our references and store their changes to the cache + } + + private boolean reject(Task task) + { + if (!(task instanceof AccordTask)) + return true; + + ExecutionContext context = ((AccordTask) task).executionContext(); + return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); + } + + void failTask(Throwable t) + { + task.fail(t); + } + + void cleanupTask() + { + try { task.cleanupExclusive(AccordExecutor.this); } + finally + { + active = 0; + owner = null; + task = super.pollMulti(); + if (DEBUG_EXECUTION) debug.onSetTask(task); + if (task != null) + { + selfTask.position = task.position; + selfTask.setStateUnsafe(WAITING_TO_RUN); + runnable.enqueue(selfTask); + } + } + } + + void enqueue(Task newTask) + { + if (task != null) + { + Invariants.require(selfTask.isInHeap() || selfTask.isReadyToCleanup()); + super.enqueueMulti(newTask); + } + else + { + Invariants.require(isEmptySingle()); + incrementDispatches(newTask); + task = newTask; + task.setQueue(this); + selfTask.position = newTask.position; + selfTask.setStateUnsafe(WAITING_TO_RUN); + runnable.enqueue(selfTask); + if (DEBUG_EXECUTION) debug.onSetTask(newTask); + } + } + + @Override + void unqueue(Task remove) + { + if (remove == task) removeCurrentTask(remove); + else super.unqueueMulti(remove); + } + + boolean tryUnqueueWaiting(Task remove) + { + if (remove == task) return tryRemoveCurrentTask(remove); + else return super.tryUnqueueWaiting(remove); + } + + private boolean tryRemoveCurrentTask(IntrusiveHeapNode remove) + { + if (runnable.isAssigned(selfTask)) + return false; + + removeCurrentTask(remove); + return true; + } + + private void removeCurrentTask(IntrusiveHeapNode remove) + { + Invariants.require(remove == task); + // cannot overwrite task while it is being executed - this cannot happen for AccordTask + // but can for other tasks that don't track their own state + + decrementDispatches(task); + task.unsetQueue(this); + task = pollMulti(); + if (DEBUG_EXECUTION) debug.onSetTask(task); + if (runnable.isWaiting(selfTask)) + { + if (task == null) runnable.unqueue(selfTask); + else + { + selfTask.position = task.position; + runnable.requeue(selfTask); + } + } + else + { + Invariants.expect(false, "%s should have been queued to run as it had the task %s pending, that has now been cancelled", this, remove); + if (task != null) + { + selfTask.position = task.position; + selfTask.setStateUnsafe(WAITING_TO_RUN); + runnable.enqueue(selfTask); + } + } + Invariants.require(task == null || runnable.isWaiting(selfTask)); + } + + public boolean inExecutor() + { + return owner == Thread.currentThread(); + } + + public boolean stopped() + { + return visibleStopped; + } + + void stop() + { + Invariants.require(inExecutor()); + this.stopped = true; + this.visibleStopped = true; + } + + void terminate() + { + Invariants.require(inExecutor()); + this.visibleStopped = this.terminated = this.stopped = true; + } + + @Override + public AsyncChain chain(Runnable run) + { + return AsyncChains.chain(this, run); + } + + @Override + public AsyncChain chain(Callable call) + { + return AsyncChains.chain(this, call); + } + + @Override + public AsyncChain flatChain(Callable> call) + { + return AsyncChains.flatChain(this, call); + } + + Task inherit() + { + Thread thread = Thread.currentThread(); + if (thread == owner) + return task; + return AccordTaskRunner.get(thread).accordActiveSelfTask(); + } + + @Override + public void execute(Runnable run) + { + Task inherit = inherit(); + PlainRunnable submit = inherit == null ? new PlainRunnable(null, run, this, ExclusiveGroup.OTHER) + : new PlainRunnable(null, run, this, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); + submitPlain(submit); + } + + @Override + public Cancellable execute(RunOrFail runOrFail) + { + Task inherit = inherit(); + PlainChain submit = inherit == null ? new PlainChain(runOrFail, ExclusiveExecutor.this, ExclusiveGroup.OTHER) + : new PlainChain(runOrFail, ExclusiveExecutor.this, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); + return submitPlain(submit); + } + + @Override + public boolean tryExecuteImmediately(Runnable run) + { + Thread thread = Thread.currentThread(); + Thread owner = this.owner; + if (owner != null && owner != thread) + return false; + + AccordTaskRunner self = AccordTaskRunner.get(thread); + AccordExecutor active = self.accordActiveExecutor(); + if (active != null && active != AccordExecutor.this) + return false; // prevent cross-executor locking/execution + + if (!ownerUpdater.compareAndSet(this, null, thread)) + return false; + + if (active == null) + self.setAccordActiveExecutor(AccordExecutor.this); + + try { run.run(); } + catch (Throwable t) { agent.onException(t); } + finally + { + if (active == null) + self.setAccordActiveExecutor(null); + + if (owner == null) + { + Thread waiting = this.waiting; + Invariants.require(waiting != self); + this.owner = waiting; + if (waiting == null) // recheck, to ensure happens-before relation with a new waiter that expects any non-null owner to notify it + waiting = this.waiting; + if (waiting != null) + LockSupport.unpark(waiting); + } + } + return true; + } + } + + // has xSingle methods to distinguish from within MultiTaskQueue whether we're invoking the multi or single variation + static class TaskQueue extends IntrusivePriorityHeap + { + final int states; + public TaskQueue(int states) {this.states = states; } + + void unqueue(T task) + { + throw new UnsupportedOperationException(); + } + + @Override + public final int compare(T o1, T o2) + { + return Long.compare(o1.position, o2.position); + } + + @Override + protected final void ensureHeapified() + { + super.ensureHeapified(); + } + + protected final boolean requeueSingle(T requeue) + { + int oldIndex = updateNode(requeue); + if (oldIndex < 0) + return false; + + int newIndex = heapIndex(requeue); + return Math.min(oldIndex, newIndex) == 0 && heapifiedSize() > 0; + } + + final T peekSingle() + { + ensureHeapified(); + return peekNode(); + } + + final T pollSingle() + { + ensureHeapified(); + return pollNode(); + } + + final boolean isEmptySingle() + { + return isEmptyInternal(); + } + + final int enqueueSingle(T enqueue) + { + return insertNode(enqueue); + } + + final boolean unqueueSingle(T unqueue) + { + int heapIndex = heapIndex(unqueue); + removeNode(unqueue); + return heapIndex == 0; + } + + final boolean tryUnqueueSingle(T remove) + { + return removeNodeIfContains(remove); + } + + final T getSingle(int index) + { + return super.getNode(index); + } + + final boolean isQueuedSingle(T test) + { + return containsNode(test); + } + } + + static final class StandaloneTaskQueue extends TaskQueue + { + StandaloneTaskQueue(int states) + { + super(states); + } + + StandaloneTaskQueue(Task.State state) + { + super(TinyEnumSet.encode(state)); + } + + void enqueue(T enqueue) + { + enqueue.setQueue(this); + enqueueSingle(enqueue); + } + + void unqueue(T unqueue) + { + unqueue.unsetQueue(this); + removeNode(unqueue); + } + + T peek() + { + return peekSingle(); + } + + T poll() + { + return pollSingle(); + } + + boolean isEmpty() + { + return isEmptySingle(); + } + } + + /** + * A {@link TaskQueue} sub-divided into up to eight per-group sub-queues (groups are phases for the exclusive + * executor, or work classes globally) plus a policy for choosing which sub-queue to serve next. The policy balances + * three competing concerns: ordering/priority (run the oldest / highest-priority work), fairness (do not let one + * group monopolise the executor), and throughput (do not let an even split leave a busy group's backlog growing + * without bound). + * + *

Packed counters

+ * Per-group state is held in {@code long}s, one 7-bit lane per group; bit 7 of each byte is a spare "guard" bit used + * to run branch-free min/compare across all eight lanes at once (see {@code minCounters}). The lanes are: + *
    + *
  • {@code positions[]} - the head task's position (HLC, or submission order) per group; drives FIFO/priority.
    • + *
  • {@code recent} - a windowed count of recently served tasks per group (service received).
    • + *
  • {@code arrivals} - a windowed, size-biased count of recent arrivals per group (demand offered).
    • + *
  • {@code current} - tasks currently in flight per group, used to enforce {@code queue_active_limits}.
    • + *
  • {@code hasWork}/{@code dirty} - guard-bit masks: which groups have queued work / need a position refresh.
    • + *
+ * + *

Windowing: a bounded memory of the past

+ * {@code recent} and {@code arrivals} are not running totals. Whenever incrementing {@code recent} tips any lane to + * its maximum (0x80), both {@code recent} and {@code arrivals} are halved (see {@code incrementRecents}), + * and each lane also saturates at 0x7f. They are therefore an exponentially-decaying window keyed on service/time. + * This is what stops a one-off burst of arrivals granting a group a permanent boost: the burst saturates the lane + * briefly and then decays away as other work is served. + * + *

The fair selection counter

+ * The fair models pick the group with the smallest {@code effective} counter, where per lane + * 
effective = min(0x7f, max(0, recent - arrivals) + bias)
+ *
    + *
  • {@code max(0, recent - arrivals)} - a group whose arrivals have outpaced its recent service clamps to 0 and + * is therefore preferred; in steady state each group's service tracks its arrival rate, which bounds backlog. + * The clamp is symmetric, so a group cannot bank "credit" during a lull and later use it to monopolise service.
    • + *
+ * + *

Choosing a group ({@code minGroup} / {@code pollByBlend})

+ *
    + *
  • {@code PRIORITY_ONLY} - always the lowest {@code positions} (oldest / highest priority), ignoring fairness.
    • + *
  • {@code PHASE_OVERRIDE} - strict phase order: the lowest-index group with work, always.
    • + *
  • {@code PHASE_FAIR} - pure fairness: the smallest {@code effective}; ties broken by index, within a group by + * position. It has no priority fallback, so it is deliberately completing-first under contention.
    • + *
  • {@code PRIORITY_FAIR} (default) - a deficit round-robin blend of two strategies, chosen per poll: flow + * (smallest {@code effective}, i.e. least fairly serviced) and age (oldest {@code positions}, i.e. FIFO). + * The flow weight ramps up with the flow imbalance ({@code max-min} of {@code effective}) over + * {@code queue_flow_imbalance_onset..+width}, trading against age zero-sum; when balanced it is pure age. The + * ramp is smooth (not a hard mode switch), so there is no boundary to oscillate around and no hysteresis is + * needed. Age also drains stale/standing backlogs for free: a backlog's items are the oldest work, so age + * clears them without a separate backlog-aware strategy (the {@code effective} flow counter tracks arrival + * rate and is deliberately blind to standing stock).
    • + *
+ * + *

Concurrency limits and eligibility

+ * A group whose in-flight {@code current} count has reached its {@code queue_active_limits} limit is + * {@code saturated} and excluded from selection ({@code disabled}), as is any group with no queued work. + * + *

Note on {@code positions} freshness

+ * {@code positions} is refreshed lazily from the sub-queue heads via the {@code dirty} mask; {@code dirty} must use + * the same guard-bit layout as {@code hasWork} so the refresh loop in {@code minGroupByPriority} actually runs - + * otherwise it silently degenerates to lowest-index-with-work and starves high-index / new-work groups. + * + *

NB it extends {@link TaskQueue} to keep the type hierarchy simple for method dispatch, and for efficiency for + * anonymous ExclusiveExecutors which do not use multiple queues, while letting ExclusiveExecutor share a parent + * class for both use cases. + */ + static abstract class MultiTaskQueue extends TaskQueue + { + private static final TaskQueue[] NO_QUEUES = new TaskQueue[0]; + private static final long[] NO_POSITIONS = new long[0]; + static final long COUNTER_OVERFLOWS = 0x8080808080808080L; + static final long COUNTER_MASKS = 0x7f7f7f7f7f7f7f7fL; + static final long COUNTER_LOWBITS = 0x0101010101010101L; + + final TaskQueue[] queues; + final long[] positions; + final byte groupShift; + final long baseBudget, limits; + + /** sets overflow bits for each counter when it needs its position updated */ + long dirty; + /** sets overflow bits for each counter when there's associated work */ + long hasWork; + /** Stores recent dequeue counts for up to 8 sub queues. */ + long dispatches; + /** Stores recent enqueue counts for up to 8 sub queues. */ + long arrivals; + /** Stores currently-active counts for up to 8 sub queues. We can use this to impose limits on specific queues. */ + long active; + /** Stores a biased budget for each task type, so that we may prefer to serve one type of task over another */ + long budget; + /** deficit-round-robin credits for the two PRIORITY_FAIR strategies (flow/age). */ + int creditFlow, creditAge; + + int waitingCount; + + MultiTaskQueue(int waitingStates, GroupKind groups, long budget, long limits) + { + super(waitingStates); + this.baseBudget = budget; + this.limits = limits; + int queueCount = groups.count; + Invariants.require(queueCount <= 8); + queues = queueCount == 0 ? NO_QUEUES : new TaskQueue[queueCount]; + positions = queueCount > 0 && BALANCE_BY_POSITION ? new long[queueCount] : NO_POSITIONS; + groupShift = groups.shift; + } + + final int group(Task task) + { + if (groupShift == 0) + return -1; + + return (task.info >>> groupShift) & Task.GROUP_MASK; + } + + final TaskQueue queue(Task task) + { + int group = group(task); + if (group < 0) + return this; + + return queue(group); + } + + final TaskQueue queue(int group) + { + TaskQueue queue = queues[group]; + if (queue == null) + queues[group] = queue = new TaskQueue<>(0); + + return queue; + } + + private int pollGroup() + { + if (hasWork == 0) + return -1; + + switch (BALANCING_MODEL) + { + default: throw new UnhandledEnum(PRIORITY_MODEL); + case PRIORITY_ONLY: return pollGroupByPriority(); + case PHASE_ONLY: return pollGroupByIndex(); + case PRIORITY_BUDGET: return pollGroupByPriorityWithBudget(); + case PHASE_BUDGET: return pollGroupByIndexWithBudget(); + case PHASE_FAIR: return pollGroupByPhaseFair(); + case PHASE_BUDGET_FAIR: return pollGroupByPhaseBudgetFair(); + case BLENDED_PRIORITY_PHASE_FAIR: return pollGroupByBlended(); + case BLENDED_PRIORITY_PHASE_BUDGET_FAIR: return pollGroupByBlendedWithBudget(); + } + } + + private int pollGroupByPriority() + { + return pollGroupByPriority(unsaturatedWithWork()); + } + + private int pollGroupByPriority(long enabled) + { + long refresh = dirty & hasWork; + while (refresh != 0) + { + int bitIndex = Long.numberOfTrailingZeros(refresh); + int group = bitIndex / 8; + positions[group] = queues[group].peekSingle().position; + refresh ^= 1L << bitIndex; + } + dirty = 0; + + long minPosition = Long.MAX_VALUE; + int minGroup = -1; + long visit = enabled >>> 7; + while (visit != 0) + { + int bitIndex = Long.numberOfTrailingZeros(visit); + int group = bitIndex / 8; + long position = positions[group]; + if (position < minPosition) + { + minGroup = group; + minPosition = position; + } + visit ^= 1L << bitIndex; + } + + return minGroup; + } + + private int pollGroupByIndex() + { + long visit = (hasWork & unsaturated()) >>> 7; + if (visit == 0) + return -1; + + int bitIndex = Long.numberOfTrailingZeros(visit); + return bitIndex / 8; + } + + private int pollGroupByPriorityWithBudget() + { + return pollGroupByPriority(unsaturatedWithWorkAndBudget()); + } + + private int pollGroupByIndexWithBudget() + { + long visit = (hasWork & unsaturated()); + if (visit == 0) + return -1; + + visit = withBudgetOrReset(visit); + visit >>>= 7; + int bitIndex = Long.numberOfTrailingZeros(visit); + return bitIndex / 8; + } + + private int pollGroupByPhaseFair() + { + return minCounterIndex(recentFlowImbalances()); + } + + private int pollGroupByPhaseBudgetFair() + { + return minCounterIndex(recentFlowImbalances(), saturatedOrWithoutWorkOrWithoutBudget()); + } + + // PRIORITY_FAIR selection: a deficit round-robin blend of two strategies, chosen per poll: + // flow -> minCounterIndex(recent - arrivals + bias) (least fairly serviced) + // age -> minGroupByPriority() (earliest-queued work) + // wFlow = ramp(flow imbalance F = max-min of the selection counters); wAge = BLEND_TOTAL - wFlow. As flow gets + // uneven, polls trade from age to flow zero-sum; when balanced it is pure age (FIFO). A ramp (not a hard + // threshold) means there is no mode cliff to oscillate around, so no anti-oscillation penalty is needed. Age is + // also what drains a stale/standing backlog -- its items are the oldest -- so no separate stock strategy is needed. + private int pollGroupByBlended() + { + return pollGroupByBlended(saturatedOrWithoutWork()); + } + + private int pollGroupByBlended(long disabled) + { + long withoutWork = hasWork ^ COUNTER_OVERFLOWS; + long counters = recentFlowImbalances(); + long minMax = minMaxCounterValue(counters, withoutWork); + long min = minMax & 0x7f; + long max = minMax >>> 8; + int flowImbalance = (int) (max - min); + + int flowWeight = flowWeight(flowImbalance); + int priorityWeight = BLEND_TOTAL - flowWeight; + + creditFlow += flowWeight; + creditAge += priorityWeight; + + if (creditFlow >= creditAge) + { + creditFlow -= BLEND_TOTAL; + if (disabled != withoutWork) + min = minCounterValue(counters, disabled); + + return minCounterIndex(counters, min, disabled); + } + else + { + creditAge -= BLEND_TOTAL; + return pollGroupByPriority(disabled ^ COUNTER_OVERFLOWS); + } + } + + private int pollGroupByBlendedWithBudget() + { + return pollGroupByBlended(saturatedOrWithoutWorkOrWithoutBudget()); + } + + private long saturated() + { + return ((active | COUNTER_OVERFLOWS) - limits) & COUNTER_OVERFLOWS; + } + + private long unsaturated() + { + return saturated() ^ COUNTER_OVERFLOWS; + } + + private long unsaturatedWithWork() + { + return hasWork & unsaturated(); + } + + private long unsaturatedWithWorkAndBudget() + { + return withBudgetOrReset(hasWork & unsaturated()); + } + + private long saturatedOrWithoutWorkOrWithoutBudget() + { + return withoutBudgetOrReset(saturatedOrWithoutWork()); + } + + private long withoutBudgetOrReset(long disabled) + { + return COUNTER_OVERFLOWS ^ withBudgetOrReset(COUNTER_OVERFLOWS ^ disabled); + } + + private long withBudgetOrReset(long enabled) + { + long hasBudget = hasBudget(); + if ((hasBudget & enabled) != 0) + return hasBudget & enabled; + + budget = baseBudget; + return enabled; + } + + private long hasBudget() + { + return setOverflowWhenLessEqual(budget, 0) ^ COUNTER_OVERFLOWS; + } + + private long saturatedOrWithoutWork() + { + return (hasWork ^ COUNTER_OVERFLOWS) | saturated(); + } + + // arrivals is a windowed measure of ARRIVAL (incremented on enqueue, size-biased; decays on recent's overflow). + // Combined with recent (service) as effective = max(0, recent - arrivals): a queue whose arrivals outpace its + // service clamps to 0 and is preferred, so service converges to arrival rate (bounding the busy queue's backlog). + private long recentFlowImbalances() + { + return clampedSubtract(dispatches, arrivals); + } + + static long minCounterValue(long counters, long disabled) + { + long mins = counters; + mins |= overflowsToLowMasks(disabled); + mins = minCounters(mins, mins >>> 8); // each slot is min of slots [i..i+1] + mins = minCounters(mins, mins >>> 16); // each slot is min of slots [i..i+3] + mins = minCounters(mins, mins >>> 32); // each slot is min of slots [i..i+7] + return mins & 0x7f; + } + + static long minMaxCounterValue(long counters, long disabled) + { + long mins = counters; + long maxs = counters ^ COUNTER_MASKS; + long overflowMasks = overflowsToLowMasks(disabled); + mins |= overflowMasks; + maxs |= overflowMasks; + mins = minCounters(mins, mins >>> 8) & 0x007f007f007f007fL; // each slot is min of slots [i..i+1] + maxs = (minCounters(maxs, maxs << 8) & 0x7f007f007f007f00L); // each slot is min of slots ~[i..i+1] + long minmaxs = mins | maxs; + minmaxs = minCounters(minmaxs, minmaxs >>> 16); // each slot is min of slots [i..i+3] + minmaxs = minCounters(minmaxs, minmaxs >>> 32); // each slot is min of slots [i..i+7] + return (minmaxs ^ 0x7f00) & 0x7f7f; + } + + /** + * If provided two counters (containing 8 7 bit counters each), + * returns the minimum of each matching counter + */ + private static long minCounters(long a, long b) + { + // set overflow bits where a <= b + long selecta = setOverflowWhenLessEqual (a, b); + return selectByOverflowBits(selecta, a, b); + } + + static long setOverflowWhenLessEqual(long a, long b) + { + return ((b | COUNTER_OVERFLOWS) - a) & COUNTER_OVERFLOWS; } - private boolean reject(Task task) + // select a if overflow bit is set; b if it is unset + static long selectByOverflowBits(long selecta, long a, long b) { - if (!(task instanceof AccordTask)) - return true; + selecta = overflowsToLowMasks(selecta); + a &= selecta; + b &= ~selecta; + return a | b; + } - ExecutionContext context = ((AccordTask) task).preLoadContext(); + static long overflowsToLowMasks(long v) + { + return v - (v >>> 7); + } - return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); + private static int flowWeight(int flowImbalance) + { + if (flowImbalance <= FLOW_ONSET) return 0; + return Math.min(BLEND_TOTAL, ((flowImbalance - FLOW_ONSET) << BLEND_SHIFT) >>> FLOW_WIDTH_SHIFT); } - void failTask(Throwable t) + // per-lane max(0, a - b), carry-free: zero both a and b in lanes where a <= b, then subtract + private static long clampedSubtract(long a, long b) { - task.fail(t); + long keep = ~overflowsToLowMasks(setOverflowWhenLessEqual(a, b)); + return (a & keep) - (b & keep); } - void cleanupTask() + private int minCounterIndex(long counters) { - try { task.cleanupExclusive(AccordExecutor.this); } - finally - { - owner = null; - task = super.poll(); - if (DEBUG_EXECUTION) debug.onSetTask(task); + return minCounterIndex(counters, saturatedOrWithoutWork()); + } - // it should only be possible for this method to be invoked once we're on the running queue - AccordExecutor.this.running.remove(selfTask); - if (task != null) - { - selfTask.queuePosition = task.queuePosition; - waitingToRun.append(selfTask); - } + private int minCounterIndex(long counters, long disabled) + { + return minCounterIndex(counters, minCounterValue(counters, disabled), disabled); + } + + private int minCounterIndex(long counters, long minCounterValue, long disabled) + { + long mins = minCounterValue * COUNTER_LOWBITS; + long select = ((mins | COUNTER_OVERFLOWS) - counters) & COUNTER_OVERFLOWS; + // now unset those overflow bits associated with disabled queues + select &= ~disabled; + if (select == 0) + return -1; + return (Long.numberOfTrailingZeros(select) - 7) / 8; + } + + final T pollMulti() + { + int group = pollGroup(); + if (group < 0) + { + // group < 0 can mean EITHER we don't have any nested queues OR those queues are either empty or DISABLED + T result = pollSingle(); + if (result != null) + --waitingCount; + return result; } + + --waitingCount; + incrementActive(group); + incrementDispatches(group); + decrementBudget(group); + + TaskQueue queue = queues[group]; + T head = queue.pollSingle(); + // NOTE: must clear dirty when emptied, symmetrically with unqueue(): the fair selection paths + // never consume the dirty bit, so a group drained during a fairness episode would otherwise + // retain a stale dirty bit and NPE in minGroupByPriority.peekSingle() when balance is restored. + if (queue.isEmptySingle()) { unsetHasWork(group); unsetDirty(group); } + else setDirty(group); + return head; } - // invoked by removeAndUpdateNext; expect to already be next - @Override - protected void append(Task newTask) + final void enqueueMulti(T task) { - if (task != null) + task.setQueue(this); + int group = group(task); + if (group < 0) { - Invariants.require(selfTask.isInHeap()); - super.append(newTask); + enqueueSingle(task); } else { - Invariants.require(isEmpty()); - task = newTask; - selfTask.queuePosition = newTask.queuePosition; - waitingToRun.append(selfTask); - if (DEBUG_EXECUTION) debug.onSetTask(newTask); + TaskQueue queue = queue(group); + int result = queue.enqueueSingle(task); + incrementArrivals(group); + if (result < 0) setHasWork(group); + if (result != 0) setDirty(group); } + ++waitingCount; } - @Override - protected void remove(Task remove) + final void requeue(T task) { - if (remove == task) removeCurrentTask(remove); - else super.remove(remove); + int group = group(task); + if (group < 0) requeueSingle(task); + else + { + TaskQueue queue = queue(group); + Invariants.require(queue != null && queue.isQueuedSingle(task)); + if (queue.requeueSingle(task)) + setDirty(group); + } } - @Override - protected boolean removeIfContains(Task remove) + final void unqueueMulti(T task) { - if (remove == task) return removeCurrentTask(remove); - else return super.removeIfContains(remove); + int group = group(task); + TaskQueue queue = group < 0 ? this : queue(task); + Invariants.require(queue.isQueuedSingle(task)); + unqueue(task, group, queue); } - private boolean removeCurrentTask(Node remove) + // if there is an active collection, we return false and do not remove ourselves from it + boolean tryUnqueueWaiting(T task) { - if (running.contains(selfTask)) + int group = group(task); + TaskQueue queue = group < 0 ? this : queue(task); + if (!queue.isQueuedSingle(task)) return false; - Invariants.require(remove == task); - // cannot overwrite task while it is being executed - this cannot happen for AccordTask - // but can for other tasks that don't track their own state + unqueue(task, group, queue); + return true; + } - task = super.poll(); - if (DEBUG_EXECUTION) debug.onSetTask(task); - if (waitingToRun.contains(selfTask)) - { - if (task == null) waitingToRun.remove(selfTask); - else - { - selfTask.queuePosition = task.queuePosition; - waitingToRun.update(selfTask); - } - } - else + private void unqueue(T task, int group, TaskQueue queue) + { + task.unsetQueue(this); + boolean dirty = queue.unqueueSingle(task); + --waitingCount; + if (group >= 0) { - Invariants.expect(false, "%s should have been queued to run as it had the task %s pending, that has now been cancelled", this, remove); - if (task != null) + if (queue.isEmptySingle()) { - selfTask.queuePosition = task.queuePosition; - waitingToRun.append(selfTask); + unsetHasWork(group); + unsetDirty(group); } + else if (dirty) setDirty(group); } - Invariants.require(task == null || waitingToRun.contains(selfTask)); - return true; } - public boolean inExecutor() + final void incrementActive(int group) { - return owner == Thread.currentThread(); + active += lowBit(group); } - public boolean stopped() + final void decrementActive(int group) { - return visibleStopped; + active -= lowBit(group); } - void stop() + final void incrementDispatches(Task task) { - Invariants.require(inExecutor()); - this.stopped = true; - this.visibleStopped = true; + int group = group(task); + if (group >= 0) + incrementDispatches(group); } - void terminate() + final void incrementDispatches(int group) { - Invariants.require(inExecutor()); - this.visibleStopped = this.terminated = this.stopped = true; + dispatches += lowBit(group); + if ((dispatches & COUNTER_OVERFLOWS) != 0) + { + dispatches = (dispatches >>> 1) & COUNTER_MASKS; + arrivals = (arrivals >>> 1) & COUNTER_MASKS; // arrivals (arrival) decays on the service/time clock + } } - @Override - protected Task poll() + final void decrementBudget(int group) { - throw new UnsupportedOperationException(); + // no need to manage underflow; if the budget is being used, + // a queue should only dispatch work when there's non-zero budget + budget -= lowBit(group); } - @Override - protected Task peek() + final void decrementDispatches(Task task) { - throw new UnsupportedOperationException(); + int group = group(task); + if (group >= 0) + decrementDispatches(group); } - @Override - protected boolean contains(Task contains) + final void decrementDispatches(int group) { - return super.contains(contains) || (task == contains && !running.contains(selfTask)); + long lowBit = lowBit(group); + dispatches -= lowBit; + dispatches += (dispatches >>> 7) & lowBit; } - @Override - public AsyncChain chain(Runnable run) + final void incrementArrivals(int group) { - long position = inheritQueuePosition(); - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - return execute(new RunAndCallback(run, callback), position); - } - }; + int shift = group * 8; + long overflowBit = 0x80L << shift; + arrivals += 1L << shift; + // if we overflow, unset the overflow bit and set all other bits for the counter + long overflow = arrivals & overflowBit; + arrivals ^= overflow; + arrivals |= overflow - (overflow >>> 7); } - @Override - public AsyncChain chain(Callable call) + final void setHasWork(int group) { - long position = inheritQueuePosition(); - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - return execute(new CallAndCallback<>(call, callback), position); - } - }; + hasWork |= overflowBit(group); } - @Override - public AsyncChain flatChain(Callable> call) + final void unsetHasWork(int group) { - long position = inheritQueuePosition(); - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - return execute(new FlatCallAndCallback<>(call, callback), position); - } - }; + hasWork &= ~overflowBit(group); } - @Override - public Cancellable execute(RunOrFail runOrFail) + final void setDirty(int group) { - return execute(runOrFail, inheritQueuePosition()); + dirty |= overflowBit(group); } - private long inheritQueuePosition() + final void unsetDirty(int group) { - return inExecutor() && task != null ? task.queuePosition : 0; + dirty &= ~overflowBit(group); } - private Cancellable execute(RunOrFail runOrFail, long queuePosition) + final boolean hasWaiting() { - PlainChain submit = new PlainChain(runOrFail, ExclusiveExecutor.this, queuePosition); - return AccordExecutor.this.submit(submit); + return waitingCount > 0; } - @Override - public void execute(Runnable run) + final boolean isWaiting(T task) { - PlainRunnable submit = new PlainRunnable(null, run, this, inheritQueuePosition()); - AccordExecutor.this.submit(submit); + return queue(task).isQueuedSingle(task); } - @Override - public boolean tryExecuteImmediately(Runnable run) + final int waitingCount() { - Thread self = Thread.currentThread(); - Thread owner = this.owner; - if (owner != null ? owner != self : !ownerUpdater.compareAndSet(this, null, self)) - return false; - - try { run.run(); } - catch (Throwable t) { agent.onException(t); } - finally - { - if (owner == null) - { - Thread waiting = this.waiting; - Invariants.require(waiting != self); - this.owner = waiting; - if (waiting == null) // recheck, to ensure happens-before relation with a new waiter that expects any non-null owner to notify it - waiting = this.waiting; - if (waiting != null) - LockSupport.unpark(waiting); - } - } - return true; + return waitingCount; } - } - - static class TaskQueue extends IntrusivePriorityHeap - { - final AccordTask.State kind; - TaskQueue(AccordTask.State kind) + final long lowBit(int group) { - this.kind = kind; + return 1L << (group * 8); } - TaskQueue(AccordTask.State kind, boolean tiny) + final long overflowBit(int group) { - super(tiny); - this.kind = kind; + return 0x80L << (group * 8); } + } - @Override - public int compare(T o1, T o2) + static final class RunnableTaskQueue extends MultiTaskQueue + { + static final int RUNNABLE = TinyEnumSet.encode(WAITING_TO_RUN, ASSIGNED, RUNNING); + + final TaskQueue assigned; + + RunnableTaskQueue() { - return Long.compare(o1.queuePosition, o2.queuePosition); + super(RUNNABLE, GroupKind.GLOBAL, GLOBAL_QUEUE_BUDGETS, GLOBAL_QUEUE_LIMITS); + this.assigned = new TaskQueue<>(0); } - protected void append(T task) + T poll() { - super.append(task); + T next = pollMulti(); + if (next == null) + return null; + + next.setState(ASSIGNED); + assigned.enqueueSingle(next); + + return next; } - protected void update(T task) + void enqueue(T enqueue) { - super.update(task); + enqueueMulti(enqueue); } - protected T poll() + void unqueue(T unqueue) { - ensureHeapified(); - return pollNode(); + if (assigned.isQueuedSingle(unqueue)) + { + int group = group(unqueue); + if (group >= 0) + decrementActive(group); + + unqueue.unsetQueue(this); + assigned.unqueueSingle(unqueue); + } + else + { + super.unqueueMulti(unqueue); + } } - protected T peek() + int waitingOrAssignedCount() { - ensureHeapified(); - return peekNode(); + return waitingCount + assigned.size(); } - protected T get(int index) + boolean hasAssignedOrWaiting() { - return super.get(index); + return waitingCount > 0 || !assigned.isEmptySingle(); } - protected void remove(T remove) + boolean hasAssigned() { - super.remove(remove); + return !assigned.isEmptySingle(); } - @Override - protected boolean removeIfContains(T node) + boolean isAssigned(T task) { - return super.removeIfContains(node); + return assigned.isQueuedSingle(task); } - protected boolean contains(T contains) + void complete(T task) { - return super.contains(contains); + if (assigned.tryUnqueueSingle(task)) + { + int group = group(task); + if (group >= 0) + decrementActive(group); + task.unsetQueue(this); + } } } static class CancelTask extends Task { final Task cancel; - private CancelTask(Task cancel) { this.cancel = cancel; } + private CancelTask(Task cancel) + { + super(GlobalGroup.OTHER, WAITING_TO_RUN); + this.cancel = cancel; + } + @Override void submitExclusive(AccordExecutor owner) { cancel.cancelExclusive(owner); } @Override protected void preRunExclusive() { throw new UnsupportedOperationException(); } @Override protected void run() { throw new UnsupportedOperationException(); } @Override protected void fail(Throwable fail) { throw new UnsupportedOperationException(); } - @Override protected void addToQueue(TaskQueue queue) { throw new UnsupportedOperationException(); } + @Override protected boolean isNewWork() { return false; } + @Override String toDescription() { return "Cancel " + cancel.toDescription(); } } static IntFunction constant(O out) @@ -1642,18 +2970,31 @@ static IntFunction constant(O out) abstract class Plain extends Task implements Cancellable { - abstract ExclusiveExecutor executor(); + Plain(GlobalGroup group, long position, int tranche) + { + super(group, WAITING_TO_RUN, position, tranche); + } - @Override - protected void preRunExclusive() {} + Plain(ExclusiveGroup group, long position, int tranche) + { + super(group, WAITING_TO_RUN, position, tranche); + } - @Override - protected final void addToQueue(TaskQueue queue) + Plain(GlobalGroup group) + { + super(group, WAITING_TO_RUN); + } + + Plain(ExclusiveGroup group) { - Invariants.require(queue.kind == WAITING_TO_RUN || queue.kind == RUNNING); - queue.append(this); + super(group, WAITING_TO_RUN); } + abstract ExclusiveExecutor executor(); + + @Override + protected void preRunExclusive() {} + @Override public void cancel() { @@ -1663,10 +3004,8 @@ public void cancel() void cancelExclusive(AccordExecutor owner) { ExclusiveExecutor executor = executor(); - TaskQueue queue = executor == null ? waitingToRun : executor; - if (queue.contains(this)) + if ((executor == null ? runnable : executor).tryUnqueueWaiting(this)) { - queue.remove(this); completeTaskExclusive(this); try { fail(new CancellationException()); } catch (Throwable t) { agent.onException(t); } @@ -1678,6 +3017,12 @@ final void submitExclusive(AccordExecutor owner) { owner.submitPlainExclusive(this); } + + @Override + protected boolean isNewWork() + { + return true; + } } class PlainRunnable extends Plain implements Cancellable @@ -1686,22 +3031,43 @@ class PlainRunnable extends Plain implements Cancellable final Runnable run; final @Nullable ExclusiveExecutor executor; - PlainRunnable(Runnable run) + PlainRunnable(AsyncPromise result, Runnable run, GlobalGroup group, long position, int tranche) + { + super(group, position, tranche); + this.result = result; + this.run = run; + this.executor = null; + } + + PlainRunnable(AsyncPromise result, Runnable run, GlobalGroup group) { - this(null, run); + super(group); + this.result = result; + this.run = run; + this.executor = null; } - PlainRunnable(AsyncPromise result, Runnable run) + PlainRunnable(AsyncPromise result, Runnable run, ExclusiveExecutor executor, ExclusiveGroup group, long position, int tranche) { - this(result, run, null, 0); + super(group, position, tranche); + this.result = result; + this.run = run; + this.executor = executor; } - PlainRunnable(AsyncPromise result, Runnable run, @Nullable ExclusiveExecutor executor, long queuePosition) + PlainRunnable(AsyncPromise result, Runnable run, ExclusiveExecutor executor, ExclusiveGroup group) { + super(group); this.result = result; this.run = run; this.executor = executor; - this.queuePosition = queuePosition; + } + + @Override + String toDescription() + { + // TODO (expected): ensure this is usefully descriptive, or accept a separate description + return run.toString(); } @Override @@ -1735,17 +3101,18 @@ ExclusiveExecutor executor() // a task that may be submitted to this executor or another abstract class IOTask extends Plain implements Cancellable, DebuggableTask { - final long createdAtNanos = MonotonicClock.Global.approxTime.now(); - long startedAtNanos; - - abstract void postRunExclusive(); + IOTask(GlobalGroup group, long position, int tranche) + { + super(group, position, tranche); + } - @Override - protected void preRunExclusive() + IOTask(GlobalGroup group) { - startedAtNanos = MonotonicClock.Global.approxTime.now(); + super(group); } + abstract void postRunExclusive(); + @Override protected void cleanupExclusive(AccordExecutor executor) { @@ -1762,13 +3129,13 @@ ExclusiveExecutor executor() @Override public long creationTimeNanos() { - return createdAtNanos; + return createdAt; } @Override public long startTimeNanos() { - return startedAtNanos; + return runningAt; } } @@ -1786,7 +3153,7 @@ static class FailureHolder LoadRunnable newLoad(AccordCacheEntry entry, boolean isForRange) { - return isForRange ? new LoadRangeRunnable<>(entry) : new LoadRunnable<>(entry); + return new LoadRunnable<>(entry, isForRange ? RANGE_LOAD : LOAD); } class LoadRunnable extends IOTask @@ -1794,12 +3161,14 @@ class LoadRunnable extends IOTask final AccordCacheEntry entry; Object result = FailureHolder.NOT_STARTED; - LoadRunnable(AccordCacheEntry entry) + LoadRunnable(AccordCacheEntry entry, GlobalGroup group) { + super(group); + Invariants.require(group == LOAD || group == RANGE_LOAD); this.entry = entry; } - boolean isForRange() { return false; } + boolean isForRange() { return is(RANGE_LOAD); } void postRunExclusive() { @@ -1807,6 +3176,12 @@ void postRunExclusive() else onLoadedExclusive(entry, null, ((FailureHolder)result).fail, isForRange()); } + @Override + String toDescription() + { + return "Load " + entry.key(); + } + @Override public void run() { @@ -1831,12 +3206,6 @@ public String description() } } - final class LoadRangeRunnable extends LoadRunnable - { - LoadRangeRunnable(AccordCacheEntry entry) { super(entry); } - @Override boolean isForRange() { return true; } - } - static abstract class AbstractIOTask { abstract protected void runInternal(); @@ -1849,11 +3218,18 @@ class WrappedIOTask extends IOTask { final AbstractIOTask wrapped; - WrappedIOTask(AbstractIOTask wrap) + WrappedIOTask(AbstractIOTask wrap, GlobalGroup group, long position, int tranche) { + super(group, position, tranche); this.wrapped = wrap; } + @Override + String toDescription() + { + return wrapped.description(); + } + @Override protected void run() { @@ -1894,6 +3270,7 @@ class SaveRunnable extends IOTask SaveRunnable(AccordCacheEntry entry, UniqueSave identity, Runnable run) { + super(SAVE); this.entry = entry; this.identity = identity; this.run = run; @@ -1905,6 +3282,12 @@ void postRunExclusive() onSavedExclusive(entry, identity, failure); } + @Override + String toDescription() + { + return "Save " + entry.key(); + } + @Override public void run() { @@ -1935,16 +3318,18 @@ class PlainChain extends Plain final RunOrFail runOrFail; final @Nullable ExclusiveExecutor executor; - PlainChain(RunOrFail runOrFail) + PlainChain(RunOrFail runOrFail, ExclusiveExecutor executor, ExclusiveGroup group) { - this(runOrFail, null, 0); + super(group); + this.runOrFail = runOrFail; + this.executor = executor; } - PlainChain(RunOrFail runOrFail, @Nullable ExclusiveExecutor executor, long queuePosition) + PlainChain(RunOrFail runOrFail, ExclusiveExecutor executor, ExclusiveGroup group, long position, int tranche) { + super(group, position, tranche); this.runOrFail = runOrFail; this.executor = executor; - this.queuePosition = queuePosition; } @Override @@ -1953,6 +3338,12 @@ ExclusiveExecutor executor() return executor; } + @Override + String toDescription() + { + return runOrFail.toString(); + } + @Override protected void run() { @@ -1986,33 +3377,30 @@ protected void fail(Throwable fail) class DebuggableChain extends PlainChain implements DebuggableTask { - final long createdAtNanos; - long startedAtNanos; final Object describe; - DebuggableChain(RunOrFail runOrFail, @Nullable ExclusiveExecutor executor, int queuePosition, Object describe) + DebuggableChain(RunOrFail runOrFail, @Nullable ExclusiveExecutor executor, Object describe) { - super(runOrFail, executor, queuePosition); - this.createdAtNanos = MonotonicClock.Global.approxTime.now(); + super(runOrFail, executor, ExclusiveGroup.OTHER); this.describe = Invariants.nonNull(describe); } - @Override - public long creationTimeNanos() + DebuggableChain(RunOrFail runOrFail, @Nullable ExclusiveExecutor executor, long position, int tranche, Object describe) { - return createdAtNanos; + super(runOrFail, executor, ExclusiveGroup.OTHER, position, tranche); + this.describe = Invariants.nonNull(describe); } @Override - public long startTimeNanos() + public long creationTimeNanos() { - return startedAtNanos; + return createdAt; } @Override - protected void preRunExclusive() + public long startTimeNanos() { - startedAtNanos = MonotonicClock.Global.approxTime.now(); + return runningAt; } @Override @@ -2058,13 +3446,13 @@ public Integer commandStoreId() public long position() { - return task.queuePosition; + return task.position; } public @Nullable String describe() { if (task instanceof AccordTask) - return ((AccordTask) task).preLoadContext().reason(); + return ((AccordTask) task).executionContext().reason(); if (task instanceof DebuggableTask) return ((DebuggableTask) task).description(); @@ -2075,7 +3463,7 @@ public long position() public @Nullable ExecutionContext preLoadContext() { if (task instanceof AccordTask) - return ((AccordTask) task).preLoadContext(); + return ((AccordTask) task).executionContext(); if (task instanceof WrappedIOTask && ((WrappedIOTask) task).wrapped instanceof AccordTask.RangeTxnScanner) return ((AccordTask.RangeTxnScanner) ((WrappedIOTask) task).wrapped).preLoadContext(); return null; @@ -2093,19 +3481,24 @@ public int compareTo(TaskInfo that) public List taskSnapshot() { List result = new ArrayList<>(); - lock(); + AccordTaskRunner self = AccordTaskRunner.get(); + lock(self); try { - addToSnapshot(result, waitingToLoad, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); - addToSnapshot(result, waitingToLoadRangeTxns, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); addToSnapshot(result, scanningRanges, TaskInfo.Status.SCANNING_RANGES, TaskInfo.Status.SCANNING_RANGES); - addToSnapshot(result, loading, TaskInfo.Status.LOADING, TaskInfo.Status.LOADING); - addToSnapshot(result, waitingToRun, TaskInfo.Status.WAITING_TO_RUN, TaskInfo.Status.WAITING_TO_RUN); - addToSnapshot(result, running, TaskInfo.Status.RUNNING, TaskInfo.Status.WAITING_TO_RUN); + addToSnapshot(result, waitingToLoadRangeTxns, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); + addToSnapshot(result, waitingToLoad, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); + addToSnapshot(result, loading, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); + for (TaskQueue queue : runnable.queues) + { + if (queue != null) + addToSnapshot(result, queue, TaskInfo.Status.WAITING_TO_RUN, TaskInfo.Status.WAITING_TO_RUN); + } + addToSnapshot(result, runnable.assigned, TaskInfo.Status.RUNNING, TaskInfo.Status.WAITING_TO_RUN); } finally { - unlock(); + unlock(self); } result.sort(TaskInfo::compareTo); return result; @@ -2115,13 +3508,19 @@ private static void addToSnapshot(List snapshot, TaskQueue queue, T { for (int i = 0 ; i < queue.size() ; ++i) { - Task t = queue.get(i); - if (t instanceof SequentialQueueTask) + Task t = queue.getSingle(i); + if (t instanceof ExclusiveExecutorTask) { - ExclusiveExecutor q = ((SequentialQueueTask) t).queue; + ExclusiveExecutor q = ((ExclusiveExecutorTask) t).queue; snapshot.add(new TaskInfo(ifCurrent, q.commandStoreId, q.task)); - for (int j = 0 ; j < q.size() ; ++j) - snapshot.add(new TaskInfo(ifQueued, q.commandStoreId, q.get(j))); + for (TaskQueue q0 : q.queues) + { + if (q0 != null) + { + for (int j = 0 ; j < q0.size() ; ++j) + snapshot.add(new TaskInfo(ifQueued, q.commandStoreId, q0.getSingle(j))); + } + } } else { @@ -2133,17 +3532,65 @@ private static void addToSnapshot(List snapshot, TaskQueue queue, T public int unsafePreparingToRunCount() { - return waitingToLoad.size() + waitingToLoadRangeTxns.size() + scanningRanges.size() + loading.size(); + return waitingToLoad.size() + loading.size() + waitingToLoadRangeTxns.size() + scanningRanges.size(); } public int unsafeWaitingToRunCount() { - return waitingToRun.size(); + return runnable.waitingCount(); } public int unsafeRunningCount() { - return running.size(); + return runnable.assigned.size(); + } + + private static long[] parseEnumParams(String input, String describe) + { + String[] specs = input.split(";"); + if (specs.length != 2) + throw new IllegalArgumentException("Invalid specifiers in " + describe + "; expect [GlobalGroup];[ExclusiveGroup] but got: " + input); + long[] result = new long[2]; + result[0] = parseEnumParams(GlobalGroup::valueOf, specs[0], describe + " for GlobalGroup"); + result[1] = parseEnumParams(ExclusiveGroup::valueOf, specs[1], describe + " for ExclusiveGroup"); + return result; + } + + private static long parseEnumParams(Function> get, String input, String describe) + { + long result = 0; + for (String spec : input.split(",")) + { + if (spec.trim().isEmpty()) continue; + + String[] split = spec.split(":"); + if (split.length != 2) + throw new IllegalArgumentException("Invalid specifier " + spec + " in " + describe + ": " + input); + + try + { + Enum queue = get.apply(split[0]); + long value = Long.parseLong(split[1]); + if (value <= 0 || value >= 128) + throw new IllegalArgumentException("Invalid limit " + value + " in queue_active_limits: " + input); + + result |= value << (queue.ordinal() * 8); + } + catch (Throwable t) + { + throw new IllegalArgumentException("Invalid queue identifier " + split[0] + " in " + describe + ": " + input); + } + } + + return result; + } + + private static long encodeCounters(Map, Long> counters) + { + long result = 0; + for (Map.Entry, Long> e : counters.entrySet()) + result |= e.getValue() << (e.getKey().ordinal() * 8); + return result; } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java index d8051f40dc2d..8e3e41f7c342 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java @@ -24,7 +24,7 @@ import accord.utils.QuadFunction; import accord.utils.QuintConsumer; -import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.concurrent.CassandraThread; import org.apache.cassandra.service.accord.AccordExecutorLoops.LoopTask; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutorLoop; @@ -33,7 +33,6 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop { - private static final int YIELD_INTERVAL = DatabaseDescriptor.getAccord().queue_yield_interval; int runningThreads; boolean shutdown; @@ -44,11 +43,10 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop abstract void notifyWork(); abstract void notifyWorkExclusive(); - void loopYieldExclusive() throws InterruptedException {} abstract void awaitExclusive() throws InterruptedException; abstract void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4); - void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + final void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) { // if we're a loop thread, we will poll the waitingToRun queue when we come around // NOTE: this assumes no synchronous blocking tasks are submitted to this executor @@ -56,7 +54,7 @@ void submit(QuintConsumer void submitExternalExclusive(QuintConsumer sync, P1s p1s, P2 p2, P3 p3, P4 p4) + final void submitExternalExclusive(QuintConsumer sync, P1s p1s, P2 p2, P3 p3, P4 p4) { try { @@ -84,6 +82,73 @@ final void notifyIfMoreWorkExclusive() notifyWorkExclusive(); } + final boolean hasWaitingToRun() + { + updateWaitingToRunExclusive(); + return hasAlreadyWaitingToRun(); + } + + final Task pollWaitingToRunExclusive() + { + updateWaitingToRunExclusive(); + return pollAlreadyWaitingToRunExclusive(); + } + + final void updateWaitingToRunExclusive() + { + drainUnqueuedExclusive(); + super.updateWaitingToRunExclusive(); + } + + final void drainUnqueuedExclusive() + { + Task cur = Task.reverse(acquireUnqueuedExclusive()); + while (cur != null) + cur = enqueueOneExclusive(cur); + } + + final void drainUnqueuedNewWorkExclusive() + { + Task cur = acquireUnqueuedExclusive(); + Task prev = null, requeue = null, requeueLast = null; + while (cur != null) + { + Task next = cur.next; + if (cur.isNewWork()) + { + cur.next = prev; + prev = cur; + } + else + { + if (requeue == null) requeue = cur; + else requeueLast.next = cur; + requeueLast = cur; + } + cur = next; + } + + if (requeue != null) + { + while (true) + { + Task next = unqueued; + requeueLast.next = next; + if (unqueuedUpdater.compareAndSet(this, next, requeue)) + break; + } + } + + cur = prev; + while (cur != null) + { + Task next = cur.next; + cur.submitExclusive(this); + cur.next = null; + cur = next; + } + } + @Override final void beforeUnlockExternal() { @@ -130,11 +195,15 @@ protected LoopTask runWithLock(String name) @Override public void run() { - Thread self = Thread.currentThread(); + Thread thread = Thread.currentThread(); + AccordTaskRunner self = AccordTaskRunner.get(thread); + self.setAccordActiveExecutor(AccordExecutorAbstractLockLoop.this); + setWrapped(self); + Task task; while (true) { - lock(); + lock(self); try { enterLockLoop(); @@ -144,7 +213,7 @@ public void run() if (task != null) { - setRunning(task); + self.setAccordActiveTask(task); try { task.preRunExclusive(); @@ -157,7 +226,7 @@ public void run() finally { completeTaskExclusive(task); - clearRunning(); + self.setAccordActiveTask(null); } } else @@ -185,7 +254,7 @@ public void run() } finally { - unlock(); + unlock(self); } } } @@ -200,12 +269,15 @@ protected LoopTask runWithoutLock(String name) @Override public void run() { - int count = 0; + CassandraThread self = (CassandraThread) Thread.currentThread(); + self.setAccordActiveExecutor(AccordExecutorAbstractLockLoop.this); + setWrapped(self); + Task task = null; while (true) { if (DEBUG_EXECUTION) debug.onLock(); - lock(); + lock(self); try { if (DEBUG_EXECUTION) debug.onEnterLock(); @@ -215,13 +287,7 @@ public void run() Task tmp = task; task = null; completeTaskExclusive(tmp); - clearRunning(); - } - - if (count >= YIELD_INTERVAL) - { - loopYieldExclusive(); - count = 0; + self.setAccordActiveTask(null); } while (true) @@ -230,7 +296,7 @@ public void run() if (task != null) { - setRunning(task); + self.setAccordActiveTask(task); task.preRunExclusive(); if (DEBUG_EXECUTION) debug.onExitLock(); exitLockLoop(); @@ -250,7 +316,6 @@ public void run() awaitExclusive(); if (DEBUG_EXECUTION) debug.onEnterLock(); resumeLoop(); - count = 0; } } catch (Throwable t) @@ -276,12 +341,11 @@ public void run() finally { if (DEBUG_EXECUTION) debug.onExitLock(); - unlock(); + unlock(self); } try { - ++count; task.run(); } catch (Throwable t) diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java index a76d1bac1809..3cbd3db14481 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java @@ -29,8 +29,8 @@ abstract class AccordExecutorAbstractLoop extends AccordExecutor { - private volatile Task unqueued; - private static final AtomicReferenceFieldUpdater unqueuedUpdater = AtomicReferenceFieldUpdater.newUpdater(AccordExecutorAbstractLoop.class, Task.class, "unqueued"); + volatile Task unqueued; + static final AtomicReferenceFieldUpdater unqueuedUpdater = AtomicReferenceFieldUpdater.newUpdater(AccordExecutorAbstractLoop.class, Task.class, "unqueued"); AccordExecutorAbstractLoop(Lock lock, int executorId, Agent agent) { @@ -67,30 +67,18 @@ public boolean hasTasks() if (hasUnqueued() || tasks > 0) return true; - lock(); + AccordTaskRunner self = AccordTaskRunner.get(); + lock(self); try { return hasUnqueued() || tasks > 0; } finally { - unlock(); + unlock(self); } } - final void updateWaitingToRunExclusive() - { - drainUnqueuedExclusive(); - super.updateWaitingToRunExclusive(); - } - - final void drainUnqueuedExclusive() - { - Task cur = Task.reverse(acquireUnqueuedExclusive()); - while (cur != null) - cur = enqueueOneExclusive(cur); - } - final Task acquireUnqueuedExclusive() { return unqueuedUpdater.getAndSet(this, null); diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java index f97e206ef5d3..67e897d47964 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java @@ -46,18 +46,6 @@ private AccordExecutorSemiSyncSubmit(ReentrantLock lock, int executorId, Mode mo this.loops = new AccordExecutorLoops(mode, threads, name, this::task); } - @Override - void loopYieldExclusive() throws InterruptedException - { - if (waiting > 0 && hasWaitingToRun()) - { - pauseLoop(); - hasWork.signal(); - awaitWork(); - resumeLoop(); - } - } - @Override void awaitExclusive() throws InterruptedException { diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java index 9ec7fa95d2ed..ce7de7fefc7c 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java @@ -100,6 +100,217 @@ private LoopTask task(int index, String id, Mode mode) return new LoopTask(index, id); } + @Override + final void drainUnqueuedNewWorkExclusive() + { + updatePendingUnqueued(); + Task requeue = null, requeueTail = null; + Task cur = pendingNewHead; + while (cur != null) + { + Task next = cur.next; + if (cur.isNewWork()) + { + cur.next = null; + cur.submitExclusive(this); + --pendingCount; + } + else + { + if (requeue == null) requeue = cur; + else requeueTail.next = cur; + requeueTail = cur; + } + cur = next; + } + + pendingNewHead = requeue; + pendingNewTail = requeueTail; + } + + private boolean enqueueOnePending() + { + if (pendingCount == 0) + return false; + + if (pendingSequentialHead != null) + { + pendingSequentialHead = enqueueOneCleanup(pendingSequentialHead); + if (pendingSequentialHead == null) + pendingSequentialTail = null; + } + else if (pendingCleanupHead != null) + { + pendingCleanupHead = enqueueOneCleanup(pendingCleanupHead); + if (pendingCleanupHead == null) + pendingCleanupTail = null; + } + else + { + pendingNewHead = enqueueOneSubmit(pendingNewHead); + if (pendingNewHead == null) + pendingNewTail = null; + } + --pendingCount; + return true; + } + + private void fetchWorkExclusive() + { + boolean hasReadyToRun = hasReadyToRun(); + boolean hadWaitingToRun = hasAlreadyWaitingToRun(); + { + int prevPendingUnqueued = pendingCount; + updateAndEnqueuePendingUntilHasWaitingToRun(prevPendingUnqueued / 2); + } + + if (hadWaitingToRun && hasReadyToRun) + { + lock.addAndGetEnabledThreadCount(1); + } + else if (hadWaitingToRun) + { + readyToRunTarget = Math.min(readyToRunLimit, readyToRunTarget + (1+readyToRunTarget)/2); + } + else if (hasReadyToRun && readyToRunTarget > 1) + { + --readyToRunTarget; + } + + boolean hasDrainedSignal = false; + while (true) + { + long state = lock.state(); + int signals = SignalLock.asyncSignalCount(state); + int waiters = SignalLock.waitingEnabledThreadCount(state); + if (signals >= readyToRunTarget) + { + if (enqueueOnePending() || (updatePendingUnqueued() && enqueueOnePending())) continue; + else if (hasDrainedSignal) + lock.signalLockWorkExclusive(); + return; + } + else if (waiters > 0 && signals > 1 && SignalLock.activeEnabledThreadCount(state) == 1) + { + // ensure at least one other thread is running if there's enough work for it; it will spin up other threads if necessary + lock.propagateAsyncWorkSignals(1); + } + + Task task = pollAlreadyWaitingToRunExclusive(); + if (task == null) + { + if (updateAndEnqueuePendingUntilHasWaitingToRun(0)) + continue; + + lock.clearLockWork(); + hasDrainedSignal = true; + if (!updateAndEnqueuePendingUntilHasWaitingToRun(0)) + { + if (tasks == 0) + notifyQuiescentExclusive(); + return; + } + } + else + { + try { task.preRunExclusive(); } + catch (Throwable t) + { + try { task.fail(t); } + catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } + try { completeTaskExclusive(task); } + catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } + continue; + } + if (DEBUG_EXECUTION) DebugTask.get(task).onPreRun(); + + addReadyToRun(task); + boolean incremented = lock.incrementAsyncWork(false); + Invariants.require(incremented); + } + } + } + + private boolean updatePendingUnqueued() + { + if (!hasUnqueued()) + return false; + + int count = 0; + Task addSequentialHead = null, addSequentialTail = null; + Task addCleanupHead = null, addCleanupTail = null; + Task addNewHead = null, addNewTail = null; + { + Task cur = Task.reverse(acquireUnqueuedExclusive()); + while (cur != null) + { + Task next = cur.next; + if (!cur.isReadyToCleanup()) + { + if (addNewHead == null) addNewHead = addNewTail = setNextNull(cur); + else addNewHead = reverseOne(addNewHead, cur); + } + else if (cur instanceof ExclusiveExecutorTask) + { + if (addSequentialHead == null) addSequentialHead = addSequentialTail = setNextNull(cur); + else addSequentialHead = reverseOne(addSequentialHead, cur); + } + else + { + if (addCleanupHead == null) addCleanupHead = addCleanupTail = setNextNull(cur); + else addCleanupHead = reverseOne(addCleanupHead, cur); + } + ++count; + cur = next; + } + } + + pendingCount += count; + if (addSequentialHead != null) + { + if (pendingSequentialHead == null) pendingSequentialHead = addSequentialHead; + else pendingSequentialTail.next = addSequentialHead; + pendingSequentialTail = addSequentialTail; + } + if (addCleanupHead != null) + { + if (pendingCleanupHead == null) pendingCleanupHead = addCleanupHead; + else pendingCleanupTail.next = addCleanupHead; + pendingCleanupTail = addCleanupTail; + } + if (addNewHead != null) + { + if (pendingNewHead == null) pendingNewHead = addNewHead; + else pendingNewTail.next = addNewHead; + pendingNewTail = addNewTail; + } + return true; + } + + private Task reverseOne(Task prev, Task cur) + { + cur.next = prev; + return cur; + } + + private Task setNextNull(Task cur) + { + cur.next = null; + return cur; + } + + private boolean updateAndEnqueuePendingUntilHasWaitingToRun(int processAtLeast) + { + updatePendingUnqueued(); + int count = 0; + while (enqueueOnePending()) + { + if (++count >= processAtLeast && hasAlreadyWaitingToRun()) + return true; + } + return hasAlreadyWaitingToRun(); + } + class LoopTask extends AccordExecutorLoops.LoopTask { final int index; @@ -110,7 +321,7 @@ class LoopTask extends AccordExecutorLoops.LoopTask this.index = index; } - private Task awaitWork() + private Task awaitWork(AccordTaskRunner self) { while (true) { @@ -123,11 +334,11 @@ private Task awaitWork() } catch (Throwable t) { - unlock(); + unlock(self); throw t; } - if (!unlockAndAcquire()) + if (!unlockAndAcquire(self)) continue; } @@ -138,11 +349,8 @@ private Task awaitWork() } } - private Task cleanupAndMaybeGetWork(@Nullable Task cleanup) + private Task cleanupAndMaybeGetWork(AccordTaskRunner self, @Nullable Task cleanup) { - if (cleanup == null) - return null; - if (lock.tryAcquireAsyncWork()) { if (shutdown) @@ -151,22 +359,21 @@ private Task cleanupAndMaybeGetWork(@Nullable Task cleanup) return pushCleanupAndReturn(cleanup, nonNull(pollReadyToRun())); } - if (!tryLock()) + if (!tryLock(self)) return pushCleanupAndReturn(cleanup, null); try { completeTaskExclusive(cleanup); - clearRunning(); fetchWorkExclusive(); } catch (Throwable t) { - unlock(); + unlock(self); throw t; } - if (unlockAndAcquire()) + if (unlockAndAcquire(self)) { if (shutdown) throw new ShutdownException(); @@ -178,225 +385,52 @@ private Task cleanupAndMaybeGetWork(@Nullable Task cleanup) private Task pushCleanupAndReturn(Task cleanup, Task result) { - clearRunning(); cleanup.setReadyToCleanup(); if (push(cleanup) == null) lock.signalLockWork(); return result; } - final boolean tryLock() + final boolean tryLock(AccordTaskRunner self) { - return onTryLock(lock.tryLock(index)); - } - - final boolean unlockAndAcquire() - { - if (Invariants.isParanoid()) paranoidUnlockExclusive(); - if (DEBUG_EXECUTION) debug.onExitLock(); - return lock.unlockAndAcquireAsyncWork(); - } - - private boolean enqueueOnePending() - { - if (pendingCount == 0) + if (self.accordLockedExecutor() != null) return false; - if (pendingSequentialHead != null) - { - pendingSequentialHead = enqueueOneCleanup(pendingSequentialHead); - if (pendingSequentialHead == null) - pendingSequentialTail = null; - } - else if (pendingCleanupHead != null) - { - pendingCleanupHead = enqueueOneCleanup(pendingCleanupHead); - if (pendingCleanupHead == null) - pendingCleanupTail = null; - } - else - { - pendingNewHead = enqueueOneSubmit(pendingNewHead); - if (pendingNewHead == null) - pendingNewTail = null; - } - --pendingCount; - return true; + return onTryLock(self, lock.tryLock(index)); } - private void fetchWorkExclusive() + final boolean unlockAndAcquire(AccordTaskRunner self) { - boolean hasReadyToRun = hasReadyToRun(); - boolean hadWaitingToRun = hasAlreadyWaitingToRun(); - { - int prevPendingUnqueued = pendingCount; - updateAndEnqueuePendingUntilHasWaitingToRun(prevPendingUnqueued / 2); - } - - if (hadWaitingToRun && hasReadyToRun) - { - lock.addAndGetEnabledThreadCount(1); - } - else if (hadWaitingToRun) - { - readyToRunTarget = Math.min(readyToRunLimit, readyToRunTarget + (1+readyToRunTarget)/2); - } - else if (hasReadyToRun && readyToRunTarget > 1) - { - --readyToRunTarget; - } - - boolean hasDrainedSignal = false; - while (true) - { - long state = lock.state(); - int signals = SignalLock.asyncSignalCount(state); - int waiters = SignalLock.waitingEnabledThreadCount(state); - if (signals >= readyToRunTarget) - { - if (enqueueOnePending() || (updatePendingUnqueued() && enqueueOnePending())) continue; - else if (hasDrainedSignal) - lock.signalLockWorkExclusive(); - return; - } - else if (waiters > 0 && signals > 1 && SignalLock.activeEnabledThreadCount(state) == 1) - { - // ensure at least one other thread is running if there's enough work for it; it will spin up other threads if necessary - lock.propagateAsyncWorkSignals(1); - } - - Task task = pollAlreadyWaitingToRunExclusive(); - if (task == null) - { - if (updateAndEnqueuePendingUntilHasWaitingToRun(0)) - continue; - - lock.clearLockWork(); - hasDrainedSignal = true; - if (!updateAndEnqueuePendingUntilHasWaitingToRun(0)) - { - if (tasks == 0) - notifyQuiescentExclusive(); - return; - } - } - else - { - try { task.preRunExclusive(); } - catch (Throwable t) - { - try { task.fail(t); } - catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } - try { completeTaskExclusive(task); } - catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } - continue; - } - if (DEBUG_EXECUTION) DebugTask.get(task).onPreRun(); - - addReadyToRun(task); - boolean incremented = lock.incrementAsyncWork(false); - Invariants.require(incremented); - } - } - } - - private boolean updatePendingUnqueued() - { - if (!hasUnqueued()) - return false; - - int count = 0; - Task addSequentialHead = null, addSequentialTail = null; - Task addCleanupHead = null, addCleanupTail = null; - Task addNewHead = null, addNewTail = null; - { - Task cur = Task.reverse(acquireUnqueuedExclusive()); - while (cur != null) - { - Task next = cur.next; - if (!cur.isReadyToCleanup()) - { - if (addNewHead == null) addNewHead = addNewTail = setNextNull(cur); - else addNewHead = reverseOne(addNewHead, cur); - } - else if (cur instanceof SequentialQueueTask) - { - if (addSequentialHead == null) addSequentialHead = addSequentialTail = setNextNull(cur); - else addSequentialHead = reverseOne(addSequentialHead, cur); - } - else - { - if (addCleanupHead == null) addCleanupHead = addCleanupTail = setNextNull(cur); - else addCleanupHead = reverseOne(addCleanupHead, cur); - } - ++count; - cur = next; - } - } - - pendingCount += count; - if (addSequentialHead != null) - { - if (pendingSequentialHead == null) pendingSequentialHead = addSequentialHead; - else pendingSequentialTail.next = addSequentialHead; - pendingSequentialTail = addSequentialTail; - } - if (addCleanupHead != null) - { - if (pendingCleanupHead == null) pendingCleanupHead = addCleanupHead; - else pendingCleanupTail.next = addCleanupHead; - pendingCleanupTail = addCleanupTail; - } - if (addNewHead != null) - { - if (pendingNewHead == null) pendingNewHead = addNewHead; - else pendingNewTail.next = addNewHead; - pendingNewTail = addNewTail; - } - return true; - } - - private Task reverseOne(Task prev, Task cur) - { - cur.next = prev; - return cur; - } - - private Task setNextNull(Task cur) - { - cur.next = null; - return cur; - } - - private boolean updateAndEnqueuePendingUntilHasWaitingToRun(int processAtLeast) - { - updatePendingUnqueued(); - int count = 0; - while (enqueueOnePending()) - { - if (++count >= processAtLeast && hasAlreadyWaitingToRun()) - return true; - } - return hasAlreadyWaitingToRun(); + self.clearAccordLockedExecutor(); + if (DEBUG_EXECUTION) debug.onExitLock(); + return lock.unlockAndAcquireAsyncWork(); } @Override public void run() { - lock.register(index, Thread.currentThread()); + Thread thread = Thread.currentThread(); + AccordTaskRunner self = AccordTaskRunner.get(thread); + self.setAccordActiveExecutor(AccordExecutorSignalLoop.this); + setWrapped(self); + + lock.register(index, thread); Task task = null; while (true) { try { - try { task = cleanupAndMaybeGetWork(task); } - catch (Throwable t) { task = null; throw t; } + if (task != null) + { + try { task = cleanupAndMaybeGetWork(self, task); } + catch (Throwable t) { task = null; throw t; } + } if (task == null) - task = awaitWork(); + task = awaitWork(self); try { - setRunning(task); + self.setAccordActiveTask(task); task.run(); } catch (Throwable t) @@ -421,6 +455,10 @@ public void run() { agent.onException(t); } + finally + { + self.setAccordActiveTask(null); + } } } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java index ab6c8c25e736..ee37744d8dfd 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java @@ -128,6 +128,18 @@ void submit(QuintConsumer void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) { - lock(); + AccordTaskRunner self = AccordTaskRunner.get(); + lock(self); try { submitExternalExclusive(sync, p1s, p2, p3, p4); } finally { - unlock(); + unlock(self); } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java index 45aeaa2f1821..e268113b8604 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java @@ -397,7 +397,7 @@ public long serializedSize(Update t, TableMetadatasAndKeys seekables, Version ve public AccordFetchCoordinator(Node node, Ranges ranges, SyncPoint syncPoint, DataStore.FetchRanges fetchRanges, CommandStore commandStore) throws TopologyException { - super(node, node.someSequentialExecutor(), ranges, syncPoint, fetchRanges, commandStore); + super(node, node.someExclusiveExecutor(), ranges, syncPoint, fetchRanges, commandStore); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java index 6a9817bc9766..d0399e1b9244 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java @@ -64,7 +64,7 @@ private AccordSafeCommandStore(AccordTask task, @Nullable CommandSummaries commandsForRanges, AccordCommandStore commandStore) { - super(task.preLoadContext(), commandStore); + super(task.executionContext(), commandStore); this.task = task; this.commandsForRanges = commandsForRanges; this.commandStore = commandStore; diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index 111e2074745c..edb0498856e3 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -964,7 +964,7 @@ public AsyncChain sync(TxnId minBound, Keys keys, DurabilityService.SyncLo return syncInternal(minBound, keys, syncLocal, syncRemote); return KeyBarriers.find(node, minBound, keys.get(0).toUnseekable(), syncLocal, syncRemote).chain() - .flatMap(found -> KeyBarriers.await(node, node.someSequentialExecutor(), found, syncLocal, syncRemote)) + .flatMap(found -> KeyBarriers.await(node, node.someExclusiveExecutor(), found, syncLocal, syncRemote)) .flatMap(success -> { if (success) return null; diff --git a/src/java/org/apache/cassandra/service/accord/AccordTask.java b/src/java/org/apache/cassandra/service/accord/AccordTask.java index fe2889e2a6e9..90781c740f25 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTask.java +++ b/src/java/org/apache/cassandra/service/accord/AccordTask.java @@ -71,7 +71,6 @@ import org.apache.cassandra.service.accord.AccordCacheEntry.Status; import org.apache.cassandra.service.accord.AccordCommandStore.Caches; import org.apache.cassandra.service.accord.AccordExecutor.Task; -import org.apache.cassandra.service.accord.AccordExecutor.TaskQueue; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; @@ -87,18 +86,17 @@ import static accord.primitives.Routable.Domain.Key; import static accord.primitives.Txn.Kind.EphemeralRead; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; -import static org.apache.cassandra.service.accord.AccordTask.State.ASSIGNED; -import static org.apache.cassandra.service.accord.AccordTask.State.CANCELLED; -import static org.apache.cassandra.service.accord.AccordTask.State.FAILED; -import static org.apache.cassandra.service.accord.AccordTask.State.FINISHED; -import static org.apache.cassandra.service.accord.AccordTask.State.INITIALIZED; -import static org.apache.cassandra.service.accord.AccordTask.State.LOADING; -import static org.apache.cassandra.service.accord.AccordTask.State.PERSISTING; -import static org.apache.cassandra.service.accord.AccordTask.State.RUNNING; -import static org.apache.cassandra.service.accord.AccordTask.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_SCAN_RANGES; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.ASSIGNED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.FAILED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.FINISHED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.PERSISTING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.SCANNING_RANGES; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_SCAN_RANGES; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; import static org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask.SANITY_CHECK; @@ -111,9 +109,9 @@ static class ForFunction extends AccordTask { private final Function function; - public ForFunction(AccordCommandStore commandStore, ExecutionContext loadCtx, Function function) + public ForFunction(AccordCommandStore commandStore, ExecutionContext context, Function function) { - super(commandStore, loadCtx); + super(commandStore, context); this.function = function; } @@ -129,9 +127,9 @@ static class ForConsumer extends AccordTask { private final Consumer consumer; - private ForConsumer(AccordCommandStore commandStore, ExecutionContext loadCtx, Consumer consumer) + private ForConsumer(AccordCommandStore commandStore, ExecutionContext context, Consumer consumer) { - super(commandStore, loadCtx); + super(commandStore, context); this.consumer = consumer; } @@ -143,63 +141,16 @@ public Void apply(SafeCommandStore commandStore) } } - public static AccordTask create(CommandStore commandStore, ExecutionContext ctx, Function function) + public static AccordTask create(CommandStore commandStore, ExecutionContext context, Function function) { - return new ForFunction<>((AccordCommandStore) commandStore, ctx, function); + return new ForFunction<>((AccordCommandStore) commandStore, context, function); } - public static AccordTask create(CommandStore commandStore, ExecutionContext ctx, Consumer consumer) + public static AccordTask create(CommandStore commandStore, ExecutionContext context, Consumer consumer) { - return new ForConsumer((AccordCommandStore) commandStore, ctx, consumer); + return new ForConsumer((AccordCommandStore) commandStore, context, consumer); } - public enum State - { - INITIALIZED(), - WAITING_TO_SCAN_RANGES(INITIALIZED), - SCANNING_RANGES(WAITING_TO_SCAN_RANGES), - WAITING_TO_LOAD(INITIALIZED, SCANNING_RANGES), - LOADING(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD), - WAITING_TO_RUN(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD, LOADING), - ASSIGNED(WAITING_TO_RUN), - RUNNING(ASSIGNED), - PERSISTING(RUNNING), - FINISHED(RUNNING, PERSISTING), - CANCELLED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, ASSIGNED), - FAILED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, ASSIGNED, RUNNING, PERSISTING); - - private final int permittedFrom; - - State() - { - this.permittedFrom = 0; - } - - State(State ... permittedFroms) - { - int permittedFrom = 0; - for (State state : permittedFroms) - permittedFrom |= 1 << state.ordinal(); - this.permittedFrom = permittedFrom; - } - - boolean isPermittedFrom(State prev) - { - return (permittedFrom & (1 << prev.ordinal())) != 0; - } - - boolean isExecuted() - { - return this.compareTo(PERSISTING) >= 0; - } - - boolean isComplete() - { - return this.compareTo(FINISHED) >= 0; - } - } - - private State state = INITIALIZED; final AccordCommandStore commandStore; private final ExecutionContext executionContext; private volatile String loggingId; @@ -214,14 +165,13 @@ boolean isComplete() // TODO (desired): collection supporting faster deletes but still fast poll (e.g. some ordered collection) @Nullable ArrayDeque> waitingToLoad; @Nullable RangeTxnScanner rangeScanner; - boolean hasRanges; @Nullable CommandSummaries commandsForRanges; - @Nullable private TaskQueue queued; private BiConsumer callback; public AccordTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext) { + super(executionContext); this.commandStore = commandStore; this.executionContext = executionContext; this.loggingId = "0x" + Long.toHexString(nextLoggingId.incrementAndGet()); @@ -250,13 +200,13 @@ public String toString() public String toBriefString() { - return '{' + loggingId() + ',' + state + '}'; + return '{' + loggingId() + ',' + state() + '}'; } public String toDescription() { return toBriefString() + ": " - + (queued == null ? "unqueued" : queued.kind) + + (queued() == null ? "unqueued" : state()) + ", primaryTxnId: " + executionContext.primaryTxnId() + ", waitingToLoad: " + summarise(waitingToLoad) + ", loading:" + summarise(loading, AccordSafeState::global) @@ -302,17 +252,6 @@ private static String summarise(Collection collection, Function no loading or waiting; found %s", this, AccordTask::toDescription); - } - } - Unseekables keys() { return executionContext.keys(); @@ -334,20 +273,6 @@ protected Cancellable start(BiConsumer callback) }; } - public AsyncChain priorityChain() - { - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - preSetup(callback); - commandStore.executor().submitPriority(AccordTask.this); - return AccordTask.this; - } - }; - } - private void preSetup(BiConsumer callback) { Invariants.require(this.callback == null); @@ -358,7 +283,7 @@ private void preSetup(BiConsumer callback) // to be invoked only by the CommandStore owning thread, to take references to objects already in use by the current execution public void presetup(AccordTask parent) { - this.queuePosition = parent.queuePosition; + this.position = parent.position; // note we use the caches "unsafely" here deliberately, as we only reference commands we already have references to // so we do not mutate anything, except the atomic counter of references if (parent.commands != null) @@ -393,8 +318,8 @@ void submitExclusive(AccordExecutor owner) public void setupExclusive() { setupInternal(commandStore.cachesExclusive()); - state(rangeScanner != null ? WAITING_TO_SCAN_RANGES - : waitingToLoad != null ? State.WAITING_TO_LOAD + setState(rangeScanner != null ? WAITING_TO_SCAN_RANGES + : waitingToLoad != null ? WAITING_TO_LOAD : loading != null ? LOADING : WAITING_TO_RUN); } @@ -455,7 +380,6 @@ private void setupRangeLoadsExclusive(Caches caches) throw new AssertionError("Incremental mode should only be used with an explicit list of keys"); case SYNC: - hasRanges = true; rangeScanner = new RangeTxnAndKeyScanner(caches.commandsForKeys()); } } @@ -535,11 +459,11 @@ public boolean onLoad(AccordCacheEntry state) return false; loading = null; - if (this.state.compareTo(State.WAITING_TO_LOAD) < 0) + if (compareTo(WAITING_TO_LOAD) < 0) return false; Invariants.require(waitingToLoad == null, "Invalid state: %s", this, AccordTask::toDescription); - state(WAITING_TO_RUN); + setState(WAITING_TO_RUN); return true; } @@ -557,14 +481,14 @@ public boolean onLoading(AccordCacheEntry state) private boolean onEmptyWaitingToLoad() { waitingToLoad = null; - if (this.state.compareTo(State.WAITING_TO_LOAD) < 0) + if (compareTo(WAITING_TO_LOAD) < 0) return false; - state(loading == null ? WAITING_TO_RUN : LOADING); + setState(loading == null ? WAITING_TO_RUN : LOADING); return true; } - public ExecutionContext preLoadContext() + public ExecutionContext executionContext() { return executionContext; } @@ -602,7 +526,7 @@ public Map ensureCommandsForKey() private ArrayDeque> ensureWaitingToLoad() { - Invariants.require(state.compareTo(WAITING_TO_LOAD) <= 0, "Expected status to be on or before WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); + Invariants.require(compareTo(WAITING_TO_LOAD) <= 0, "Expected status to be on or before WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); if (waitingToLoad == null) waitingToLoad = new ArrayDeque<>(); return waitingToLoad; @@ -610,7 +534,7 @@ public Map ensureCommandsForKey() public AccordCacheEntry pollWaitingToLoad() { - Invariants.require(state == State.WAITING_TO_LOAD, "Expected status to be WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); + Invariants.require(is(WAITING_TO_LOAD), "Expected status to be WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); if (waitingToLoad == null) return null; @@ -658,8 +582,7 @@ private void save(List diffs, Runnable onFlush) @Override protected void preRunExclusive() { - state(ASSIGNED); - queued = null; + setState(ASSIGNED); if (rangeScanner != null) { commandsForRanges = rangeScanner.finish(commandStore.cachesExclusive()); @@ -675,14 +598,13 @@ protected void preRunExclusive() public void run() { onRunning(); - logger.trace("Running {} with state {}", this, state); AccordSafeCommandStore safeStore = null; try (Closeable close = resources.get()) { if (Tracing.isTracing()) Tracing.trace(executionContext.describe()); - state(RUNNING); + setState(RUNNING); safeStore = commandStore.begin(this, commandsForRanges); R result = apply(safeStore); @@ -710,7 +632,7 @@ public void run() boolean flush = changes != null || safeStore.fieldUpdates() != null; if (flush) { - state(PERSISTING); + setState(PERSISTING); Runnable onFlush = () -> finish(result, null); safeStore.persistFieldUpdatesInternal(changes == null ? onFlush : null); if (changes != null) save(changes, onFlush); @@ -735,12 +657,12 @@ public void run() public void fail(Throwable throwable) { - if (state.isComplete()) + if (state().isComplete()) return; try { - state(FAILED); + setState(FAILED); commandStore.agent().onException(throwable); } finally @@ -750,6 +672,12 @@ public void fail(Throwable throwable) } } + @Override + protected boolean isNewWork() + { + return true; + } + public void failExclusive(Throwable throwable) { fail(throwable); @@ -758,7 +686,7 @@ public void failExclusive(Throwable throwable) @Override protected void cleanupExclusive(AccordExecutor executor) { - Invariants.expect(state.isExecuted()); + Invariants.expect(state().isExecuted()); releaseResources(commandStore.cachesExclusive()); super.cleanupExclusive(executor); executor.keys.increment(commandsForKey == null ? 0 : commandsForKey.size(), runningAt); @@ -775,22 +703,17 @@ public RangeTxnScanner rangeScanner() return rangeScanner; } - public boolean hasRanges() - { - return hasRanges; - } - @Override public void cancel() { - if (!state.isComplete()) + if (!state().isComplete()) commandStore.executor().cancel(this); } void cancelExclusive() { logger.info("Cancelling {}", executionContext); - state(CANCELLED); + setState(CANCELLED); if (rangeScanner != null) rangeScanner.cancelled = true; if (callback != null) @@ -805,14 +728,9 @@ void cancelExclusive(AccordExecutor owner) owner.cancelExclusive(this); } - public State state() - { - return state; - } - private void finish(R result, Throwable failure) { - state(failure == null ? FINISHED : FAILED); + setState(failure == null ? FINISHED : FAILED); if (callback != null) callback.accept(result, failure); } @@ -921,38 +839,6 @@ void revert() commandsForKey.forEach((k, v) -> v.revert()); } - protected void addToQueue(TaskQueue queue) - { - if (state == CANCELLED) - return; - - Invariants.require(queue.kind == state || (queue.kind == State.WAITING_TO_LOAD && state == WAITING_TO_SCAN_RANGES), "Invalid queue type: %s vs %s", queue.kind, this, AccordTask::toDescription); - Invariants.require(this.queued == null, "Already queued with state: %s", this, AccordTask::toDescription); - queued = queue; - queue.append(this); - } - - @Nullable - TaskQueue queued() - { - return queued; - } - - TaskQueue unqueue() - { - TaskQueue wasQueued = queued; - queued.remove(this); - queued = null; - return wasQueued; - } - - TaskQueue unqueueIfQueued() - { - if (queued == null) - return null; - return unqueue(); - } - public class RangeTxnAndKeyScanner extends RangeTxnScanner { class KeyWatcher implements AccordCache.Listener @@ -965,7 +851,6 @@ public void onUpdate(AccordCacheEntry state) } } - // TODO (expected): produce key summaries to avoid locking all in memory final Set intersectingKeys = new ObjectHashSet<>(); final KeyWatcher keyWatcher = new KeyWatcher(); final Ranges ranges = ((AbstractRanges) executionContext.keys()).toRanges(); @@ -1102,9 +987,9 @@ protected void fail(Throwable t) public void start(AccordExecutor executor) { Caches caches = commandStore.cachesExclusive(); - state(SCANNING_RANGES); + setState(SCANNING_RANGES); startInternal(caches); - executor.submitPlainExclusive(AccordTask.this, this); + executor.submitPlainExclusive(AccordTask.this, GlobalGroup.RANGE_SCAN, this); } void startInternal(Caches caches) @@ -1115,12 +1000,12 @@ void startInternal(Caches caches) public void scannedExclusive() { - Invariants.require(state == SCANNING_RANGES, "Expected SCANNING_RANGES; found %s", AccordTask.this, AccordTask::toDescription); + Invariants.require(is(SCANNING_RANGES), "Expected SCANNING_RANGES; found %s", AccordTask.this, AccordTask::toDescription); scanned = true; scannedInternal(); - if (loading == null) state(WAITING_TO_RUN); - else if (waitingToLoad == null) state(LOADING); - else state(State.WAITING_TO_LOAD); + if (loading == null) setState(WAITING_TO_RUN); + else if (waitingToLoad == null) setState(LOADING); + else setState(WAITING_TO_LOAD); } void scannedInternal() diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java index d42f9bb3383c..e01f877d14d4 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java @@ -131,11 +131,11 @@ public void onExitLock() } } - public static class DebugSequentialExecutor + public static class DebugExclusiveExecutor { - public static DebugSequentialExecutor maybeDebug(DebugExecutor owner, int commandStoreId) + public static DebugExclusiveExecutor maybeDebug(DebugExecutor owner, int commandStoreId) { - return DEBUG_EXECUTION ? new DebugSequentialExecutor(owner, commandStoreId) : null; + return DEBUG_EXECUTION ? new DebugExclusiveExecutor(owner, commandStoreId) : null; } final DebugExecutor owner; @@ -144,7 +144,7 @@ public static DebugSequentialExecutor maybeDebug(DebugExecutor owner, int comman long setTaskAt, waitingAt; Task prev; - public DebugSequentialExecutor(DebugExecutor owner, int commandStoreId) + public DebugExclusiveExecutor(DebugExecutor owner, int commandStoreId) { this.owner = owner; this.commandStoreId = commandStoreId; diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java index b79fc5f841b2..697a094e608f 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java @@ -23,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Result; import accord.api.Update; import accord.coordinate.CoordinationAdapter; @@ -30,7 +31,6 @@ import accord.coordinate.ExecuteFlag.CoordinationFlags; import accord.coordinate.ExecutePath; import accord.local.Node; -import accord.local.ExclusiveAsyncExecutor; import accord.messages.Apply; import accord.primitives.Ballot; import accord.primitives.Deps; diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java index 8b215a903ffd..262e98af4e62 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java @@ -28,12 +28,12 @@ import java.util.function.BiConsumer; import accord.api.Data; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Result; import accord.coordinate.CoordinationAdapter; import accord.coordinate.ExecuteFlag.CoordinationFlags; import accord.local.Node; import accord.local.Node.Id; -import accord.local.ExclusiveAsyncExecutor; import accord.messages.Commit; import accord.messages.Commit.Kind; import accord.primitives.AbstractRanges; diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java index 95551f709293..ac17755bb87c 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java @@ -20,13 +20,13 @@ import java.util.function.BiConsumer; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Result; import accord.coordinate.ExecuteFlag; import accord.coordinate.Persist; import accord.coordinate.tracking.AllTracker; import accord.coordinate.tracking.QuorumTracker; import accord.local.Node; -import accord.local.ExclusiveAsyncExecutor; import accord.messages.Apply; import accord.primitives.Ballot; import accord.primitives.Deps; diff --git a/src/java/org/apache/cassandra/utils/concurrent/SignalLock.java b/src/java/org/apache/cassandra/utils/concurrent/SignalLock.java index b84cb78a3289..9a95f38a0360 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/SignalLock.java +++ b/src/java/org/apache/cassandra/utils/concurrent/SignalLock.java @@ -424,12 +424,6 @@ private boolean tryAcquireAsyncInAwaitLoop(long cur, int thread, long threadBitI public boolean incrementAsyncWork(boolean signalIfWaiting) { - return incrementAsyncWork(signalIfWaiting, 1); - } - - public boolean incrementAsyncWork(boolean signalIfWaiting, int increment) - { - Invariants.require(increment >= 1 && increment <= MAX_SIGNAL_COUNT); while (true) { long cur = state; diff --git a/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java b/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java index de181ef6881d..e1a50740effc 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java +++ b/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java @@ -401,10 +401,13 @@ public Object next() advance(scheduled.poll()); } - Action perform = runnable.poll(); + Action perform = runnableByDeadline.peek(); if (perform == null) throw new NoSuchElementException(); + if (perform.deadline() < now - currentJitter) runnable.remove(perform); + else perform = runnable.poll(); + if (!runnableByDeadline.remove(perform) && perform.deadline() > 0) throw new IllegalStateException(); time.tick(perform.deadline()); diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java index eadb7a4f5546..52a2b85409ee 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; @@ -28,14 +30,19 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.LockSupport; import java.util.function.BooleanSupplier; +import java.util.function.Consumer; + +import javax.annotation.Nullable; import org.junit.Test; import accord.api.AsyncExecutor; -import accord.local.ExclusiveAsyncExecutor; +import accord.api.ExclusiveAsyncExecutor; +import accord.utils.async.Cancellable; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; @@ -44,37 +51,144 @@ import org.apache.cassandra.service.accord.AccordExecutorAsyncSubmit; import org.apache.cassandra.service.accord.AccordExecutorSignalLoop; import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.SignalLock; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; -import static org.apache.cassandra.service.accord.AccordService.toFuture; // TODO (required): have simulator intercept ReentantLock so we can test SyncSubmit and SemiSyncSubmit public class AccordExecutorTest extends SimulationTestBase { - static final int EXECUTOR_THREAD_COUNT = 44; - @Test public void signalLoopTest() { - executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, EXECUTOR_THREAD_COUNT, -1, -1, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent()), + executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, SignalLock.MAX_THREADS, -1, -1, TimeUnit.MICROSECONDS, i -> "Loop" + i, new AccordAgent()), 16); } @Test public void signalSpinLoopTest() { - executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, EXECUTOR_THREAD_COUNT, 10, 100, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent()), + executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, SignalLock.MAX_THREADS, 10, 100, TimeUnit.MICROSECONDS, i ->"Loop" + i, new AccordAgent()), 16); } @Test public void ayncSubmitTest() { - executorTest(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, EXECUTOR_THREAD_COUNT, i -> "Loop" + i, new AccordAgent()), + int threads = 32; + executorTest(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, threads, i -> "Loop" + i, new AccordAgent()), 16); } + static class Submitted + { + final AtomicInteger nextId = new AtomicInteger(); + final AtomicInteger doneBefore = new AtomicInteger(); + final ConcurrentHashMap>> consequences = new ConcurrentHashMap<>(); + + boolean isDone() + { + return isDoneBetween(0, nextId.get()); + } + + boolean isDoneBefore(int before) + { + if (!isDoneBetween(doneBefore.get(), before)) + return false; + doneBefore.accumulateAndGet(before, Integer::max); + return true; + } + + boolean isDoneBetween(int from, int before) + { + for (int id = from ; id < before ; ++id) + { + for (Future future : consequences.get(id)) + { + if (!future.isDone()) + return false; + } + } + return true; + } + + Collection> start() + { + ConcurrentLinkedQueue> result = new ConcurrentLinkedQueue<>(); + + while (true) + { + int id = nextId.get(); + Object prev = consequences.putIfAbsent(id, result); + nextId.compareAndSet(id, id + 1); + if (prev == null) + return result; + } + } + } + + static class Control extends ConcurrentLinkedQueue + { + final Submitted submitted; + final AtomicInteger count = new AtomicInteger(); + final float cancelChance; + float processChance; + + Control(float cancelChance, Submitted submitted) + { + this(submitted, cancelChance, ThreadLocalRandom.current().nextFloat() * 0.5f); + } + + Control(Submitted submitted, float cancelChance, float processChance) + { + this.submitted = submitted; + this.cancelChance = cancelChance; + this.processChance = processChance; + } + + void submit(AsyncExecutor executor, Collection> consequences, Consumer>> run) + { + AsyncPromise future = new AsyncPromise<>(); + Cancellable cancel = executor.chain(() -> run.accept(consequences)).begin((success, fail) -> { + if (fail == null) future.trySuccess(null); + else + { + future.tryFailure(fail); + if (fail instanceof CancellationException) + run.accept(submitted.start()); + } + }); + consequences.add(future); + + if (cancel != null && ThreadLocalRandom.current().nextFloat() <= cancelChance) + { + add(cancel); + count.incrementAndGet(); + } + + if (count.get() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance) + { + int cancelCount = 0; + do + { + ++cancelCount; + + float delta = ThreadLocalRandom.current().nextFloat() - 0.5f; + if (delta < 0) processChance /= delta; + else processChance *= -delta; + if (processChance < 0.001f || processChance > 0.999f) + processChance = cancelChance; + } while (count.decrementAndGet() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance); + + // do outside of loop to avoid reentry + while (cancelCount-- > 0) + remove().cancel(); + } + } + } + public void executorTest(SerializableSupplier supplier, int submissionThreads) { simulate(arr(() -> { @@ -84,30 +198,38 @@ public void executorTest(SerializableSupplier supplier, int subm ExecutorPlus submit = executorFactory().pooled("submit-test", submissionThreads); AccordExecutor executor = supplier.get(); Lock lock = executor.unsafeLock(); - ExclusiveAsyncExecutor sequentialExecutor = executor.newSequentialExecutor(); + ExclusiveAsyncExecutor sequentialExecutor = executor.newExclusiveExecutor(); Executor lockExecutor = executorFactory().sequential("lock"); for (float sleepChance : new float[] { 0f, 0.01f, 0.1f }) { for (float lockChance : new float[] { 0f, 0.01f, 0.1f }) { - List> done = new ArrayList<>(); - for (int i = 0 ; i < submissionThreads ; ++i) + for (float cancelChance : new float[] { 0f, 0.01f, 0.1f }) { - int id = i; - done.add(submit.submit(() -> { - try - { - submitLoop(id, lock, executor, sequentialExecutor, lockExecutor, 20, 10, sleepChance, lockChance); - } - catch (ExecutionException | InterruptedException e) - { - throw new RuntimeException(e); - } - })); + System.out.println(String.format("sleepChance %.2f, lockChance %.2f, cancelChance %.2f", sleepChance, lockChance, cancelChance)); + List> done = new ArrayList<>(); + Submitted submitted = new Submitted(); + for (int i = 0; i < submissionThreads; ++i) + { + int id = i; + done.add(submit.submit(() -> { + try + { + submitLoop(id, lock, executor, sequentialExecutor, lockExecutor, 20, 10, sleepChance, lockChance, new Control(cancelChance, submitted), submitted); + } + catch (ExecutionException | InterruptedException e) + { + throw new RuntimeException(e); + } + })); + } + for (Future f : done) + f.get(); + + if (!submitted.isDone()) + throw new AssertionError(); } - for (Future f : done) - f.get(); } } } @@ -119,27 +241,34 @@ public void executorTest(SerializableSupplier supplier, int subm () -> {}, 1L); } - private static void submitLoop(int id, Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance) throws ExecutionException, InterruptedException + private static void submitLoop(int id, Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance, Control control, Submitted submitted) throws ExecutionException, InterruptedException { - ConcurrentLinkedQueue> await = new ConcurrentLinkedQueue<>(); + ConcurrentLinkedQueue> awaitConsequences = new ConcurrentLinkedQueue<>(); while (outerLoop-- > 0) { + List>> allAwaitSubmitted = new ArrayList<>(); for (int i = 0; i < innerLoop; ++i) - submitRecursive(lock, executor, sequentialExecutor, 1 + i, await, sleepChance, lockChance); + { + Collection> awaitSubmitted = submitted.start(); + allAwaitSubmitted.add(awaitSubmitted); + submitRecursive(lock, executor, sequentialExecutor, 1 + i, awaitSubmitted, awaitConsequences, submitted, sleepChance, lockChance, control); + } AtomicBoolean done = new AtomicBoolean(); submitUntil(lock, lockExecutor, sleepChance, done::get); - while (!await.isEmpty()) - await.poll().get(); + for (Collection> awaitSubmitted : allAwaitSubmitted) + await(awaitSubmitted, CancellationException.class); + await(awaitConsequences, null); done.set(true); System.out.println("Loop " + id + '.' + (1 + outerLoop)); } } - private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Collection> await, float sleepChance, float lockChance) + private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Collection> consequences, Collection> awaitConsequences, Submitted submitted, float sleepChance, float lockChance, Control control) { AsyncExecutor submitTo = ThreadLocalRandom.current().nextBoolean() ? executor : sequentialExecutor; - await.add(toFuture(submitTo.chain(() -> { + + control.submit(submitTo, consequences, nextConsequences -> { ThreadLocalRandom rnd = ThreadLocalRandom.current(); boolean locked = false; if (rnd.nextFloat() < lockChance) @@ -147,10 +276,21 @@ private static void submitRecursive(Lock lock, AccordExecutor executor, Exclusiv if (rnd.nextBoolean()) locked = lock.tryLock(); else { locked = true; lock.lock(); } } + if (ThreadLocalRandom.current().nextFloat() < 0.01f) + { + int expectDoneBefore = submitted.nextId.get(); + AsyncPromise afterConsequences = new AsyncPromise<>(); + executor.afterSubmittedAndConsequences(() -> { + if (!submitted.isDoneBefore(expectDoneBefore)) + throw new AssertionError(); + afterConsequences.setSuccess(null); + }); + awaitConsequences.add(afterConsequences); + } try { if (count > 1) - submitRecursive(lock, executor, sequentialExecutor, count -1, await, sleepChance, lockChance); + submitRecursive(lock, executor, sequentialExecutor, count -1, nextConsequences, awaitConsequences, submitted, sleepChance, lockChance, control); if (rnd.nextFloat() < sleepChance) LockSupport.parkNanos(rnd.nextInt(10000, 100000)); } @@ -159,7 +299,7 @@ private static void submitRecursive(Lock lock, AccordExecutor executor, Exclusiv if (locked) lock.unlock(); } - }).beginAsResult())); + }); } private static void submitUntil(Lock lock, Executor executor, float sleepChance, BooleanSupplier done) @@ -188,4 +328,17 @@ private static void submitUntil(Lock lock, Executor executor, float sleepChance, } }); } + + private static void await(Collection> await, @Nullable Class ignore) throws InterruptedException, ExecutionException + { + for (Future future : await) + { + try { future.get(); } + catch (ExecutionException e) + { + if (ignore == null || !(ignore.isInstance(e.getCause()))) + throw e; + } + } + } } diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java index a9a5c6db580f..5d6408f676d0 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java @@ -90,9 +90,9 @@ import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.utils.FBUtilities; +import static accord.local.ExecutionContext.contextFor; import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; -import static accord.local.ExecutionContext.contextFor; import static accord.local.RedundantStatus.SomeStatus.GC_BEFORE_AND_LOCALLY_DURABLE; import static accord.primitives.Routable.Domain.Range; import static accord.primitives.Timestamp.Flag.HLC_BOUND; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java index dcd348685283..3509c2e50612 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java @@ -27,9 +27,9 @@ import accord.api.Key; import accord.api.RoutingKey; import accord.local.Command; +import accord.local.ExecutionContext; import accord.local.LoadKeys; import accord.local.Node; -import accord.local.ExecutionContext; import accord.local.SafeCommand; import accord.local.SafeCommandStore; import accord.local.StoreParticipants; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java index ede89a4d77fa..54dee77c04cd 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java @@ -85,9 +85,9 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Condition; +import static accord.local.ExecutionContext.contextFor; import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; -import static accord.local.ExecutionContext.contextFor; import static accord.utils.Property.qt; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; import static org.apache.cassandra.service.accord.AccordService.getBlocking; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index fc4fdd436755..ae436768517f 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -37,6 +37,7 @@ import accord.api.AsyncExecutor; import accord.api.Data; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Journal; import accord.api.ProgressLog.NoOpProgressLog; import accord.api.RemoteListeners.NoOpRemoteListeners; @@ -56,7 +57,6 @@ import accord.local.Node.Id; import accord.local.NodeCommandStoreService; import accord.local.SafeCommandStore; -import accord.local.ExclusiveAsyncExecutor; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.durability.DurabilityService; @@ -132,7 +132,7 @@ public class AccordTestUtils public static class Commands { - public static Command notDefined(TxnId txnId, PartialTxn txn) + public static Command notDefined(TxnId txnId) { return Command.NotDefined.notDefined(txnId, NotDefined, NotDurable, StoreParticipants.empty(txnId), Ballot.ZERO); } @@ -371,7 +371,7 @@ public AsyncExecutor someExecutor() } @Override - public ExclusiveAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someExclusiveExecutor() { return null; } diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java index 228e2e564a3d..00705915dd5c 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java @@ -35,6 +35,7 @@ import org.assertj.core.api.Assertions; import accord.api.AsyncExecutor; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Journal; import accord.api.LocalListeners; import accord.api.ProgressLog; @@ -59,7 +60,6 @@ import accord.local.NodeCommandStoreService; import accord.local.SafeCommand; import accord.local.SafeCommandStore; -import accord.local.ExclusiveAsyncExecutor; import accord.local.TimeService; import accord.local.durability.DurabilityService; import accord.messages.BeginRecovery; @@ -182,7 +182,7 @@ public AsyncExecutor someExecutor() } @Override - public ExclusiveAsyncExecutor someSequentialExecutor() + public ExclusiveAsyncExecutor someExclusiveExecutor() { return null; } diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java index 5130dce55339..9975fd6b2bc3 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java @@ -49,6 +49,7 @@ import accord.api.Agent; import accord.api.AsyncExecutor; import accord.api.DataStore; +import accord.api.ExclusiveAsyncExecutor; import accord.api.Journal; import accord.api.Key; import accord.api.OwnershipEventListener; @@ -72,7 +73,6 @@ import accord.local.RedundantBefore; import accord.local.SafeCommand; import accord.local.SafeCommandStore; -import accord.local.ExclusiveAsyncExecutor; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.cfk.CommandsForKey; @@ -718,7 +718,7 @@ public TestSafeCommandStore(ExecutionContext context) @Override public NodeCommandStoreService node() { return new NodeCommandStoreService() { @Override public AsyncExecutor someExecutor() { throw new UnsupportedOperationException(); } - @Override public ExclusiveAsyncExecutor someSequentialExecutor() { throw new UnsupportedOperationException(); } + @Override public ExclusiveAsyncExecutor someExclusiveExecutor() { throw new UnsupportedOperationException(); } @Override public long epoch() { return 0;} @Override public Node.Id id() { return Node.Id.NONE; } @Override public Timeouts timeouts() { return null; } diff --git a/test/unit/org/apache/cassandra/utils/AccordGenerators.java b/test/unit/org/apache/cassandra/utils/AccordGenerators.java index 0e7d7a3685b8..a05b644b53f4 100644 --- a/test/unit/org/apache/cassandra/utils/AccordGenerators.java +++ b/test/unit/org/apache/cassandra/utils/AccordGenerators.java @@ -175,7 +175,7 @@ public static Gen commands() switch (targetType) { case notDefined: - return AccordTestUtils.Commands.notDefined(id, txn); + return AccordTestUtils.Commands.notDefined(id); case preaccepted: return AccordTestUtils.Commands.preaccepted(id, txn, executeAt); case committed: From 37b4fbf08fc21ff849774b89dd5bc978fec1bd81 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Wed, 15 Jul 2026 22:50:50 +0100 Subject: [PATCH 04/10] Accord Executor QoS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fairness: tasks are grouped by kind of work, and for each group we track work arrival and service via decaying counters. If work for some group(s) begin to be served at a much lower rate than some other group (e.g. because that group has a large backlog, that by simple priority comes first), then we begin processing some fraction of the work by service-ratio rather than priority. This means that we cannot starve request processing because, e.g., a sync point has produced thousands of state save tasks. Incremental processing: tasks that process many keys may be declared INCR or ASYNC. ASYNC indicates the keys are needed (so should be loaded), but may be used in follow-up work so the task may be executed if the data is not yet loaded. An ASYNC task may be followed by an INCR task that processes keys as they are loaded, with limits to the batch sizes. This limits the latency impact of processing larger transactions such as sync points. INCR tasks may be declared to be processed in either an arbitrary order with no isolation, or “atomically” with its parent task. In the latter case, the complete unit of work appears to execute as a single execution to external observers (blocking progress on unprocessed keys). Key-level ordering: AccordCacheEntry can inflate a special Queue object that can be used to impose ordering constraints: FIFO for atomic work (once it has been part processed, or submitted as part of a parent task); prioritised for tasks that have a key-level priority to impose (i.e. PreAccept/Accept/Commit/Stable/Apply want to be ordered by TxnId); and unsequenced tasks for those that have no sequencing restrictions. Additional improvements: - AccordCacheEntry are now explicitly LOCKED for the duration of their usage, which may span multiple executions for long running INCR tasks. - ExclusiveExecutor releases its ownership lock immediately, since AccordCacheEntry locks guarantee safety (as they will not be released until the task is cleaned up) - CassandraThread tracks active and locked AccordExecutor, so that we may safely and more aggressively use executeMaybeImmediately, safe in the knowledge it will never permit cross-executor executions or multiple executor locks to be held. Also: - Rename PreLoadContext -> ExecutionContext - Break out AccordExecutor/AccordTask into separate package --- modules/accord | 2 +- .../cassandra/concurrent/CassandraThread.java | 14 +- .../apache/cassandra/config/AccordConfig.java | 47 +- .../db/virtual/AccordDebugKeyspace.java | 4 +- .../cassandra/service/accord/AccordCache.java | 240 ++-- .../service/accord/AccordCacheEntry.java | 1149 ++++++++++++++-- .../service/accord/AccordCommandStore.java | 39 +- .../service/accord/AccordCommandStores.java | 34 +- .../service/accord/AccordExecutor.java | 1151 ++++++++-------- .../AccordExecutorAbstractLockLoop.java | 19 +- .../accord/AccordExecutorAbstractLoop.java | 21 +- .../accord/AccordExecutorSignalLoop.java | 119 +- .../service/accord/AccordExecutorSimple.java | 22 +- .../service/accord/AccordSafeCommand.java | 91 +- .../accord/AccordSafeCommandStore.java | 142 +- .../accord/AccordSafeCommandsForKey.java | 86 +- .../service/accord/AccordSafeState.java | 42 +- .../service/accord/AccordService.java | 1 + .../cassandra/service/accord/AccordTask.java | 1202 +++++++++++------ .../service/accord/InMemoryRangeIndex.java | 3 +- .../cassandra/service/accord/RangeIndex.java | 2 +- .../accord/debug/DebugBlockedTxns.java | 6 +- .../service/accord/debug/DebugExecution.java | 54 +- .../service/accord/debug/DebugTxnGraph.java | 6 +- .../accord/journal/JournalRangeIndex.java | 8 +- .../test/accord/AccordDropTableBase.java | 12 +- .../test/accord/AccordLoadTest.java | 16 +- .../AccordJournalConsistentExpungeTest.java | 3 +- .../test/AccordExecutorAndCacheTest.java | 329 +++++ .../CompactionAccordIteratorsTest.java | 13 +- .../service/accord/AccordCacheEntryTest.java | 14 +- .../service/accord/AccordCacheTest.java | 78 +- .../accord/AccordCommandStoreTest.java | 17 +- .../service/accord/AccordCommandTest.java | 2 +- .../service/accord/AccordTaskTest.java | 13 +- .../service/accord/AccordTestUtils.java | 21 +- ...SimpleSimulatedAccordCommandStoreTest.java | 2 +- .../accord/SimulatedAccordTaskTest.java | 26 +- .../CommandsForKeySerializerTest.java | 32 +- 39 files changed, 3213 insertions(+), 1869 deletions(-) create mode 100644 test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java diff --git a/modules/accord b/modules/accord index fff32de2e915..84ac4db03251 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit fff32de2e915772fbc70d16b1c32346313877838 +Subproject commit 84ac4db03251c3b842c98a29868e072792662f51 diff --git a/src/java/org/apache/cassandra/concurrent/CassandraThread.java b/src/java/org/apache/cassandra/concurrent/CassandraThread.java index d66adc24423a..b013a5f0ad54 100644 --- a/src/java/org/apache/cassandra/concurrent/CassandraThread.java +++ b/src/java/org/apache/cassandra/concurrent/CassandraThread.java @@ -31,6 +31,7 @@ public class CassandraThread extends FastThreadLocalThread implements AccordExec private ExecutorLocals executorLocals; private AccordExecutor accordActiveExecutor; private AccordExecutor accordLockedExecutor; + private int accordLockedExecutorDepth; private volatile AccordExecutor.Task accordActiveTask; private static final AtomicReferenceFieldUpdater accordActiveTaskUpdater = AtomicReferenceFieldUpdater.newUpdater(CassandraThread.class, AccordExecutor.Task.class, "accordActiveTask"); @@ -113,18 +114,19 @@ public final AccordExecutor accordLockedExecutor() } @Override - public final boolean trySetAccordLockedExecutor(AccordExecutor newLockedExecutor) + public final boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor) { - if (accordLockedExecutor != null) - return false; - accordLockedExecutor = newLockedExecutor; + if (accordLockedExecutor == null) accordLockedExecutor = newLockedExecutor; + else if (accordLockedExecutor != newLockedExecutor) return false; + ++accordLockedExecutorDepth; return true; } @Override - public final void clearAccordLockedExecutor() + public final void exitAccordLockedExecutor() { - accordLockedExecutor = null; + if (--accordLockedExecutorDepth == 0) + accordLockedExecutor = null; } public final AccordExecutor.Task accordActiveTask() diff --git a/src/java/org/apache/cassandra/config/AccordConfig.java b/src/java/org/apache/cassandra/config/AccordConfig.java index fe226fe92f53..9813f3ffa016 100644 --- a/src/java/org/apache/cassandra/config/AccordConfig.java +++ b/src/java/org/apache/cassandra/config/AccordConfig.java @@ -156,31 +156,12 @@ public enum QueueBalancingModel */ PHASE_ONLY, - /** - * Always pick the task by priority. - */ - PRIORITY_BUDGET, - - /** - * Pick by phase first, so the highest phase with budget ALWAYS runs. - * Within a phase, pick by priority. - */ - PHASE_BUDGET, - /** * Pick by phase first, selecting the phase that has processed the least work recently relative to arrivals. * Within a phase, pick by priority. */ PHASE_FAIR, - /** - * Pick by phase first, selecting the phase that has processed the least work recently relative to arrivals. - * However, each phase has a budget that is consumed on dispatch, and we pick only from those phases with budget. - * If there is no phase with budget, the budget resets. - * Within a phase, pick by priority. - */ - PHASE_BUDGET_FAIR, - /** * While phases are within a threshold of imbalance, pick tasks by priority. * Once the threshold is crossed, over-processed phases have a small penalty applied @@ -190,8 +171,6 @@ public enum QueueBalancingModel * Within a phase, pick by priority. */ BLENDED_PRIORITY_PHASE_FAIR, - - BLENDED_PRIORITY_PHASE_BUDGET_FAIR, } public QueueShardModel queue_shard_model = THREAD_POOL_PER_SHARD; @@ -212,8 +191,29 @@ public enum QueueBalancingModel public Integer queue_flow_imbalance_onset = null; public Integer queue_flow_imbalance_width_shift = null; + public String queue_active_limits; - public String queue_budgets; + + public Boolean queue_nonsync_enabled; + + /** + * Size at which we will begin processing a task that is ASYNC, INCR OR INCR_ATOMIC. + * Note that a size of zero will effectively give implicit priority to INCR_ATOMIC tasks, as they may immediately + * take a FIFO queue slot (which is processed preferentially). + */ + public Integer queue_nonsync_min_batch_size; + + /** + * If there are more than min_batch_size keys ready for an ASYNC, INCR or INCR_ATOMIC task, + * process up to this many keys at once. + */ + public Integer queue_nonsync_max_batch_size; + + /** + * An ASYNC, INCR or INCR_ATOMIC task that is ready to run but waiting for batch_size work will proceed + * once this number of tasks are blocked behind it, regardless of batch_size. + */ + public Integer queue_nonsync_blocked_limit; /** * If set, the signal loop does not match park/unpark pairs, but instead consumers perform timed-park spin waits @@ -240,9 +240,6 @@ public enum QueueBalancingModel */ public volatile OptionaldPositiveInt command_store_shard_count = OptionaldPositiveInt.UNDEFINED; - public volatile OptionaldPositiveInt max_queued_loads = OptionaldPositiveInt.UNDEFINED; - public volatile OptionaldPositiveInt max_queued_range_loads = OptionaldPositiveInt.UNDEFINED; - public volatile OptionaldPositiveInt progress_log_concurrency = OptionaldPositiveInt.UNDEFINED; public DurationSpec.IntMillisecondsBound progress_log_query_fallback_timeout = new DurationSpec.IntMillisecondsBound("1m"); diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 86238688f5bf..3a8c6e32bd8b 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -1650,7 +1650,7 @@ Command get(AccordCommandStore commandStore, TxnId txnId) { try (AccordCommandStore.ExclusiveCaches caches = commandStore.lockCaches()) { - AccordCacheEntry entry = caches.commands().getUnsafe(txnId); + AccordCacheEntry entry = caches.commands().getUnsafe(txnId); return entry == null ? null : entry.getExclusive(); } } @@ -1895,7 +1895,7 @@ private void run(TxnId txnId, int commandStoreId, Function i)); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCache.java b/src/java/org/apache/cassandra/service/accord/AccordCache.java index 0d679b2f519f..a75e6a6e27f8 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCache.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCache.java @@ -44,6 +44,7 @@ import accord.api.RoutingKey; import accord.local.Command; +import accord.local.SafeState; import accord.local.cfk.CommandsForKey; import accord.local.cfk.Serialize; import accord.primitives.Routable; @@ -66,6 +67,7 @@ import org.apache.cassandra.metrics.ShardedHitRate; import org.apache.cassandra.service.accord.AccordCache.Adapter.Shrink; import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; +import org.apache.cassandra.service.accord.AccordCacheEntry.Loading; import org.apache.cassandra.service.accord.AccordCacheEntry.Status; import org.apache.cassandra.service.accord.AccordSafeCommandsForKey.CommandsForKeyCacheEntry; import org.apache.cassandra.service.accord.events.CacheEvents; @@ -79,6 +81,8 @@ import static accord.utils.Invariants.illegalState; import static accord.utils.Invariants.require; +import static org.apache.cassandra.service.accord.AccordCacheEntry.AGE_MASK; +import static org.apache.cassandra.service.accord.AccordCacheEntry.GENERATION_MASK; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADED; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.MODIFIED; @@ -106,7 +110,7 @@ public static void validateLoadOnEvict(boolean value) VALIDATE_LOAD_ON_EVICT = value; } - public interface Adapter + public interface Adapter & AccordSafeState> { enum Shrink { EVICT, DONE, PERFORM_WITHOUT_LOCK } @@ -122,10 +126,10 @@ enum Shrink { EVICT, DONE, PERFORM_WITHOUT_LOCK } long estimateHeapSize(V value); long estimateShrunkHeapSize(Object shrunk); boolean validate(AccordCommandStore commandStore, K key, V value); - S safeRef(AccordCacheEntry node); + S safeRef(AccordCacheEntry node); default Comparator keyComparator() { return null; } - default AccordCacheEntry newEntry(K key, AccordCache.Type.Instance owner) + default AccordCacheEntry newEntry(K key, AccordCache.Type.Instance owner) { return AccordCacheEntry.createReadyToLoad(key, owner); } @@ -151,8 +155,8 @@ public ImmutableStats(Stats stats) private final List> types = new CopyOnWriteArrayList<>(); final AccordCacheEntry.SaveExecutor saveExecutor; - private final IntrusiveLinkedList> evictQueue = new IntrusiveLinkedList<>(); - private final IntrusiveLinkedList> noEvictQueue = new IntrusiveLinkedList<>(); + private final IntrusiveLinkedList> evictQueue = new IntrusiveLinkedList<>(); + private final IntrusiveLinkedList> noEvictQueue = new IntrusiveLinkedList<>(); private long unreferencedBytes; private int unreferenced; @@ -192,16 +196,16 @@ public long capacity() */ void processNoEvictQueue() { - noEvictGeneration = (noEvictGeneration + 1) & 0xffff; + noEvictGeneration = (noEvictGeneration + 1) & GENERATION_MASK; if (noEvictQueue.isEmpty()) return; - Iterator> iter = noEvictQueue.iterator(); + Iterator> iter = noEvictQueue.iterator(); int skipCount = 3; while (skipCount > 0 && iter.hasNext()) { - AccordCacheEntry entry = iter.next(); - int age = (noEvictGeneration - entry.noEvictGeneration()) & 0xffff; + AccordCacheEntry entry = iter.next(); + int age = (noEvictGeneration - entry.noEvictGeneration()) & GENERATION_MASK; if (age >= entry.noEvictMaxAge()) { evictNoEvict.warn(entry, age, entry.noEvictMaxAge()); @@ -226,14 +230,14 @@ void tryShrinkOrEvict(Lock lock) while (bytesCached > maxSizeInBytes && !evictQueue.isEmpty()) { - AccordCacheEntry node = evictQueue.peek(); + AccordCacheEntry node = evictQueue.peek(); shrinkOrEvict(lock, node); } tryShrinkOrEvict = false; } @VisibleForTesting - private void shrinkOrEvict(Lock lock, AccordCacheEntry node) + private void shrinkOrEvict(Lock lock, AccordCacheEntry node) { require(node.references() == 0); @@ -244,7 +248,7 @@ private void shrinkOrEvict(Lock lock, AccordCacheEntry node) } else { - IntrusiveLinkedList> queue; + IntrusiveLinkedList> queue; queue = node.isNoEvict() ? noEvictQueue : evictQueue; node.unlink(); if (shrink == Shrink.DONE) @@ -272,7 +276,7 @@ private void shrinkOrEvict(Lock lock, AccordCacheEntry node) } @VisibleForTesting - public void tryEvict(AccordCacheEntry node) + public void tryEvict(AccordCacheEntry node) { require(node.references() == 0); @@ -290,7 +294,6 @@ public void tryEvict(AccordCacheEntry node) case LOADING: node.loading().loading.cancel(); case WAITING_TO_LOAD: - Invariants.paranoid(node.loadingOrWaiting().waiters == null); case LOADED: node.unlink(); evict(node, true); @@ -305,7 +308,7 @@ public void tryEvict(AccordCacheEntry node) } } - public void saveWhenReadyExclusive(AccordCacheEntry entry, Runnable onSuccess) + public void saveWhenReadyExclusive(AccordCacheEntry entry, Runnable onSuccess) { if (!entry.isSavingOrWaiting() && !entry.saveWhenReady()) onSuccess.run(); @@ -313,7 +316,7 @@ public void saveWhenReadyExclusive(AccordCacheEntry entry, Runnable onSucc entry.savingOrWaitingToSave().identity.onSuccess(onSuccess); } - private void evict(AccordCacheEntry node, boolean updateUnreferenced) + private void evict(AccordCacheEntry node, boolean updateUnreferenced) { if (logger.isTraceEnabled()) logger.trace("Evicting {}", node); @@ -336,25 +339,25 @@ private void evict(AccordCacheEntry node, boolean updateUnreferenced) if (node.status() == LOADED && VALIDATE_LOAD_ON_EVICT) owner.validateLoadEvicted(node); - AccordCacheEntry self = node.owner.remove(node.key()); + AccordCacheEntry self = node.owner.remove(node.key()); Invariants.require(self.references() == 0); require(self == node, "Leaked node detected; was attempting to remove %s but cache had %s", node, self); node.notifyListeners(Listener::onEvict); node.evicted(); } - Collection> load(LoadExecutor loadExecutor, P1 p1, P2 p2, AccordCacheEntry node) + Loading load(LoadExecutor loadExecutor, P1 p1, P2 p2, AccordCacheEntry node) { - return node.load(loadExecutor, p1, p2).waiters(); + return node.load(loadExecutor, p1, p2); } - void loaded(AccordCacheEntry node, V value) + void loaded(AccordCacheEntry node, V value) { node.loaded(value); node.notifyListeners(Listener::onUpdate); } - void failedToLoad(AccordCacheEntry node) + void failedToLoad(AccordCacheEntry node) { Invariants.require(node.references() == 0); if (node.isUnqueued()) @@ -367,38 +370,38 @@ void failedToLoad(AccordCacheEntry node) evict(node, true); } - void saved(AccordCacheEntry node, Object identity, Throwable fail) + void saved(AccordCacheEntry node, Object identity, Throwable fail) { if (node.saved(identity, fail) && node.references() == 0 && node.isUnqueued()) evictQueue.addFirst(node); // add to front since we have just saved, so we were eligible for eviction } - public > void release(S safeRef, AccordTask owner) + public & AccordSafeState> void release(S safeRef, AccordTask owner) { safeRef.global().owner.release(safeRef, owner); } - public > Type newType(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) + public & AccordSafeState> Type newType(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) { Type instance = new Type<>(keyClass, adapter, metrics); types.add(instance); return instance; } - public > Type newType( + public & AccordSafeState> Type newType( Class keyClass, BiFunction loadFunction, QuadFunction saveFunction, Function quickShrink, TriFunction validateFunction, ToLongFunction heapEstimator, - Function, S> safeRefFactory, + Function, S> safeRefFactory, AccordCacheMetrics.Shard metrics) { return newType(keyClass, loadFunction, saveFunction, quickShrink, (i, j) -> j, (c, i, j) -> (V)j, validateFunction, heapEstimator, i -> 0, safeRefFactory, metrics); } - public > Type newType( + public & AccordSafeState> Type newType( Class keyClass, BiFunction loadFunction, QuadFunction saveFunction, @@ -408,7 +411,7 @@ public > Type newType( TriFunction validateFunction, ToLongFunction heapEstimator, ToLongFunction shrunkHeapEstimator, - Function, S> safeRefFactory, + Function, S> safeRefFactory, AccordCacheMetrics.Shard metrics) { return newType(keyClass, new FunctionalAdapter<>(loadFunction, saveFunction, quickShrink, @@ -425,18 +428,18 @@ public > Type newType( public interface Listener { - default void onAdd(AccordCacheEntry state) {} - default void onUpdate(AccordCacheEntry state) {} - default void onEvict(AccordCacheEntry state) {} + default void onAdd(AccordCacheEntry state) {} + default void onUpdate(AccordCacheEntry state) {} + default void onEvict(AccordCacheEntry state) {} } - public class Type> implements CacheSize + public class Type & AccordSafeState> implements CacheSize { - public class Instance implements Iterable> + public class Instance implements Iterable> { final AccordCommandStore commandStore; // TODO (desired): don't need to store key separately as stored in node; ideally use a hash set that allows us to get the current entry - private final Map> cache = new Object2ObjectHashMap<>(); + private final Map> cache = new Object2ObjectHashMap<>(); private List> listeners = null; // TODO (expected): update this after releasing the lock private OrderedKeys orderedKeys; @@ -446,50 +449,51 @@ public Instance(AccordCommandStore commandStore) this.commandStore = commandStore; } - public S acquire(K key) + public final S acquire(K key) { - AccordCacheEntry node = acquire(key, false); + AccordCacheEntry node = acquire(key, false); return adapter.safeRef(node); } - public S acquireIfLoaded(K key) + public final S acquireIfLoadedAndPermitted(K key) { - AccordCacheEntry node = acquire(key, true); + AccordCacheEntry node = acquire(key, true); if (node == null) return null; return adapter.safeRef(node); } - public S acquire(AccordCacheEntry node) + public final S acquire(AccordCacheEntry node) { Invariants.require(node.owner == this); acquireExisting(node, false); return adapter.safeRef(node); } - public void recordPreAcquired(AccordSafeState ref) + public final void recordPreAcquired(AccordCacheEntry entry) { - Invariants.require(ref.global().owner == this); - incrementCacheHits(); + Invariants.require(entry.owner == this); + if (entry.isLoaded()) incrementCacheHits(); + else incrementCacheMisses(); } - private AccordCacheEntry acquire(K key, boolean onlyIfLoaded) + private AccordCacheEntry acquire(K key, boolean onlyIfLoadedAndPermitted) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node == null - ? acquireAbsent(key, onlyIfLoaded) - : acquireExisting(node, onlyIfLoaded); + ? acquireAbsent(key, onlyIfLoadedAndPermitted) + : acquireExisting(node, onlyIfLoadedAndPermitted); } /* * Can only return a LOADING Node (or null) */ - private AccordCacheEntry acquireAbsent(K key, boolean onlyIfLoaded) + private AccordCacheEntry acquireAbsent(K key, boolean onlyIfLoaded) { incrementCacheMisses(); if (onlyIfLoaded) return null; - AccordCacheEntry node = adapter.newEntry(key, this); + AccordCacheEntry node = adapter.newEntry(key, this); node.increment(); Object prev = cache.put(key, node); @@ -506,7 +510,7 @@ private AccordCacheEntry acquireAbsent(K key, boolean onlyIfLoaded) /* * Can't return EVICTED or INITIALIZED */ - private AccordCacheEntry acquireExisting(AccordCacheEntry node, boolean onlyIfLoaded) + private AccordCacheEntry acquireExisting(AccordCacheEntry node, boolean onlyIfLoadedAndPermitted) { boolean isLoaded = node.isLoaded(); if (isLoaded) @@ -514,8 +518,11 @@ private AccordCacheEntry acquireExisting(AccordCacheEntry node, bool else incrementCacheMisses(); - if (onlyIfLoaded && !isLoaded) - return null; + if (onlyIfLoadedAndPermitted) + { + if (!isLoaded || node.hasFifoOrLocked()) + return null; + } if (node.increment() == 1) { @@ -527,21 +534,21 @@ private AccordCacheEntry acquireExisting(AccordCacheEntry node, bool return node; } - public void release(AccordSafeState safeRef, AccordTask owner) + public final void release(S safeRef, AccordTask owner) { K key = safeRef.global().key(); logger.trace("Releasing resources for {}: {}", key, safeRef); - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); - require(!safeRef.isUnsafe()); + require(!safeRef.isReleased()); require(safeRef.global() != null, "safeRef node is null for %s", key); require(safeRef.global() == node, "safeRef node not in map: %s != %s", safeRef.global(), node); require(node.references() > 0, "references (%d) are zero for %s (%s)", node.references(), key, node); require(node.isUnqueued()); boolean evict = false; - if (safeRef.hasUpdate()) + if (safeRef.isModified()) { V update = safeRef.current(); if (update != null) @@ -555,15 +562,12 @@ public void release(AccordSafeState safeRef, AccordTask owner) } node.notifyListeners(Listener::onUpdate); } - else if (node.isLoadingOrWaiting()) - { - node.loadingOrWaiting().remove(owner); - } else { evict = node.is(LOADED) && node.isNull(); } - safeRef.markUnsafe(); + node.remove(owner, safeRef.isSafe()); + safeRef.setReleased(); if (node.decrement() == 0) { @@ -596,9 +600,9 @@ else if (node.isLoadingOrWaiting()) tryShrinkOrEvict = true; } - AccordCacheEntry remove(K key) + final AccordCacheEntry remove(K key) { - AccordCacheEntry result = cache.remove(key); + AccordCacheEntry result = cache.remove(key); if (orderedKeys != null && result != null) orderedKeys.remove(key); return result; @@ -609,7 +613,7 @@ final Type parent() return Type.this; } - public Iterable keysBetween(K start, boolean startInclusive, K end, boolean endInclusive) + public final Iterable keysBetween(K start, boolean startInclusive, K end, boolean endInclusive) { if (orderedKeys == null) orderedKeys = new OrderedKeys<>(adapter.keyComparator(), cache.keySet()); @@ -618,15 +622,15 @@ public Iterable keysBetween(K start, boolean startInclusive, K end, boolean e } @Override - public Iterator> iterator() + public final Iterator> iterator() { return cache.values().iterator(); } - void validateLoadEvicted(AccordCacheEntry node) + final void validateLoadEvicted(AccordCacheEntry node) { @SuppressWarnings("unchecked") - AccordCacheEntry state = (AccordCacheEntry) node; + AccordCacheEntry state = (AccordCacheEntry) node; K key = state.key(); V evicted = state.tryGetFull(); if (evicted == null) @@ -649,46 +653,46 @@ void validateLoadEvicted(AccordCacheEntry node) } @VisibleForTesting - public AccordCacheEntry getUnsafe(K key) + public final AccordCacheEntry getUnsafe(K key) { return cache.get(key); } @VisibleForTesting - public boolean isReferenced(K key) + public final boolean isReferenced(K key) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node != null && node.references() > 0; } @VisibleForTesting - boolean keyIsReferenced(Object key, Class> valClass) + final boolean keyIsReferenced(Object key, Class> valClass) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node != null && node.references() > 0; } @VisibleForTesting - boolean keyIsCached(Object key, Class> valClass) + final boolean keyIsCached(Object key, Class> valClass) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node != null; } @VisibleForTesting - int references(Object key, Class> valClass) + final int references(Object key, Class> valClass) { - AccordCacheEntry node = cache.get(key); + AccordCacheEntry node = cache.get(key); return node != null ? node.references() : 0; } - void notifyListeners(BiConsumer, AccordCacheEntry> notify, AccordCacheEntry node) + final void notifyListeners(BiConsumer, AccordCacheEntry> notify, AccordCacheEntry node) { notifyListeners(listeners, notify, node); notifyListeners(typeListeners, notify, node); } - void notifyListeners(List> listeners, BiConsumer, AccordCacheEntry> notify, AccordCacheEntry node) + final void notifyListeners(List> listeners, BiConsumer, AccordCacheEntry> notify, AccordCacheEntry node) { if (listeners != null) { @@ -698,20 +702,20 @@ void notifyListeners(List> listeners, BiConsumer, } } - public void register(Listener l) + public final void register(Listener l) { if (listeners == null) listeners = new ArrayList<>(); listeners.add(l); } - public void unregister(Listener l) + public final void unregister(Listener l) { if (!tryUnregister(l)) throw illegalState("Listener was not registered"); } - public boolean tryUnregister(Listener l) + public final boolean tryUnregister(Listener l) { if (listeners == null || !listeners.remove(l)) return false; @@ -719,9 +723,23 @@ public boolean tryUnregister(Listener l) listeners = null; return true; } + + final boolean isCommandsForKey() + { + return getClass() == KeyInstance.class; + } + } + + // KeyInstance exists to provide us slightly easier discrimination about the Type an AccordCacheEntry is associated with + public final class KeyInstance extends Instance + { + public KeyInstance(AccordCommandStore commandStore) + { + super(commandStore); + } } - private final Class keyClass; + private final Class keyClass; // type of key, useful primarily for toString(), but also piggyback for deciding Instance type private Adapter adapter; private long bytesCached; private int size; @@ -734,6 +752,8 @@ public boolean tryUnregister(Listener l) public Type(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) { + // Integer and String permitted for testing, but the Invariant exists only to enforce that we construct the right kind of Instance + Invariants.require(keyClass == RoutingKey.class || keyClass == TxnId.class || keyClass == String.class || keyClass == Integer.class); this.keyClass = keyClass; this.adapter = adapter; this.objectSize = metrics.objectSize; @@ -751,7 +771,7 @@ void updateSize(long newSize, long delta, boolean isUnreferenced, boolean update // can be safely garbage collected if empty Instance newInstance(AccordCommandStore commandStore) { - return new Instance(commandStore); + return keyClass == RoutingKey.class ? new KeyInstance(commandStore) : new Instance(commandStore); } private void incrementCacheHits() @@ -867,17 +887,17 @@ public String toString() } @VisibleForTesting - AccordCacheEntry head() + AccordCacheEntry head() { - Iterator> iter = evictQueue.iterator(); + Iterator> iter = evictQueue.iterator(); return iter.hasNext() ? iter.next() : null; } @VisibleForTesting - AccordCacheEntry tail() + AccordCacheEntry tail() { - AccordCacheEntry last = null; - Iterator> iter = evictQueue.iterator(); + AccordCacheEntry last = null; + Iterator> iter = evictQueue.iterator(); while (iter.hasNext()) last = iter.next(); return last; @@ -888,7 +908,7 @@ public boolean isEmpty() return size() == 0; } - Iterable> evictionQueue() + Iterable> evictionQueue() { return evictQueue::iterator; } @@ -937,10 +957,10 @@ static void registerJfrListener(int shardId, AccordCache.Type ty return; type.register(new AccordCache.Listener<>() { - private final IdentityHashMap, CacheEvents.Evict> pendingEvicts = new IdentityHashMap<>(); + private final IdentityHashMap, CacheEvents.Evict> pendingEvicts = new IdentityHashMap<>(); @Override - public void onAdd(AccordCacheEntry state) + public void onAdd(AccordCacheEntry state) { CacheEvents.Add add = new CacheEvents.Add(); CacheEvents.Evict evict = new CacheEvents.Evict(); @@ -957,7 +977,7 @@ public void onAdd(AccordCacheEntry state) } @Override - public void onEvict(AccordCacheEntry state) + public void onEvict(AccordCacheEntry state) { CacheEvents.Evict event = pendingEvicts.remove(state); if (event == null) return; @@ -967,7 +987,7 @@ public void onEvict(AccordCacheEntry state) }); } - private static void updateMutable(AccordCache.Type type, AccordCacheEntry state, CacheEvents event) + private static void updateMutable(AccordCache.Type type, AccordCacheEntry state, CacheEvents event) { event.status = state.status().name(); @@ -987,7 +1007,7 @@ private static void updateMutable(AccordCache.Type type, AccordCacheEnt event.update(); } - static class FunctionalAdapter implements Adapter + static class FunctionalAdapter & AccordSafeState> implements Adapter { final BiFunction load; final QuadFunction save; @@ -997,8 +1017,8 @@ static class FunctionalAdapter implements Adapter final TriFunction validate; final ToLongFunction estimateHeapSize; final ToLongFunction estimateShrunkHeapSize; - final Function, S> newSafeRef; - final BiFunction.Instance, AccordCacheEntry> newNode; + final Function, S> newSafeRef; + final BiFunction.Instance, AccordCacheEntry> newNode; FunctionalAdapter(BiFunction load, QuadFunction save, @@ -1007,8 +1027,8 @@ static class FunctionalAdapter implements Adapter TriFunction validate, ToLongFunction estimateHeapSize, ToLongFunction estimateShrunkHeapSize, - Function, S> newSafeRef, - BiFunction.Instance, AccordCacheEntry> newNode) + Function, S> newSafeRef, + BiFunction.Instance, AccordCacheEntry> newNode) { this.load = load; this.save = save; @@ -1082,13 +1102,13 @@ public long estimateShrunkHeapSize(Object shrunk) } @Override - public S safeRef(AccordCacheEntry node) + public S safeRef(AccordCacheEntry node) { return newSafeRef.apply(node); } @Override - public AccordCacheEntry newEntry(K key, Type.Instance owner) + public AccordCacheEntry newEntry(K key, Type.Instance owner) { return newNode.apply(key, owner); } @@ -1100,7 +1120,7 @@ public Comparator keyComparator() } } - static class SettableWrapper extends FunctionalAdapter + static class SettableWrapper & AccordSafeState> extends FunctionalAdapter { volatile BiFunction load; @@ -1110,9 +1130,9 @@ static class SettableWrapper extends FunctionalAdapter this.load = super.load; } - public static Adapter loadOnly(BiFunction load) + public static & AccordSafeState> Adapter loadOnly(BiFunction load) { - SettableWrapper result = new SettableWrapper<>(new NoOpAdapter<>()); + SettableWrapper result = new SettableWrapper<>(new NoOpAdapter()); result.load = load; return result; } @@ -1124,7 +1144,7 @@ public V load(AccordCommandStore commandStore, K key) } } - static class NoOpAdapter implements Adapter + static class NoOpAdapter & AccordSafeState> implements Adapter { @Override public V load(AccordCommandStore commandStore, K key) { return null; } @Override public Runnable save(AccordCommandStore commandStore, K key, @Nullable V value, @Nullable Object shrunk) { return null; } @@ -1135,7 +1155,7 @@ static class NoOpAdapter implements Adapter @Override public long estimateHeapSize(V value) { return 0; } @Override public long estimateShrunkHeapSize(Object shrunk) { return 0; } @Override public boolean validate(AccordCommandStore commandStore, K key, V value) { return false; } - @Override public S safeRef(AccordCacheEntry node) { return null; } + @Override public S safeRef(AccordCacheEntry node) { return null; } } public static class CommandsForKeyAdapter implements Adapter @@ -1239,7 +1259,7 @@ public boolean validate(AccordCommandStore commandStore, RoutingKey key, Command } @Override - public AccordSafeCommandsForKey safeRef(AccordCacheEntry node) + public AccordSafeCommandsForKey safeRef(AccordCacheEntry node) { return new AccordSafeCommandsForKey(node); } @@ -1251,7 +1271,7 @@ public Comparator keyComparator() } @Override - public AccordCacheEntry newEntry(RoutingKey key, Type.Instance owner) + public AccordCacheEntry newEntry(RoutingKey key, Type.Instance owner) { CommandsForKeyCacheEntry entry = new CommandsForKeyCacheEntry(key, owner); entry.readyToLoad(); @@ -1376,19 +1396,19 @@ public boolean validate(AccordCommandStore commandStore, TxnId key, Command valu } @Override - public AccordSafeCommand safeRef(AccordCacheEntry node) + public AccordSafeCommand safeRef(AccordCacheEntry node) { return new AccordSafeCommand(node); } @Override - public AccordCacheEntry newEntry(TxnId txnId, Type.Instance owner) + public AccordCacheEntry newEntry(TxnId txnId, Type.Instance owner) { - AccordCacheEntry node = new AccordCacheEntry<>(txnId, owner); + AccordCacheEntry node = new AccordCacheEntry<>(txnId, owner); if (txnId.is(Txn.Kind.EphemeralRead)) { node.initialize(null); - int maxAge = (int)Math.min(0xff, 1 + DatabaseDescriptor.getReadRpcTimeout(TimeUnit.SECONDS)); + int maxAge = (int)Math.min(AGE_MASK, 1 + DatabaseDescriptor.getReadRpcTimeout(TimeUnit.SECONDS)); node.markNoEvict(owner.parent().parent().noEvictGeneration, maxAge); } else diff --git a/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java b/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java index 77b56f3203bb..11d8a2b105cd 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java @@ -18,27 +18,43 @@ package org.apache.cassandra.service.accord; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.function.BiConsumer; +import java.util.function.BiPredicate; import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import com.google.common.primitives.Ints; +import accord.local.SafeState; import accord.utils.ArrayBuffers.BufferList; import accord.utils.IntrusiveLinkedList; import accord.utils.IntrusiveLinkedListNode; import accord.utils.Invariants; +import accord.utils.SortedArrays; +import accord.utils.TriConsumer; +import accord.utils.UnhandledEnum; import accord.utils.async.Cancellable; import org.apache.cassandra.service.accord.AccordCache.Adapter; import org.apache.cassandra.service.accord.AccordCache.Adapter.Shrink; +import org.apache.cassandra.service.accord.AccordExecutor.IOTask; import org.apache.cassandra.utils.ObjectSizes; +import static accord.utils.Invariants.nonNull; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.HOLD_QUEUE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.UNLOCKED; +import static org.apache.cassandra.service.accord.AccordCacheEntry.Queue.compare; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NEWLY_BLOCKING_RUNNABLE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NEWLY_RUNNABLE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_LOAD; import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_SAVE; @@ -52,7 +68,7 @@ /** * Global (per CommandStore) state of a cached entity (Command or CommandsForKey). */ -public class AccordCacheEntry extends IntrusiveLinkedListNode +public class AccordCacheEntry & AccordSafeState> extends IntrusiveLinkedListNode { public enum Status { @@ -128,9 +144,12 @@ public enum Status } } - static final int STATUS_MASK = 0x0000001F; - static final int SHRUNK = 0x00000040; - static final int NO_EVICT = 0x00000020; + static final int STATUS_MASK = 0x0000001F; + static final int NO_EVICT = 0x00000020; + static final int SHRUNK = 0x00000040; + static final int LOCKED_MASK = 0x00000180; + static final int LOCKED_SHIFT = Integer.numberOfTrailingZeros(LOCKED_MASK); + static final int LOCKED_HOLDING_QUEUE = HOLD_QUEUE.ordinal() << LOCKED_SHIFT; static final int IS_NOT_EVICTED = 0xF; static final int IS_LOADED = 0x8; static final int IS_NESTED = 0x4; @@ -138,117 +157,1045 @@ public enum Status static final int IS_LOADING_OR_WAITING = 0x2; static final int IS_SAVING_OR_WAITING_MASK = 0xE; static final int IS_SAVING_OR_WAITING = 0xC; + static final int GENERATION_SHIFT = 9; + static final int GENERATION_MASK = 0x7fff; + static final int AGE_SHIFT = 24; + static final int AGE_MASK = 0xff; + + static class Queue + { + private static final int DEFAULT_CAPACITY = 4; + private static final int LOCKED_INDEX = 0; + private static final int PRIORITY_START_INDEX = LOCKED_INDEX + 1; + /** + * [priorityHead..priorityTail) stores a priority-sorted list of tasks + * (fifoTail...fifoHead] stores a fifo queue that runs ahead of any priority tasks + * (fifoTail-unsequencedCount...fifoTail] stores unsequenced tasks that are waiting for a queued incremental task. + * This only happens for TxnId cache entries, since they may lockAndHoldQueue. Once the lock is released, + * any pending unsequenced tasks are notified and immediately made (irrevocably) runnable for this entry. + */ + private AccordTask[] tasks; + // TODO (expected): use bytes/shorts for indexes to keep size down, and have an expanded version of the Queue + // with better algorithmic complexity (e.g. Hash -> IntrusivePriorityHeap) + private int priorityHead, priorityTail, fifoHead, fifoTail; + private int unsequencedSize; + + public Queue() + { + tasks = new AccordTask[DEFAULT_CAPACITY]; + priorityHead = priorityTail = PRIORITY_START_INDEX; + fifoHead = fifoTail = DEFAULT_CAPACITY - 1; + } + + private Queue(Queue copy) + { + tasks = copy.tasks.clone(); + priorityHead = copy.priorityHead; + priorityTail = copy.priorityTail; + fifoHead = copy.fifoHead; + fifoTail = copy.fifoTail; + } + + // returns true if no fifo tasks already queue (i.e. so we become head) + boolean addFifo(AccordTask task) + { + ensureCapacity(); + boolean isHead = fifoHead == fifoTail; + if (unsequencedSize > 0) // simply displace the unsequence task, as they're an unordered list + tasks[fifoTail - unsequencedSize] = tasks[fifoTail]; + tasks[fifoTail--] = task; + validate(); + return isHead; + } + + boolean addUnsequenced(AccordTask task) + { + Invariants.require(task.isUnsequenced()); + ensureCapacity(); + tasks[fifoTail - unsequencedSize++] = task; + return unsequencedSize == 1 && sequencedSize() == 1; + } + + boolean isLocked(AccordTask task) + { + return tasks[LOCKED_INDEX] == task; + } + + AccordTask lockedBy() + { + return tasks[LOCKED_INDEX]; + } + + boolean removeIfHead(AccordTask task) + { + return removeIfFifoHead(task) || removeIfPriorityHead(task); + } + + boolean removeIfFifoHead(AccordTask task) + { + if (!hasFifo()) + return false; + + if (task != tasks[fifoHead]) + return false; + tasks[fifoHead--] = null; + return true; + } + + boolean removeIfPriorityHead(AccordTask task) + { + if (!hasPriority()) + return false; + + if (task != tasks[priorityHead]) + return false; + tasks[priorityHead++] = null; + return true; + } + + void lock(AccordTask task) + { + tasks[LOCKED_INDEX] = task; + } + + void unlock(AccordTask task) + { + Invariants.require(tasks[LOCKED_INDEX] == task); + tasks[LOCKED_INDEX] = null; + } + + // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue + boolean addPrioritised(AccordTask task) + { + ensureCapacity(); + int insertPos = Arrays.binarySearch(tasks, priorityHead, priorityTail, task, Queue::compare); + if (insertPos < 0) + insertPos = -1 - insertPos; + + if (priorityHead == PRIORITY_START_INDEX || insertPos > (priorityTail + priorityHead)/2) + { + System.arraycopy(tasks, insertPos, tasks, insertPos + 1, priorityTail - insertPos); + tasks[insertPos] = task; + priorityTail++; + } + else + { + System.arraycopy(tasks, priorityHead, tasks, priorityHead - 1, insertPos - priorityHead); + tasks[insertPos - 1] = task; + priorityHead--; + } + + validate(); + return fifoHead == fifoTail && tasks[priorityHead] == task; + } + + // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue + void addWaitingToLoad(AccordTask task) + { + ensureCapacity(); + tasks[priorityTail++] = task; + } + + private boolean hasTailRoom() + { + if (priorityTail + unsequencedSize <= fifoTail) + return true; + Invariants.require(priorityTail + unsequencedSize == 1 + fifoTail); + return false; + } + + private void ensureCapacity() + { + if (!hasTailRoom()) + { + if (fifoHead == fifoTail && unsequencedSize == 0 && fifoTail < tasks.length - 1) fifoHead = fifoTail = tasks.length - 1; + else if (priorityHead == priorityTail && priorityHead > PRIORITY_START_INDEX) priorityHead = priorityTail = PRIORITY_START_INDEX; + else if (totalSize() >= (tasks.length - 1) / 2) compact(new AccordTask[tasks.length * 2]); + else compact(tasks); + Invariants.require(hasTailRoom()); + } + } + + private void compact(AccordTask[] into) + { + if (priorityHead == priorityTail) priorityHead = priorityTail = PRIORITY_START_INDEX; + else + { + int priorityLength = priorityTail - priorityHead; + System.arraycopy(tasks, priorityHead, into, PRIORITY_START_INDEX, priorityLength); + int newTail = PRIORITY_START_INDEX + priorityLength; + Invariants.require(newTail <= priorityTail); + if (into == tasks) + Arrays.fill(into, newTail, priorityTail, null); + priorityHead = PRIORITY_START_INDEX; + priorityTail = newTail; + } + + if (fifoHead == fifoTail && unsequencedSize == 0) fifoHead = fifoTail = into.length - 1; + else + { + int fifoLength = fifoHead - fifoTail; + int copyLength = fifoLength + unsequencedSize; + int copyFrom = (fifoTail - unsequencedSize) + 1; + int copyTo = into.length - copyLength; + Invariants.require(copyTo >= copyFrom); + System.arraycopy(tasks, copyFrom, into, copyTo, copyLength); + if (into == tasks) + Arrays.fill(into, copyFrom, copyTo, null); + fifoHead = into.length - 1; + fifoTail = fifoHead - fifoLength; + } + + if (tasks != into) + { + into[LOCKED_INDEX] = tasks[LOCKED_INDEX]; + tasks = into; + } + validate(); + } + + private void validate() + { + for (int i = PRIORITY_START_INDEX ; i < priorityHead ; ++i) + Invariants.require(tasks[i] == null); + for (int i = priorityHead ; i < priorityTail ; ++i) + Invariants.require(tasks[i] != null); + for (int i = priorityTail; i <= fifoTail - unsequencedSize; ++i) + Invariants.require(tasks[i] == null); + for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoHead ; ++i) + Invariants.require(tasks[i] != null); + for (int i = fifoHead + 1 ; i < tasks.length ; ++i) + Invariants.require(tasks[i] == null); + } + + AccordTask peek() + { + if (hasFifo()) return tasks[fifoHead]; + if (hasPriority()) return tasks[priorityHead]; + return null; + } + + AccordTask peekFifo() + { + return hasFifo() ? tasks[fifoHead] : null; + } + + // second task + AccordTask peekBehind() + { + int fifoSize = fifoSize(); + if (fifoSize > 1) + return tasks[fifoHead - 1]; + int priorityIndex = priorityHead + (1 - fifoSize); + if (priorityIndex < priorityTail) + return tasks[priorityIndex]; + return null; + } + + boolean hasFifo() + { + return fifoHead != fifoTail; + } + + boolean hasPriority() + { + return priorityHead != priorityTail; + } + + boolean hasUnsequenced() + { + return unsequencedSize > 0; + } + + int sequencedSize() + { + return prioritySize() + fifoSize(); + } + + int unsequencedSize() + { + return unsequencedSize; + } + + int totalSize() + { + return sequencedSize() + unsequencedSize; + } + + int prioritySize() + { + return priorityTail - priorityHead; + } + + int fifoSize() + { + return fifoHead - fifoTail; + } + + // true iff was head + boolean removeFifoOrPriority(AccordTask task, boolean permitMissing) + { + int fifoIndex = fifoIndexOf(task); + if (fifoIndex >= 0) + { + if (fifoIndex == fifoHead) + { + tasks[fifoHead--] = null; + validate(); + return true; + } + else + { + if (remove(fifoIndex, fifoTail + 1, fifoHead + 1)) ++fifoTail; + else --fifoHead; + validate(); + return false; + } + } + + int priorityIndex = priorityIndexOf(task); + Invariants.require(priorityIndex >= 0 || permitMissing); + if (priorityIndex >= 0) + { + if (priorityIndex == priorityHead) + { + tasks[priorityHead++] = null; + return !hasFifo(); + } + + if (remove(priorityIndex, priorityHead, priorityTail)) ++priorityHead; + else --priorityTail; + return false; + } + + return false; + } + + boolean removeUnsequenced(AccordTask task) + { + int unsequencedIndex = unsequencedIndexOf(task); + if (unsequencedIndex < 0) + return false; + + --unsequencedSize; + tasks[unsequencedIndex] = tasks[fifoTail - unsequencedSize]; + tasks[fifoTail - unsequencedSize] = null; + return true; + } + + // return true IFF was head + private boolean removePriority(AccordTask task, boolean permitAbsent) + { + int i = priorityIndexOf(task); + if (i < 0) + { + Invariants.require(permitAbsent); + return false; + } + + boolean wasHead = i == priorityHead; + if (remove(i, priorityHead, priorityTail)) ++priorityHead; + else --priorityTail; + return wasHead; + } + + // return true if we move the start forwards, false if we moved the end back + private boolean remove(int i, int start, int end) + { + if (i < (start + end)/2) + { + System.arraycopy(tasks, start, tasks, start + 1, i - start); + tasks[start] = null; + return true; + } + else + { + System.arraycopy(tasks, i + 1, tasks, i, end - (i + 1)); + tasks[end - 1] = null; + return false; + } + } + + boolean contains(AccordTask task) + { + return indexOf(task) >= 0; + } + + private int indexOf(AccordTask task) + { + if (tasks[priorityHead] == task) + return priorityHead; + + if (tasks[fifoHead] == task) + return fifoHead; + + int i = priorityIndexOf(task); + if (i >= 0) + return i; + + return fifoIndexOf(task); + } + + private int priorityIndexOf(AccordTask task) + { + if (priorityTail - priorityHead > 16) + { + if (tasks[priorityHead] == task) + return priorityHead; + + int i = SortedArrays.binarySearch(tasks, priorityHead + 1, priorityTail, task, Queue::compare, SortedArrays.Search.CEIL); + if (i < 0) + return -1; + + while (i < priorityTail) + { + if (tasks[i] == task) + return i; + if (compare(task, tasks[i]) != 0) + break; + ++i; + } + } + + for (int i = priorityHead ; i < priorityTail ; ++i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + private int fifoIndexOf(AccordTask task) + { + for (int i = fifoHead ; i > fifoTail ; --i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + private int unsequencedIndexOf(AccordTask task) + { + for (int i = (fifoTail - unsequencedSize) + 1 ; i <= fifoTail ; ++i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + int drainUnsequenced(TriConsumer, P1, P2> forEach, P1 p1, P2 p2) + { + for (int i = (fifoTail - unsequencedSize) + 1 ; i <= fifoTail ; ++i) + { + AccordTask task = tasks[i]; + tasks[i] = null; + // should not be reentrant + forEach.accept(task, p1, p2); + } + int count = unsequencedSize; + unsequencedSize = 0; + return count; + } + + RunnableStatus ensureHeadFifo(AccordTask task) + { + if (hasFifo()) + { + Invariants.require(tasks[fifoHead] == task); + return NOT_RUNNABLE; + } + + if (tasks[priorityHead] == task) + { + tasks[priorityHead++] = null; + addFifo(task); + return STILL_RUNNABLE; + } + else + { + boolean wasPriorityHead = removePriority(task, false); + boolean isFifoHead = addFifo(task); + if (!isFifoHead) + return NOT_RUNNABLE; + if (wasPriorityHead) + return STILL_RUNNABLE; + if (hasPriority() || hasUnsequenced()) + return NEWLY_BLOCKING_RUNNABLE; + return NEWLY_RUNNABLE; + } + } + + static int compare(AccordTask a, AccordTask b) + { + Invariants.require(a != null && b != null); + int c = Long.compare(a.position, b.position); + if (c == 0) + c = a.executionContext().executionKind().compareTo(b.executionContext().executionKind()); + if (c == 0) + c = Long.compare(a.createdAt, b.createdAt); + if (c == 0) + c = Long.compare(a.loadedAt, b.loadedAt); + if (c == 0) + c = a.loggingId().compareTo(b.loggingId()); + return c; + } + + public boolean hasQueued() + { + return hasFifo() || hasPriority(); + } + } + static final long EMPTY_SIZE = ObjectSizes.measure(new AccordCacheEntry<>(null, null)); private final K key; - final AccordCache.Type.Instance owner; + final AccordCache.Type.Instance owner; private Object state; + /** + * Either a single AccordTask or a Queue object. The meaning of a single task is defined by various flags. + * If locked, then the task is not logically part of the queue unless LOCKED_HOLDING_QUEUE. + * If unlocked, or LOCKED_HOLDING_QUEUE, the task represents a single-item queue. + * If the task forms a single-item queue, whether it is FIFO or prioritised is determined by the task's isCacheQueuedFifo flag. + */ + private Object queue; // private int status; + private int unsequenced; int sizeOnHeap; private volatile int references; private static final AtomicIntegerFieldUpdater referencesUpdater = AtomicIntegerFieldUpdater.newUpdater(AccordCacheEntry.class, "references"); - AccordCacheEntry(K key, AccordCache.Type.Instance owner) + AccordCacheEntry(K key, AccordCache.Type.Instance owner) { this.key = key; this.owner = owner; } - void unlink() + private RunnableStatus validate(RunnableStatus status) + { + Invariants.require(queue != null); + AccordTask head = queue instanceof Queue ? ((Queue) queue).peek() : (AccordTask) queue; + Invariants.require(isRunnable(head) || status == NOT_RUNNABLE); + return status; + } + + // TODO (expected): don't unwrap when only one entry, since this may cause us to flap when locking unsequenced tasks + private void maybeUnwrap(Queue q) + { + int size = q.sequencedSize(); + switch (size) + { + case 0: + Invariants.require(q.unsequencedSize() == 0); + queue = isLocked() ? nonNull(q.lockedBy()) : null; + break; + + case 1: + if (isLocked() || q.unsequencedSize() > 0) + break; + queue = q.peek(); + } + } + + private boolean maybeUnwrap(Queue q, AccordTask lockedBy) + { + if (q.sequencedSize() == 0) + { + Invariants.require(q.unsequencedSize() == 0); + queue = lockedBy; + return true; + } + return false; + } + + // assumes already queued with priority + final RunnableStatus moveToFifo(AccordTask task) + { + if (queue != task) + { + Queue q = (Queue) queue; + RunnableStatus status = q.ensureHeadFifo(task); + if (status == NEWLY_BLOCKING_RUNNABLE && isLoaded()) + onChangedHead(q, null, q.peekBehind()); + return validate(isRunnable(task) ? status : NOT_RUNNABLE); + } + return validate(isRunnable(task) ? STILL_RUNNABLE : NOT_RUNNABLE); + } + + // drains ONLY those queued with addWaitingToLoad; addFifo are included in the result but are not removed from the collection + public final BufferList> drainWaitingToLoad() + { + Invariants.require(isLoading()); + Invariants.require(!isLocked()); + BufferList> list = new BufferList<>(); + if (queue != null) + { + if (queue instanceof Queue) + { + Queue q = (Queue) queue; + for (int i = q.priorityHead ; i < q.priorityTail ; ++i) + { + list.add(q.tasks[i]); + q.tasks[i] = null; + } + q.priorityHead = q.priorityTail = Queue.PRIORITY_START_INDEX; + for (int i = q.fifoHead ; i > q.fifoTail ; --i) + list.add(q.tasks[i]); + + maybeUnwrap(q); + } + else + { + AccordTask task = (AccordTask) queue; + list.add(task); + Invariants.require(!isLocked()); + if (!task.isCacheQueuedFifo()) + queue = null; + } + } + return list; + } + + final void remove(AccordTask task, boolean ownsLock) + { + if (queue instanceof Queue) + { + Queue q = (Queue) queue; + boolean remove; + boolean isLocked = isLocked() && q.isLocked(task); + Invariants.require(isLocked == ownsLock); + if (isLocked) + { + // if locked, we've already released unsequenced/pririty/fifo positions unless isLockedHoldingQueue + remove = isLockedHoldingQueue(); + status &= ~LOCKED_MASK; + q.unlock(task); + } + else if (task.isUnsequenced(this)) + { + if (task.isCacheQueued()) + { + if (!q.removeUnsequenced(task)) + releaseUnsequenced(q, task); + } + remove = false; + } + else remove = task.isCacheQueued(); + + if (remove) + { + boolean wasHead = remove && q.removeFifoOrPriority(task, false); + if (isLoaded() && wasHead) + { + unsequenced += q.drainUnsequenced(AccordTask::onChangeHeadStatus, this, NEWLY_RUNNABLE); + onChangedHead(q, q.peek(), null); + } + } + + if (remove || isLocked) + maybeUnwrap(q); + } + else if (queue == task) + { + boolean isLocked = isLocked(); + Invariants.require(isLocked == ownsLock); + if (isLocked) + { + status &= ~LOCKED_MASK; + } + else if (task.isUnsequenced(this)) + { + if (task.isCacheQueued()) --unsequenced; // nothing to release if we hit zero + else Invariants.require(isLoading()); + } + queue = null; + } + else + { + Invariants.require(!ownsLock); + if (task.isUnsequenced(this) && task.isCacheQueued()) + --unsequenced; // nothing to release if we hit zero + } + } + + final boolean isCommandsForKey() + { + return getClass() == AccordSafeCommandsForKey.CommandsForKeyCacheEntry.class; + } + + final RunnableStatus headStatus(AccordTask task) + { + if (queue == task) + return validate(isRunnable(task) ? NEWLY_RUNNABLE : NOT_RUNNABLE); + + Queue q = (Queue) queue; + if (q.peek() != task || !isRunnable(task)) + return NOT_RUNNABLE; + + return validate(q.totalSize() == 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); + } + + private Queue ensureQueue() + { + if (queue instanceof Queue) + return (Queue) queue; + + AccordTask head = (AccordTask) this.queue; + Queue q = new Queue(); + if (isLocked()) + { + if (isLockedHoldingQueue()) + q.addFifo(head); + q.lock(head); + } + else if (head.isCacheQueuedFifo()) q.addFifo(head); + else q.addPrioritised(head); + this.queue = q; + return q; + } + + final void addWaitingToLoad(AccordTask task) + { + Invariants.require(isLoading()); + if (queue == null) queue = task; + else ensureQueue().addWaitingToLoad(task); + } + + final AccordTask head() + { + if (queue == null) + return null; + + if (queue instanceof Queue) + return ((Queue) queue).peek(); + + if (isLocked() && !isLockedHoldingQueue()) + return null; + + return (AccordTask) queue; + } + + final RunnableStatus addUnsequenced(AccordTask task) + { + Invariants.require(isLoaded()); + + AccordTask head = head(); + if (head != null && head.holdsLocksBetweenRuns()) + { + boolean wait = compare(task, head) > 0 || (unsequenced == 0 && head.hasIncrementalStarted()); + if (wait) + { + if (ensureQueue().addUnsequenced(task) && isLoaded() && unsequenced == 0) + head.onChangeHeadStatus(this, STILL_RUNNABLE_NEWLY_BLOCKING); + return NOT_RUNNABLE; + } + else + { + ++unsequenced; + if (unsequenced == 1 && isLoaded()) + head.onChangeHeadStatus(this, NOT_RUNNABLE); + return NEWLY_RUNNABLE; + } + } + + ++unsequenced; + return NEWLY_RUNNABLE; + } + + final int waitingCount() + { + Invariants.require(isLoading()); + return queue == null ? 0 : queue instanceof Queue + ? ((Queue)queue).sequencedSize() + : isLocked() == isLockedHoldingQueue() ? 1 : 0; + } + + public enum RunnableStatus + { + NOT_RUNNABLE, STILL_RUNNABLE, NEWLY_RUNNABLE, NEWLY_BLOCKING_RUNNABLE, STILL_RUNNABLE_NEWLY_BLOCKING + } + + private boolean isRunnable(AccordTask head) + { + return !head.holdsLocksBetweenRuns() || unsequenced == 0; + } + + private RunnableStatus add(AccordTask task, BiPredicate> add) + { + Object prev = this.queue; + if (prev == null) + { + queue = task; + return validate(isRunnable(task) ? NEWLY_RUNNABLE : NOT_RUNNABLE); + } + + Queue q = ensureQueue(); + if (!add.test(q, task)) + { + if (isLoaded() && q.totalSize() == 2) + { + AccordTask head = q.peek(); + if (isRunnable(head)) + head.onChangeHeadStatus(this, STILL_RUNNABLE_NEWLY_BLOCKING); + } + return NOT_RUNNABLE; + } + + boolean isRunnable = isRunnable(task); + int sequencedSize = q.sequencedSize(); + int unsequencedSize = q.unsequencedSize(); + if (sequencedSize + unsequencedSize == 1) // could have one locked and one waiting + return validate(isRunnable ? NEWLY_RUNNABLE : NOT_RUNNABLE); + + if (isLoaded() && sequencedSize > 1) + onChangedHead(q, null, q.peekBehind()); + + return validate(isRunnable ? NEWLY_BLOCKING_RUNNABLE : NOT_RUNNABLE); + } + + final RunnableStatus addPrioritised(AccordTask task) + { + Invariants.require(!isLoading()); + return add(task, Queue::addPrioritised); + } + + final RunnableStatus addFifo(AccordTask task) + { + return add(task, Queue::addFifo); + } + + private void onChangedHead(Queue q, @Nullable AccordTask notifyNewHead, @Nullable AccordTask notifyPrevHead) + { + if (notifyNewHead != null && isRunnable(notifyNewHead)) + notifyNewHead.onChangeHeadStatus(this, q.totalSize() == 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); + if (notifyPrevHead != null && isRunnable(notifyPrevHead)) + notifyPrevHead.onChangeHeadStatus(this, NOT_RUNNABLE); + } + + public enum LockMode + { + /** + * Invalid as a parameter to lock methods, but represents the unlocked state + */ + UNLOCKED, + + /** + * If we're sequenced, remove ourselves from the relevant queue so that the next task can queue itself up. + */ + RELEASE_QUEUE, + + /** + * Hold onto our queue position (which we expect to be the head position, as we're queued to execute). + * This is used exclusively for INCR tasks that may hold onto their TxnId for multiple rounds of execution, + * and prevents later tasks from being scheduled when they will be unable to obtain the lock. + */ + HOLD_QUEUE, + + /** + * Skip all queue accounting (sequenced or unsequenced). This mode is used by optimistic referencing via + * tryLockCaches. + */ + UNQUEUED + } + + /** + * On lock we remove ourselves from the priority/fifo queues and notify the new head + */ + public final V lockExclusive(AccordTask owner, LockMode lockMode) + { + Invariants.require(!isLocked()); + Invariants.require(isRunnable(owner) || owner.isUnsequenced(this)); + + if (queue == owner) + { + Invariants.require(lockMode != UNLOCKED); + } + else if (queue == null) + { + queue = owner; + switch (lockMode) + { + default: throw UnhandledEnum.unknown(lockMode); + case UNLOCKED: throw UnhandledEnum.invalid(UNLOCKED); + case HOLD_QUEUE: throw UnhandledEnum.invalid(HOLD_QUEUE, "Must already be head of the queue"); + case RELEASE_QUEUE: + Invariants.require(owner.isUnsequenced(this) && owner.isCacheQueued(), "Must already be head of the queue"); + --unsequenced; + case UNQUEUED: + } + } + else + { + Queue q = ensureQueue(); + switch (lockMode) + { + default: throw UnhandledEnum.unknown(lockMode); + case UNLOCKED: throw UnhandledEnum.invalid(UNLOCKED); + case HOLD_QUEUE: + Invariants.require(!owner.isUnsequenced(this)); + if (q.hasFifo()) Invariants.require(q.peekFifo() == owner); + else + { + boolean wasHead = q.removeIfPriorityHead(owner); + Invariants.require(wasHead); + q.addFifo(owner); + } + q.lock(owner); + break; + case RELEASE_QUEUE: + if (owner.isUnsequenced(this)) releaseUnsequenced(q, owner); + else + { + boolean wasHead = q.removeIfHead(owner); + Invariants.require(wasHead); + if (isLoaded()) + { + unsequenced += q.drainUnsequenced(AccordTask::onChangeHeadStatus, this, NEWLY_RUNNABLE); + onChangedHead(q, q.peek(), null); + } + if (maybeUnwrap(q, owner)) + break; + } + case UNQUEUED: + q.lock(owner); + } + } + + status |= lockMode.ordinal() << LOCKED_SHIFT; + return getExclusive(); + } + + private void releaseUnsequenced(Queue q, AccordTask release) + { + Invariants.require(release.isCacheQueued()); + if (--unsequenced == 0) + { + AccordTask head = q.peek(); + if (head != null && head.holdsLocksBetweenRuns()) + onChangedHead(q, head, null); + } + } + + final boolean hasFifoOrLocked() + { + if (isLocked()) + return true; + + if (queue == null) + return false; + + if (queue instanceof AccordTask) + return ((AccordTask) queue).isCacheQueuedFifo(); + + return ((Queue)queue).hasFifo(); + } + + final void unlink() { remove(); } - boolean isUnqueued() + final boolean isUnqueued() { return isFree(); } - public K key() + public final K key() { return key; } - public int references() + public final int references() { return references; } - public int increment() + public final int increment() { return referencesUpdater.incrementAndGet(this); } - public int decrement() + public final int decrement() { return referencesUpdater.decrementAndGet(this); } - boolean isLoaded() + final boolean isLocked() + { + return (status & LOCKED_MASK) != 0; + } + + final boolean isLockedHoldingQueue() + { + return (status & LOCKED_MASK) == LOCKED_HOLDING_QUEUE; + } + + final boolean isLoaded() { return (status & IS_LOADED) != 0; } - boolean isModified() + final boolean isModified() { return (status & IS_NOT_EVICTED) >= MODIFIED.ordinal(); } - boolean isNested() + final boolean isNested() { Invariants.require(isLoaded()); return (status & IS_NESTED) != 0; } - boolean isShrunk() + final boolean isShrunk() { return (status & SHRUNK) != 0; } - public boolean is(Status status) + public final boolean is(Status status) { return (this.status & STATUS_MASK) == status.ordinal(); } - boolean isLoadingOrWaiting() + final boolean isLoading() { return (status & IS_LOADING_OR_WAITING_MASK) == IS_LOADING_OR_WAITING; } - boolean isSavingOrWaiting() + final boolean isSavingOrWaiting() { return (status & IS_SAVING_OR_WAITING_MASK) == IS_SAVING_OR_WAITING; } - public boolean isComplete() + public final boolean isComplete() { return !is(LOADING) && !is(SAVING); } - int noEvictGeneration() + final int noEvictGeneration() { Invariants.require(isNoEvict()); - return (status >>> 8) & 0xffff; + return (status >>> GENERATION_SHIFT) & GENERATION_MASK; } - int noEvictMaxAge() + final int noEvictMaxAge() { Invariants.require(isNoEvict()); - return status >>> 24; + return status >>> AGE_SHIFT; } - boolean isNoEvict() + final boolean isNoEvict() { return (status & NO_EVICT) != 0; } - int sizeOnHeap() + final int sizeOnHeap() { return sizeOnHeap; } - void updateSize(AccordCache.Type parent) + final void updateSize(AccordCache.Type parent) { // TODO (expected): we aren't weighing the keys int newSizeOnHeap = Ints.saturatedCast(EMPTY_SIZE + estimateOnHeapSize(parent.adapter())); @@ -256,7 +1203,7 @@ void updateSize(AccordCache.Type parent) sizeOnHeap = newSizeOnHeap; } - void initSize(AccordCache.Type parent) + final void initSize(AccordCache.Type parent) { // TODO (expected): we aren't weighing the keys sizeOnHeap = Ints.saturatedCast(EMPTY_SIZE); @@ -265,7 +1212,7 @@ void initSize(AccordCache.Type parent) } @Override - public String toString() + public final String toString() { return "Node{" + status() + ", key=" + key() + @@ -273,7 +1220,7 @@ public String toString() "}@" + Integer.toHexString(System.identityHashCode(this)); } - public Status status() + public final Status status() { return Status.VALUES[(status & STATUS_MASK)]; } @@ -290,42 +1237,36 @@ private void setStatusUnsafe(Status newStatus) status |= newStatus.ordinal(); } - public void initialize(V value) + public final void initialize(V value) { Invariants.require(state == null); setStatus(LOADED); state = value; } - public void readyToLoad() + public final void readyToLoad() { Invariants.require(state == null); setStatus(WAITING_TO_LOAD); - state = new WaitingToLoad(); } - public void markNoEvict(int generation, int maxAge) + public final void markNoEvict(int generation, int maxAge) { - Invariants.require((maxAge & ~0xff) == 0); - Invariants.require((generation & ~0xffff) == 0); + Invariants.require((maxAge & ~AGE_MASK) == 0); + Invariants.require((generation & ~GENERATION_MASK) == 0); status |= NO_EVICT; - status |= generation << 8; - status |= maxAge << 24; - } - - public LoadingOrWaiting loadingOrWaiting() - { - return (LoadingOrWaiting)state; + status |= generation << GENERATION_SHIFT; + status |= maxAge << AGE_SHIFT; } - void notifyListeners(BiConsumer, AccordCacheEntry> notify) + final void notifyListeners(BiConsumer, AccordCacheEntry> notify) { owner.notifyListeners(notify, this); } - public interface LoadExecutor + interface LoadExecutor { - Cancellable load(P1 p1, P2 p2, AccordCacheEntry entry); + IOTask load(P1 p1, P2 p2, AccordCacheEntry entry); } // functions as both an identity object, and a register of listeners @@ -355,32 +1296,31 @@ static void notify(List onSuccess) } } - public interface SaveExecutor + interface SaveExecutor { - Cancellable save(AccordCacheEntry saving, UniqueSave identity, Runnable save); + Cancellable save(AccordCacheEntry saving, UniqueSave identity, Runnable save); } - public Loading load(LoadExecutor loadExecutor, P1 p1, P2 p2) + final Loading load(LoadExecutor loadExecutor, P1 p1, P2 p2) { Invariants.require(is(WAITING_TO_LOAD), "%s", this); - WaitingToLoad cur = (WaitingToLoad)state; - Loading loading = cur.load(loadExecutor.load(p1, p2, this)); + Loading loading = new Loading(loadExecutor.load(p1, p2, this)); setStatus(LOADING); state = loading; return loading; } - public Loading testLoad() + public final Loading testLoad() { Invariants.require(is(WAITING_TO_LOAD)); - Loading loading = ((WaitingToLoad)state).load(() -> {}); + Loading loading = new Loading(null); setStatus(LOADING); state = loading; return loading; } - public Loading loading() + public final Loading loading() { Invariants.require(is(LOADING), "%s", this); return (Loading) state; @@ -388,7 +1328,7 @@ public Loading loading() // must own the cache's lock when invoked. this is true of most methods in the class, // but this one is less obvious so named as to draw attention - public V getExclusive() + public final V getExclusive() { Invariants.require(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread()); Invariants.require(isLoaded(), "%s", this); @@ -399,14 +1339,19 @@ public V getExclusive() updateSize(parent); } - return (V)unwrap(); + return (V) maybeUnwrap(); + } + + public final void releaseExclusive(S safeState, AccordTask task) + { + owner.release(safeState, task); } - public Object getOrShrunkExclusive() + public final Object getOrShrunkExclusive() { Invariants.require(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread()); Invariants.require(isLoaded(), "%s", this); - return unwrap(); + return maybeUnwrap(); } public V tryGetExclusive() @@ -414,10 +1359,10 @@ public V tryGetExclusive() Invariants.require(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread()); if (!isLoaded() || isShrunk()) return null; - return (V)unwrap(); + return (V) maybeUnwrap(); } - private Object unwrap() + private Object maybeUnwrap() { return isNested() ? ((Nested)state).state : state; } @@ -475,7 +1420,7 @@ Shrink tryShrink() if (isShrunk() || state == null) return Shrink.EVICT; - V cur = (V)unwrap(); + V cur = (V) maybeUnwrap(); Shrink shrink = adapter.decideFullShrink(key, cur); if (shrink == Shrink.PERFORM_WITHOUT_LOCK) return Shrink.PERFORM_WITHOUT_LOCK; @@ -489,12 +1434,12 @@ Shrink tryShrink() V tryGetFull() { - return isShrunk() ? null : (V)unwrap(); + return isShrunk() ? null : (V) maybeUnwrap(); } Object tryGetShrunk() { - return isShrunk() ? unwrap() : null; + return isShrunk() ? maybeUnwrap() : null; } boolean isNull() @@ -583,7 +1528,7 @@ public SavingOrWaitingToSave savingOrWaitingToSave() return (SavingOrWaitingToSave) state; } - public AccordCacheEntry evicted() + public AccordCacheEntry evicted() { if (isNoEvict()) setStatusUnsafe(EVICTED); @@ -597,12 +1542,12 @@ public Throwable failure() return ((FailedToSave)state).cause; } - void tryApplyShrink(Object cur, Object upd, IntrusiveLinkedList> queue) + void tryApplyShrink(Object cur, Object upd, IntrusiveLinkedList> queue) { if (references() > 0 || !isUnqueued()) return; - if (isLoaded() && unwrap() == cur && upd != cur && upd != null) + if (isLoaded() && maybeUnwrap() == cur && upd != cur && upd != null) applyShrink(owner.parent(), cur, upd); queue.addLast(this); } @@ -632,74 +1577,18 @@ private void inflate(AccordCommandStore commandStore, K key, Adapter ad private long estimateOnHeapSize(Adapter adapter) { - Object current = unwrap(); + Object current = maybeUnwrap(); if (current == null) return 0; else if (isShrunk()) return adapter.estimateShrunkHeapSize(current); return adapter.estimateHeapSize((V)current); } - public static abstract class LoadingOrWaiting - { - Collection> waiters; - - public LoadingOrWaiting() - { - } - - public LoadingOrWaiting(Collection> waiters) - { - this.waiters = waiters; - } - - public Collection> waiters() - { - return waiters != null ? waiters : Collections.emptyList(); - } - - public BufferList> copyWaiters() - { - BufferList> list = new BufferList<>(); - if (waiters != null) - list.addAll(waiters); - return list; - } - - public void add(AccordTask waiter) - { - if (waiters == null) - waiters = new ArrayList<>(); - waiters.add(waiter); - } - - public void remove(AccordTask waiter) - { - if (waiters != null) - { - waiters.remove(waiter); - if (waiters.isEmpty()) - waiters = null; - } - } - } - - static class WaitingToLoad extends LoadingOrWaiting - { - public Loading load(Cancellable loading) - { - Invariants.paranoid(waiters == null || !waiters.isEmpty()); - Loading result = new Loading(waiters, loading); - waiters = Collections.emptyList(); - return result; - } - } - - static class Loading extends LoadingOrWaiting + public static class Loading { - public final Cancellable loading; + final IOTask loading; - public Loading(Collection> waiters, Cancellable loading) + Loading(IOTask loading) { - super(waiters); this.loading = loading; } } @@ -755,9 +1644,9 @@ public Throwable failure() } } - public static AccordCacheEntry createReadyToLoad(K key, AccordCache.Type.Instance owner) + public static & AccordSafeState> AccordCacheEntry createReadyToLoad(K key, AccordCache.Type.Instance owner) { - AccordCacheEntry node = new AccordCacheEntry<>(key, owner); + AccordCacheEntry node = new AccordCacheEntry<>(key, owner); node.readyToLoad(); return node; } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index 60205bf890c9..aeae9844e581 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -56,7 +56,6 @@ import accord.local.Command; import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; -import accord.local.CommandSummaries; import accord.local.ExecutionContext; import accord.local.ExecutionContext.Empty; import accord.local.MaxConflicts; @@ -178,13 +177,16 @@ public ExclusiveCaches(AccordExecutor owner, AccordCache global, AccordCache.Typ @Override public AccordSafeCommand acquireIfLoaded(TxnId txnId) { - return commands().acquireIfLoaded(txnId); + // note: we must return false if the entry is locked to enforce ordering. + // note importantly that this is also coupled to the safety of synchronously releasing ExclusiveExecutor.owner, + // rather than waiting until the (potentially asynchronous) cleanup of the task completes + return commands().acquireIfLoadedAndPermitted(txnId); } @Override public AccordSafeCommandsForKey acquireIfLoaded(RoutingKey key) { - return commandsForKeys().acquireIfLoaded(key); + return commandsForKeys().acquireIfLoadedAndPermitted(key); } @Override @@ -299,7 +301,7 @@ public boolean inStore() void tryPreSetup(AccordTask task) { if (inStore() && current != null) - task.presetup(current.task); + task.preSetup(current.task); } public final TableId tableId() @@ -432,13 +434,19 @@ public boolean tryExecuteImmediately(Runnable run) return taskExecutor().tryExecuteImmediately(run); } - public AccordSafeCommandStore begin(AccordTask operation, @Nullable CommandSummaries commandsForRanges) + public AccordSafeCommandStore begin(AccordSafeCommandStore safeStore) { require(current == null); - current = AccordSafeCommandStore.create(operation, commandsForRanges, this); + current = safeStore; return current; } + public void complete(AccordSafeCommandStore store) + { + require(current == store); + current = null; + } + public boolean hasSafeStore() { return current != null; @@ -454,19 +462,6 @@ ProgressLog progressLog() return progressLog; } - public void complete(AccordSafeCommandStore store) - { - require(current == store); - current.postExecute(); - current = null; - } - - public void abort(AccordSafeCommandStore store) - { - Invariants.require(store == current); - current = null; - } - @Override public void shutdown() { @@ -650,7 +645,7 @@ class Ready extends CountingResult implements Runnable public Ready() { super(1); } @Override public void run() { decrement(); } - void maybeFlush(ExclusiveCaches caches, AccordCacheEntry e) + void maybeFlush(ExclusiveCaches caches, AccordCacheEntry e) { if (e.isModified()) { @@ -665,7 +660,7 @@ void maybeFlush(ExclusiveCaches caches, AccordCacheEntry e : caches.commandsForKeys()) + for (AccordCacheEntry e : caches.commandsForKeys()) ready.maybeFlush(caches, e); } else @@ -717,7 +712,7 @@ public AsyncChain replay(TxnId txnId) if (!maybeShouldReplay(txnId)) return AsyncChains.success(null); - return commandStore.chain(ExecutionContext.contextFor(txnId, "Replay"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Replay"), safeStore -> { Replay replay = shouldReplay(txnId, safeStore.unsafeGet(txnId).current().participants()); if (replay == Replay.NONE) return null; diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index f82215919e1e..14accb0fb885 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -55,7 +55,6 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordCommandStore.DurablyAppliedTo; import org.apache.cassandra.service.accord.AccordExecutor.AccordExecutorFactory; -import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_PER_SHARD; import static org.apache.cassandra.config.DatabaseDescriptor.getAccord; @@ -73,7 +72,6 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu private final int mask; private long cacheSize, workingSetSize; - private int maxQueuedLoads, maxQueuedRangeLoads; private boolean shrinkingOn; AccordCommandStores(NodeCommandStoreService node, Agent agent, DataStore store, RandomSource random, @@ -87,12 +85,9 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu cacheSize = DatabaseDescriptor.getAccordCacheSizeInMiB() << 20; workingSetSize = DatabaseDescriptor.getAccordWorkingSetSizeInMiB() << 20; - AccordConfig config = DatabaseDescriptor.getAccord(); - maxQueuedLoads = maxQueuedLoads(config); - maxQueuedRangeLoads = maxQueuedRangeLoads(config); shrinkingOn = DatabaseDescriptor.getAccordCacheShrinkingOn(); refreshCapacities(); - ScheduledExecutors.scheduledFastTasks.scheduleWithFixedDelay(() -> { + ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(() -> { for (AccordExecutor executor : executors) { executor.executeDirectlyWithLock(() -> { @@ -173,13 +168,6 @@ public synchronized void setCapacityAndWorkingSetSize(long newCacheSize, long ne refreshCapacities(); } - public synchronized void setMaxQueuedLoads(int total, int range) - { - maxQueuedLoads = total; - maxQueuedRangeLoads = range; - refreshCapacities(); - } - public synchronized void setShrinking(boolean on) { shrinkingOn = on; @@ -213,14 +201,11 @@ synchronized void refreshCapacities() { long capacityPerExecutor = cacheSize / executors.length; long workingSetPerExecutor = workingSetSize < 0 ? Long.MAX_VALUE : workingSetSize / executors.length; - int maxLoadsPerExecutor = Math.max(1, (maxQueuedLoads + executors.length - 1) / executors.length); - int maxRangeLoadsPerExecutor = Math.max(1, (maxQueuedRangeLoads + executors.length - 1) / executors.length); for (AccordExecutor executor : executors) { executor.executeDirectlyWithLock(() -> { executor.setCapacity(capacityPerExecutor); executor.setWorkingSetSize(workingSetPerExecutor); - executor.setMaxQueuedLoads(maxLoadsPerExecutor, maxRangeLoadsPerExecutor); executor.cacheExclusive().setShrinkingOn(shrinkingOn); }); } @@ -341,21 +326,4 @@ private static int executorShards(AccordConfig config) return Math.max(1, config.queue_shard_count.or(DatabaseDescriptor.getAvailableProcessors() / 8)); } } - - private static int threads(AccordConfig config) - { - return config.queue_thread_count.or(2 * FBUtilities.getAvailableProcessors()); - } - - public static int maxQueuedLoads(AccordConfig config) - { - return config.max_queued_loads.or(FBUtilities.getAvailableProcessors()); - } - - public static int maxQueuedRangeLoads(AccordConfig config) - { - return config.max_queued_range_loads.or(maxQueuedLoads(config) / 4); - } - - } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java index c60017851344..67a4d815c9dc 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java @@ -26,6 +26,7 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.LockSupport; @@ -46,6 +47,7 @@ import accord.impl.AbstractAsyncExecutor; import accord.local.Command; import accord.local.ExecutionContext; +import accord.local.ExecutionContext.ExecutionSequence; import accord.local.cfk.CommandsForKey; import accord.messages.Accept; import accord.messages.Commit; @@ -96,17 +98,20 @@ import org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup; import org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup; import org.apache.cassandra.service.accord.AccordExecutor.Task.GroupKind; +import org.apache.cassandra.service.accord.AccordExecutor.Task.State; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExclusiveExecutor; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutor; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.WithResources; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Condition; import org.apache.cassandra.utils.concurrent.Future; +import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY; +import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY_ATOMIC; import static accord.primitives.Routable.Domain.Range; -import static accord.utils.Invariants.createIllegalState; import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.ORIG_HLC_FIFO; import static org.apache.cassandra.service.accord.AccordCache.CommandAdapter.COMMAND_ADAPTER; import static org.apache.cassandra.service.accord.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER; @@ -131,14 +136,17 @@ import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_SCAN; import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.SAVE; import static org.apache.cassandra.service.accord.AccordExecutor.Task.MAX_TRANCHE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.ASSIGNED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.INITIALIZED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.EXECUTED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.FAILED_TO_LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_OPTIONAL; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_REQUIRED; import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING_OR_EXECUTED; import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_OPTIONAL; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_REQUIRED; import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_SCAN_RANGES; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; import static org.apache.cassandra.utils.Clock.Global.nanoTime; @@ -158,37 +166,39 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor= 0 && FLOW_WIDTH_SHIFT >= 0); switch (BALANCING_MODEL) { default: throw new UnhandledEnum(BALANCING_MODEL); case PRIORITY_ONLY: case BLENDED_PRIORITY_PHASE_FAIR: - case BLENDED_PRIORITY_PHASE_BUDGET_FAIR: - case PRIORITY_BUDGET: BALANCE_BY_POSITION = true; break; case PHASE_ONLY: case PHASE_FAIR: - case PHASE_BUDGET: - case PHASE_BUDGET_FAIR: BALANCE_BY_POSITION = false; } { + // TODO (required): pick default max loads/saves/range loads based on number of threads long global = COUNTER_MASKS, exclusive = COUNTER_MASKS; - global ^= (0x7f ^ 1) << RANGE_SCAN.ordinal(); + global ^= (0x7fL ^ 1) << (RANGE_SCAN.ordinal() * 8); if (config.queue_active_limits != null) { long[] limits = parseEnumParams(config.queue_active_limits, "queue_active_limits"); @@ -199,30 +209,6 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor 1) - { - mins ^= (min ^ 1) << (RANGE_SCAN.ordinal() * 8); // set the default RANGE_SCAN budget to 1 - mins ^= (min ^ 2) << (RANGE_LOAD.ordinal() * 8); // set the default RANGE_LOAD budget to 2 - } - global = selectByOverflowBits(setOverflowWhenLessEqual(limits[0], 0), mins, limits[0]); - } - if (limits[1] != 0) - exclusive = selectByOverflowBits(setOverflowWhenLessEqual(limits[1], 0), minCounterValue(limits[1], 0) * COUNTER_LOWBITS, limits[1]); - } - GLOBAL_QUEUE_BUDGETS = global; - EXCLUSIVE_QUEUE_BUDGETS = exclusive; - } } public static final ShardedDecayingHistograms HISTOGRAMS = new ShardedDecayingHistograms(); @@ -240,15 +226,14 @@ public interface AccordTaskRunner void setAccordActiveExecutor(AccordExecutor newExecutor); AccordExecutor accordLockedExecutor(); - boolean trySetAccordLockedExecutor(AccordExecutor newLockedExecutor); - void clearAccordLockedExecutor(); + boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor); + void exitAccordLockedExecutor(); AccordExecutor.Task accordActiveTask(); // to be called only by the thread itself, so can (eventually) avoid any memory barriers AccordExecutor.Task accordActiveSelfTask(); void setAccordActiveTask(AccordExecutor.Task newActiveTask); - static AccordTaskRunner get() { return get(Thread.currentThread()); @@ -287,7 +272,7 @@ public AccordExecutor accordLockedExecutor() } @Override - public boolean trySetAccordLockedExecutor(AccordExecutor newLockedExecutor) + public boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor) { if (lockedExecutor != null) return false; @@ -296,7 +281,7 @@ public boolean trySetAccordLockedExecutor(AccordExecutor newLockedExecutor) } @Override - public void clearAccordLockedExecutor() + public void exitAccordLockedExecutor() { lockedExecutor = null; } @@ -576,28 +561,16 @@ private void compact(long[] newMins, int[] newCounts, Runnable[] newRuns) final int executorId; private final AccordCache cache; private final ExclusiveGlobalCaches caches; + final AtomicLong uniqueCreatedAt = new AtomicLong(); final DebugExecutor debug = DebugExecutor.maybeDebug(); - // TODO (expected): remove this - /** - * The maximum total number of loads we can queue at once - this includes loads for range transactions, - * which are subject to this limit as well as that imposed by {@link #maxQueuedRangeLoads} - */ - private int maxQueuedLoads = 64; - - /** - * The maximum number of loads exclusively for range transactions we can queue at once; the {@link #maxQueuedLoads} limit also applies. - */ - private int maxQueuedRangeLoads = 8; - private long maxWorkingSetSizeInBytes; private long maxWorkingCapacityInBytes; - private final StandaloneTaskQueue> scanningRanges = new StandaloneTaskQueue<>(TinyEnumSet.encode(SCANNING_RANGES)); // never queried, just parked here while scanning - private final StandaloneTaskQueue> waitingToLoadRangeTxns = new StandaloneTaskQueue<>(TinyEnumSet.encode(WAITING_TO_LOAD)); - private final StandaloneTaskQueue> waitingToLoad = new StandaloneTaskQueue<>(TinyEnumSet.encode(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD)); - private final StandaloneTaskQueue> loading = new StandaloneTaskQueue<>(LOADING); - private final RunnableTaskQueue runnable = new RunnableTaskQueue<>(); + final StandaloneTaskQueue> scanningRanges = new StandaloneTaskQueue<>(TinyEnumSet.encode(SCANNING_RANGES)); // never queried, just parked here while scanning + final StandaloneTaskQueue> loading = new StandaloneTaskQueue<>(TinyEnumSet.encode(LOADING_REQUIRED, LOADING_OPTIONAL)); + final StandaloneTaskQueue> waitingOnCacheQueues = new StandaloneTaskQueue<>(TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL)); + final RunnableTaskQueue runnable = new RunnableTaskQueue<>(); private final Tranches tranches = new Tranches(this); @@ -610,7 +583,6 @@ private void compact(long[] newMins, int[] newCounts, Runnable[] newRuns) private long nextPosition = 1; int tasks; - private int activeLoads, activeRangeLoads; private boolean hasPausedLoading; private List waitingForQuiescence; @@ -662,16 +634,17 @@ public final Lock unsafeLock() final void lock(AccordTaskRunner self) { - if (!self.trySetAccordLockedExecutor(this)) + long startAt = DEBUG_EXECUTION ? 0 : Clock.Global.nanoTime(); + if (!self.tryEnterAccordLockedExecutor(this)) throw new UnsupportedOperationException("To ensure system performance, it is not permitted to lock multiple AccordExecutor simultaneously with the same thread"); //noinspection LockAcquiredButNotSafelyReleased lock.lock(); - if (DEBUG_EXECUTION) debug.onEnterLock(); + if (DEBUG_EXECUTION) debug.onEnterLock(startAt); } final void unlock(AccordTaskRunner self) { - self.clearAccordLockedExecutor(); + self.exitAccordLockedExecutor(); if (DEBUG_EXECUTION) debug.onExitLock(); lock.unlock(); } @@ -687,7 +660,7 @@ final boolean tryLock(AccordTaskRunner self) final boolean onTryLock(AccordTaskRunner self, boolean result) { - if (result && !self.trySetAccordLockedExecutor(this)) + if (result && !self.tryEnterAccordLockedExecutor(this)) { // shouldn't be possible, we should have checked this already lock.unlock(); @@ -709,7 +682,7 @@ public AccordCache cacheUnsafe() final boolean hasAlreadyWaitingToRun() { - return runnable.hasWaiting(); + return runnable.hasWaitingToRun(); } void updateWaitingToRunExclusive() @@ -797,152 +770,33 @@ final void maybeUnpauseLoading() if (!hasPausedLoading) return; - if (cache.weightedSize() < maxWorkingCapacityInBytes || (loading.isEmpty() || runnable.hasWaiting())) + if (cache.weightedSize() < maxWorkingCapacityInBytes && !runnable.hasWaitingToRun()) { hasPausedLoading = false; - enqueueLoadsExclusive(); - } - } - - public abstract boolean hasTasks(); - abstract void beforeUnlockExternal(); - abstract boolean isOwningThread(); - - private void enqueueLoadsExclusive() - { - outer: while (true) - { - StandaloneTaskQueue> queue = waitingToLoadRangeTxns.isEmpty() || activeRangeLoads >= maxQueuedRangeLoads ? waitingToLoad : waitingToLoadRangeTxns; - AccordTask next = queue.peek(); - if (next == null) - return; - - if (hasPausedLoading || cache.weightedSize() >= maxWorkingCapacityInBytes) - { - // we have too much in memory already, and we have work waiting to run, so let that complete before queueing more - if (!loading.isEmpty() || runnable.hasWaiting()) - { - AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); - hasPausedLoading = true; - return; - } - } - - switch (next.state()) - { - default: - { - failExclusive(next, createIllegalState("Unexpected state: " + next.toDescription())); - break; - } - case WAITING_TO_SCAN_RANGES: - if (activeRangeLoads >= maxQueuedRangeLoads) - { - parkRangeLoad(next); - } - else - { - ++activeRangeLoads; - ++activeLoads; - next.rangeScanner().start(this); - updateQueue(next); - } - break; - - case WAITING_TO_LOAD: - while (true) - { - AccordCacheEntry load = next.peekWaitingToLoad(); - boolean isForRange = isForRange(next, load); - if (isForRange && activeRangeLoads >= maxQueuedRangeLoads) - { - parkRangeLoad(next); - continue outer; - } - - Invariants.require(load != null); - ++activeLoads; - if (isForRange) - ++activeRangeLoads; - - for (AccordTask task : cache.load(this, next, isForRange, load)) - { - if (task == next) continue; - if (task.onLoading(load)) - updateQueue(task); - } - Object prev = next.pollWaitingToLoad(); - Invariants.require(prev == load); - if (next.peekWaitingToLoad() == null) - break; - - Invariants.require(next.state() == WAITING_TO_LOAD, "Invalid state: %s", next); - if (activeLoads >= maxQueuedLoads) - return; - } - Invariants.require(next.state().compareTo(LOADING) >= 0, "Invalid state: %s", next); - updateQueue(next); - } + runnable.restart(RANGE_SCAN.ordinal()); + runnable.restart(LOAD.ordinal()); + runnable.restart(RANGE_LOAD.ordinal()); } } - private boolean isForRange(AccordTask task, AccordCacheEntry load) + final void maybePauseLoading() { - boolean isForRangeTxn = task.isRange(); - if (!isForRangeTxn) - return false; - - for (AccordTask t : load.loadingOrWaiting().waiters()) - { - if (!t.isRange()) - return false; - } - return true; - } + if (hasPausedLoading) + return; - private void parkRangeLoad(AccordTask task) - { - if (task.queued() != waitingToLoadRangeTxns) + if (cache.weightedSize() >= maxWorkingCapacityInBytes || !runnable.hasWaitingToRun()) { - task.unqueue(); - waitingToLoadRangeTxns.enqueue(task); + AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); + hasPausedLoading = true; + runnable.stop(RANGE_SCAN.ordinal()); + runnable.stop(LOAD.ordinal()); + runnable.stop(RANGE_LOAD.ordinal()); } } - private void updateQueue(AccordTask task) - { - task.unqueueIfQueued(); - switch (task.state()) - { - default: throw new AssertionError("Unexpected state: " + task.toDescription()); - case WAITING_TO_SCAN_RANGES: - case WAITING_TO_LOAD: - waitingToLoad.enqueue(task); - break; - case SCANNING_RANGES: - scanningRanges.enqueue(task); - break; - case LOADING: - loading.enqueue(task); - break; - case WAITING_TO_RUN: - waitingToRun(task); - break; - } - } - - private void waitingToRun(AccordTask task) - { - task.onWaitingToRun(); - task.commandStore.exclusiveExecutor.enqueue(task); - } - - private void waitingToRun(Task task, @Nullable ExclusiveExecutor queue) - { - task.onWaitingToRun(); - if (queue == null) runnable.enqueue(task); - else queue.enqueue(task); - } + public abstract boolean hasTasks(); + abstract void beforeUnlockExternal(); + abstract boolean isOwningThread(); public ExclusiveExecutor executor() { @@ -968,13 +822,14 @@ public void cancel(AccordTask task) } @Override - public Cancellable load(AccordTask parent, Boolean isForRange, AccordCacheEntry entry) + public LoadRunnable load(AccordTask parent, Boolean isForRange, AccordCacheEntry entry) { - return submitPlainExclusive(parent, newLoad(entry, isForRange)); + LoadRunnable load = newLoad(entry, isForRange); + return submitPlainExclusive(parent, load); } @Override - public Cancellable save(AccordCacheEntry entry, UniqueSave identity, Runnable save) + public Cancellable save(AccordCacheEntry entry, UniqueSave identity, Runnable save) { return submitPlainExclusive(null, new SaveRunnable(entry, identity, save)); } @@ -1003,15 +858,7 @@ private void submit(QuintConsumer void submit(AccordTask operation) { - submit(AccordExecutor::submitExclusive, i -> i, operation); - } - - void submitExclusive(AccordTask task) - { - registerExclusive(task); - task.setupExclusive(); - updateQueue(task); - enqueueLoadsExclusive(); + submit((self, task) -> task.submitExclusive(self), i -> i, operation); } public void submitExclusive(Runnable runnable) @@ -1028,7 +875,10 @@ private Cancellable submitPlain(Plain task) private void submitPlainExclusive(Plain task) { registerExclusive(task); - waitingToRun(task, task.executor()); + task.onLoaded(); + ExclusiveExecutor executor = task.executor(); + if (executor == null) runnable.enqueue(task, true); + else executor.enqueue(task, true); } Cancellable submitPlainExclusive(Task parent, GlobalGroup group, AbstractIOTask task) @@ -1039,10 +889,11 @@ Cancellable submitPlainExclusive(Task parent, GlobalGroup group, AbstractIOTask final T submitPlainExclusive(Task parent, T task) { Invariants.require(isOwningThread()); + task.setStateExclusive(WAITING_TO_RUN); if (parent == null) registerExclusive(task); else registerConsequenceExclusive(parent, task); - task.onWaitingToRun(); - runnable.enqueue(task); + task.onLoaded(); + runnable.enqueue(task, true); return task; } @@ -1052,10 +903,10 @@ private void registerConsequenceExclusive(Task parent, Task task) int tranche = parent.tranche(); tranches.addInherited(tranche, parent.position); task.position = parent.position; - task.setInheritedTranche(tranche); + task.setInheritedWithTranche(tranche); } - private void registerExclusive(Task task) + void registerExclusive(Task task) { ++tasks; if (task.hasInherited()) @@ -1083,11 +934,15 @@ else if (position < minPosition) } } - final void completeTaskExclusive(Task task) + final void cleanupTaskExclusive(Task task, boolean executed) { - runnable.complete(task); - try { task.cleanupExclusive(this); } - finally { cache.tryShrinkOrEvict(lock); } + runnable.cleanup(task); + try { task.cleanupExclusive(this, executed); } + finally + { + cache.tryShrinkOrEvict(lock); + maybeUnpauseLoading(); + } } final void unregisterExclusive(Task task) @@ -1099,103 +954,84 @@ final void unregisterExclusive(Task task) final void cancelExclusive(AccordTask task) { - AccordTask.State state = task.state(); + State state = task.state(); switch (state) { default: throw new UnhandledEnum(state); case SCANNING_RANGES: - case LOADING: - case WAITING_TO_LOAD: - case WAITING_TO_SCAN_RANGES: + case LOADING_REQUIRED: + case LOADING_OPTIONAL: + case WAITING_ON_REQUIRED: + case WAITING_ON_OPTIONAL: case WAITING_TO_RUN: - task.unqueueIfQueued(); - try { task.cancelExclusive(); } - finally { completeTaskExclusive(task); } - break; - - case INITIALIZED: // TODO (expected): preferable to be able to cancel at this stage, even if unlikely to trigger at this phase - case ASSIGNED: + if (!task.hasIncrementalStarted()) + { + task.unqueueIfQueued(); + try { task.cancelExclusive(); } + finally { cleanupTaskExclusive(task, false); } + break; + } + case UNINITIALIZED: // TODO (expected): preferable to be able to cancel at this stage, even if unlikely to trigger at this phase case RUNNING: - case PERSISTING: - case FINISHED: + case INCOMPLETE: + case EXECUTED: case CANCELLED: - case FAILED: + case FAILED_TO_LOAD: // cannot safely cancel } } void onScannedRangesExclusive(AccordTask task, Throwable fail) { - --activeLoads; - --activeRangeLoads; // the task may have already been cancelled, in which case we don't need to fail it - if (!task.state().isExecuted()) - { - if (fail != null) - { - failExclusive(task, fail); - } - else - { - task.rangeScanner().scannedExclusive(); - updateQueue(task); - } - } - enqueueLoadsExclusive(); + if (task.state().isExecuted()) + return; + + if (fail != null) failExclusive(task, fail, FAILED_TO_LOAD); + else task.rangeScanner().scannedExclusive(); } - private void failExclusive(AccordTask task, Throwable fail) + private void failExclusive(AccordTask task, Throwable fail, State newState) { if (task.state().isExecuted()) return; - try { task.failExclusive(fail); } + try { task.failExclusive(fail, newState); } catch (Throwable t) { agent.onException(t); } finally { task.unqueueIfQueued(); - completeTaskExclusive(task); + cleanupTaskExclusive(task, false); } } - private void onSavedExclusive(AccordCacheEntry state, Object identity, Throwable fail) + private void onSavedExclusive(AccordCacheEntry state, Object identity, Throwable fail) { cache.saved(state, identity, fail); } - private void onLoadedExclusive(AccordCacheEntry loaded, V value, Throwable fail, boolean isForRange) + private void onLoadedExclusive(AccordCacheEntry loaded, V value, Throwable fail) { - --activeLoads; - if (isForRange) - --activeRangeLoads; + if (loaded.status() == EVICTED) + return; - if (loaded.status() != EVICTED) + try (ArrayBuffers.BufferList> tasks = loaded.drainWaitingToLoad()) { - try (ArrayBuffers.BufferList> tasks = loaded.loading().copyWaiters()) + if (fail != null) { - if (fail != null) - { - for (AccordTask task : tasks) - failExclusive(task, fail); - cache.failedToLoad(loaded); - } - else - { - cache.loaded(loaded, value); - for (AccordTask task : tasks) - { - if (task.onLoad(loaded)) - { - Invariants.require(task.queued() == loading); - task.unqueue(); - waitingToRun(task); - } - } - } + for (AccordTask task : tasks) + failExclusive(task, fail, FAILED_TO_LOAD); + cache.failedToLoad(loaded); + } + else + { + cache.loaded(loaded, value); + for (AccordTask task : tasks) + task.onLoadOneExclusive(loaded); } } - enqueueLoadsExclusive(); + maybePauseLoading(); } private Task inherit() @@ -1264,7 +1100,7 @@ public void executeDirectlyWithLock(Runnable command) finally { beforeUnlockExternal(); - self.clearAccordLockedExecutor(); + self.exitAccordLockedExecutor(); unlock(self); } } @@ -1286,15 +1122,6 @@ public void setWorkingSetSize(long bytes) maxWorkingCapacityInBytes = Long.MAX_VALUE; } - public void setMaxQueuedLoads(int total, int range) - { - Invariants.require(isOwningThread()); - Invariants.requireArgument(total >= 1, "Must permit at least one load"); - Invariants.requireArgument(range >= 1, "Must permit at least one range load"); - maxQueuedLoads = total; - maxQueuedRangeLoads = range; - } - @Override public long capacity() { @@ -1332,24 +1159,39 @@ public DebuggableTask running() public static abstract class Task extends IntrusiveHeapNode { + private static final int WAITING_ON_OPTIONAL_BIT = 1 << 5; + private static final int WAITING_TO_RUN_BIT = 1 << 6; + private static final int INCOMPLETE_BIT = 1 << 9; + enum State { - INITIALIZED(), - WAITING_TO_SCAN_RANGES(INITIALIZED), - SCANNING_RANGES(WAITING_TO_SCAN_RANGES), - WAITING_TO_LOAD(INITIALIZED, SCANNING_RANGES), - LOADING(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD), - WAITING_TO_RUN(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD, LOADING), - ASSIGNED(WAITING_TO_RUN), - RUNNING(ASSIGNED), - PERSISTING(RUNNING), - FINISHED(RUNNING, PERSISTING), - CANCELLED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, ASSIGNED), - FAILED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, ASSIGNED, RUNNING, PERSISTING); + UNINITIALIZED(), + SCANNING_RANGES(UNINITIALIZED), + LOADING_REQUIRED(UNINITIALIZED, SCANNING_RANGES), + LOADING_OPTIONAL(UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED), + WAITING_ON_REQUIRED(WAITING_ON_OPTIONAL_BIT | WAITING_TO_RUN_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), + WAITING_ON_OPTIONAL(WAITING_TO_RUN_BIT | INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED), + WAITING_TO_RUN(INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL), + RUNNING(WAITING_TO_RUN), + EXECUTED(RUNNING), + INCOMPLETE(RUNNING), + FAILED_TO_LOAD(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), + FAILED_OTHER(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN), + CANCELLED(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN, RUNNING), + ; private final int permittedFrom; + public static final int WAITING = TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN); + public static final int WAITING_OR_RUNNING = WAITING | TinyEnumSet.encode(RUNNING); + public static final int RUNNING_OR_EXECUTED = WAITING | TinyEnumSet.encode(RUNNING, EXECUTED); static final State[] VALUES = values(); + static + { + // hack to allow us to create loops in our enum transition declarations + Invariants.require(INCOMPLETE_BIT == 1 << INCOMPLETE.ordinal()); + } + State() { this.permittedFrom = 0; @@ -1357,7 +1199,12 @@ enum State State(State ... permittedFroms) { - int permittedFrom = 0; + this(0, permittedFroms); + } + + State(int additional, State ... permittedFroms) + { + int permittedFrom = additional; for (State state : permittedFroms) permittedFrom |= 1 << state.ordinal(); this.permittedFrom = permittedFrom; @@ -1370,12 +1217,12 @@ boolean isPermittedFrom(int prevOrdinal) boolean isExecuted() { - return this.compareTo(PERSISTING) >= 0; + return this.compareTo(EXECUTED) >= 0; } - boolean isComplete() + boolean hasStarted() { - return this.compareTo(FINISHED) >= 0; + return this.compareTo(RUNNING) >= 0; } static State forOrdinal(int ordinal) @@ -1384,27 +1231,25 @@ static State forOrdinal(int ordinal) } } + enum RunState + { + NONE, PERSISTING, SUCCESS, FAILED; + + private static final RunState[] VALUES = values(); + static RunState forOrdinal(int ordinal) + { + return VALUES[ordinal]; + } + } + enum GlobalGroup { COMMAND_STORE, LOAD, SAVE, OTHER, - RANGE_LOAD(true), - RANGE_SCAN(true), - ; - - final int bits; - - GlobalGroup() - { - this(false); - } - - GlobalGroup(boolean isRange) - { - this.bits = (ordinal() << GLOBAL_GROUP_SHIFT) | (isRange ? RANGE_BIT : 0); - } + RANGE_LOAD, + RANGE_SCAN, } enum ExclusiveGroup @@ -1414,21 +1259,9 @@ enum ExclusiveGroup COMMIT, ACCEPT, OTHER, + RECOVER, PREACCEPT, - RANGE(true), - ; - - final int bits; - - ExclusiveGroup() - { - this(false); - } - - ExclusiveGroup(boolean isRange) - { - this.bits = (ordinal() << EXCLUSIVE_GROUP_SHIFT) | (isRange ? RANGE_BIT : 0); - } + RANGE, } public enum GroupKind @@ -1448,15 +1281,39 @@ public enum GroupKind private static final int STATE_MASK = 0xf; private static final int GROUP_MASK = 0x7; - private static final int GLOBAL_GROUP_SHIFT = 7; private static final int EXCLUSIVE_GROUP_SHIFT = 4; - private static final int RANGE_BIT = 1 << 10; - private static final int CLEANUP_BIT = 1 << 11; - private static final int HAS_TRANCHE_BIT = 1 << 12; - private static final int HAS_INHERITED_BIT = 1 << 13; - private static final int TRANCHE_SHIFT = 14; + private static final int GLOBAL_GROUP_SHIFT = 7; + + private static final int NONSYNC_BIT = 1 << 10; + private static final int CACHE_QUEUED_BIT = 1 << 11; + + private static final int INCREMENTAL_MASK = 0x3 << 12; + private static final int INCREMENTAL = 0x1 << 12; + private static final int INCREMENTAL_STARTED = 0x2 << 12; + private static final int INCREMENTAL_FINISHING = 0x3 << 12; + + private static final int SEQUENCED_SHIFT = 14; + private static final int SEQUENCED_MASK = 0x3 << SEQUENCED_SHIFT; + private static final int SEQUENCED_PRIORITY = 0x1 << SEQUENCED_SHIFT; + private static final int SEQUENCED_ATOMIC = 0x2 << SEQUENCED_SHIFT; + private static final int SEQUENCED_ATOMIC_AND_QUEUED = 0x3 << SEQUENCED_SHIFT; + + // spare two bits + + private static final int HAS_TRANCHE_BIT = 1 << 18; + private static final int HAS_INHERITED_BIT = 1 << 19; + private static final int HAS_INHERITED_RANGE_SCAN_BIT = 1 << 20; + + private static final int TRANCHE_SHIFT = 22; static final int MAX_TRANCHE = 0x3ff; + static + { + Invariants.require(SEQUENCED_PRIORITY == BY_PRIORITY.ordinal() << SEQUENCED_SHIFT); + Invariants.require(SEQUENCED_ATOMIC == BY_PRIORITY_ATOMIC.ordinal() << SEQUENCED_SHIFT); + Invariants.require(ExecutionSequence.values().length <= 3); + } + public final WithResources resources; Task next; @@ -1466,39 +1323,44 @@ public enum GroupKind // TODO (expected): do we need this? we should be able to determine the queue from state() if needed for e.g. cancellation private TaskQueue queued; + public final long createdAt; // TODO (expected): expose via executors vtable // TODO (expected): use just one long and some flag bits to indicate which point it represents, and report incrementally - public long createdAt = nanoTime(), waitingToRunAt, runningAt, cleanupAt; + public long loadedAt, runningAt, completeAt; + private byte runState; - protected Task(GlobalGroup group, State state) + Task(GlobalGroup group) { resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); - info = init(group, ExclusiveGroup.OTHER, state); + info = init(group, ExclusiveGroup.OTHER); + createdAt = nanoTime(); } - protected Task(ExclusiveGroup group, State state) + Task(ExclusiveGroup group) { resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); - info = init(GlobalGroup.OTHER, group, state); + info = init(GlobalGroup.OTHER, group); + createdAt = nanoTime(); } - protected Task(GlobalGroup group, State state, long position, int tranche) + Task(GlobalGroup group, long position, int tranche) { - this(group, state); + this(group); this.position = position; - setInheritedTranche(tranche); + setInheritedWithTranche(tranche); } - protected Task(ExclusiveGroup group, State state, long position, int tranche) + Task(ExclusiveGroup group, long position, int tranche) { - this(group, state); + this(group); this.position = position; - setInheritedTranche(tranche); + setInheritedWithTranche(tranche); } - protected Task(ExecutionContext context) + protected Task(ExecutionContext context, AtomicLong lastCreatedAt) { resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + createdAt = lastCreatedAt.accumulateAndGet(nanoTime(), (prev, next) -> next < prev ? prev + 1 : next); ExclusiveGroup group = ExclusiveGroup.OTHER; TxnId txnId = context.primaryTxnId(); if (txnId != null) @@ -1511,7 +1373,7 @@ protected Task(ExecutionContext context) case HLC_FIFO: case ORIG_HLC_FIFO: { - // TODO (expected): we should process messages for a TxnId together, to avoid processing delayed messages out of order + // TODO (expected): port to ExecutionKind; also we aren't consistent about using Ballot if (context instanceof Request) { MessageType type = ((Request) context).type(); @@ -1576,7 +1438,7 @@ protected Task(ExecutionContext context) } } - this.info = init(GlobalGroup.OTHER, group, INITIALIZED); + this.info = init(GlobalGroup.OTHER, group); if (txnId != null) this.position = txnId.hlc(); } @@ -1588,44 +1450,9 @@ public final Task unwrap() return this; } - final void setReadyToCleanup() - { - info |= CLEANUP_BIT; - } - - final boolean isReadyToCleanup() - { - return 0 != (info & CLEANUP_BIT); - } - - final void onRunning() - { - runningAt = nanoTime(); - if (DEBUG_EXECUTION) ((DebugTask)resources).onRunning(); - } - - final void onRunComplete() + static Task unwrap(Task task) { - if (DEBUG_EXECUTION) ((DebugTask)resources).onRunComplete(); - } - - final void onWaitingToRun() - { - waitingToRunAt = nanoTime(); - } - - static Task reverse(Task unqueued) - { - Task prev = null; - Task cur = unqueued; - while (cur != null) - { - Task next = cur.next; - cur.next = prev; - prev = cur; - cur = next; - } - return prev; + return task == null ? null : task.unwrap(); } public DebuggableTask debuggable() { return null; } @@ -1637,43 +1464,129 @@ static Task reverse(Task unqueued) /** * Prepare to run while holding the state cache lock */ - abstract protected void preRunExclusive(); + void preRunExclusive() { setStateExclusive(RUNNING); } /** * Run the command; the state cache lock may or may not be held depending on the executor implementation */ - protected abstract void run(); + abstract void run(); /** * Fail the command; the state cache lock may or may not be held depending on the executor implementation */ - abstract protected void fail(Throwable fail); + abstract void reportFailure(Throwable fail); + + final void failExclusive(Throwable fail, State newState) + { + try { setStateExclusive(newState); } + finally { reportFailure(fail); } + + } - abstract protected boolean isNewWork(); + final void failExecution(Throwable fail) + { + Invariants.require(is(RUNNING)); + try { setRunState(RunState.FAILED); } + finally { reportFailure(fail); } + + } + + abstract boolean isNewWork(); /** * Cleanup the command while holding the state cache lock */ - protected void cleanupExclusive(AccordExecutor executor) + void cleanupExclusive(AccordExecutor executor, boolean executed) { + if (executed) setStateExclusive(EXECUTED); + else Invariants.require(state().isExecuted()); executor.unregisterExclusive(this); - cleanupAt = nanoTime(); + completeAt = nanoTime(); if (runningAt != 0) { - if (waitingToRunAt == 0) - waitingToRunAt = runningAt; - executor.elapsedWaitingToRun.increment(runningAt - waitingToRunAt, runningAt); - executor.elapsedPreparingToRun.increment(waitingToRunAt - createdAt, runningAt); - executor.elapsedRunning.increment(cleanupAt - runningAt, cleanupAt); - executor.elapsed.increment(cleanupAt - createdAt, cleanupAt); + if (loadedAt == 0) + loadedAt = runningAt; + executor.elapsedWaitingToRun.increment(runningAt - loadedAt, runningAt); + executor.elapsedPreparingToRun.increment(loadedAt - createdAt, runningAt); + executor.elapsedRunning.increment(completeAt - runningAt, completeAt); + executor.elapsed.increment(completeAt - createdAt, completeAt); } if (DEBUG_EXECUTION) DebugTask.get(this).onCompleted(executor.debug); } void cancelExclusive(AccordExecutor owner) {} - public final State state() + @Nullable + final TaskQueue queued() { + return queued; + } + + final void unqueueIfQueued() + { + if (queued != null) + { + queued.unqueue(this); + queued = null; + } + } + + final void unqueue(TaskQueue expected) + { + Invariants.require(queued == expected, "%s != %s", queued, expected); + queued.unqueue(this); + queued = null; + } + + final void unsetQueue(TaskQueue expected) + { + Invariants.require(queued == expected, "%s != %s", queued, expected); + queued = null; + } + + final void setQueue(TaskQueue queue) + { + Invariants.require(queued == null); + Invariants.require(isCompatible(queue)); + queued = queue; + } + + final void onRunning() + { + runningAt = nanoTime(); + if (DEBUG_EXECUTION) ((DebugTask)resources).onRunning(); + } + + final void onRunComplete() + { + if (DEBUG_EXECUTION) ((DebugTask)resources).onRunComplete(); + } + + final void onLoaded() + { + loadedAt = nanoTime(); + } + + final State state() + { + return State.forOrdinal(stateOrdinal()); + } + + final RunState runState() + { + return RunState.forOrdinal(runState); + } + + final Enum describeState() + { + State state = state(); + if (state == RUNNING || state == EXECUTED) + { + RunState runState = runState(); + if (runState == RunState.NONE) + return state; + return runState; + } return State.forOrdinal(stateOrdinal()); } @@ -1687,14 +1600,24 @@ final boolean is(State state) return stateOrdinal() == state.ordinal(); } + final boolean isState(int stateBitSet) + { + return TinyEnumSet.contains(stateBitSet, stateOrdinal()); + } + final boolean is(GlobalGroup group) { return globalGroupOrdinal() == group.ordinal(); } - boolean isRange() + final boolean is(ExclusiveGroup group) { - return 0 != (info & RANGE_BIT); + return exclusiveGroupOrdinal() == group.ordinal(); + } + + final void override(GlobalGroup group) + { + info = (info & ~(GROUP_MASK << GLOBAL_GROUP_SHIFT)) | (group.ordinal() << GLOBAL_GROUP_SHIFT); } final int compareTo(State state) @@ -1702,12 +1625,18 @@ final int compareTo(State state) return stateOrdinal() - state.ordinal(); } - final void setState(State state) + final void setStateExclusive(State state) { Invariants.require(state.isPermittedFrom(stateOrdinal()), "%s forbidden from %s", state, this, Task::reportBadStateTransition); setStateUnsafe(state); } + final void setRunState(RunState state) + { + Invariants.require(isState(RUNNING_OR_EXECUTED)); + runState = (byte) state.ordinal(); + } + private static String reportBadStateTransition(Task task) { return task.state() + " for " + task.toDescription(); @@ -1728,59 +1657,101 @@ final int exclusiveGroupOrdinal() return (info >>> EXCLUSIVE_GROUP_SHIFT) & GROUP_MASK; } - @Nullable - final TaskQueue queued() + private boolean isCompatible(TaskQueue queue) { - return queued; + int self = stateOrdinal(); + return TinyEnumSet.contains(queue.states, self); } - final void unqueueIfQueued() + final boolean isSync() { - if (queued != null) - unqueue(); + return 0 == (info & NONSYNC_BIT); } - final void unqueue() + final boolean isNonSync() { - Invariants.require(queued != null); - queued.unqueue(this); - queued = null; + return !isSync(); } - final void unsetQueue(TaskQueue queue) + final void setNonSyncExclusive() { - Invariants.require(queued == queue); - queued = null; + info |= NONSYNC_BIT; } - final void setQueue(TaskQueue queue) + final boolean isIncremental() { - Invariants.require(queued == null); - Invariants.require(isCompatible(queue)); - queued = queue; + return 0 != (info & INCREMENTAL_MASK); } - private boolean isCompatible(TaskQueue queue) + final void setIncrementalExclusive() { - int self = stateOrdinal(); - return TinyEnumSet.contains(queue.states, self); + info |= INCREMENTAL | NONSYNC_BIT; } - private static int init(GlobalGroup global, ExclusiveGroup exclusive, State state) + final boolean hasIncrementalStarted() { - return global.bits | exclusive.bits | state.ordinal(); + return (info & INCREMENTAL_MASK) >= INCREMENTAL_STARTED; } - final void setTranche(int tranche) + final void setIncrementalStartedExclusive() { - Invariants.require(tranche <= MAX_TRANCHE); - info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; + Invariants.require(isIncremental()); + if (!isIncrementalFinishing()) + info = (info & ~INCREMENTAL_MASK) | INCREMENTAL_STARTED; } - final void setInheritedTranche(int tranche) + final boolean isIncrementalFinishing() { - Invariants.require(tranche <= MAX_TRANCHE); - info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT | HAS_INHERITED_BIT; + return (info & INCREMENTAL_MASK) >= INCREMENTAL_FINISHING; + } + + final void setIncrementalFinishingExclusive() + { + Invariants.require(isIncremental()); + info |= INCREMENTAL_FINISHING; + } + + final void setSequencedExclusive(ExecutionSequence sequence) + { + Invariants.require(isUnsequenced()); + info |= sequence.ordinal() << SEQUENCED_SHIFT; + } + + final boolean isUnsequenced() + { + return (info & SEQUENCED_MASK) == 0; + } + + final boolean isSequencedByPriority() + { + return (info & SEQUENCED_MASK) == SEQUENCED_PRIORITY; + } + + final boolean isSequencedByPriorityAtomic() + { + return (info & SEQUENCED_MASK) >= SEQUENCED_ATOMIC; + } + + final boolean isCacheQueuedFifo() + { + return (info & SEQUENCED_MASK) == SEQUENCED_ATOMIC_AND_QUEUED; + } + + final boolean isCacheQueued() + { + return 0 != (info & CACHE_QUEUED_BIT); + } + + // supersedes priority, in whichever order they're called + final void setCacheQueuedFifoExclusive() + { + Invariants.require(isSequencedByPriorityAtomic()); + info |= SEQUENCED_ATOMIC_AND_QUEUED | CACHE_QUEUED_BIT; + } + + final void setCacheQueuedExclusive() + { + info |= CACHE_QUEUED_BIT; } final int tranche() @@ -1789,10 +1760,51 @@ final int tranche() return info >>> TRANCHE_SHIFT; } + final void setTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; + } + + final void setInheritedWithTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT | HAS_INHERITED_BIT; + } + final boolean hasInherited() { return (info & HAS_INHERITED_BIT) != 0; } + + final void setInheritedRangeScan() + { + info = info | HAS_INHERITED_RANGE_SCAN_BIT; + } + + final boolean hasInheritedRangeScan() + { + return (info & HAS_INHERITED_RANGE_SCAN_BIT) != 0; + } + + static int init(GlobalGroup global, ExclusiveGroup exclusive) + { + return (global.ordinal() << GLOBAL_GROUP_SHIFT) | (exclusive.ordinal() << EXCLUSIVE_GROUP_SHIFT); + } + + static Task reverse(Task unqueued) + { + Task prev = null; + Task cur = unqueued; + while (cur != null) + { + Task next = cur.next; + cur.next = prev; + prev = cur; + cur = next; + } + return prev; + } } // run the task even on a stopped commandStore @@ -1811,7 +1823,7 @@ static final class ExclusiveExecutorTask extends Task ExclusiveExecutorTask(ExclusiveExecutor queue) { - super(COMMAND_STORE, INITIALIZED); + super(COMMAND_STORE); this.queue = queue; } @@ -1824,33 +1836,33 @@ String toDescription() @Override void submitExclusive(AccordExecutor owner) { throw new UnsupportedOperationException(); } @Override - protected void preRunExclusive() + void preRunExclusive() { queue.preRunTask(); } @Override - protected void run() + void run() { queue.runTask(); } @Override - protected void fail(Throwable t) + void reportFailure(Throwable t) { - queue.failTask(t); + queue.task.reportFailure(t); } @Override - protected boolean isNewWork() + boolean isNewWork() { throw new UnsupportedOperationException(); } @Override - protected void cleanupExclusive(AccordExecutor executor) + void cleanupExclusive(AccordExecutor executor, boolean executed) { - queue.cleanupTask(); + queue.cleanupTask(executed); } protected boolean isInHeap() @@ -1879,7 +1891,7 @@ public final class ExclusiveExecutor extends MultiTaskQueue implements Exc ExclusiveExecutor(AccordExecutor executor, int commandStoreId) { - super(RUNNABLE, commandStoreId < 0 ? GroupKind.NONE : GroupKind.EXCLUSIVE, EXCLUSIVE_QUEUE_BUDGETS, EXCLUSIVE_QUEUE_LIMITS); + super(RUNNABLE, commandStoreId < 0 ? GroupKind.NONE : GroupKind.EXCLUSIVE, EXCLUSIVE_QUEUE_LIMITS); this.commandStoreId = commandStoreId; this.selfTask = new ExclusiveExecutorTask(this); this.debug = DebugExclusiveExecutor.maybeDebug(executor.debug, commandStoreId); @@ -1913,10 +1925,13 @@ void runTask() waiting = null; if (stopped && reject(task)) - task.fail(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((AccordTask) task).executionContext())); + task.reportFailure(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((AccordTask) task).executionContext())); else task.run(); - // NOTE: cannot safely release owner here, in case an immediate-execution runs before we can release our references and store their changes to the cache + + // NOTE: we can ONLY safely release owner here due to AccordCacheEntry locking, which remains in place until AccordTask.releaseResourcesExclusive + // this also relies on AccordSafeCommandStore$ExclusiveCaches.acquireIfLoaded returning false when the entry is locked + owner = null; } private boolean reject(Task task) @@ -1928,45 +1943,49 @@ private boolean reject(Task task) return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); } - void failTask(Throwable t) + void cleanupTask(boolean executed) { - task.fail(t); - } - - void cleanupTask() - { - try { task.cleanupExclusive(AccordExecutor.this); } + try + { + task.unsetQueue(this); + task.cleanupExclusive(AccordExecutor.this, executed); + } finally { active = 0; - owner = null; task = super.pollMulti(); if (DEBUG_EXECUTION) debug.onSetTask(task); if (task != null) { selfTask.position = task.position; selfTask.setStateUnsafe(WAITING_TO_RUN); - runnable.enqueue(selfTask); + runnable.enqueue(selfTask, false); } } } - void enqueue(Task newTask) + void enqueue(Task newTask, boolean incrementArrivals) { + if (task != null) { - Invariants.require(selfTask.isInHeap() || selfTask.isReadyToCleanup()); - super.enqueueMulti(newTask); + if (incrementArrivals) + runnable.incrementArrivals(selfTask); + // TODO (expected): restore some invariant here +// Invariants.require(selfTask.isInHeap() || selfTask.is(RUNNING)); + super.enqueueMulti(newTask, incrementArrivals); } else { Invariants.require(isEmptySingle()); + if (incrementArrivals) + incrementArrivals(newTask); incrementDispatches(newTask); task = newTask; task.setQueue(this); selfTask.position = newTask.position; selfTask.setStateUnsafe(WAITING_TO_RUN); - runnable.enqueue(selfTask); + runnable.enqueue(selfTask, incrementArrivals); if (DEBUG_EXECUTION) debug.onSetTask(newTask); } } @@ -2019,7 +2038,7 @@ private void removeCurrentTask(IntrusiveHeapNode remove) { selfTask.position = task.position; selfTask.setStateUnsafe(WAITING_TO_RUN); - runnable.enqueue(selfTask); + runnable.enqueue(selfTask, false); } } Invariants.require(task == null || runnable.isWaiting(selfTask)); @@ -2070,8 +2089,8 @@ Task inherit() { Thread thread = Thread.currentThread(); if (thread == owner) - return task; - return AccordTaskRunner.get(thread).accordActiveSelfTask(); + return Task.unwrap(task); + return Task.unwrap(AccordTaskRunner.get(thread).accordActiveSelfTask()); } @Override @@ -2105,7 +2124,7 @@ public boolean tryExecuteImmediately(Runnable run) if (active != null && active != AccordExecutor.this) return false; // prevent cross-executor locking/execution - if (!ownerUpdater.compareAndSet(this, null, thread)) + if (owner == null && !ownerUpdater.compareAndSet(this, null, thread)) return false; if (active == null) @@ -2209,6 +2228,12 @@ final boolean isQueuedSingle(T test) { return containsNode(test); } + + @Override + public String toString() + { + return TinyEnumSet.toString(states, State::forOrdinal); + } } static final class StandaloneTaskQueue extends TaskQueue @@ -2218,7 +2243,7 @@ static final class StandaloneTaskQueue extends TaskQueue super(states); } - StandaloneTaskQueue(Task.State state) + StandaloneTaskQueue(State state) { super(TinyEnumSet.encode(state)); } @@ -2325,29 +2350,30 @@ static abstract class MultiTaskQueue extends TaskQueue final TaskQueue[] queues; final long[] positions; final byte groupShift; - final long baseBudget, limits; + final long limits; + /** sets overflow bits for a queue that has been stopped */ + long stopped; /** sets overflow bits for each counter when it needs its position updated */ long dirty; /** sets overflow bits for each counter when there's associated work */ long hasWork; /** Stores recent dequeue counts for up to 8 sub queues. */ long dispatches; + // TODO (required): increment arrivals based on internal queue for ExclusiveExecutors + // also: experiment with decaying on arrival schedule rather than poll schedule, since this should respond to work growth more accurately /** Stores recent enqueue counts for up to 8 sub queues. */ long arrivals; /** Stores currently-active counts for up to 8 sub queues. We can use this to impose limits on specific queues. */ long active; - /** Stores a biased budget for each task type, so that we may prefer to serve one type of task over another */ - long budget; /** deficit-round-robin credits for the two PRIORITY_FAIR strategies (flow/age). */ int creditFlow, creditAge; int waitingCount; - MultiTaskQueue(int waitingStates, GroupKind groups, long budget, long limits) + MultiTaskQueue(int waitingStates, GroupKind groups, long limits) { super(waitingStates); - this.baseBudget = budget; this.limits = limits; int queueCount = groups.count; Invariants.require(queueCount <= 8); @@ -2364,6 +2390,16 @@ final int group(Task task) return (task.info >>> groupShift) & Task.GROUP_MASK; } + void stop(int group) + { + stopped |= overflowBit(group); + } + + void restart(int group) + { + stopped &= ~overflowBit(group); + } + final TaskQueue queue(Task task) { int group = group(task); @@ -2392,12 +2428,8 @@ private int pollGroup() default: throw new UnhandledEnum(PRIORITY_MODEL); case PRIORITY_ONLY: return pollGroupByPriority(); case PHASE_ONLY: return pollGroupByIndex(); - case PRIORITY_BUDGET: return pollGroupByPriorityWithBudget(); - case PHASE_BUDGET: return pollGroupByIndexWithBudget(); case PHASE_FAIR: return pollGroupByPhaseFair(); - case PHASE_BUDGET_FAIR: return pollGroupByPhaseBudgetFair(); case BLENDED_PRIORITY_PHASE_FAIR: return pollGroupByBlended(); - case BLENDED_PRIORITY_PHASE_BUDGET_FAIR: return pollGroupByBlendedWithBudget(); } } @@ -2447,33 +2479,11 @@ private int pollGroupByIndex() return bitIndex / 8; } - private int pollGroupByPriorityWithBudget() - { - return pollGroupByPriority(unsaturatedWithWorkAndBudget()); - } - - private int pollGroupByIndexWithBudget() - { - long visit = (hasWork & unsaturated()); - if (visit == 0) - return -1; - - visit = withBudgetOrReset(visit); - visit >>>= 7; - int bitIndex = Long.numberOfTrailingZeros(visit); - return bitIndex / 8; - } - private int pollGroupByPhaseFair() { return minCounterIndex(recentFlowImbalances()); } - private int pollGroupByPhaseBudgetFair() - { - return minCounterIndex(recentFlowImbalances(), saturatedOrWithoutWorkOrWithoutBudget()); - } - // PRIORITY_FAIR selection: a deficit round-robin blend of two strategies, chosen per poll: // flow -> minCounterIndex(recent - arrivals + bias) (least fairly serviced) // age -> minGroupByPriority() (earliest-queued work) @@ -2516,11 +2526,6 @@ private int pollGroupByBlended(long disabled) } } - private int pollGroupByBlendedWithBudget() - { - return pollGroupByBlended(saturatedOrWithoutWorkOrWithoutBudget()); - } - private long saturated() { return ((active | COUNTER_OVERFLOWS) - limits) & COUNTER_OVERFLOWS; @@ -2533,42 +2538,12 @@ private long unsaturated() private long unsaturatedWithWork() { - return hasWork & unsaturated(); - } - - private long unsaturatedWithWorkAndBudget() - { - return withBudgetOrReset(hasWork & unsaturated()); - } - - private long saturatedOrWithoutWorkOrWithoutBudget() - { - return withoutBudgetOrReset(saturatedOrWithoutWork()); - } - - private long withoutBudgetOrReset(long disabled) - { - return COUNTER_OVERFLOWS ^ withBudgetOrReset(COUNTER_OVERFLOWS ^ disabled); - } - - private long withBudgetOrReset(long enabled) - { - long hasBudget = hasBudget(); - if ((hasBudget & enabled) != 0) - return hasBudget & enabled; - - budget = baseBudget; - return enabled; - } - - private long hasBudget() - { - return setOverflowWhenLessEqual(budget, 0) ^ COUNTER_OVERFLOWS; + return hasWork & unsaturated() & ~stopped; } private long saturatedOrWithoutWork() { - return (hasWork ^ COUNTER_OVERFLOWS) | saturated(); + return (hasWork ^ COUNTER_OVERFLOWS) | saturated() | stopped; } // arrivals is a windowed measure of ARRIVAL (incremented on enqueue, size-biased; decays on recent's overflow). @@ -2683,7 +2658,6 @@ final T pollMulti() --waitingCount; incrementActive(group); incrementDispatches(group); - decrementBudget(group); TaskQueue queue = queues[group]; T head = queue.pollSingle(); @@ -2695,7 +2669,7 @@ final T pollMulti() return head; } - final void enqueueMulti(T task) + final void enqueueMulti(T task, boolean incrementArrivals) { task.setQueue(this); int group = group(task); @@ -2707,7 +2681,8 @@ final void enqueueMulti(T task) { TaskQueue queue = queue(group); int result = queue.enqueueSingle(task); - incrementArrivals(group); + if (incrementArrivals) + incrementArrivals(group); if (result < 0) setHasWork(group); if (result != 0) setDirty(group); } @@ -2790,13 +2765,6 @@ final void incrementDispatches(int group) } } - final void decrementBudget(int group) - { - // no need to manage underflow; if the budget is being used, - // a queue should only dispatch work when there's non-zero budget - budget -= lowBit(group); - } - final void decrementDispatches(Task task) { int group = group(task); @@ -2811,6 +2779,13 @@ final void decrementDispatches(int group) dispatches += (dispatches >>> 7) & lowBit; } + final void incrementArrivals(Task task) + { + int group = group(task); + if (group >= 0) + incrementArrivals(group); + } + final void incrementArrivals(int group) { int shift = group * 8; @@ -2842,9 +2817,9 @@ final void unsetDirty(int group) dirty &= ~overflowBit(group); } - final boolean hasWaiting() + final boolean hasWaitingToRun() { - return waitingCount > 0; + return unsaturatedWithWork() != 0; } final boolean isWaiting(T task) @@ -2870,13 +2845,13 @@ final long overflowBit(int group) static final class RunnableTaskQueue extends MultiTaskQueue { - static final int RUNNABLE = TinyEnumSet.encode(WAITING_TO_RUN, ASSIGNED, RUNNING); + static final int RUNNABLE = TinyEnumSet.encode(WAITING_TO_RUN, RUNNING); final TaskQueue assigned; RunnableTaskQueue() { - super(RUNNABLE, GroupKind.GLOBAL, GLOBAL_QUEUE_BUDGETS, GLOBAL_QUEUE_LIMITS); + super(RUNNABLE, GroupKind.GLOBAL, GLOBAL_QUEUE_LIMITS); this.assigned = new TaskQueue<>(0); } @@ -2886,15 +2861,13 @@ T poll() if (next == null) return null; - next.setState(ASSIGNED); assigned.enqueueSingle(next); - return next; } - void enqueue(T enqueue) + void enqueue(T enqueue, boolean incrementArrivals) { - enqueueMulti(enqueue); + enqueueMulti(enqueue, incrementArrivals); } void unqueue(T unqueue) @@ -2934,7 +2907,7 @@ boolean isAssigned(T task) return assigned.isQueuedSingle(task); } - void complete(T task) + void cleanup(T task) { if (assigned.tryUnqueueSingle(task)) { @@ -2951,15 +2924,15 @@ static class CancelTask extends Task final Task cancel; private CancelTask(Task cancel) { - super(GlobalGroup.OTHER, WAITING_TO_RUN); + super(GlobalGroup.OTHER); this.cancel = cancel; } @Override void submitExclusive(AccordExecutor owner) { cancel.cancelExclusive(owner); } - @Override protected void preRunExclusive() { throw new UnsupportedOperationException(); } - @Override protected void run() { throw new UnsupportedOperationException(); } - @Override protected void fail(Throwable fail) { throw new UnsupportedOperationException(); } - @Override protected boolean isNewWork() { return false; } + @Override void preRunExclusive() { throw new UnsupportedOperationException(); } + @Override void run() { throw new UnsupportedOperationException(); } + @Override void reportFailure(Throwable fail) { throw new UnsupportedOperationException(); } + @Override boolean isNewWork() { return false; } @Override String toDescription() { return "Cancel " + cancel.toDescription(); } } @@ -2972,29 +2945,26 @@ abstract class Plain extends Task implements Cancellable { Plain(GlobalGroup group, long position, int tranche) { - super(group, WAITING_TO_RUN, position, tranche); + super(group, position, tranche); } Plain(ExclusiveGroup group, long position, int tranche) { - super(group, WAITING_TO_RUN, position, tranche); + super(group, position, tranche); } Plain(GlobalGroup group) { - super(group, WAITING_TO_RUN); + super(group); } Plain(ExclusiveGroup group) { - super(group, WAITING_TO_RUN); + super(group); } abstract ExclusiveExecutor executor(); - @Override - protected void preRunExclusive() {} - @Override public void cancel() { @@ -3006,15 +2976,16 @@ void cancelExclusive(AccordExecutor owner) ExclusiveExecutor executor = executor(); if ((executor == null ? runnable : executor).tryUnqueueWaiting(this)) { - completeTaskExclusive(this); - try { fail(new CancellationException()); } + try { failExclusive(new CancellationException(), CANCELLED); } catch (Throwable t) { agent.onException(t); } + finally { cleanupTaskExclusive(this, false); } } } @Override final void submitExclusive(AccordExecutor owner) { + setStateExclusive(WAITING_TO_RUN); owner.submitPlainExclusive(this); } @@ -3084,7 +3055,7 @@ protected void run() } @Override - protected void fail(Throwable t) + protected void reportFailure(Throwable t) { if (result != null) result.tryFailure(t); @@ -3099,7 +3070,7 @@ ExclusiveExecutor executor() } // a task that may be submitted to this executor or another - abstract class IOTask extends Plain implements Cancellable, DebuggableTask + public abstract class IOTask extends Plain implements Cancellable, DebuggableTask { IOTask(GlobalGroup group, long position, int tranche) { @@ -3114,9 +3085,9 @@ abstract class IOTask extends Plain implements Cancellable, DebuggableTask abstract void postRunExclusive(); @Override - protected void cleanupExclusive(AccordExecutor executor) + void cleanupExclusive(AccordExecutor executor, boolean executed) { - super.cleanupExclusive(executor); + super.cleanupExclusive(executor, executed); postRunExclusive(); } @@ -3151,29 +3122,27 @@ static class FailureHolder } } - LoadRunnable newLoad(AccordCacheEntry entry, boolean isForRange) + LoadRunnable newLoad(AccordCacheEntry entry, boolean isForRange) { return new LoadRunnable<>(entry, isForRange ? RANGE_LOAD : LOAD); } - class LoadRunnable extends IOTask + public class LoadRunnable extends IOTask { - final AccordCacheEntry entry; + final AccordCacheEntry entry; Object result = FailureHolder.NOT_STARTED; - LoadRunnable(AccordCacheEntry entry, GlobalGroup group) + LoadRunnable(AccordCacheEntry entry, GlobalGroup group) { super(group); Invariants.require(group == LOAD || group == RANGE_LOAD); this.entry = entry; } - boolean isForRange() { return is(RANGE_LOAD); } - void postRunExclusive() { - if (!(result instanceof FailureHolder)) onLoadedExclusive(entry, (V)result, null, isForRange()); - else onLoadedExclusive(entry, null, ((FailureHolder)result).fail, isForRange()); + if (!(result instanceof FailureHolder)) onLoadedExclusive(entry, (V)result, null); + else onLoadedExclusive(entry, null, ((FailureHolder)result).fail); } @Override @@ -3194,7 +3163,7 @@ public void run() } @Override - protected void fail(Throwable t) + void reportFailure(Throwable t) { result = new FailureHolder(t); } @@ -3208,10 +3177,10 @@ public String description() static abstract class AbstractIOTask { - abstract protected void runInternal(); - abstract protected void postRunExclusive(); - abstract protected void fail(Throwable t); - abstract protected String description(); + abstract void runInternal(); + abstract void postRunExclusive(); + abstract void fail(Throwable t); + abstract String description(); } class WrappedIOTask extends IOTask @@ -3254,7 +3223,7 @@ public String description() } @Override - protected void fail(Throwable fail) + protected void reportFailure(Throwable fail) { wrapped.fail(fail); } @@ -3263,12 +3232,12 @@ protected void fail(Throwable fail) private static final Throwable NOT_STARTED = new Throwable(); class SaveRunnable extends IOTask { - final AccordCacheEntry entry; + final AccordCacheEntry entry; final UniqueSave identity; final Runnable run; Throwable failure = NOT_STARTED; - SaveRunnable(AccordCacheEntry entry, UniqueSave identity, Runnable run) + SaveRunnable(AccordCacheEntry entry, UniqueSave identity, Runnable run) { super(SAVE); this.entry = entry; @@ -3301,7 +3270,7 @@ public void run() } @Override - protected void fail(Throwable t) + protected void reportFailure(Throwable t) { failure = t; } @@ -3361,7 +3330,7 @@ protected void run() } @Override - protected void fail(Throwable fail) + protected void reportFailure(Throwable fail) { try { @@ -3486,8 +3455,6 @@ public List taskSnapshot() try { addToSnapshot(result, scanningRanges, TaskInfo.Status.SCANNING_RANGES, TaskInfo.Status.SCANNING_RANGES); - addToSnapshot(result, waitingToLoadRangeTxns, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); - addToSnapshot(result, waitingToLoad, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); addToSnapshot(result, loading, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); for (TaskQueue queue : runnable.queues) { @@ -3532,7 +3499,7 @@ private static void addToSnapshot(List snapshot, TaskQueue queue, T public int unsafePreparingToRunCount() { - return waitingToLoad.size() + loading.size() + waitingToLoadRangeTxns.size() + scanningRanges.size(); + return loading.size() + scanningRanges.size(); } public int unsafeWaitingToRunCount() diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java index 8e3e41f7c342..d7594025cd18 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java @@ -26,7 +26,6 @@ import org.apache.cassandra.concurrent.CassandraThread; import org.apache.cassandra.service.accord.AccordExecutorLoops.LoopTask; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutorLoop; import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; @@ -214,18 +213,21 @@ public void run() if (task != null) { self.setAccordActiveTask(task); + boolean executed = false; try { task.preRunExclusive(); + executed = true; task.run(); } catch (Throwable t) { - task.fail(t); + executed = false; + task.failExecution(t); } finally { - completeTaskExclusive(task); + cleanupTaskExclusive(task, executed); self.setAccordActiveTask(null); } } @@ -265,7 +267,6 @@ protected LoopTask runWithoutLock(String name) { return new LoopTask(name) { - final DebugExecutorLoop debug = DEBUG_EXECUTION ? new DebugExecutorLoop(AccordExecutorAbstractLockLoop.this.debug) : null; @Override public void run() { @@ -276,17 +277,15 @@ public void run() Task task = null; while (true) { - if (DEBUG_EXECUTION) debug.onLock(); lock(self); try { - if (DEBUG_EXECUTION) debug.onEnterLock(); enterLockLoop(); if (task != null) { Task tmp = task; task = null; - completeTaskExclusive(tmp); + cleanupTaskExclusive(tmp, true); self.setAccordActiveTask(null); } @@ -322,9 +321,9 @@ public void run() { if (task != null) { - try { task.fail(t); } + try { task.failExecution(t); } catch (Throwable t2) { t.addSuppressed(t2); } - try { completeTaskExclusive(task); } + try { cleanupTaskExclusive(task, false); } catch (Throwable t2) { t.addSuppressed(t2); } try { agent.onException(t); } catch (Throwable t2) { /* nothing we can sensibly do after already reporting */ } @@ -350,7 +349,7 @@ public void run() } catch (Throwable t) { - try { task.fail(t); } + try { task.failExecution(t); } catch (Throwable t2) { try diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java index 3cbd3db14481..5c93a8a0cce4 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLoop.java @@ -44,11 +44,6 @@ boolean hasUnqueued() return unqueued != null; } - Task unqueued() - { - return unqueued; - } - final Task push(Task submit) { Invariants.require(submit.next == null); @@ -89,26 +84,16 @@ final Task enqueueOneExclusive(Task cur) Invariants.require(cur != null); Task next = cur.next; cur.next = null; - if (cur.isReadyToCleanup()) completeTaskExclusive(cur); - else cur.submitExclusive(this); - return next; - } - - final Task enqueueOneCleanup(Task cur) - { - Invariants.require(cur != null); - Task next = cur.next; - cur.next = null; - completeTaskExclusive(cur); + if (cur.is(Task.State.UNINITIALIZED)) cur.submitExclusive(this); + else cleanupTaskExclusive(cur, true); return next; } - final Task enqueueOneSubmit(Task cur) + final Task destructiveNext(Task cur) { Invariants.require(cur != null); Task next = cur.next; cur.next = null; - cur.submitExclusive(this); return next; } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java index ce7de7fefc7c..392ce2a0154b 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java @@ -33,12 +33,14 @@ import org.apache.cassandra.utils.concurrent.SignalLock; import static accord.utils.Invariants.nonNull; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.UNINITIALIZED; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop { private static class ShutdownException extends RuntimeException {} + private static final int MAX_LOOPS = 1000; // limit the amount of time we hold the lock for private final SignalLock lock; private final AccordExecutorLoops loops; private int readyToRunTarget = 1; @@ -46,8 +48,7 @@ private static class ShutdownException extends RuntimeException {} // TODO (desired): intrusive queue using Task.next, but a little challenging because we reuse SequentialQueueTask so have ABA problem private final ConcurrentLinkedQueue readyToRun = new ConcurrentLinkedQueue<>(); - private Task pendingSequentialHead, pendingSequentialTail; - private Task pendingCleanupHead, pendingCleanupTail; + private Task pendingExecutedHead, pendingExecutedTail; private Task pendingNewHead, pendingNewTail; private int pendingCount; @@ -105,14 +106,17 @@ final void drainUnqueuedNewWorkExclusive() { updatePendingUnqueued(); Task requeue = null, requeueTail = null; + Task submit = null, submitTail = null; Task cur = pendingNewHead; while (cur != null) { Task next = cur.next; + cur.next = null; if (cur.isNewWork()) { - cur.next = null; - cur.submitExclusive(this); + if (submit == null) submit = cur; + else submitTail.next = cur; + submitTail = cur; --pendingCount; } else @@ -126,6 +130,13 @@ final void drainUnqueuedNewWorkExclusive() pendingNewHead = requeue; pendingNewTail = requeueTail; + while (submit != null) + { + Task next = submit.next; + submit.next = null; + submit.submitExclusive(this); + submit = next; + } } private boolean enqueueOnePending() @@ -133,25 +144,23 @@ private boolean enqueueOnePending() if (pendingCount == 0) return false; - if (pendingSequentialHead != null) - { - pendingSequentialHead = enqueueOneCleanup(pendingSequentialHead); - if (pendingSequentialHead == null) - pendingSequentialTail = null; - } - else if (pendingCleanupHead != null) + --pendingCount; + if (pendingExecutedHead != null) { - pendingCleanupHead = enqueueOneCleanup(pendingCleanupHead); - if (pendingCleanupHead == null) - pendingCleanupTail = null; + Task executed = pendingExecutedHead; + pendingExecutedHead = destructiveNext(executed); + if (pendingExecutedHead == null) + pendingExecutedTail = null; + cleanupTaskExclusive(executed, true); } else { - pendingNewHead = enqueueOneSubmit(pendingNewHead); + Task submit = pendingNewHead; + pendingNewHead = destructiveNext(submit); if (pendingNewHead == null) pendingNewTail = null; + submit.submitExclusive(this); } - --pendingCount; return true; } @@ -178,22 +187,29 @@ else if (hasReadyToRun && readyToRunTarget > 1) } boolean hasDrainedSignal = false; + int loops = 0; while (true) { long state = lock.state(); int signals = SignalLock.asyncSignalCount(state); int waiters = SignalLock.waitingEnabledThreadCount(state); - if (signals >= readyToRunTarget) - { - if (enqueueOnePending() || (updatePendingUnqueued() && enqueueOnePending())) continue; - else if (hasDrainedSignal) - lock.signalLockWorkExclusive(); - return; - } - else if (waiters > 0 && signals > 1 && SignalLock.activeEnabledThreadCount(state) == 1) + if (signals > 0) { - // ensure at least one other thread is running if there's enough work for it; it will spin up other threads if necessary - lock.propagateAsyncWorkSignals(1); + if (++loops > MAX_LOOPS) + return; + + if (signals >= readyToRunTarget) + { + if (enqueueOnePending() || (updatePendingUnqueued() && enqueueOnePending())) continue; + else if (hasDrainedSignal) + lock.signalLockWorkExclusive(); + return; + } + else if (waiters > 0 && signals > 1 && SignalLock.activeEnabledThreadCount(state) == 1) + { + // ensure at least one other thread is running if there's enough work for it; it will spin up other threads if necessary + lock.propagateAsyncWorkSignals(1); + } } Task task = pollAlreadyWaitingToRunExclusive(); @@ -216,9 +232,9 @@ else if (waiters > 0 && signals > 1 && SignalLock.activeEnabledThreadCount(state try { task.preRunExclusive(); } catch (Throwable t) { - try { task.fail(t); } + try { task.failExclusive(t, Task.State.FAILED_OTHER); } catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } - try { completeTaskExclusive(task); } + try { cleanupTaskExclusive(task, false); } catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } continue; } @@ -237,28 +253,22 @@ private boolean updatePendingUnqueued() return false; int count = 0; - Task addSequentialHead = null, addSequentialTail = null; - Task addCleanupHead = null, addCleanupTail = null; + Task addExecutedHead = null, addExecutedTail = null; Task addNewHead = null, addNewTail = null; { Task cur = Task.reverse(acquireUnqueuedExclusive()); while (cur != null) { Task next = cur.next; - if (!cur.isReadyToCleanup()) + if (cur.is(UNINITIALIZED)) { if (addNewHead == null) addNewHead = addNewTail = setNextNull(cur); else addNewHead = reverseOne(addNewHead, cur); } - else if (cur instanceof ExclusiveExecutorTask) - { - if (addSequentialHead == null) addSequentialHead = addSequentialTail = setNextNull(cur); - else addSequentialHead = reverseOne(addSequentialHead, cur); - } else { - if (addCleanupHead == null) addCleanupHead = addCleanupTail = setNextNull(cur); - else addCleanupHead = reverseOne(addCleanupHead, cur); + if (addExecutedHead == null) addExecutedHead = addExecutedTail = setNextNull(cur); + else addExecutedHead = reverseOne(addExecutedHead, cur); } ++count; cur = next; @@ -266,17 +276,11 @@ else if (cur instanceof ExclusiveExecutorTask) } pendingCount += count; - if (addSequentialHead != null) - { - if (pendingSequentialHead == null) pendingSequentialHead = addSequentialHead; - else pendingSequentialTail.next = addSequentialHead; - pendingSequentialTail = addSequentialTail; - } - if (addCleanupHead != null) + if (addExecutedHead != null) { - if (pendingCleanupHead == null) pendingCleanupHead = addCleanupHead; - else pendingCleanupTail.next = addCleanupHead; - pendingCleanupTail = addCleanupTail; + if (pendingExecutedHead == null) pendingExecutedHead = addExecutedHead; + else pendingExecutedTail.next = addExecutedHead; + pendingExecutedTail = addExecutedTail; } if (addNewHead != null) { @@ -349,22 +353,22 @@ private Task awaitWork(AccordTaskRunner self) } } - private Task cleanupAndMaybeGetWork(AccordTaskRunner self, @Nullable Task cleanup) + private Task executedAndMaybeGetWork(AccordTaskRunner self, @Nullable Task executed) { if (lock.tryAcquireAsyncWork()) { if (shutdown) throw new ShutdownException(); - return pushCleanupAndReturn(cleanup, nonNull(pollReadyToRun())); + return pushExecutedAndReturn(executed, nonNull(pollReadyToRun())); } if (!tryLock(self)) - return pushCleanupAndReturn(cleanup, null); + return pushExecutedAndReturn(executed, null); try { - completeTaskExclusive(cleanup); + cleanupTaskExclusive(executed, true); fetchWorkExclusive(); } catch (Throwable t) @@ -383,10 +387,9 @@ private Task cleanupAndMaybeGetWork(AccordTaskRunner self, @Nullable Task cleanu return null; } - private Task pushCleanupAndReturn(Task cleanup, Task result) + private Task pushExecutedAndReturn(Task complete, Task result) { - cleanup.setReadyToCleanup(); - if (push(cleanup) == null) + if (push(complete) == null) lock.signalLockWork(); return result; } @@ -401,7 +404,7 @@ final boolean tryLock(AccordTaskRunner self) final boolean unlockAndAcquire(AccordTaskRunner self) { - self.clearAccordLockedExecutor(); + self.exitAccordLockedExecutor(); if (DEBUG_EXECUTION) debug.onExitLock(); return lock.unlockAndAcquireAsyncWork(); } @@ -422,7 +425,7 @@ public void run() { if (task != null) { - try { task = cleanupAndMaybeGetWork(self, task); } + try { task = executedAndMaybeGetWork(self, task); } catch (Throwable t) { task = null; throw t; } } if (task == null) @@ -435,7 +438,7 @@ public void run() } catch (Throwable t) { - try { task.fail(t); } + try { task.failExecution(t); } catch (Throwable t2) { try diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java index ee37744d8dfd..cadd7a766eb3 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java +++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java @@ -82,7 +82,8 @@ void beforeUnlockExternal() protected void run() { - Thread self = Thread.currentThread(); + AccordTaskRunner self = AccordTaskRunner.get(); + self.setAccordActiveExecutor(AccordExecutorSimple.this); lock.lock(); try { @@ -95,11 +96,24 @@ protected void run() return; } - try { task.preRunExclusive(); task.run(); } - catch (Throwable t) { task.fail(t); } + // TODO (expected): dedup with AbstractLockLoop, and cleanup (executed flag is a bit ugly) + self.setAccordActiveTask(task); + boolean executed = false; + try + { + task.preRunExclusive(); + executed = true; + task.run(); + } + catch (Throwable t) + { + executed = false; + task.failExecution(t); + } finally { - completeTaskExclusive(task); + cleanupTaskExclusive(task, executed); + self.setAccordActiveTask(null); } } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java index 488ec61cb0a8..1263a3e1ec2d 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java +++ b/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java @@ -20,51 +20,22 @@ import java.util.Objects; -import com.google.common.annotations.VisibleForTesting; - import accord.api.Journal; import accord.local.Command; import accord.local.SafeCommand; import accord.primitives.TxnId; -import org.apache.cassandra.utils.concurrent.Ref; +import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; -public class AccordSafeCommand extends SafeCommand implements AccordSafeState +public class AccordSafeCommand extends SafeCommand implements AccordSafeState { - public static class DebugAccordSafeCommand extends AccordSafeCommand - { - final Ref selfRef; - public DebugAccordSafeCommand(AccordCacheEntry global) - { - super(global); - selfRef = new Ref<>(this, null); - selfRef.debug(global.key().toString()); - } - - @Override - public void markUnsafe() - { - super.markUnsafe(); - selfRef.release(); - } - - public static void trace(AccordSafeCommand safeCommand, String message) - { - ((DebugAccordSafeCommand)safeCommand).selfRef.debug(message); - } - } - - private boolean unsafe; - private final AccordCacheEntry global; + private final AccordCacheEntry global; private Command original; - private Command current; - public AccordSafeCommand(AccordCacheEntry global) + public AccordSafeCommand(AccordCacheEntry global) { super(global.key()); this.global = global; - this.original = null; - this.current = null; } @Override @@ -73,7 +44,7 @@ public boolean equals(Object o) if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; AccordSafeCommand that = (AccordSafeCommand) o; - return Objects.equals(original, that.original) && Objects.equals(current, that.current); + return Objects.equals(this.original, that.original) && Objects.equals(this.current(), that.current()); } @Override @@ -86,66 +57,36 @@ public int hashCode() public String toString() { return "AccordSafeCommand{" + - "invalidated=" + unsafe + + "status=" + statusString() + ", global=" + global + ", original=" + original + ", current=" + current + '}'; } - @Override - public AccordCacheEntry global() + public AccordCacheEntry global() { - checkNotInvalidated(); return global; } @Override - public Command current() - { - checkNotInvalidated(); - return current; - } - - @Override - @VisibleForTesting - public void set(Command command) - { - checkNotInvalidated(); - this.current = command; - } - - @Override - public Command original() + public void postExecute(AccordTask owner) { - checkNotInvalidated(); - return original; + global.releaseExclusive(this, owner); } public Journal.CommandUpdate update() { - return new Journal.CommandUpdate(original, current); + return new Journal.CommandUpdate(original, current()); } - @Override - public void preExecute() + public void preExecute(AccordTask owner, LockMode lockMode) { - checkNotInvalidated(); - original = global.getExclusive(); + requireUninitialised(); + original = global.lockExclusive(owner, lockMode); current = original; - if (isUnset()) - uninitialised(); - } - - @Override - public void markUnsafe() - { - unsafe = true; - } - - @Override - public boolean isUnsafe() - { - return unsafe; + if (current == null) + initialise(); + setSafe(); } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java index d0399e1b9244..6b02b5ab9df6 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java @@ -18,13 +18,8 @@ package org.apache.cassandra.service.accord; -import java.util.Collections; -import java.util.Map; -import java.util.Set; import java.util.function.Predicate; -import javax.annotation.Nullable; - import com.google.common.annotations.VisibleForTesting; import accord.api.Agent; @@ -35,16 +30,20 @@ import accord.impl.AbstractSafeCommandStore; import accord.local.Command; import accord.local.CommandStores; -import accord.local.CommandSummaries; +import accord.local.ExecutionContext; import accord.local.NodeCommandStoreService; import accord.local.RedundantBefore; +import accord.local.SafeState; import accord.local.cfk.CommandsForKey; import accord.local.cfk.SafeCommandsForKey; +import accord.primitives.AbstractUnseekableKeys; +import accord.primitives.Routables; import accord.primitives.Timestamp; import accord.primitives.Txn.Kind.Kinds; import accord.primitives.TxnId; import accord.primitives.Unseekables; import accord.utils.Invariants; +import accord.utils.UnhandledEnum; import org.apache.cassandra.metrics.LogLinearDecayingHistograms; import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches; @@ -53,69 +52,53 @@ import org.apache.cassandra.service.paxos.PaxosState; import static accord.utils.Invariants.illegalState; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.UNQUEUED; -public class AccordSafeCommandStore extends AbstractSafeCommandStore +public final class AccordSafeCommandStore extends AbstractSafeCommandStore { final AccordTask task; - private final @Nullable CommandSummaries commandsForRanges; - private final AccordCommandStore commandStore; - private AccordSafeCommandStore(AccordTask task, - @Nullable CommandSummaries commandsForRanges, - AccordCommandStore commandStore) + AccordSafeCommandStore(AccordTask task, ExecutionContext context) { - super(task.executionContext(), commandStore); + super(context); this.task = task; - this.commandsForRanges = commandsForRanges; - this.commandStore = commandStore; - } - - @Override - public CommandStores.RangesForEpoch ranges() - { - CommandStores.RangesForEpoch ranges = super.ranges(); - if (ranges != null) - return ranges; - - return commandStore.unsafeGetRangesForEpoch(); - } - - public static AccordSafeCommandStore create(AccordTask task, - @Nullable CommandSummaries commandsForRanges, - AccordCommandStore commandStore) - { - return new AccordSafeCommandStore(task, commandsForRanges, commandStore); } @VisibleForTesting - public Set commandsForKeysKeys() + public Iterable safeCommandsForKeys() { - if (task.commandsForKey() == null) - return Collections.emptySet(); - return task.commandsForKey().keySet(); + return task.refs.values().stream() + .filter(v -> v instanceof SafeCommandsForKey) + .map(v -> (SafeCommandsForKey)v)::iterator; } @Override protected AccordSafeCommand getInternal(TxnId txnId) { - Map commands = task.commands(); - if (commands == null) - return null; - return commands.get(txnId); + return (AccordSafeCommand) task.refs.get(txnId); } @Override protected ExclusiveCaches tryGetCaches() { - return commandStore.tryLockCaches(); + if (task.isIncremental()) + { + // We could relax this, but requires careful consideration of semantics when: + // - touching keys we have scheduled to process later + // - touching txnIds we may have to logically lock until the whole processing completes to + // ensure persistent state machine updates are consistent with incremental updates to cache + return null; + } + return commandStore().tryLockCaches(); } protected AccordSafeCommand add(AccordSafeCommand safeCommand, ExclusiveCaches caches) { - Object check = task.ensureCommands().putIfAbsent(safeCommand.txnId(), safeCommand); + Object check = task.refs.putIfAbsent(safeCommand.txnId(), safeCommand); if (check == null) { - safeCommand.preExecute(); + Invariants.require(!task.isIncremental()); // if isIncremental we'll have to supply holdQueue==true, but for now we forbid it + safeCommand.preExecute(task, UNQUEUED); return safeCommand; } else @@ -131,7 +114,7 @@ protected void persistFieldUpdates() // Field persistence is handled by AccordTask } - protected void persistFieldUpdatesInternal(Runnable onDone) + void persistFieldUpdatesInternal(Runnable onDone) { FieldUpdates updates = fieldUpdates(); if (updates == null) @@ -142,7 +125,7 @@ protected void persistFieldUpdatesInternal(Runnable onDone) long ticket = AccordCommandStore.nextSafeRedundantBeforeTicket.incrementAndGet(); SafeRedundantBefore update = new SafeRedundantBefore(ticket, updates.newRedundantBefore); Runnable reportRedundantBefore = () -> { - AccordCommandStore.safeRedundantBeforeUpdater.accumulateAndGet(commandStore, update, SafeRedundantBefore::max); + AccordCommandStore.safeRedundantBeforeUpdater.accumulateAndGet(commandStore(), update, SafeRedundantBefore::max); }; Runnable prevOnDone = onDone; onDone = prevOnDone == null ? reportRedundantBefore : () -> { @@ -150,15 +133,15 @@ protected void persistFieldUpdatesInternal(Runnable onDone) finally { prevOnDone.run(); } }; } - commandStore.persistFieldUpdates(updates, onDone); + commandStore().persistFieldUpdates(updates, onDone); } protected AccordSafeCommandsForKey add(AccordSafeCommandsForKey safeCfk, ExclusiveCaches caches) { - Object check = task.ensureCommandsForKey().putIfAbsent(safeCfk.key(), safeCfk); + Object check = task.refs.putIfAbsent(safeCfk.key(), safeCfk); if (check == null) { - safeCfk.preExecute(); + safeCfk.preExecute(task, UNQUEUED); return safeCfk; } else @@ -171,17 +154,17 @@ protected AccordSafeCommandsForKey add(AccordSafeCommandsForKey safeCfk, Exclusi @Override protected AccordSafeCommandsForKey getInternal(RoutingKey key) { - Map commandsForKey = task.commandsForKey(); - if (commandsForKey == null) + AccordSafeCommandsForKey safeCfk = (AccordSafeCommandsForKey) task.refs.get(key); + if (safeCfk == null || safeCfk.isUninitialised()) return null; - return commandsForKey.get(key); + return safeCfk; } @Override public void setRangesForEpoch(CommandStores.RangesForEpoch rangesForEpoch) { super.setRangesForEpoch(rangesForEpoch); - commandStore.updateMinHlc(PaxosState.ballotTracker().getLowBound().unixMicros() + 1); + commandStore().updateMinHlc(PaxosState.ballotTracker().getLowBound().unixMicros() + 1); } @Override @@ -194,13 +177,13 @@ public void updateCommandsForRanges(Command prev, Command updated, boolean force public void reportDurable(RedundantBefore addRedundantBefore, int flags) { upsertRedundantBefore(addRedundantBefore); - commandStore.maybeTerminated(ReportDurable.isCommandStoreFlush(flags), ReportDurable.isDataStoreFlush(flags)); + commandStore().maybeTerminated(ReportDurable.isCommandStoreFlush(flags), ReportDurable.isDataStoreFlush(flags)); } @Override public AccordCommandStore commandStore() { - return commandStore; + return task.commandStore; } @Override @@ -212,7 +195,7 @@ public DataStore dataStore() @Override public Agent agent() { - return commandStore.agent(); + return commandStore().agent(); } @Override @@ -224,16 +207,16 @@ public ProgressLog progressLog() @Override public NodeCommandStoreService node() { - return commandStore.node(); + return commandStore().node(); } public LogLinearDecayingHistograms.Buffer histogramBuffer() { if (task.histogramBuffer == null) { - task.histogramBuffer = commandStore.metricsBuffer; + task.histogramBuffer = commandStore().metricsBuffer; if (task.histogramBuffer == null) - task.histogramBuffer = commandStore.metricsBuffer = new LogLinearDecayingHistograms.Buffer(commandStore.executor().histograms); + task.histogramBuffer = commandStore().metricsBuffer = new LogLinearDecayingHistograms.Buffer(commandStore().executor().histograms); Invariants.require(task.histogramBuffer.isEmpty()); } return task.histogramBuffer; @@ -241,35 +224,48 @@ public LogLinearDecayingHistograms.Buffer histogramBuffer() private boolean visitForKey(Unseekables keysOrRanges, Predicate forEach) { - Map commandsForKey = task.commandsForKey; - if (commandsForKey == null) - return true; - - Unseekables skip = context.keys().without(keysOrRanges); - for (SafeCommandsForKey safeCfk : commandsForKey.values()) + Unseekables unseekables = context.keys(); + switch (unseekables.domain()) { - if (skip.contains(safeCfk.key())) - continue; - - if (!forEach.test(safeCfk.current())) - return false; + default: throw new UnhandledEnum(unseekables.domain()); + case Key: + AbstractUnseekableKeys keys = (AbstractUnseekableKeys) context.keys(); + return Routables.foldl(keys, keysOrRanges, (self, f, key, v, index) -> { + SafeCommandsForKey safeCfk = (SafeCommandsForKey) self.task.refs.get(key); + return f.test(safeCfk.current()); + }, this, forEach, Boolean.TRUE, cont -> !cont); + + case Range: + Unseekables skip = context.keys().without(keysOrRanges); + for (SafeState safeState : task.refs.values()) + { + if (!(safeState instanceof AccordSafeCommandsForKey)) + continue; + + SafeCommandsForKey safeCfk = (SafeCommandsForKey) safeState; + if (skip.contains(safeCfk.key())) + continue; + + if (!forEach.test(safeCfk.current())) + return false; + } + return true; } - return true; } @Override public void visit(Unseekables keysOrRanges, Timestamp startedBefore, Kinds testKind, ActiveCommandVisitor visitor, P1 p1, P2 p2) { visitForKey(keysOrRanges, cfk -> { cfk.visit(startedBefore, testKind, visitor, p1, p2); return true; }); - if (commandsForRanges != null) - commandsForRanges.visit(keysOrRanges, startedBefore, testKind, visitor, p1, p2); + if (task.commandsForRanges != null) + task.commandsForRanges.visit(keysOrRanges, startedBefore, testKind, visitor, p1, p2); } @Override public boolean visit(Unseekables keysOrRanges, TxnId testTxnId, Kinds testKind, SupersedingCommandVisitor visit) { return visitForKey(keysOrRanges, cfk -> cfk.visit(testTxnId, testKind, visit)) - && (commandsForRanges == null || commandsForRanges.visit(keysOrRanges, testTxnId, testKind, visit)); + && (task.commandsForRanges == null || task.commandsForRanges.visit(keysOrRanges, testTxnId, testKind, visit)); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java index 74201823b10b..fc38874caae4 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java +++ b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java @@ -20,38 +20,32 @@ import java.util.Objects; -import com.google.common.annotations.VisibleForTesting; - import accord.api.RoutingKey; import accord.local.cfk.CommandsForKey; import accord.local.cfk.NotifySink; import accord.local.cfk.SafeCommandsForKey; +import accord.utils.Invariants; + +import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; -public class AccordSafeCommandsForKey extends SafeCommandsForKey implements AccordSafeState +public class AccordSafeCommandsForKey extends SafeCommandsForKey implements AccordSafeState { - public static class CommandsForKeyCacheEntry extends AccordCacheEntry + public static class CommandsForKeyCacheEntry extends AccordCacheEntry { private NotifySink overrideSink; - CommandsForKeyCacheEntry(RoutingKey key, AccordCache.Type.Instance owner) + CommandsForKeyCacheEntry(RoutingKey key, AccordCache.Type.Instance owner) { super(key, owner); } } - private boolean invalidated; - private final AccordCacheEntry global; - private CommandsForKey original; - private CommandsForKey current; + private final AccordCacheEntry global; - public AccordSafeCommandsForKey(AccordCacheEntry global) + public AccordSafeCommandsForKey(AccordCacheEntry global) { super(global.key()); this.global = global; - this.original = null; - this.current = null; -// if (overrideSink() == null) -// overrideSink(new RecordingNotifySink()); } @Override @@ -60,7 +54,7 @@ public boolean equals(Object o) if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; AccordSafeCommandsForKey that = (AccordSafeCommandsForKey) o; - return Objects.equals(original, that.original) && Objects.equals(current, that.current); + return Objects.equals(current, that.current); } @Override @@ -73,47 +67,21 @@ public int hashCode() public String toString() { return "AccordSafeCommandsForKey{" + - "invalidated=" + invalidated + + "state=" + statusString() + ", global=" + global + - ", original=" + original + ", current=" + current + '}'; } - @Override - public boolean hasUpdate() - { - boolean hasUpdate = AccordSafeState.super.hasUpdate(); - - // cfk initialization is legal, but doesn't need to be propagated to the cache (and would - // cause an exception to be thrown if it were). Making an exception on the cache side could - // throw away applied cfk updates as well, so it's special cased here - if (hasUpdate && original == null && current != null && current.size() == 0) - return false; - - return hasUpdate; - } - - @Override - public AccordCacheEntry global() + public final AccordCacheEntry global() { - checkNotInvalidated(); return global; } @Override - public CommandsForKey current() - { - checkNotInvalidated(); - return current; - } - - @Override - @VisibleForTesting - public void set(CommandsForKey cfk) + public void postExecute(AccordTask owner) { - checkNotInvalidated(); - this.current = cfk; + global.releaseExclusive(this, owner); } @Override @@ -128,31 +96,13 @@ public NotifySink overrideSink() return ((CommandsForKeyCacheEntry)global).overrideSink; } - public CommandsForKey original() - { - checkNotInvalidated(); - return original; - } - - @Override - public void preExecute() + public void preExecute(AccordTask owner, LockMode lockMode) { - checkNotInvalidated(); - original = global.getExclusive(); - current = original; - if (isUnset()) + requireUninitialised(); + current = global.lockExclusive(owner, lockMode); + if (current == null) initialize(); + setSafe(); } - @Override - public void markUnsafe() - { - invalidated = true; - } - - @Override - public boolean isUnsafe() - { - return invalidated; - } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeState.java b/src/java/org/apache/cassandra/service/accord/AccordSafeState.java index a254c1154d08..ca8baafbedbe 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeState.java +++ b/src/java/org/apache/cassandra/service/accord/AccordSafeState.java @@ -17,41 +17,31 @@ */ package org.apache.cassandra.service.accord; -import accord.impl.SafeState; +import accord.local.SafeState; -public interface AccordSafeState extends SafeState -{ - void set(V update); - V original(); - void markUnsafe(); - boolean isUnsafe(); - void preExecute(); - - AccordCacheEntry global(); - - default boolean hasUpdate() - { - return original() != current(); - } +import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; - default void revert() - { - set(original()); - } +public interface AccordSafeState & AccordSafeState> +{ + AccordCacheEntry global(); + void postExecute(AccordTask owner); + void preExecute(AccordTask owner, LockMode lockMode); - default K key() + static AccordCacheEntry global(SafeState safeState) { - return global().key(); + return safeState.getClass() == AccordSafeCommand.class ? ((AccordSafeCommand) safeState).global() + : ((AccordSafeCommandsForKey) safeState).global(); } - default Throwable failure() + static void postExecute(SafeState safeState, AccordTask owner) { - return global().failure(); + if (safeState.getClass() == AccordSafeCommand.class) ((AccordSafeCommand) safeState).postExecute(owner); + else ((AccordSafeCommandsForKey) safeState).postExecute(owner); } - default void checkNotInvalidated() + static void preExecute(SafeState safeState, AccordTask owner, LockMode lockMode) { - if (isUnsafe()) - throw new IllegalStateException("Cannot access invalidated " + this); + if (safeState.getClass() == AccordSafeCommand.class) ((AccordSafeCommand) safeState).preExecute(owner, lockMode); + else ((AccordSafeCommandsForKey) safeState).preExecute(owner, lockMode); } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index edb0498856e3..98e1b239f4d9 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -464,6 +464,7 @@ public AccordService(Id localId) agent.setup(localId); AccordTimeService time = new AccordTimeService(); this.scheduler = new AccordScheduler(); + // TODO (expected): can we pass ImmediateExecutor rather than Scheduler? final RequestCallbacks callbacks = new RequestCallbacks(time, scheduler); this.dataStore = new AccordDataStore(); this.journal = new AccordJournal(DatabaseDescriptor.getAccord().journal); diff --git a/src/java/org/apache/cassandra/service/accord/AccordTask.java b/src/java/org/apache/cassandra/service/accord/AccordTask.java index 90781c740f25..30e6998396f2 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTask.java +++ b/src/java/org/apache/cassandra/service/accord/AccordTask.java @@ -18,19 +18,17 @@ package org.apache.cassandra.service.accord; import java.nio.ByteBuffer; -import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.CancellationException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; @@ -52,14 +50,17 @@ import accord.local.ExecutionContext; import accord.local.LoadKeys; import accord.local.SafeCommandStore; +import accord.local.SafeState; import accord.local.cfk.CommandsForKey; import accord.primitives.AbstractRanges; import accord.primitives.AbstractUnseekableKeys; import accord.primitives.Range; import accord.primitives.Ranges; +import accord.primitives.RoutingKeys; import accord.primitives.Timestamp; import accord.primitives.TxnId; import accord.primitives.Unseekables; +import accord.utils.ArrayBuffers.BufferList; import accord.utils.Invariants; import accord.utils.UnhandledEnum; import accord.utils.async.AsyncChain; @@ -68,6 +69,9 @@ import org.apache.cassandra.concurrent.DebuggableTask; import org.apache.cassandra.metrics.LogLinearDecayingHistograms; +import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus; +import org.apache.cassandra.service.accord.AccordCacheEntry.Loading; import org.apache.cassandra.service.accord.AccordCacheEntry.Status; import org.apache.cassandra.service.accord.AccordCommandStore.Caches; import org.apache.cassandra.service.accord.AccordExecutor.Task; @@ -76,142 +80,259 @@ import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; import org.apache.cassandra.service.accord.serializers.CommandSerializers; import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.NoSpamLogger; import org.apache.cassandra.utils.concurrent.Condition; +import static accord.local.LoadKeys.INCR; +import static accord.local.LoadKeys.NONE; +import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.RECOVERY; import static accord.local.LoadKeysFor.WRITE; -import static accord.primitives.Routable.Domain.Key; -import static accord.primitives.Txn.Kind.EphemeralRead; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.ASSIGNED; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.HOLD_QUEUE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; +import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; +import static org.apache.cassandra.service.accord.AccordExecutor.NONSYNC_BLOCKED_LIMIT; +import static org.apache.cassandra.service.accord.AccordExecutor.NONSYNC_ENABLED; +import static org.apache.cassandra.service.accord.AccordExecutor.NONSYNC_MIN_BATCH_SIZE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.RunState.PERSISTING; import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.CANCELLED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.FAILED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.FINISHED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.PERSISTING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.INCOMPLETE; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_OPTIONAL; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_REQUIRED; import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING; import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_LOAD; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_OPTIONAL; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_REQUIRED; +import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_OR_RUNNING; import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_SCAN_RANGES; +import static org.apache.cassandra.service.accord.AccordSafeState.global; +import static org.apache.cassandra.service.accord.AccordSafeState.postExecute; +import static org.apache.cassandra.service.accord.AccordSafeState.preExecute; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; import static org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask.SANITY_CHECK; -public abstract class AccordTask extends Task implements Function, Cancellable, DebuggableTask +public final class AccordTask extends Task implements Cancellable, DebuggableTask { private static final Logger logger = LoggerFactory.getLogger(AccordTask.class); private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES); - static class ForFunction extends AccordTask + public static AccordTask create(CommandStore commandStore, ExecutionContext context, Consumer consumer) + { + return new AccordTask<>((AccordCommandStore) commandStore, context, safeStore -> { + consumer.accept(safeStore); + return null; + }); + } + + public static AccordTask create(CommandStore commandStore, ExecutionContext context, Function function) + { + return new AccordTask<>((AccordCommandStore) commandStore, context, function); + } + + final class NonSyncState extends ExecutionContext.Wrapped implements ExecutionContext { - private final Function function; + RoutingKeys active; + ObjectHashSet notBlocking; + ObjectHashSet blocking; // cache entries on which we're blocking work + int loaded, processed; + int ready; - public ForFunction(AccordCommandStore commandStore, ExecutionContext context, Function function) + public NonSyncState() { - super(commandStore, context); - this.function = function; + super(executionContext); } @Override - public R apply(SafeCommandStore commandStore) + public Unseekables keys() { - return function.apply(commandStore); + return active; } - } - // TODO (desired): these anonymous ops are somewhat tricky to debug. We may want to at least give them names. - static class ForConsumer extends AccordTask - { - private final Consumer consumer; + void addLoaded() + { + ++loaded; + } - private ForConsumer(AccordCommandStore commandStore, ExecutionContext context, Consumer consumer) + void onNotHead(AccordCacheEntry entry) { - super(commandStore, context); - this.consumer = consumer; + if ((notBlocking == null || !notBlocking.remove((RoutingKey) entry.key())) && blocking != null) + blocking.remove((RoutingKey) entry.key()); } - @Override - public Void apply(SafeCommandStore commandStore) + void onNewHead(AccordCacheEntry entry) { - consumer.accept(commandStore); - return null; + ensureNotBlocking().add((RoutingKey) entry.key()); } - } - public static AccordTask create(CommandStore commandStore, ExecutionContext context, Function function) - { - return new ForFunction<>((AccordCommandStore) commandStore, context, function); - } + void onNewBlockingHead(AccordCacheEntry entry) + { + ensureBlocking().add((RoutingKey) entry.key()); + } - public static AccordTask create(CommandStore commandStore, ExecutionContext context, Consumer consumer) - { - return new ForConsumer((AccordCommandStore) commandStore, context, consumer); + void onStillHeadNewBlocking(AccordCacheEntry entry) + { + notBlocking.remove((RoutingKey) entry.key()); + ensureBlocking().add((RoutingKey) entry.key()); + } + + private ObjectHashSet ensureBlocking() + { + if (blocking == null) + blocking = new ObjectHashSet<>(); + return blocking; + } + + private ObjectHashSet ensureNotBlocking() + { + if (notBlocking == null) + notBlocking = new ObjectHashSet<>(); + return notBlocking; + } + + private int readyCount() + { + return (blocking == null ? 0 : blocking.size()) + (notBlocking == null ? 0 : notBlocking.size()); + } + + boolean isLoaded() + { + return loaded >= Math.min(keys, NONSYNC_MIN_BATCH_SIZE); + } + + boolean isWaitReady() + { + if (readyCount() >= Math.min(keys - processed, NONSYNC_MIN_BATCH_SIZE)) + return true; + + return blocking != null && blocking.size() >= NONSYNC_BLOCKED_LIMIT; + } + + void preRunExclusive() + { + try (BufferList keys = new BufferList<>()) + { + if ((blocking == null || !populate(keys, blocking)) && notBlocking != null) + populate(keys, notBlocking); + + keys.forEach(key -> preExecute(refs.get(key), AccordTask.this, RELEASE_QUEUE)); + keys.sort(RoutingKey::compareTo); + active = RoutingKeys.of(keys); + } + processed += active.size(); + if (processed == keys && isIncremental()) + setIncrementalFinishingExclusive(); + } + + private boolean populate(List keys, ObjectHashSet from) + { + if (keys.size() + from.size() <= AccordExecutor.NONSYNC_MAX_BATCH_SIZE) + { + keys.addAll(from); + from.clear(); + return keys.size() == AccordExecutor.NONSYNC_MAX_BATCH_SIZE; + } + + Iterator iterator = from.iterator(); + while (iterator.hasNext()) + { + if (keys.size() == AccordExecutor.NONSYNC_MAX_BATCH_SIZE) + return true; + + RoutingKey key = iterator.next(); + keys.add(key); + iterator.remove(); + } + + return false; + } + + void postRunExclusive() + { + if (active != null) + { + for (RoutingKey key : active) + postExecute(refs.remove(key), AccordTask.this); + active = null; + } + ready = readyCount(); + } } final AccordCommandStore commandStore; private final ExecutionContext executionContext; - private volatile String loggingId; - private static final AtomicLong nextLoggingId = new AtomicLong(Clock.Global.currentTimeMillis()); - private static final AtomicReferenceFieldUpdater loggingIdUpdater = AtomicReferenceFieldUpdater.newUpdater(AccordTask.class, String.class, "loggingId"); - - // TODO (desired): merge all of these maps into one - @Nullable Object2ObjectHashMap commands; - @Nullable Object2ObjectHashMap commandsForKey; - @Nullable Object2ObjectHashMap> loading; + private final Function function; + + // TODO (expected): simple custom map that allows (at least): + // - efficient putIfAbsent + // - efficient small collections (2-4 entries) + // - forEach with parameters to avoid boxing lambdas + // - destructive forEach + // - forEach over specific SafeState types + Object2ObjectHashMap> refs = new Object2ObjectHashMap<>(); + + /** + * if is(LOADING), this is the number of cache entries we're waiting to complete loading before we can transition to WAITING_ON_CACHE_QUEUES; + * if is(WAITING_ON_CACHE_QUEUES), it's the number we're waiting to be head of before we can run + *

+ * if isNonSync(), this counts only txnId; otherwise it counts keys and txnId + */ + int waitingForState; + + /** + * Only set when isNonSync() + *

+ * if is(LOADING), this is the cache entries that have finished loading + * otherwise it's the cache entries for which we're at the head of the queue and are ready to run with + */ + @Nullable NonSyncState nonSync; + + int keys; // TODO (expected): not counting keys we add during execution LogLinearDecayingHistograms.Buffer histogramBuffer; - // TODO (desired): collection supporting faster deletes but still fast poll (e.g. some ordered collection) - @Nullable ArrayDeque> waitingToLoad; @Nullable RangeTxnScanner rangeScanner; @Nullable CommandSummaries commandsForRanges; + byte runState; private BiConsumer callback; - public AccordTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext) + public AccordTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext, Function function) { - super(executionContext); + super(executionContext, commandStore.executor().uniqueCreatedAt); this.commandStore = commandStore; this.executionContext = executionContext; - this.loggingId = "0x" + Long.toHexString(nextLoggingId.incrementAndGet()); - + this.function = function; if (logger.isTraceEnabled()) logger.trace("Created {} on {}", this, commandStore); } - private String loggingId() + String loggingId() { - String id = loggingId; - if (id == null) - { - id = "0x" + Long.toHexString(nextLoggingId.incrementAndGet()); - if (!loggingIdUpdater.compareAndSet(this, null, id)) - id = loggingId; - } - return id; + return executor().executorId + "/" + Long.toHexString(createdAt); } @Override public String toString() { - return executionContext.describe() + ' ' + toBriefString(); + return "@[" + commandStore.id() + ',' + commandStore.node().id() + "] " + executionContext.describe() + ' ' + toBriefString(); } public String toBriefString() { - return '{' + loggingId() + ',' + state() + '}'; + return '{' + loggingId() + ',' + describeState() + '}'; } public String toDescription() { return toBriefString() + ": " - + (queued() == null ? "unqueued" : state()) + + (queued() == null ? "unqueued" : describeState()) + ", primaryTxnId: " + executionContext.primaryTxnId() - + ", waitingToLoad: " + summarise(waitingToLoad) - + ", loading:" + summarise(loading, AccordSafeState::global) - + ", cfks:" + summarise(commandsForKey, AccordSafeState::global) - + ", txns:" + summarise(commands, AccordSafeState::global); + + ", state: " + summarise(refs, AccordSafeState::global); } @@ -252,11 +373,6 @@ private static String summarise(Collection collection, Function keys() - { - return executionContext.keys(); - } - // TODO (expected): try to execute immediately BUT consider ordering requirements // esp. with deferred actions on e.g. CommandsForKey (not yet supported but also important for performance) public AsyncChain chain() @@ -267,7 +383,7 @@ public AsyncChain chain() protected Cancellable start(BiConsumer callback) { preSetup(callback); - commandStore.executor().submit(AccordTask.this); + executor().submit(AccordTask.this); return AccordTask.this; } }; @@ -281,86 +397,147 @@ private void preSetup(BiConsumer callback) } // to be invoked only by the CommandStore owning thread, to take references to objects already in use by the current execution - public void presetup(AccordTask parent) + public void preSetup(AccordTask parent) { this.position = parent.position; + setInheritedWithTranche(parent.tranche()); + // note we use the caches "unsafely" here deliberately, as we only reference commands we already have references to // so we do not mutate anything, except the atomic counter of references - if (parent.commands != null) - { - for (TxnId txnId : executionContext.txnIds()) - presetupExclusive(txnId, AccordTask::ensureCommands, parent.commands, commandStore.cachesUnsafe().commands()); - } - if (parent.commandsForKey == null) return; - if (executionContext.keys().domain() != Key) return; - switch (executionContext.loadKeys()) + LoadKeys loadKeys = loadKeys(executionContext); + if (loadKeys != NONE) { - default: throw new UnhandledEnum(executionContext.loadKeys()); - case NONE: - break; + Unseekables parentKeysOrRanges = parent.executionContext.keys(); + Unseekables keysOrRanges = executionContext.keys(); - case ASYNC: - case INCR: - case SYNC: - for (RoutingKey key : (AbstractUnseekableKeys) executionContext.keys()) - presetupExclusive(key, AccordTask::ensureCommandsForKey, parent.commandsForKey, commandStore.cachesUnsafe().commandsForKeys()); - break; + boolean isKeySubset = parent.isIncremental() ? parent.nonSync.active.containsAll(keysOrRanges) : parentKeysOrRanges.containsAll(keysOrRanges); + if (isKeySubset) + setInheritedRangeScan(); + + setSequencedExclusive(executionContext.executionSequence()); + if (isSequencedByPriorityAtomic()) + { + boolean isTxnIdSubset = executionContext.isTxnIdSubsetOf(parent.executionContext); + Invariants.require(isKeySubset, "Must start ATOMIC tasks from a task declaring a superset of the required keys (for ASYNC/INCR tasks this means the keys active for the batch in question)"); + Invariants.require(isTxnIdSubset, "Must start ATOMIC tasks from a task declaring a superset of the required TxnIds"); + // TODO (required): we're appending to the fifo queue - does this maintain correct order? + setCacheQueuedFifoExclusive(); + } + + if (loadKeys != SYNC) + { + setNonSyncExclusive(); + nonSync = new NonSyncState(); + if (loadKeys == INCR) + { + setIncrementalExclusive(); + // forbid BY_PRIORITY sequencing to avoid priority inversion deadlocks on INCR tasks that lock a TxnId but await some key that has a higher priority task (that is waiting on our locked TxnId) - solvable in future if necessary + Invariants.require(isSequencedByPriorityAtomic() || isUnsequenced(), "INCR tasks may currently only be ATOMIC or UNSEQUENCED"); + } + } + + if (keysOrRanges.equals(parentKeysOrRanges)) + { + // TODO (desired): custom map we can more cheaply fork/copy + parent.refs.forEach((key, val) -> { + if (val instanceof AccordSafeCommandsForKey) + preSetup((RoutingKey) key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); + }); + } + else + { + switch (keysOrRanges.domain()) + { + case Key: + for (RoutingKey key : (AbstractUnseekableKeys) keysOrRanges) + preSetup(key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); + break; + + case Range: + AbstractRanges ranges = (AbstractRanges) keysOrRanges; + parent.refs.forEach((key, val) -> { + if (val instanceof AccordSafeCommandsForKey && ranges.contains((RoutingKey) key)) + preSetup((RoutingKey) key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); + }); + break; + } + } } + + for (TxnId txnId : executionContext.txnIds()) + preSetup(txnId, parent.refs, commandStore.cachesUnsafe().commands()); } @Override void submitExclusive(AccordExecutor owner) { - owner.submitExclusive(this); - } - - public void setupExclusive() - { + owner.registerExclusive(this); setupInternal(commandStore.cachesExclusive()); - setState(rangeScanner != null ? WAITING_TO_SCAN_RANGES - : waitingToLoad != null ? WAITING_TO_LOAD - : loading != null ? LOADING : WAITING_TO_RUN); } private void setupInternal(Caches caches) { + boolean hasPreSetup = hasInherited(); + LoadKeys loadKeys = loadKeys(executionContext); + if (loadKeys != NONE) { - boolean hasPreSetup = commands != null; - for (TxnId txnId : executionContext.txnIds()) + if (loadKeys != SYNC && !hasPreSetup) { - if (hasPreSetup && completePresetupExclusive(txnId, commands, caches.commands())) - continue; - setupExclusive(txnId, AccordTask::ensureCommands, caches.commands()); + setNonSyncExclusive(); + nonSync = new NonSyncState(); + if (loadKeys == INCR) + setIncrementalExclusive(); } - } - if (executionContext.keys().isEmpty()) - return; + Unseekables keysOrRanges = executionContext.keys(); + switch (keysOrRanges.domain()) + { + case Range: + if (!hasInheritedRangeScan()) setupRangeLoadsExclusive(caches); + else refs.forEach((k, v) -> { + if (v instanceof AccordSafeCommandsForKey) + completePresetupExclusive((AccordSafeCommandsForKey)v); + }); + break; + case Key: + setupKeyLoadsExclusive(hasPreSetup, caches, (AbstractUnseekableKeys) keysOrRanges, hasInheritedRangeScan()); + break; + } + } - switch (executionContext.keys().domain()) + for (TxnId txnId : executionContext.txnIds()) { - case Key: setupKeyLoadsExclusive(caches, (AbstractUnseekableKeys) executionContext.keys(), false); break; - case Range: setupRangeLoadsExclusive(caches); + if (hasPreSetup && completePresetupExclusive(txnId)) + continue; + setupExclusive(txnId, caches.commands(), 1); } + + if (is(SCANNING_RANGES)) + return; + + onSetupOrScannedExclusive(); } - private void setupKeyLoadsExclusive(Caches caches, Iterable keys, boolean isToCompleteRangeScan) + private void setupKeyLoadsExclusive(boolean hasPreSetup, Caches caches, Iterable setupKeys, boolean doNotScanRanges) { - if (executionContext.loadKeys() == LoadKeys.NONE) + if (executionContext.loadKeys() == NONE) return; - if (!isToCompleteRangeScan && executionContext.loadKeysFor() == RECOVERY) + if (!doNotScanRanges && executionContext.loadKeysFor() == RECOVERY) { Invariants.require(rangeScanner == null); rangeScanner = new RangeTxnScanner(); + rangeScanner.start(); } - boolean hasPreSetup = commandsForKey != null; - for (RoutingKey key : keys) + int waitsForIncrement = isSync() ? 1 : 0; + for (RoutingKey setupKey : setupKeys) { - if (hasPreSetup && completePresetupExclusive(key, commandsForKey, caches.commandsForKeys())) continue; - setupExclusive(key, AccordTask::ensureCommandsForKey, caches.commandsForKeys()); + if (hasPreSetup && completePresetupExclusive(setupKey)) + continue; + + setupExclusive(setupKey, caches.commandsForKeys(), waitsForIncrement); } } @@ -369,229 +546,433 @@ private void setupRangeLoadsExclusive(Caches caches) if (executionContext.loadKeysFor() == WRITE) return; - switch (executionContext.loadKeys()) - { - default: throw new UnhandledEnum(executionContext.loadKeys()); - case NONE: - case ASYNC: - break; - - case INCR: - throw new AssertionError("Incremental mode should only be used with an explicit list of keys"); - - case SYNC: - rangeScanner = new RangeTxnAndKeyScanner(caches.commandsForKeys()); - } + rangeScanner = new RangeTxnAndKeyScanner(caches.commandsForKeys()); + rangeScanner.start(); } // expects mutual exclusivity only on the command store - private > void presetupExclusive(K k, Function, Map> loaded, Map parentMap, AccordCache.Type.Instance cache) + private & AccordSafeState> void preSetup(K k, Map> parentMap, AccordCache.Type.Instance cache) { - AccordSafeState ref = parentMap.get(k); + S ref = (S) parentMap.get(k); if (ref == null) return; - AccordCacheEntry node = ref.global(); + AccordCacheEntry node = ref.global(); int refs = node.increment(); Invariants.require(refs > 1); - loaded.apply(this).put(k, cache.parent().adapter().safeRef(node)); + S safeState = cache.parent().adapter().safeRef(node); + this.refs.put(k, safeState); + if (cache.isCommandsForKey()) + keys++; } - // expects to hold lock - private > boolean completePresetupExclusive(K k, Map map, AccordCache.Type.Instance cache) + private & AccordSafeState> boolean completePresetupExclusive(K k) { - AccordSafeState preacquired = map.get(k); + S preacquired = (S) refs.get(k); if (preacquired != null) { - cache.recordPreAcquired(preacquired); + completePresetupExclusive(preacquired); return true; } return false; } + private & AccordSafeState> void completePresetupExclusive(S preacquired) + { + AccordCacheEntry entry = preacquired.global(); + if (entry.isLoaded()) completeSetupOfLoaded(entry); + else completeSetupOfLoading(entry, true); + } + // expects to hold lock - private > void setupExclusive(K k, Function, Map> loaded, AccordCache.Type.Instance cache) + private & AccordSafeState> void setupExclusive(K k, AccordCache.Type.Instance cache, int waitForIncrement) { S safeRef = cache.acquire(k); - Status entryStatus = safeRef.global().status(); - Map map; + AccordCacheEntry entry = safeRef.global(); + Status entryStatus = entry.status(); + boolean submitLoad = false; + boolean isLoaded; switch (entryStatus) { default: throw new UnhandledEnum(entryStatus); case WAITING_TO_LOAD: + submitLoad = true; case LOADING: - map = ensureLoading(); + isLoaded = false; + waitingForState += waitForIncrement; break; case WAITING_TO_SAVE: case SAVING: case LOADED: case MODIFIED: case FAILED_TO_SAVE: - map = loaded.apply(this); + isLoaded = true; } - Object prev = map.putIfAbsent(k, safeRef); + Object prev = refs.putIfAbsent(k, safeRef); if (prev != null) { - noSpamLogger.warn("PreLoadContext {} contained key {} more than once", map, k); + noSpamLogger.warn("ExecutionContext {} contained key {} more than once", refs, k); cache.release(safeRef, this); + waitingForState -= waitForIncrement; } - else if (map == loading) + else { - if (entryStatus == Status.WAITING_TO_LOAD) - ensureWaitingToLoad().add(safeRef.global()); - safeRef.global().loadingOrWaiting().add(this); - Invariants.paranoid(safeRef.global().loadingOrWaiting().waiters().size() == safeRef.global().references()); + if (entry.isCommandsForKey()) + keys++; + + if (isLoaded) completeSetupOfLoaded(entry); + else + { + if (submitLoad) executor().cacheUnsafe().load(executor(), this, is(ExclusiveGroup.RANGE), entry); + completeSetupOfLoading(entry, !submitLoad); + } } } - // expects to hold lock - public boolean onLoad(AccordCacheEntry state) + private void completeSetupOfLoaded(AccordCacheEntry entry) { - AccordSafeState safeRef = loading == null ? null : loading.remove(state.key()); - Invariants.require(safeRef != null && safeRef.global() == state, "Expected to find %s loading; found %s", state, this, AccordTask::toDescription); - if (safeRef.getClass() == AccordSafeCommand.class) - ensureCommands().put((TxnId)state.key(), (AccordSafeCommand) safeRef); - else - ensureCommandsForKey().put((RoutingKey) state.key(), (AccordSafeCommandsForKey) safeRef); - - if (!loading.isEmpty()) - return false; - - loading = null; - if (compareTo(WAITING_TO_LOAD) < 0) - return false; - - Invariants.require(waitingToLoad == null, "Invalid state: %s", this, AccordTask::toDescription); - setState(WAITING_TO_RUN); - return true; + if (isOptional(entry)) + { + nonSync.addLoaded(); + if (isCacheQueuedFifo()) + addQueuedOptionalKey(entry, entry.addFifo(this)); + } + else if (isCacheQueuedFifo()) + { + entry.addFifo(this); + } } - // expects to hold lock - public boolean onLoading(AccordCacheEntry state) + private void completeSetupOfLoading(AccordCacheEntry entry, boolean alreadyLoading) { - boolean removed = waitingToLoad != null && waitingToLoad.remove(state); - Invariants.require(removed, "%s not found in waitingToLoad %s", state, this, AccordTask::toDescription); - if (!waitingToLoad.isEmpty()) - return false; + if (alreadyLoading) + { + Loading loading = entry.loading(); + if (loading.loading != null && loading.loading.is(RANGE_LOAD) && loading.loading.is(WAITING_TO_RUN) && !is(ExclusiveGroup.RANGE)) + { + // requeue anything setup as a range load that's now needed for a key-based operation, so it can use the correct the queue limits + loading.loading.unqueue(executor().runnable); + loading.loading.override(LOAD); + executor().runnable.enqueue(loading.loading, false); + } + } - return onEmptyWaitingToLoad(); + if (isCacheQueuedFifo()) entry.addFifo(this); + else entry.addWaitingToLoad(this); + Invariants.paranoid(entry.waitingCount() == entry.references()); } - private boolean onEmptyWaitingToLoad() + private void onSetupOrScannedExclusive() { - waitingToLoad = null; - if (compareTo(WAITING_TO_LOAD) < 0) - return false; + if (waitingForState > 0) + { + setStateExclusive(LOADING_REQUIRED); + executor().loading.enqueue(this); + } + else onLoadedRequiredExclusive(); + } - setState(loading == null ? WAITING_TO_RUN : LOADING); - return true; + private void onLoadedRequiredExclusive() + { + if (isSync() || nonSync.isLoaded()) + { + waitOnCacheQueuesExclusive(); + } + else + { + setStateExclusive(LOADING_OPTIONAL); + executor().loading.enqueue(this); + } } - public ExecutionContext executionContext() + boolean isUnsequenced(AccordCacheEntry entry) { - return executionContext; + return isUnsequenced() && (entry.isCommandsForKey() || !isIncremental()); } - public Map commands() + boolean isOptional(AccordCacheEntry entry) { - return commands; + return isNonSync() && entry.isCommandsForKey(); + } + + boolean holdsLocksBetweenRuns() + { // TODO (desired): encode as a state bit + return isIncremental() && executionContext.primaryTxnId() != null; } - public Map ensureCommands() + private void waitOnCacheQueuesExclusive() { - if (commands == null) - commands = new Object2ObjectHashMap<>(); - return commands; + Invariants.require(waitingForState == 0); + onLoaded(); + executor().runnable.incrementArrivals(this); + commandStore.exclusiveExecutor.incrementArrivals(this); + + this.refs.forEach((key, safeState) -> { + AccordCacheEntry entry = global(safeState); + boolean optional = isOptional(entry); + if (entry.isLoaded()) + { + RunnableStatus status = addToCacheQueue(entry, false); + if (optional) addQueuedOptionalKey(entry, status); + else if (status == NOT_RUNNABLE) + ++waitingForState; + } + else Invariants.require(optional); + }); + + // TODO (desired): exception-safe rollback for addUnsequenced + setCacheQueuedExclusive(); + if (waitingForState == 0) waitOnOptionalCacheQueuesExclusive(); + else + { + setStateExclusive(WAITING_ON_REQUIRED); + executor().waitingOnCacheQueues.enqueue(this); + } } - public Map commandsForKey() + private void waitOnOptionalCacheQueuesExclusive() { - return commandsForKey; + if (isSync() || nonSync.isWaitReady()) waitToRunExclusive(); + else + { + setStateExclusive(WAITING_ON_OPTIONAL); + executor().waitingOnCacheQueues.enqueue(this); + } } - public Map ensureCommandsForKey() + RunnableStatus addToCacheQueue(AccordCacheEntry loaded, boolean addIfFifo) { - if (commandsForKey == null) - commandsForKey = new Object2ObjectHashMap<>(); - return commandsForKey; + if (isCacheQueuedFifo()) return addIfFifo ? loaded.addFifo(this) : loaded.headStatus(this); + else if (isUnsequenced(loaded)) return loaded.addUnsequenced(this); + else return loaded.addPrioritised(this); } - private Map> ensureLoading() + void onLoadOneExclusive(AccordCacheEntry loaded) { - if (loading == null) - loading = new Object2ObjectHashMap<>(); - return loading; + if (isOptional(loaded)) + { + // if we're incremental/async we don't block on keys loading, so we don't need to decrement anything + // however, if we're in fifo mode this loaded key might be ready for us to run with + State state = state(); + switch (state) + { + default: throw new UnhandledEnum(state); + case WAITING_ON_REQUIRED: + case WAITING_ON_OPTIONAL: + case WAITING_TO_RUN: + case RUNNING: + RunnableStatus status = addToCacheQueue(loaded, false); + if (status != NOT_RUNNABLE) + addQueuedOptionalKey(loaded, status); + // fall-through + case LOADING_REQUIRED: + nonSync.addLoaded(); + break; + + case LOADING_OPTIONAL: + addLoadedOptionalKey(); + break; + } + } + else + { + if (--waitingForState == 0) + { + if (is(LOADING_REQUIRED)) + { + unqueue(executor().loading); + onLoadedRequiredExclusive(); + } + else Invariants.require(is(SCANNING_RANGES)); + } + } } - private ArrayDeque> ensureWaitingToLoad() + void addLoadedOptionalKey() { - Invariants.require(compareTo(WAITING_TO_LOAD) <= 0, "Expected status to be on or before WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); - if (waitingToLoad == null) - waitingToLoad = new ArrayDeque<>(); - return waitingToLoad; + nonSync.addLoaded(); + if (is(LOADING_OPTIONAL) && nonSync.isLoaded()) + { + unqueue(executor().loading); + waitOnCacheQueuesExclusive(); + } } - public AccordCacheEntry pollWaitingToLoad() + // TODO (expected): add vs setup vs onChange; some callers don't need to try + void addQueuedOptionalKey(AccordCacheEntry loaded, RunnableStatus status) { - Invariants.require(is(WAITING_TO_LOAD), "Expected status to be WAITING_TO_LOAD; found %s", this, AccordTask::toDescription); - if (waitingToLoad == null) - return null; + switch (status) + { + default: throw UnhandledEnum.unknown(status); + case NOT_RUNNABLE: break; + case STILL_RUNNABLE: + case NEWLY_RUNNABLE: + nonSync.onNewHead(loaded); + break; + case STILL_RUNNABLE_NEWLY_BLOCKING: + case NEWLY_BLOCKING_RUNNABLE: + nonSync.onNewBlockingHead(loaded); + break; + } - AccordCacheEntry next = waitingToLoad.poll(); - if (waitingToLoad.isEmpty()) - onEmptyWaitingToLoad(); - return next; + if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady()) + { + unqueue(executor().waitingOnCacheQueues); + waitToRunExclusive(); + } } - public AccordCacheEntry peekWaitingToLoad() + void onChangeHeadStatus(AccordCacheEntry entry, RunnableStatus status) { - return waitingToLoad == null ? null : waitingToLoad.peek(); + if (isSync() || !entry.isCommandsForKey()) onChangeRequiredHeadStatus(entry, status); + if (isNonSync() && entry.isCommandsForKey()) onChangeOptionalHeadStatus(entry, status); } - private void maybeSanityCheck(AccordSafeCommand safeCommand) + private void incrementWaitingWhileAlreadyWaiting() { - if (SANITY_CHECK) + Invariants.require(isState(WAITING)); + if (waitingForState == 0) { - DebugTask debug = DebugTask.get(this); - if (debug.sanityCheck == null) - debug.sanityCheck = new ArrayList<>(commands.size()); - debug.sanityCheck.add(safeCommand.current()); + if (is(WAITING_ON_OPTIONAL)) setStateExclusive(WAITING_ON_REQUIRED); + else + { + // TODO (expected): this is potentially costly; maybe we don't want to swap these in and out (but harder to maintain invariants) + unqueue(commandStore.exclusiveExecutor); + setStateExclusive(WAITING_ON_REQUIRED); + executor().waitingOnCacheQueues.enqueue(this); + } } + Invariants.require(waitingForState < refs.size()); + ++waitingForState; } - private void save(List diffs, Runnable onFlush) + void onChangeRequiredHeadStatus(AccordCacheEntry entry, RunnableStatus newStatus) { - if (SANITY_CHECK && DebugTask.get(this).sanityCheck != null) + if (newStatus == NOT_RUNNABLE) { - Condition condition = Condition.newOneTimeCondition(); - this.commandStore.appendCommands(diffs, condition::signal); - condition.awaitUninterruptibly(); + incrementWaitingWhileAlreadyWaiting(); + } + else if (newStatus != STILL_RUNNABLE_NEWLY_BLOCKING) + { + Invariants.require(is(WAITING_ON_REQUIRED)); + if (--waitingForState == 0) + { + unqueue(executor().waitingOnCacheQueues); + waitOnOptionalCacheQueuesExclusive(); + } + } + } - for (Command check : DebugTask.get(this).sanityCheck) - this.commandStore.sanityCheckCommand(commandStore.unsafeGetRedundantBefore(), check); + void onChangeOptionalHeadStatus(AccordCacheEntry entry, RunnableStatus status) + { + Invariants.require(isState(WAITING_OR_RUNNING)); + switch (status) + { + default: throw UnhandledEnum.unknown(status); + case STILL_RUNNABLE: throw UnhandledEnum.invalid(STILL_RUNNABLE); // onChange -> changed (but this means no change) + case NOT_RUNNABLE: + nonSync.onNotHead(entry); + if (is(WAITING_TO_RUN) && !nonSync.isWaitReady()) + { + unqueue(commandStore.exclusiveExecutor); + setStateExclusive(WAITING_ON_OPTIONAL); + executor().waitingOnCacheQueues.enqueue(this); + } + return; - if (onFlush != null) onFlush.run(); + case STILL_RUNNABLE_NEWLY_BLOCKING: + nonSync.onStillHeadNewBlocking(entry); + break; + + case NEWLY_RUNNABLE: + nonSync.onNewHead(entry); + break; + + case NEWLY_BLOCKING_RUNNABLE: + nonSync.onNewBlockingHead(entry); + break; } - else + + if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady()) { - this.commandStore.appendCommands(diffs, onFlush); + unqueue(executor().waitingOnCacheQueues); + waitToRunExclusive(); } } + void waitToRunExclusive() + { + setStateExclusive(WAITING_TO_RUN); + commandStore.exclusiveExecutor.enqueue(this, false); + } + + public ExecutionContext executionContext() + { + return executionContext; + } + @Override protected void preRunExclusive() { - setState(ASSIGNED); + super.preRunExclusive(); if (rangeScanner != null) { commandsForRanges = rangeScanner.finish(commandStore.cachesExclusive()); rangeScanner = null; } - if (commands != null) - commands.forEach((k, v) -> v.preExecute()); - if (commandsForKey != null) - commandsForKey.forEach((k, v) -> v.preExecute()); + + if (isSync()) + { + refs.forEach((k, v) -> { + preExecute(v, this, RELEASE_QUEUE); + }); + } + else + { + if (!hasIncrementalStarted()) + { + TxnId primaryTxnId = executionContext.primaryTxnId(); + if (primaryTxnId != null) + { + LockMode lockMode = holdsLocksBetweenRuns() ? HOLD_QUEUE : RELEASE_QUEUE; + preExecute(refs.get(primaryTxnId), this, lockMode); + TxnId additionalTxnId = executionContext.additionalTxnId(); + if (additionalTxnId != null) + preExecute(refs.get(additionalTxnId), this, lockMode); + } + + if (isIncremental() && isSequencedByPriorityAtomic() && !isCacheQueuedFifo()) + { + setCacheQueuedFifoExclusive(); + refs.forEach((key, safeState) -> { + AccordCacheEntry entry = global(safeState); + RunnableStatus status = entry.moveToFifo(this); + if (entry.isLoaded()) + { + switch (status) + { + default: throw UnhandledEnum.unknown(status); + case NOT_RUNNABLE: + case STILL_RUNNABLE: + case STILL_RUNNABLE_NEWLY_BLOCKING: + break; + case NEWLY_RUNNABLE: + nonSync.onNewHead(entry); + break; + case NEWLY_BLOCKING_RUNNABLE: + nonSync.onNewBlockingHead(entry); + break; + } + } + }); + } + + if (isIncremental()) + setIncrementalStartedExclusive(); + } + nonSync.preRunExclusive(); + } } @Override @@ -604,122 +985,146 @@ public void run() if (Tracing.isTracing()) Tracing.trace(executionContext.describe()); - setState(RUNNING); + commandStore.begin(safeStore = new AccordSafeCommandStore(this, isSync() ? executionContext : nonSync)); - safeStore = commandStore.begin(this, commandsForRanges); - R result = apply(safeStore); + R result = function.apply(safeStore); - List changes = null; - if (commands != null) + boolean finished = !isIncremental() || isIncrementalFinishing(); + if (finished) { - for (AccordSafeCommand safeCommand : commands.values()) - { - if (safeCommand.txnId().is(EphemeralRead)) - continue; - - Journal.CommandUpdate diff = safeCommand.update(); - if (diff == null) - continue; - - if (changes == null) - changes = new ArrayList<>(commands.size()); - changes.add(diff); + List changes = new ArrayList<>(); + // TODO (expected): save any TxnId we add so that we don't need to iterate all of refs + refs.forEach((key, value) -> { + if (value instanceof AccordSafeCommand) + { + AccordSafeCommand safeCommand = (AccordSafeCommand) value; + Journal.CommandUpdate diff = safeCommand.update(); + if (diff != null) + { + changes.add(diff); + maybeSanityCheck(safeCommand); + } + } + }); - maybeSanityCheck(safeCommand); + boolean flush = !changes.isEmpty() || safeStore.fieldUpdates() != null; + if (flush) + { + setRunState(PERSISTING); + Runnable onFlush = () -> finish(result, null); + safeStore.persistFieldUpdatesInternal(changes.isEmpty() ? onFlush : null); + if (!changes.isEmpty()) + save(changes, onFlush); + finished = false; } } - boolean flush = changes != null || safeStore.fieldUpdates() != null; - if (flush) - { - setState(PERSISTING); - Runnable onFlush = () -> finish(result, null); - safeStore.persistFieldUpdatesInternal(changes == null ? onFlush : null); - if (changes != null) save(changes, onFlush); - } - + // TODO (required): exception handling here needs improving + safeStore.postExecute(); commandStore.complete(safeStore); safeStore = null; - onRunComplete(); - if (!flush) + + if (finished) finish(result, null); } catch (Throwable t) { if (safeStore != null) - { - revert(); - commandStore.abort(safeStore); - } + refs.forEach((k, v) -> v.setAbandoned()); throw t; } + finally + { + if (safeStore != null) + commandStore.complete(safeStore); + onRunComplete(); + } } - public void fail(Throwable throwable) + private void save(List diffs, Runnable onFlush) { - if (state().isComplete()) - return; - - try + if (SANITY_CHECK && DebugTask.get(this).sanityCheck != null) { - setState(FAILED); - commandStore.agent().onException(throwable); + Condition condition = Condition.newOneTimeCondition(); + this.commandStore.appendCommands(diffs, condition::signal); + condition.awaitUninterruptibly(); + + for (Command check : DebugTask.get(this).sanityCheck) + this.commandStore.sanityCheckCommand(commandStore.unsafeGetRedundantBefore(), check); + + if (onFlush != null) onFlush.run(); } - finally + else { - if (callback != null) - callback.accept(null, throwable); + this.commandStore.appendCommands(diffs, onFlush); } } - @Override - protected boolean isNewWork() + private void maybeSanityCheck(AccordSafeCommand safeCommand) { - return true; + if (SANITY_CHECK) + { + DebugTask debug = DebugTask.get(this); + if (debug.sanityCheck == null) + debug.sanityCheck = new ArrayList<>(2); + debug.sanityCheck.add(safeCommand.current()); + } } - public void failExclusive(Throwable throwable) + public void reportFailure(Throwable throwable) { - fail(throwable); + try + { + if (callback != null) + callback.accept(null, throwable); + } + finally + { + commandStore.agent().onException(throwable); + } } @Override - protected void cleanupExclusive(AccordExecutor executor) + protected void cleanupExclusive(AccordExecutor executor, boolean executed) { - Invariants.expect(state().isExecuted()); - releaseResources(commandStore.cachesExclusive()); - super.cleanupExclusive(executor); - executor.keys.increment(commandsForKey == null ? 0 : commandsForKey.size(), runningAt); + if (is(RUNNING) && isNonSync()) + { + nonSync.postRunExclusive(); + if (isIncremental() && !isIncrementalFinishing()) + { + setStateExclusive(INCOMPLETE); + waitOnOptionalCacheQueuesExclusive(); + return; + } + } + + executor.keys.increment(keys, runningAt); + releaseResourcesExclusive(commandStore.cachesExclusive()); + super.cleanupExclusive(executor, executed); if (histogramBuffer != null) { - histogramBuffer.flush(cleanupAt); + histogramBuffer.flush(completeAt); histogramBuffer = null; } } - @Nullable - public RangeTxnScanner rangeScanner() - { - return rangeScanner; - } - @Override public void cancel() { - if (!state().isComplete()) - commandStore.executor().cancel(this); + if (!state().hasStarted()) + executor().cancel(this); } void cancelExclusive() { logger.info("Cancelling {}", executionContext); - setState(CANCELLED); + setStateExclusive(CANCELLED); if (rangeScanner != null) rangeScanner.cancelled = true; if (callback != null) { - if (commandStore.executor().isInLoop()) callback.accept(null, new CancellationException()); - else commandStore.executor().submit(() -> callback.accept(null, new CancellationException())); + if (executor().isInLoop()) callback.accept(null, new CancellationException()); + else executor().submit(() -> callback.accept(null, new CancellationException())); } } @@ -730,113 +1135,54 @@ void cancelExclusive(AccordExecutor owner) private void finish(R result, Throwable failure) { - setState(failure == null ? FINISHED : FAILED); + setRunState(failure == null ? RunState.SUCCESS : RunState.FAILED); if (callback != null) callback.accept(result, failure); } - void releaseResources(Caches caches) + void releaseResourcesExclusive(Caches caches) { + if (refs == null) + return; + try { - // TODO (expected): we should destructively iterate to avoid invoking second time in fail; or else read and set to null if (rangeScanner != null) { rangeScanner.cleanup(caches); rangeScanner = null; if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedRangeScanner(); } - if (commands != null) - { - commands.forEach((k, v) -> caches.commands().release(v, this)); - commands.clear(); - commands = null; - if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedCommands(); - } - if (commandsForKey != null) - { - commandsForKey.forEach((k, v) -> caches.commandsForKeys().release(v, this)); - commandsForKey.clear(); - commandsForKey = null; - if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedCommandsForKeys(); - } - if (waitingToLoad != null) - { - while (!waitingToLoad.isEmpty()) - waitingToLoad.poll().loadingOrWaiting().remove(this); - waitingToLoad = null; - } - if (loading != null) - { - loading.forEach((k, v) -> caches.global().release(v, this)); - loading.clear(); - loading = null; - } + + refs.forEach((key, safeState) -> { + AccordSafeState.postExecute(safeState, this); + }); + if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedState(); } catch (Throwable t) { - releaseResourcesSlow(caches, t); + releaseResourcesSlowExclusive(t); commandStore.agent().onException(t); } - } - - private void releaseResourcesSlow(Caches caches, Throwable suppressedBy) - { - if (commands != null) - { - safeRelease(commands, caches.commands(), suppressedBy); - commands.clear(); - commands = null; - } - if (commandsForKey != null) - { - safeRelease(commandsForKey, caches.commandsForKeys(), suppressedBy); - commandsForKey.clear(); - commandsForKey = null; - } - if (waitingToLoad != null) - { - while (!waitingToLoad.isEmpty()) - { - try { waitingToLoad.poll().loadingOrWaiting().remove(this); } - catch (Throwable t) { suppressedBy.addSuppressed(t); } - } - waitingToLoad = null; - } - if (loading != null) - { - safeRelease(loading, caches.global(), suppressedBy); - loading.clear(); - loading = null; - } - } - - private void safeRelease(Map> map, AccordCache.Type.Instance cache, Throwable suppressedBy) - { - for (AccordSafeState safeState : map.values()) + finally { - if (safeState.isUnsafe()) continue; - try { cache.release(safeState, this); } - catch (Throwable t) { suppressedBy.addSuppressed(t); } + refs = null; } } - private void safeRelease(Map> map, AccordCache cache, Throwable suppressedBy) + private void releaseResourcesSlowExclusive(Throwable suppressedBy) { - for (AccordSafeState safeState : map.values()) - { - if (safeState.isUnsafe()) continue; - try { cache.release(safeState, this); } - catch (Throwable t) { suppressedBy.addSuppressed(t); } - } - } + if (refs == null) + return; - void revert() - { - if (commands != null) - commands.forEach((k, v) -> v.revert()); - if (commandsForKey != null) - commandsForKey.forEach((k, v) -> v.revert()); + refs.forEach((k, safeState) -> { + if (!safeState.isReleased()) + { + try { AccordSafeState.postExecute(safeState, this); } + catch (Throwable t) { suppressedBy.addSuppressed(t); } + } + }); + refs = null; } public class RangeTxnAndKeyScanner extends RangeTxnScanner @@ -844,17 +1190,17 @@ public class RangeTxnAndKeyScanner extends RangeTxnScanner class KeyWatcher implements AccordCache.Listener { @Override - public void onUpdate(AccordCacheEntry state) + public void onUpdate(AccordCacheEntry state) { if (ranges.contains(state.key())) - reference(state); + reference((AccordCacheEntry) state); } } final Set intersectingKeys = new ObjectHashSet<>(); - final KeyWatcher keyWatcher = new KeyWatcher(); final Ranges ranges = ((AbstractRanges) executionContext.keys()).toRanges(); final AccordCache.Type.Instance commandsForKeyCache; + KeyWatcher keyWatcher = new KeyWatcher(); public RangeTxnAndKeyScanner(AccordCache.Type.Instance commandsForKeyCache) { @@ -877,11 +1223,8 @@ protected void runInternal() super.runInternal(); } - private void reference(AccordCacheEntry entry) + private void reference(AccordCacheEntry entry) { - if (loading != null && loading.containsKey(entry.key())) - return; - switch (entry.status()) { default: throw new AssertionError("Unhandled Status: " + entry.status()); @@ -894,7 +1237,7 @@ private void reference(AccordCacheEntry entry) case SAVING: case LOADED: case FAILED_TO_SAVE: - if (commandsForKey != null && commandsForKey.containsKey(entry.key())) + if (refs.containsKey(entry.key())) return; Object v = entry.getOrShrunkExclusive(); @@ -913,7 +1256,20 @@ else if (v instanceof CommandsForKey) return; } - ensureCommandsForKey().putIfAbsent(entry.key(), commandsForKeyCache.acquire(entry)); + refs.put(entry.key(), commandsForKeyCache.acquire(entry)); + ++keys; + + Invariants.require(!isCacheQueuedFifo(), "Unsafe to addFifo in listener as no ordering guarantees"); + if (isOptional(entry)) + addLoadedOptionalKey(); + + if (isCacheQueued()) + { + Invariants.require(isState(WAITING)); + RunnableStatus status = addToCacheQueue(entry, false); + if (isOptional(entry)) addQueuedOptionalKey(entry, status); + else if (status == NOT_RUNNABLE) incrementWaitingWhileAlreadyWaiting(); + } } } @@ -930,23 +1286,22 @@ void startInternal(Caches caches) void scannedInternal() { - if (commandsForKey != null) - intersectingKeys.removeAll(commandsForKey.keySet()); - if (loading != null) - intersectingKeys.removeAll(loading.keySet()); - setupKeyLoadsExclusive(commandStore.cachesExclusive(), intersectingKeys, true); + intersectingKeys.removeAll(refs.keySet()); + setupKeyLoadsExclusive(false, commandStore.cachesExclusive(), intersectingKeys, true); super.scannedInternal(); } void cleanup(Caches caches) { - caches.commandsForKeys().tryUnregister(keyWatcher); + if (keyWatcher != null) + caches.commandsForKeys().tryUnregister(keyWatcher); super.cleanup(caches); } CommandSummaries finish(Caches caches) { caches.commandsForKeys().unregister(keyWatcher); + keyWatcher = null; return super.finish(caches); } } @@ -975,7 +1330,7 @@ ExecutionContext preLoadContext() @Override protected void postRunExclusive() { - commandStore.executor().onScannedRangesExclusive(AccordTask.this, failure); + executor().onScannedRangesExclusive(AccordTask.this, failure); } @Override @@ -984,12 +1339,13 @@ protected void fail(Throwable t) this.failure = t; } - public void start(AccordExecutor executor) + public void start() { Caches caches = commandStore.cachesExclusive(); - setState(SCANNING_RANGES); + setStateExclusive(SCANNING_RANGES); + executor().scanningRanges.enqueue(AccordTask.this); startInternal(caches); - executor.submitPlainExclusive(AccordTask.this, GlobalGroup.RANGE_SCAN, this); + executor().submitPlainExclusive(AccordTask.this, GlobalGroup.RANGE_SCAN, this); } void startInternal(Caches caches) @@ -1003,9 +1359,8 @@ public void scannedExclusive() Invariants.require(is(SCANNING_RANGES), "Expected SCANNING_RANGES; found %s", AccordTask.this, AccordTask::toDescription); scanned = true; scannedInternal(); - if (loading == null) setState(WAITING_TO_RUN); - else if (waitingToLoad == null) setState(LOADING); - else setState(WAITING_TO_LOAD); + unqueue(executor().scanningRanges); + onSetupOrScannedExclusive(); } void scannedInternal() @@ -1021,6 +1376,8 @@ void cleanup(Caches caches) CommandSummaries finish(Caches caches) { loader.finish(summaries); + loader.cleanupExclusive(caches); + loader = null; TreeMap byId = new TreeMap<>(summaries); return (CommandSummaries.ByTxnIdSnapshot) () -> byId; } @@ -1061,4 +1418,29 @@ public String description() { return executionContext.describe(); } + + private AccordExecutor executor() + { + return commandStore.executor(); + } + + @Override + protected boolean isNewWork() + { + return true; + } + + @Nullable + public RangeTxnScanner rangeScanner() + { + return rangeScanner; + } + + private static LoadKeys loadKeys(ExecutionContext context) + { + LoadKeys loadKeys = context.loadKeys(); + if (NONSYNC_ENABLED) + return loadKeys; + return loadKeys == NONE ? NONE : SYNC; + } } diff --git a/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java b/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java index b4b607ded359..1af4dc96a00c 100644 --- a/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java @@ -88,7 +88,7 @@ public void load(Map into, BooleanSupplier abort) { for (TxnId txnId : load) { - AccordCacheEntry entry = caches.commands().getUnsafe(txnId); + AccordCacheEntry entry = caches.commands().getUnsafe(txnId); if (entry == null) { loadFromDisk.add(txnId); @@ -115,7 +115,6 @@ public void load(Map into, BooleanSupplier abort) public void finish(Map into) { - cleanupExclusive(null); owner.search(this, into::put, null); } diff --git a/src/java/org/apache/cassandra/service/accord/RangeIndex.java b/src/java/org/apache/cassandra/service/accord/RangeIndex.java index a31179a4ebaa..228aad3781bc 100644 --- a/src/java/org/apache/cassandra/service/accord/RangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/RangeIndex.java @@ -82,7 +82,7 @@ protected CommandSummaries.Summary loadFromDisk(TxnId txnId) return null; } - public CommandSummaries.Summary ifRelevant(AccordCacheEntry state) + public CommandSummaries.Summary ifRelevant(AccordCacheEntry state) { if (state.key().domain() != Routable.Domain.Range) return null; diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java b/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java index 15cbd81ff279..e5ff023c80a2 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java @@ -178,7 +178,7 @@ private Future> drainToFuture(Queue> drain, List visitRootTxnAsync(CommandStore commandStore, TxnId txnId) { - return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_blocked_by"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return null; @@ -188,7 +188,7 @@ private AsyncChain visitRootTxnAsync(CommandStore commandStore, TxnId txnId private AsyncChain visitTxnAsync(CommandStore commandStore, TxnId txnId, Timestamp rootExecuteAt, @Nullable TokenKey byKey, int depth, boolean recurse) { - return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_blocked_by"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return null; @@ -231,7 +231,7 @@ private Txn visitTxnSync(SafeCommandStore safeStore, Command command, Timestamp private AsyncChain visitKeysAsync(CommandStore commandStore, TokenKey key, Timestamp rootExecuteAt, int depth) { - return commandStore.chain(ExecutionContext.contextFor(RoutingKeys.of(key.toUnseekable()), SYNC, READ_WRITE, "Populate txn_blocked_by"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequencedReadWrite(RoutingKeys.of(key.toUnseekable()), "Populate txn_blocked_by"), safeStore -> { visitKeysSync(safeStore, key, rootExecuteAt, depth); }); } diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java index e01f877d14d4..037a09f1c4be 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java @@ -40,10 +40,10 @@ public class DebugExecution { private static final Logger logger = LoggerFactory.getLogger(DebugExecution.class); public static final boolean DEBUG_EXECUTION = CassandraRelevantProperties.ACCORD_DEBUG_EXECUTION.getBoolean(false); - private static final long REPORT_MIN_LATENCY_MICROS = 20_000; + private static final long REPORT_MIN_LATENCY_MICROS = 50_000; private static final long REPORT_CPU_RATIO = 2; - private static final long REPORT_MAX_LATENCY_MICROS = 50_000; - private static final long REPORT_CPU_MICROS = 10_000; + private static final long REPORT_MAX_LATENCY_MICROS = 100_000; + private static final long REPORT_CPU_MICROS = 50_000; // TODO (expected): use sharded histogram so we can report global stats public static class DebugExecutor @@ -94,43 +94,16 @@ public void onExitLock() long lockedForCpuMicros = (unlockedAtCpu - lockedAtCpu)/1000; if (lockedForMicros >= REPORT_MAX_LATENCY_MICROS) { - report("Held lock for {}us (cpu:{}us)\n", lockedForMicros, lockedForCpuMicros); + report("Held lock for {}us (cpu:{}us)", lockedForMicros, lockedForCpuMicros); } else if (lockedForMicros >= REPORT_MIN_LATENCY_MICROS && (lockedForMicros / lockedForCpuMicros) >= REPORT_CPU_RATIO) { - report("Held lock for {}us with cpu time only {}us\n", lockedForMicros, lockedForCpuMicros); + report("Held lock for {}us with cpu time only {}us", lockedForMicros, lockedForCpuMicros); } locked.increment(lockedForMicros); } } - public static class DebugExecutorLoop - { - final DebugExecutor owner; - long lockAt; - - public DebugExecutorLoop(DebugExecutor owner) - { - this.owner = owner; - } - - public void onLock() - { - lockAt = nanoTime(); - } - - public void onEnterLock() - { - owner.onEnterLock(lockAt); - lockAt = 0; - } - - public void onExitLock() - { - owner.onExitLock(); - } - } - public static class DebugExclusiveExecutor { public static DebugExclusiveExecutor maybeDebug(DebugExecutor owner, int commandStoreId) @@ -200,7 +173,7 @@ public DebugTask(WithResources resources, Task task) public List sanityCheck; // for AccordTask only long polledAt, preRunAt, runCompleteAt, completedAt; - long releasedRangeScannerAt, releasedCommandsAt, releasedCommandsForKeyAt; + long releasedRangeScannerAt, releasedStateAt; long runningAtCpu, runCompleteAtCpu; Thread thread; @@ -231,14 +204,9 @@ public void onReleasedRangeScanner() releasedRangeScannerAt = nanoTime(); } - public void onReleasedCommands() - { - releasedCommandsAt = nanoTime(); - } - - public void onReleasedCommandsForKeys() + public void onReleasedState() { - releasedCommandsForKeyAt = nanoTime(); + releasedStateAt = nanoTime(); } public void onCompleted(DebugExecutor owner) @@ -254,9 +222,9 @@ public void onCompleted(DebugExecutor owner) runningMicros = (runCompleteAt - task.runningAt) / 1000; owner.running.increment(runningMicros); } - long runToCleanMicros = (task.cleanupAt - runCompleteAt)/1000; + long runToCleanMicros = (task.completeAt - runCompleteAt) / 1000; owner.runToCleanup.increment(runToCleanMicros); - long cleanupMicros = (completedAt - task.cleanupAt)/1000; + long cleanupMicros = (completedAt - task.completeAt) / 1000; owner.cleanup.increment(cleanupMicros); long totalMicros = (completedAt - polledAt)/1000; owner.taskTotal.increment(totalMicros); @@ -266,7 +234,7 @@ public void onCompleted(DebugExecutor owner) String reason = ""; if (totalMicros > REPORT_MAX_LATENCY_MICROS) reason += "LONG TIME "; if (totalCpu > REPORT_CPU_MICROS) reason += "HIGH CPU "; - if ((totalMicros > REPORT_MIN_LATENCY_MICROS && (totalMicros/totalCpu) >= REPORT_CPU_RATIO)) reason += "LOW RATIO "; + if ((totalMicros > REPORT_MIN_LATENCY_MICROS && (totalCpu == 0 || (totalMicros/totalCpu) >= REPORT_CPU_RATIO))) reason += "LOW RATIO "; report("{}{}: total {}us cpu:{}us ({}), pollToRun {}us, running {}us, runToClean {}us, cleanup {}us", reason, task, totalMicros, totalCpu, thread, pollToRunMicros, runningMicros, runToCleanMicros, cleanupMicros); } diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java b/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java index a821db634033..7807eff9bc5b 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java @@ -221,7 +221,7 @@ static Future> drainToFuture(Queue> drain, List> submitRoot(CommandStore commandStore, TxnId txnId) { - return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return AsyncChains.>success(null); @@ -232,7 +232,7 @@ private AsyncChain> submitRoot(CommandStore commandStore, TxnId txnI private AsyncChain> submitParent(CommandStore commandStore, TxnId txnId, P param, Map infos, Set visitedParent, int depth) { - return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); if (command == null || command.saveStatus() == SaveStatus.Uninitialised) return AsyncChains.>success(null); @@ -344,7 +344,7 @@ protected void visitLatestCommitted(List sortedInfos, Command parent, private AsyncChain populateTxnAsync(CommandStore commandStore, TxnId txnId, Map visited) { - return commandStore.chain(ExecutionContext.contextFor(txnId, "Populate txn_graph"), safeStore -> { + return commandStore.chain(ExecutionContext.unsequenced(txnId, "Populate txn_graph"), safeStore -> { Command command = safeStore.unsafeGetNoCleanup(txnId).current(); visited.putIfAbsent(txnId, command == null || command.saveStatus() == SaveStatus.Uninitialised ? SaveInfo.NONE : new SaveInfo(command.saveStatus(), command.executeAtIfKnown())); }); diff --git a/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java b/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java index eba2e534b4b4..c3be7c87a0ed 100644 --- a/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java @@ -107,7 +107,7 @@ static class CommandWatcher implements AccordCache.Listener } @Override - public void onUpdate(AccordCacheEntry state) + public void onUpdate(AccordCacheEntry state) { Summary summary = loader.ifRelevant(state); if (summary != null) @@ -162,7 +162,7 @@ public void forEachInCache(Unseekables keysOrRanges, Consumer

forEac if (isMaybeRelevant(i)) { TxnId txnId = i.txnId; - AccordCacheEntry entry = c.getUnsafe(txnId); + AccordCacheEntry entry = c.getUnsafe(txnId); Invariants.expect(entry != null, "%s found interval %s but no matching transaction in cache", owner.commandStore, i); if (entry != null) { @@ -267,7 +267,7 @@ public JournalRangeIndex(AccordCommandStore commandStore) } @Override - public void onUpdate(AccordCacheEntry state) + public void onUpdate(AccordCacheEntry state) { TxnId txnId = state.key(); if (txnId.is(Routable.Domain.Range)) @@ -334,7 +334,7 @@ protected void onRemainingEdits() } @Override - public void onEvict(AccordCacheEntry state) + public void onEvict(AccordCacheEntry state) { TxnId txnId = state.key(); if (txnId.is(Routable.Domain.Range)) diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java index 04db46248330..edff67507ac0 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java @@ -22,12 +22,12 @@ import org.assertj.core.api.Assertions; -import accord.api.RoutingKey; import accord.local.CommandStores; import accord.local.ExecutionContext; import accord.local.LoadKeys; import accord.local.Node; import accord.local.cfk.CommandsForKey; +import accord.local.cfk.SafeCommandsForKey; import accord.primitives.Ranges; import accord.primitives.Routable; import accord.primitives.Txn; @@ -43,7 +43,6 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordCommandStore; import org.apache.cassandra.service.accord.AccordSafeCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommandsForKey; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.TokenRange; @@ -135,19 +134,16 @@ protected static void validateAccord(Cluster cluster, TableId id) TableId tableId = TableId.fromString(s); AccordService accord = (AccordService) AccordService.instance(); TxnId syntheticTxnId = new TxnId(TxnId.MAX_EPOCH, 0, Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range, new Node.Id(1)); - ExecutionContext ctx = ExecutionContext.contextFor(syntheticTxnId, Ranges.single(TokenRange.fullRange(tableId, getPartitioner())), LoadKeys.SYNC, READ_WRITE, "Test"); + ExecutionContext ctx = ExecutionContext.unsequencedReadWrite(syntheticTxnId, Ranges.single(TokenRange.fullRange(tableId, getPartitioner())), "Test"); CommandStores stores = accord.node().commandStores(); for (int storeId : stores.ids()) { AccordCommandStore store = (AccordCommandStore) stores.forId(storeId); getBlocking(store.chain(ctx, input -> { AccordSafeCommandStore safe = (AccordSafeCommandStore) input; - for (RoutingKey key : safe.commandsForKeysKeys()) + for (SafeCommandsForKey safeCfk : safe.safeCommandsForKeys()) { - AccordSafeCommandsForKey safeCFK = (AccordSafeCommandsForKey) safe.ifLoadedAndInitialised(key); - if (safeCFK == null) // we read and found a key, but its null at load time... so ignore it - continue; - CommandsForKey cfk = safeCFK.current(); + CommandsForKey cfk = safeCfk.current(); CommandsForKey.TxnInfo minUndecided = cfk.minUndecidedManaged(); if (minUndecided != null) throw new AssertionError("Undecided txn: " + minUndecided); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java index 8270da184e37..f7e52f108ab8 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java @@ -355,6 +355,12 @@ private static SettingsBuilder withArtificialLatencies(SettingsBuilder builder) return builder.setArtificialLatencies(LATENCIES); } + private static SettingsBuilder populate(SettingsBuilder builder, int keyCount) + { + return builder.setKeySelector(roundrobin(keyCount)) + .setReadRatio(0f); + } + private static SettingsBuilder ycsbA(SettingsBuilder builder, int keyCount) { return builder.setKeySelector(ycsbZipfian(keyCount)) @@ -832,7 +838,7 @@ else if (random.nextFloat() < settings.readRatio) if (storeId.get() >= 0) { CommandStore commandStore = service.node().commandStores().forId(storeId.get()); - List> result = AccordService.getBlocking(commandStore.submit(ExecutionContext.contextFor(candidate, "LoadTest"), safeStore -> { + List> result = AccordService.getBlocking(commandStore.submit(ExecutionContext.unsequenced(candidate, "LoadTest"), safeStore -> { SafeCommand safeCommand = safeStore.unsafeGet(candidate); PartialDeps deps = safeCommand.current().partialDeps(); if (deps == null) @@ -855,7 +861,7 @@ else if (random.nextFloat() < settings.readRatio) for (List info : result) { TxnId txnId = TxnId.parse(info.get(0)); - AccordService.getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, "LoadTest"), safeStore -> { + AccordService.getBlocking(commandStore.execute(ExecutionContext.unsequenced(txnId, "LoadTest"), safeStore -> { SafeCommand safeCommand = safeStore.unsafeGet(txnId); if (safeCommand.current().executeAt != null) info.add(safeCommand.current().executeAt.toString()); @@ -915,7 +921,7 @@ else if (random.nextFloat() < settings.readRatio) cluster.forEach(() -> { refresh(AccordExecutorMetrics.INSTANCE.elapsedRunning); refresh(AccordExecutorMetrics.INSTANCE.elapsed); - System.out.printf("%tT.%tL (%d %d %d %d %d %d)ms (%d %d %d %d %d %d)ms (%d %d %d %d %.0f, %d %d %d)us %d %d %d\n", nowMillis, nowMillis, + System.out.printf("%tT.%tL (%d %d %d %d %d %d)ms (%d %d %d %d %d %d)ms (%d %d %d)us %d %.0f (%d %d %d)us %d %d %d\n", nowMillis, nowMillis, getLatency(AccordCoordinatorMetrics.readMetrics.preacceptLatency, 0.5), getLatency(AccordCoordinatorMetrics.readMetrics.executeLatency, 0.5), getLatency(AccordCoordinatorMetrics.readMetrics.applyLatency, 0.5), @@ -1092,13 +1098,13 @@ public static void main(String[] args) throws Throwable try { test.setup(); - test.testLoad(withArtificialLatencies(ycsbA(new SettingsBuilder(), 100_000) + test.testLoad(populate(new SettingsBuilder(), 1_000_000) // .setRatePerSecond(400).setMinRatePerSecond(200) // .setRatePerSecond(800).setMinRatePerSecond(200) .setRatePerSecond(1600).setMinRatePerSecond(200) .setIncreaseRatePerSecondInterval(5000) // .setTraceLast(5000) - ).build()); + .build()); } finally { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java index 8770df01c96f..1b864aea1b04 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java @@ -38,6 +38,7 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordCacheEntry; import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.AccordSafeCommand; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.utils.ByteBufferUtil; @@ -86,7 +87,7 @@ public void loadCommandErasedTest() throws Throwable Node node = service.node(); AccordCommandStore commandStore = (AccordCommandStore) node.commandStores().unsafeForKey(key.toUnseekable()); - Iterator> iterator = commandStore.cachesUnsafe().commands().iterator(); + Iterator> iterator = commandStore.cachesUnsafe().commands().iterator(); TxnId txnId = TxnId.NONE; diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java new file mode 100644 index 000000000000..6bf1f46dfd97 --- /dev/null +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.simulator.test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.LockSupport; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; + +import javax.annotation.Nullable; + +import org.junit.Test; + +import accord.api.AsyncExecutor; +import accord.api.ExclusiveAsyncExecutor; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; +import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.AccordExecutorAsyncSubmit; +import org.apache.cassandra.service.accord.AccordExecutorSignalLoop; +import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.SignalLock; + +import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; + +// TODO (required): have simulator intercept ReentantLock so we can test SyncSubmit and SemiSyncSubmit +public class AccordExecutorAndCacheTest extends SimulationTestBase +{ + @Test + public void signalLoopTest() + { + executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, SignalLock.MAX_THREADS, -1, -1, TimeUnit.MICROSECONDS, i -> "Loop" + i, new AccordAgent()), + 16); + } + + static class Submitted + { + final AtomicInteger nextId = new AtomicInteger(); + final AtomicInteger doneBefore = new AtomicInteger(); + final ConcurrentHashMap>> consequences = new ConcurrentHashMap<>(); + + boolean isDone() + { + return isDoneBetween(0, nextId.get()); + } + + boolean isDoneBefore(int before) + { + if (!isDoneBetween(doneBefore.get(), before)) + return false; + doneBefore.accumulateAndGet(before, Integer::max); + return true; + } + + boolean isDoneBetween(int from, int before) + { + for (int id = from ; id < before ; ++id) + { + for (Future future : consequences.get(id)) + { + if (!future.isDone()) + return false; + } + } + return true; + } + + Collection> start() + { + ConcurrentLinkedQueue> result = new ConcurrentLinkedQueue<>(); + + while (true) + { + int id = nextId.get(); + Object prev = consequences.putIfAbsent(id, result); + nextId.compareAndSet(id, id + 1); + if (prev == null) + return result; + } + } + } + + static class Control extends ConcurrentLinkedQueue + { + final Submitted submitted; + final AtomicInteger count = new AtomicInteger(); + final float cancelChance; + float processChance; + + Control(float cancelChance, Submitted submitted) + { + this(submitted, cancelChance, ThreadLocalRandom.current().nextFloat() * 0.5f); + } + + Control(Submitted submitted, float cancelChance, float processChance) + { + this.submitted = submitted; + this.cancelChance = cancelChance; + this.processChance = processChance; + } + + void submit(AsyncExecutor executor, Collection> consequences, Consumer>> run) + { + AsyncPromise future = new AsyncPromise<>(); + Cancellable cancel = executor.chain(() -> run.accept(consequences)).begin((success, fail) -> { + if (fail == null) future.trySuccess(null); + else + { + future.tryFailure(fail); + if (fail instanceof CancellationException) + run.accept(submitted.start()); + } + }); + consequences.add(future); + + if (cancel != null && ThreadLocalRandom.current().nextFloat() <= cancelChance) + { + add(cancel); + count.incrementAndGet(); + } + + if (count.get() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance) + { + int cancelCount = 0; + do + { + ++cancelCount; + + float delta = ThreadLocalRandom.current().nextFloat() - 0.5f; + if (delta < 0) processChance /= delta; + else processChance *= -delta; + if (processChance < 0.001f || processChance > 0.999f) + processChance = cancelChance; + } while (count.decrementAndGet() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance); + + // do outside of loop to avoid reentry + while (cancelCount-- > 0) + remove().cancel(); + } + } + } + + public void executorTest(SerializableSupplier supplier, int submissionThreads) + { + simulate(arr(() -> { + try + { + DatabaseDescriptor.daemonInitialization(); + ExecutorPlus submit = executorFactory().pooled("submit-test", submissionThreads); + AccordExecutor executor = supplier.get(); + Lock lock = executor.unsafeLock(); + ExclusiveAsyncExecutor sequentialExecutor = executor.newExclusiveExecutor(); + Executor lockExecutor = executorFactory().sequential("lock"); + + for (float sleepChance : new float[] { 0f, 0.01f, 0.1f }) + { + for (float lockChance : new float[] { 0f, 0.01f, 0.1f }) + { + for (float cancelChance : new float[] { 0f, 0.01f, 0.1f }) + { + System.out.println(String.format("sleepChance %.2f, lockChance %.2f, cancelChance %.2f", sleepChance, lockChance, cancelChance)); + List> done = new ArrayList<>(); + Submitted submitted = new Submitted(); + for (int i = 0; i < submissionThreads; ++i) + { + int id = i; + done.add(submit.submit(() -> { + try + { + submitLoop(id, lock, executor, sequentialExecutor, lockExecutor, 20, 10, sleepChance, lockChance, new Control(cancelChance, submitted), submitted); + } + catch (ExecutionException | InterruptedException e) + { + throw new RuntimeException(e); + } + })); + } + for (Future f : done) + f.get(); + + if (!submitted.isDone()) + throw new AssertionError(); + } + } + } + } + catch (Throwable t) + { + throw new RuntimeException(t); + } + }), + () -> {}, 1L); + } + + private static void submitLoop(int id, Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance, Control control, Submitted submitted) throws ExecutionException, InterruptedException + { + ConcurrentLinkedQueue> awaitConsequences = new ConcurrentLinkedQueue<>(); + while (outerLoop-- > 0) + { + List>> allAwaitSubmitted = new ArrayList<>(); + for (int i = 0; i < innerLoop; ++i) + { + Collection> awaitSubmitted = submitted.start(); + allAwaitSubmitted.add(awaitSubmitted); + submitRecursive(lock, executor, sequentialExecutor, 1 + i, awaitSubmitted, awaitConsequences, submitted, sleepChance, lockChance, control); + } + + AtomicBoolean done = new AtomicBoolean(); + submitUntil(lock, lockExecutor, sleepChance, done::get); + for (Collection> awaitSubmitted : allAwaitSubmitted) + await(awaitSubmitted, CancellationException.class); + await(awaitConsequences, null); + done.set(true); + System.out.println("Loop " + id + '.' + (1 + outerLoop)); + } + } + + private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Collection> consequences, Collection> awaitConsequences, Submitted submitted, float sleepChance, float lockChance, Control control) + { + AsyncExecutor submitTo = ThreadLocalRandom.current().nextBoolean() ? executor : sequentialExecutor; + + control.submit(submitTo, consequences, nextConsequences -> { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + boolean locked = false; + if (rnd.nextFloat() < lockChance) + { + if (rnd.nextBoolean()) locked = lock.tryLock(); + else { locked = true; lock.lock(); } + } + if (ThreadLocalRandom.current().nextFloat() < 0.01f) + { + int expectDoneBefore = submitted.nextId.get(); + AsyncPromise afterConsequences = new AsyncPromise<>(); + executor.afterSubmittedAndConsequences(() -> { + if (!submitted.isDoneBefore(expectDoneBefore)) + throw new AssertionError(); + afterConsequences.setSuccess(null); + }); + awaitConsequences.add(afterConsequences); + } + try + { + if (count > 1) + submitRecursive(lock, executor, sequentialExecutor, count -1, nextConsequences, awaitConsequences, submitted, sleepChance, lockChance, control); + if (rnd.nextFloat() < sleepChance) + LockSupport.parkNanos(rnd.nextInt(10000, 100000)); + } + finally + { + if (locked) + lock.unlock(); + } + }); + } + + private static void submitUntil(Lock lock, Executor executor, float sleepChance, BooleanSupplier done) + { + if (done.getAsBoolean()) + return; + + executor.execute(() -> { + + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + boolean tryLock = rnd.nextBoolean(); + boolean locked = !tryLock; + if (tryLock) locked = lock.tryLock(); + else lock.lock(); + try + { + if (rnd.nextFloat() < sleepChance) + LockSupport.parkNanos(rnd.nextInt(10000, 100000)); + + submitUntil(lock, executor, sleepChance, done); + } + finally + { + if (locked) + lock.unlock(); + } + }); + } + + private static void await(Collection> await, @Nullable Class ignore) throws InterruptedException, ExecutionException + { + for (Future future : await) + { + try { future.get(); } + catch (ExecutionException e) + { + if (ignore == null || !(ignore.isInstance(e.getCause()))) + throw e; + } + } + } +} diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java index 5d6408f676d0..69f3722bd0f2 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java @@ -91,6 +91,7 @@ import org.apache.cassandra.utils.FBUtilities; import static accord.local.ExecutionContext.contextFor; +import static accord.local.ExecutionContext.unsequencedReadWrite; import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; import static accord.local.RedundantStatus.SomeStatus.GC_BEFORE_AND_LOCALLY_DURABLE; @@ -322,28 +323,28 @@ private void testWithCommandStoreInternal(TestWithCommandStore test, boolean add PartialDeps partialDeps = Deps.NONE.intersecting(AccordTestUtils.fullRange(txn)); PartialTxn partialTxn = txn.slice(commandStore.unsafeGetRangesForEpoch().currentRanges(), true); Route partialRoute = route.overlapping(commandStore.unsafeGetRangesForEpoch().currentRanges()); - getBlocking(commandStore.execute(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + getBlocking(commandStore.execute(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.preaccept(safe, txnId, partialTxn, route, (a, b) -> {}); })); flush(commandStore); - getBlocking(commandStore.execute(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + getBlocking(commandStore.execute(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.accept(safe, txnId, Ballot.ZERO, partialRoute, txnId, partialDeps, (a, b) -> {}); })); flush(commandStore); - getBlocking(commandStore.execute(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + getBlocking(commandStore.execute(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.commit(safe, SaveStatus.Stable, Ballot.ZERO, txnId, route, partialTxn, txnId, partialDeps, (a, b) -> {}); })); flush(commandStore); - getBlocking(commandStore.chain(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + getBlocking(commandStore.chain(unsequencedReadWrite(txnId, route, "Test"), safe -> { return AccordTestUtils.processTxnResultDirect(safe, txnId, partialTxn, txnId); - }).flatMap(i -> i).flatMap(result -> commandStore.chain(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + }).flatMap(i -> i).flatMap(result -> commandStore.chain(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.apply(safe, txnId, route, txnId, partialDeps, partialTxn, result.left, result.right, (a, b) -> {}); }))); flush(commandStore); // The apply chain is asychronous, so it is easiest to just spin until it is applied // in order to have the updated state in the system table spinAssertEquals(true, 5, () -> { - return getBlocking(commandStore.submit(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + return getBlocking(commandStore.submit(unsequencedReadWrite(txnId, route, "Test"), safe -> { StoreParticipants participants = StoreParticipants.all(route); Command command = safe.get(txnId, participants).current(); return command.hasBeen(Status.Applied); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java index a91c08203947..4c174a37b725 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java @@ -20,14 +20,24 @@ import org.junit.Assert; import org.junit.Test; +import accord.local.SafeState; + import org.apache.cassandra.service.accord.AccordCache.Type; +import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; import org.apache.cassandra.service.accord.AccordCacheEntry.Status; public class AccordCacheEntryTest { - static class CacheEntry extends AccordCacheEntry + static class TestSafeState extends SafeState implements AccordSafeState + { + @Override public AccordCacheEntry global() { return null; } + @Override public void preExecute(AccordTask owner, LockMode lockMode) {} + @Override public void postExecute(AccordTask owner) {} + } + + static class CacheEntry extends AccordCacheEntry { - public CacheEntry(String key, Type.Instance instance) + public CacheEntry(String key, Type.Instance instance) { super(key, instance); } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java index 5cb919a66e30..aedf100df066 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java @@ -23,13 +23,18 @@ import org.junit.Assert; import org.junit.Test; +import accord.local.ExecutionContext; +import accord.local.SafeState; + import org.apache.cassandra.cache.CacheSize; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.concurrent.ManualExecutor; import org.apache.cassandra.metrics.AccordCacheMetrics; +import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; import org.apache.cassandra.service.accord.AccordCacheEntry.SaveExecutor; import org.apache.cassandra.service.accord.AccordCacheEntry.Status; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; import static org.apache.cassandra.service.accord.AccordTestUtils.testLoad; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; @@ -40,84 +45,55 @@ public class AccordCacheTest { private static final long DEFAULT_NODE_SIZE = nodeSize(0); - private static abstract class TestSafeState implements AccordSafeState + private static abstract class TestSafeState & AccordSafeState> extends SafeState implements AccordSafeState { - protected boolean invalidated = false; - protected final AccordCacheEntry global; - private T original = null; + protected final AccordCacheEntry global; - public TestSafeState(AccordCacheEntry global) + public TestSafeState(AccordCacheEntry global) { this.global = global; } - public AccordCacheEntry global() + public AccordCacheEntry global() { return global; } - @Override - public T key() - { - return global.key(); - } + public final T key() { return global.key(); } - @Override - public T current() + public void preExecute(AccordTask owner, LockMode lockMode) { - return global.getExclusive(); - } - - @Override - public void set(T update) - { - global.setExclusive(update); - } - - @Override - public T original() - { - return original; - } - - @Override - public void preExecute() - { - original = global.getExclusive(); - } - - @Override - public Throwable failure() - { - return global.failure(); + requireUninitialised(); + current = global.lockExclusive(owner, lockMode); + setSafe(); } + } - @Override - public void markUnsafe() + private static class SafeString extends TestSafeState + { + public SafeString(AccordCacheEntry global) { - invalidated = true; + super(global); } @Override - public boolean isUnsafe() + public void postExecute(AccordTask owner) { - return invalidated; + global.releaseExclusive(this, owner); } } - private static class SafeString extends TestSafeState + private static class SafeInt extends TestSafeState { - public SafeString(AccordCacheEntry global) + public SafeInt(AccordCacheEntry global) { super(global); } - } - private static class SafeInt extends TestSafeState - { - public SafeInt(AccordCacheEntry global) + @Override + public void postExecute(AccordTask owner) { - super(global); + global.releaseExclusive(this, owner); } } @@ -264,6 +240,7 @@ public void testRotation() assertCacheMetrics(cacheMetrics, 0, 3, 3, 3); SafeString safeString = instance.acquire("1"); + safeString.preExecute(new AccordTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); Assert.assertEquals(Status.LOADED, safeString.global.status()); assertCacheState(cache, 1, 3, nodeSize(1) * 3); @@ -392,6 +369,7 @@ public void testMultiAcquireRelease() assertCacheState(cache, 1, 1, nodeSize(1)); SafeString safeString2 = instance.acquire("0"); + safeString2.preExecute(new AccordTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); Assert.assertEquals("0", safeString2.current()); Assert.assertEquals(Status.LOADED, safeString1.global.status()); Assert.assertEquals(2, instance.references("0", SafeString.class)); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java index fc691f60fce0..a8bbec8fdf7e 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java @@ -18,8 +18,6 @@ package org.apache.cassandra.service.accord; -import java.util.NavigableMap; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicLong; import org.junit.Assert; @@ -73,6 +71,7 @@ import static accord.primitives.Status.Durability.AllQuorums; import static com.google.common.collect.Iterables.getOnlyElement; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; import static org.apache.cassandra.service.accord.AccordService.getBlocking; import static org.apache.cassandra.service.accord.AccordTestUtils.Commands.preaccepted; import static org.apache.cassandra.service.accord.AccordTestUtils.ballot; @@ -140,6 +139,7 @@ public void commandLoadSave() throws Throwable promised, executeAt, txn, dependencies, accepted, waitingOn, result.left, TxnDataResult.PERSISTABLE); AccordSafeCommand safeCommand = new AccordSafeCommand(loaded(txnId, null)); + safeCommand.preExecute(new AccordTask<>(null, ExecutionContext.unsequenced(txnId, "Test"), null), RELEASE_QUEUE); safeCommand.set(expected); // In practice we should never need to save it with the condition boolean set // Not sure why this test does that @@ -168,10 +168,10 @@ public void commandsForKeyLoadSave() Command command2 = preaccepted(txnId2, txn, timestamp(1, clock.incrementAndGet(), 1)); AccordSafeCommandsForKey cfk = new AccordSafeCommandsForKey(loaded(key, null)); - cfk.initialize(); + cfk.preExecute(new AccordTask<>(null, ExecutionContext.unsequenced(txnId1, "Test"), null), RELEASE_QUEUE); - cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.contextFor(command1.txnId(), "Test")), command1).cfk()); - cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.contextFor(command1.txnId(), "Test")), command2).cfk()); + cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.unsequenced(command1.txnId(), "Test")), command1).cfk()); + cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.unsequenced(command1.txnId(), "Test")), command2).cfk()); CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null, commandStore.nextSystemTimestampMicros()).run(); logger.info("E: {}", cfk); @@ -180,11 +180,4 @@ public void commandsForKeyLoadSave() Assert.assertEquals(cfk.current(), actual); } - - private static > NavigableMap toNavigableMap(V safeState) - { - TreeMap map = new TreeMap<>(); - map.put(safeState.key(), safeState); - return map; - } } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java index 3509c2e50612..189b08ed0dea 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java @@ -174,7 +174,7 @@ public void basicCycleTest() throws Throwable Commit commit = Commit.SerializerSupport.create(txnId, route, 1, 1, Commit.Kind.StableWithTxnAndDeps, Ballot.ZERO, executeAt, partialTxn, deps, fullRoute); getBlocking(commandStore.execute(commit, commit::apply)); - getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, Keys.of(key).toParticipants(), LoadKeys.SYNC, READ_WRITE, "Test"), safeStore -> { + getBlocking(commandStore.execute(ExecutionContext.unsequencedReadWrite(txnId, Keys.of(key).toParticipants(), "Test"), safeStore -> { Command before = safeStore.ifInitialised(txnId).current(); Assert.assertEquals(commit.executeAt, before.executeAt()); Assert.assertTrue(before.hasBeen(Status.Committed)); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java index 54dee77c04cd..9ca2b116785c 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java @@ -86,6 +86,7 @@ import org.apache.cassandra.utils.concurrent.Condition; import static accord.local.ExecutionContext.contextFor; +import static accord.local.ExecutionContext.unsequencedReadWrite; import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.READ_WRITE; import static accord.utils.Property.qt; @@ -127,7 +128,7 @@ public void optionalCommandTest() throws Throwable AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); - getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, "Test"), instance -> { + getBlocking(commandStore.execute(ExecutionContext.unsequenced(txnId, "Test"), instance -> { // TODO review: This change to `ifInitialized` was done in a lot of places and it doesn't preserve this property // I fixed this reference to point to `ifLoadedAndInitialised` and but didn't update other places Assert.assertNull(instance.ifInitialised(txnId)); @@ -141,7 +142,7 @@ public void touchUnknownTxn() throws Throwable AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); - getBlocking(commandStore.execute(ExecutionContext.contextFor(txnId, "Test"), safe -> { + getBlocking(commandStore.execute(ExecutionContext.unsequenced(txnId, "Test"), safe -> { StoreParticipants participants = StoreParticipants.empty(txnId); SafeCommand command = safe.get(txnId, participants); Assert.assertNotNull(command); @@ -198,7 +199,7 @@ private static Command createStableUsingFastLifeCycle(AccordCommandStore command route.overlapping(ranges); PartialDeps deps = PartialDeps.builder(ranges, true).build(); - Command command = getBlocking(commandStore.submit(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + Command command = getBlocking(commandStore.submit(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.preaccept(safe, txnId, partialTxn, route, appendDiffToLog(commandStore)); CheckedCommands.commit(safe, SaveStatus.Stable, Ballot.ZERO, txnId, route, partialTxn, executeAt, deps, appendDiffToLog(commandStore)); return safe.ifInitialised(txnId).current(); @@ -246,7 +247,7 @@ private static Command createStableUsingSlowLifeCycle(AccordCommandStore command Route partialRoute = route.overlapping(ranges); PartialDeps deps = PartialDeps.builder(ranges, true).build(); - Command command = getBlocking(commandStore.submit(contextFor(txnId, route, SYNC, READ_WRITE, "Test"), safe -> { + Command command = getBlocking(commandStore.submit(unsequencedReadWrite(txnId, route, "Test"), safe -> { CheckedCommands.preaccept(safe, txnId, partialTxn, route, appendDiffToLog(commandStore)); CheckedCommands.accept(safe, txnId, Ballot.ZERO, partialRoute, executeAt, deps, appendDiffToLog(commandStore)); CheckedCommands.commit(safe, SaveStatus.Committed, Ballot.ZERO, txnId, route, partialTxn, executeAt, deps, appendDiffToLog(commandStore)); @@ -441,7 +442,7 @@ private static void assertNoReferences(AccordCache.Type.Instance ca AssertionError error = null; for (T key : keys) { - AccordCacheEntry node = cache.getUnsafe(key); + AccordCacheEntry node = cache.getUnsafe(key); if (node == null) continue; try { @@ -476,7 +477,7 @@ private static void awaitDone(AccordCache.Type.Instance cache, Iter { for (T key : keys) { - AccordCacheEntry node = cache.getUnsafe(key); + AccordCacheEntry node = cache.getUnsafe(key); if (node == null) continue; Awaitility.await("For node " + node.key() + " to complete") .atMost(Duration.ofMinutes(1)) diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index ae436768517f..297d13a79d96 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -57,6 +57,7 @@ import accord.local.Node.Id; import accord.local.NodeCommandStoreService; import accord.local.SafeCommandStore; +import accord.local.SafeState; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.durability.DurabilityService; @@ -82,7 +83,6 @@ import accord.utils.SortedArrays.SortedArrayList; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; -import accord.utils.async.Cancellable; import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.concurrent.ExecutorPlus; @@ -104,6 +104,7 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; +import org.apache.cassandra.service.accord.AccordExecutor.IOTask; import org.apache.cassandra.service.accord.api.AccordAgent; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.journal.AccordJournal; @@ -114,7 +115,6 @@ import org.apache.cassandra.service.accord.txn.TxnRead; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Condition; -import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static accord.primitives.Routable.Domain.Key; @@ -123,6 +123,7 @@ import static accord.primitives.Status.Durability.NotDurable; import static accord.primitives.Txn.Kind.Write; import static java.lang.String.format; +import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; import static org.apache.cassandra.service.accord.AccordService.getBlocking; @@ -162,16 +163,16 @@ private static FullRoute route(PartialTxn txn) } } - public static AccordCacheEntry loaded(K key, V value) + public static & AccordSafeState> AccordCacheEntry loaded(K key, V value) { - AccordCacheEntry global = new AccordCacheEntry<>(key, null); + AccordCacheEntry global = new AccordCacheEntry<>(key, null); global.initialize(value); return global; } public static AccordSafeCommand safeCommand(Command command) { - AccordCacheEntry global = loaded(command.txnId(), command); + AccordCacheEntry global = loaded(command.txnId(), command); return new AccordSafeCommand(global); } @@ -188,9 +189,9 @@ private static LoadExecutor loadExecutor(ExecutorPlus executor) return new LoadExecutor<>() { @Override - public Cancellable load(P1 p1, P2 p2, AccordCacheEntry entry) + public IOTask load(P1 p1, P2 p2, AccordCacheEntry entry) { - Future future = executor.submit(() -> { + executor.submit(() -> { V v; try { v = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); } catch (Throwable t) @@ -200,19 +201,19 @@ public Cancellable load(P1 p1, P2 p2, AccordCacheEntry entry) } entry.loaded(v); }); - return () -> future.cancel(true); + return null; } }; } - public static void testLoad(ManualExecutor executor, AccordSafeState safeState, V val) + public static & AccordSafeState> void testLoad(ManualExecutor executor, S safeState, V val) { Assert.assertEquals(AccordCacheEntry.Status.WAITING_TO_LOAD, safeState.global().status()); safeState.global().load(loadExecutor(executor), null, null); Assert.assertEquals(AccordCacheEntry.Status.LOADING, safeState.global().status()); executor.runOne(); Assert.assertEquals(AccordCacheEntry.Status.LOADED, safeState.global().status()); - safeState.preExecute(); + safeState.preExecute(new AccordTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); Assert.assertEquals(val, safeState.current()); } diff --git a/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java index fae6c912fa4e..f62a784f1e5c 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java @@ -41,7 +41,7 @@ public void emptyTxns() for (int i = 0, examples = 100; i < examples; i++) { TxnId id = AccordGens.txnIds().next(rs); - instance.process(ExecutionContext.contextFor(id, "Test"), (safe) -> { + instance.process(ExecutionContext.unsequenced(id, "Test"), (safe) -> { var safeCommand = safe.get(id, StoreParticipants.empty(id)); var command = safeCommand.current(); Assertions.assertThat(command.saveStatus()).isEqualTo(SaveStatus.Uninitialised); diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java index ce061f40bf30..fcd7ccdde6cd 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java @@ -23,6 +23,7 @@ import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.BooleanSupplier; +import java.util.function.Function; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -180,7 +181,10 @@ private enum Action { SUCCESS, FAILURE, LOAD_FAILURE } private static AccordTask operation(SimulatedAccordCommandStore instance, ExecutionContext ctx, Action action, BooleanSupplier delay) { - return new SimulatedOperation(instance.commandStore, ctx, action == Action.FAILURE ? SimulatedOperation.Action.FAILURE : SimulatedOperation.Action.SUCCESS); + + Function function = action == Action.FAILURE ? safeStore -> { throw new SimulatedFault("Operation failed for keys " + ctx.keys()); } + : safeStore -> null; + return AccordTask.create(instance.commandStore, ctx, function); } private static class Counter implements BiConsumer @@ -196,26 +200,6 @@ public void accept(Object o, Throwable failure) } } - private static class SimulatedOperation extends AccordTask - { - enum Action { SUCCESS, FAILURE} - private final Action action; - - public SimulatedOperation(AccordCommandStore commandStore, ExecutionContext executionContext, Action action) - { - super(commandStore, executionContext); - this.action = action; - } - - @Override - public Void apply(SafeCommandStore safe) - { - if (action == Action.FAILURE) - throw new SimulatedFault("Operation failed for keys " + keys()); - return null; - } - } - private static class SimulatedLoadFunctionWrapper implements FunctionWrapper { final Supplier actions; diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java index 9975fd6b2bc3..448583ac00c7 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java @@ -505,8 +505,8 @@ private static void testOne(long seed) { int next = source.nextInt(commands.size()); Command command = commands.get(next); - if (command.txnId.isSyncPoint()) cfk = cfk.registerUnmanaged(new TestSafeCommandStore(ExecutionContext.contextFor(command.txnId(), "Test")), new TestSafeCommand(command), REGISTER).cfk(); - else cfk = cfk.update(new TestSafeCommandStore(ExecutionContext.contextFor(command.txnId(), "Test")), command).cfk(); + if (command.txnId.isSyncPoint()) cfk = cfk.registerUnmanaged(new TestSafeCommandStore(ExecutionContext.unsequenced(command.txnId(), "Test")), new TestSafeCommand(command), REGISTER).cfk(); + else cfk = cfk.update(new TestSafeCommandStore(ExecutionContext.unsequenced(command.txnId(), "Test")), command).cfk(); commands.set(next, commands.get(commands.size() - 1)); commands.remove(commands.size() - 1); } @@ -547,33 +547,10 @@ private static void testOne(long seed) static class TestSafeCommand extends SafeCommand { - final Command command; TestSafeCommand(Command command) { super(command.txnId); - this.command = command; - } - - @Override - public Command current() - { - return command; - } - - @Override - public void markUnsafe() - { - } - - @Override - public boolean isUnsafe() - { - return false; - } - - @Override - protected void set(Command command) - { + current = command; } } @@ -704,7 +681,7 @@ public static class TestSafeCommandStore extends AbstractSafeCommandStore { public TestSafeCommandStore(ExecutionContext context) { - super(context, TestCommandStore.INSTANCE); + super(context); } @Override protected CommandStoreCaches tryGetCaches() { return null; } @@ -712,6 +689,7 @@ public TestSafeCommandStore(ExecutionContext context) @Override protected SafeCommandsForKey add(SafeCommandsForKey safeCfk, CommandStoreCaches caches) { return null; } @Override protected SafeCommand getInternal(TxnId txnId) { return null; } @Override protected SafeCommandsForKey getInternal(RoutingKey key) { return null; } + @Override public CommandStore commandStore() { return TestCommandStore.INSTANCE; } @Override public DataStore dataStore() { return null; } @Override public Agent agent() { return null; } @Override public ProgressLog progressLog() { return null; } From 15bede148a02dffe16b43a8c05e0157156e19582 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Mon, 27 Jul 2026 17:27:22 +0100 Subject: [PATCH 05/10] split AccordExecutor et al into own package --- .../architecture/accord-architecture.adoc | 4 +- modules/accord | 2 +- .../cassandra/concurrent/CassandraThread.java | 16 +- .../db/virtual/AccordDebugKeyspace.java | 11 +- .../cassandra/db/virtual/QueriesTable.java | 2 +- .../apache/cassandra/io/util/PathUtils.java | 1 - .../cassandra/metrics/AccordCacheMetrics.java | 2 +- .../metrics/AccordExecutorMetrics.java | 4 +- .../metrics/AccordReplicaMetrics.java | 6 +- .../service/accord/AccordCommandStore.java | 131 +- .../service/accord/AccordCommandStores.java | 14 +- .../service/accord/AccordDurableOnFlush.java | 10 +- .../service/accord/AccordExecutor.java | 3563 ----------------- .../accord/AccordFetchCoordinator.java | 2 +- .../service/accord/AccordKeyspace.java | 12 +- .../service/accord/AccordSafeState.java | 47 - .../service/accord/AccordService.java | 1 + .../service/accord/IAccordService.java | 1 + .../service/accord/InMemoryRangeIndex.java | 1 + .../cassandra/service/accord/RangeIndex.java | 9 +- .../accord/debug/DebugBlockedTxns.java | 2 - .../service/accord/debug/DebugExecution.java | 2 +- .../AbstractLockLoop.java} | 36 +- .../AbstractLoop.java} | 14 +- .../AbstractSemiSyncSubmit.java} | 14 +- .../accord/{ => execution}/AccordCache.java | 114 +- .../{ => execution}/AccordCacheEntry.java | 644 +-- .../execution/AccordCacheEntryQueue.java | 518 +++ .../accord/execution/AccordExecutor.java | 823 ++++ .../AccordExecutorAsyncSubmit.java | 12 +- .../AccordExecutorSemiSyncSubmit.java | 12 +- .../AccordExecutorSignalLoop.java | 38 +- .../{ => execution}/AccordExecutorSimple.java | 16 +- .../AccordExecutorSyncSubmit.java | 22 +- .../service/accord/execution/CancelTask.java | 53 + .../accord/execution/ExclusiveExecutor.java | 390 ++ .../service/accord/execution/IOTask.java | 64 + .../service/accord/execution/IOTaskLoad.java | 89 + .../service/accord/execution/IOTaskSave.java | 77 + .../accord/execution/IOTaskWrapper.java | 75 + .../Loops.java} | 14 +- .../service/accord/execution/Plain.java | 110 + .../service/accord/execution/PlainChain.java | 87 + .../execution/PlainChainDebuggable.java | 67 + .../accord/execution/PlainRunnable.java | 99 + .../SafeTask.java} | 219 +- .../SaferCommand.java} | 18 +- .../SaferCommandStore.java} | 33 +- .../SaferCommandsForKey.java} | 23 +- .../service/accord/execution/SaferState.java | 47 + .../service/accord/execution/Task.java | 724 ++++ .../service/accord/execution/TaskInfo.java | 89 + .../service/accord/execution/TaskQueue.java | 110 + .../accord/execution/TaskQueueMulti.java | 612 +++ .../accord/execution/TaskQueueRunnable.java | 100 + .../accord/execution/TaskQueueStandalone.java | 61 + .../service/accord/execution/TaskRunner.java | 124 + .../service/accord/execution/Tranches.java | 254 ++ .../service/accord/execution/Unstoppable.java | 26 + .../accord/execution/Unterminatable.java | 24 + .../accord/journal/JournalRangeIndex.java | 8 +- .../service/accord/txn/TxnNamedRead.java | 2 +- .../cassandra/service/accord/txn/TxnRead.java | 2 +- .../service/accord/txn/TxnWrite.java | 2 +- .../accord/AccordExecutorBurnTest.java | 1 + .../distributed/test/TestBaseImpl.java | 2 +- .../test/accord/AccordBootstrapTest.java | 4 +- .../test/accord/AccordDropTableBase.java | 6 +- .../accord/AccordIncrementalRepairTest.java | 4 +- .../test/accord/AccordMetricsTest.java | 2 +- .../test/accord/AccordMoveTest.java | 4 +- ...AccordRecoverFromAvailabilityLossTest.java | 4 +- .../AccordJournalConsistentExpungeTest.java | 6 +- .../fuzz/topology/AccordRebootstrapTest.java | 2 +- ...actPairOfSequencesPaxosSimulationTest.java | 8 +- .../test/AccordExecutorAndCacheTest.java | 329 -- .../simulator/test/AccordExecutorTest.java | 55 +- .../org/apache/cassandra/cql3/CQLTester.java | 2 +- .../CompactionAccordIteratorsTest.java | 5 +- .../accord/AccordCommandStoreTest.java | 17 +- .../service/accord/AccordExpungeTest.java | 3 +- .../service/accord/AccordKeyspaceTest.java | 4 +- .../service/accord/AccordTestUtils.java | 59 +- ...{AccordTaskTest.java => SafeTaskTest.java} | 21 +- .../accord/SimulatedAccordCommandStore.java | 4 + .../accord/SimulatedAccordTaskTest.java | 5 +- .../{ => execution}/AccordCacheEntryTest.java | 14 +- .../{ => execution}/AccordCacheTest.java | 24 +- .../execution/AccordExecutionTestUtils.java | 72 + 89 files changed, 5330 insertions(+), 5036 deletions(-) delete mode 100644 src/java/org/apache/cassandra/service/accord/AccordExecutor.java delete mode 100644 src/java/org/apache/cassandra/service/accord/AccordSafeState.java rename src/java/org/apache/cassandra/service/accord/{AccordExecutorAbstractLockLoop.java => execution/AbstractLockLoop.java} (88%) rename src/java/org/apache/cassandra/service/accord/{AccordExecutorAbstractLoop.java => execution/AbstractLoop.java} (84%) rename src/java/org/apache/cassandra/service/accord/{AccordExecutorAbstractSemiSyncSubmit.java => execution/AbstractSemiSyncSubmit.java} (65%) rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordCache.java (90%) rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordCacheEntry.java (60%) create mode 100644 src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordExecutorAsyncSubmit.java (88%) rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordExecutorSemiSyncSubmit.java (89%) rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordExecutorSignalLoop.java (92%) rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordExecutorSimple.java (90%) rename src/java/org/apache/cassandra/service/accord/{ => execution}/AccordExecutorSyncSubmit.java (79%) create mode 100644 src/java/org/apache/cassandra/service/accord/execution/CancelTask.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/IOTask.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java rename src/java/org/apache/cassandra/service/accord/{AccordExecutorLoops.java => execution/Loops.java} (87%) create mode 100644 src/java/org/apache/cassandra/service/accord/execution/Plain.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/PlainChain.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java rename src/java/org/apache/cassandra/service/accord/{AccordTask.java => execution/SafeTask.java} (84%) rename src/java/org/apache/cassandra/service/accord/{AccordSafeCommand.java => execution/SaferCommand.java} (77%) rename src/java/org/apache/cassandra/service/accord/{AccordSafeCommandStore.java => execution/SaferCommandStore.java} (85%) rename src/java/org/apache/cassandra/service/accord/{AccordSafeCommandsForKey.java => execution/SaferCommandsForKey.java} (72%) create mode 100644 src/java/org/apache/cassandra/service/accord/execution/SaferState.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/Task.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/Tranches.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/Unstoppable.java create mode 100644 src/java/org/apache/cassandra/service/accord/execution/Unterminatable.java delete mode 100644 test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java rename test/unit/org/apache/cassandra/service/accord/{AccordTaskTest.java => SafeTaskTest.java} (95%) rename test/unit/org/apache/cassandra/service/accord/{ => execution}/AccordCacheEntryTest.java (88%) rename test/unit/org/apache/cassandra/service/accord/{ => execution}/AccordCacheTest.java (95%) create mode 100644 test/unit/org/apache/cassandra/service/accord/execution/AccordExecutionTestUtils.java diff --git a/doc/modules/cassandra/pages/architecture/accord-architecture.adoc b/doc/modules/cassandra/pages/architecture/accord-architecture.adoc index ac7c6b718013..ebe8631b7769 100644 --- a/doc/modules/cassandra/pages/architecture/accord-architecture.adoc +++ b/doc/modules/cassandra/pages/architecture/accord-architecture.adoc @@ -103,11 +103,11 @@ to ensure transactional integrity, changes to commands are tracked and are recorded into `Journal` for crash-recovery. `ProgressLog` and `CommandsForKey` are up -On Cassandra side, concurrent execution is controlled by `AccordTask`, +On Cassandra side, concurrent execution is controlled by `SafeTask`, which contains cache loading logic and persistence callbacks. Since Accord may potentially hold a large number of command states in memory, their states may be _shrunk_ to their binary representation to save some -memory, or they can get fully evicted. This also means that `AccordTask` +memory, or they can get fully evicted. This also means that `SafeTask` will have to reload relevant dependencies from preload context before command execution can begin. diff --git a/modules/accord b/modules/accord index 84ac4db03251..8d2cdb35ae96 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 84ac4db03251c3b842c98a29868e072792662f51 +Subproject commit 8d2cdb35ae96205b44da16582797bf42aa6fddcc diff --git a/src/java/org/apache/cassandra/concurrent/CassandraThread.java b/src/java/org/apache/cassandra/concurrent/CassandraThread.java index b013a5f0ad54..f2702ed7c0d8 100644 --- a/src/java/org/apache/cassandra/concurrent/CassandraThread.java +++ b/src/java/org/apache/cassandra/concurrent/CassandraThread.java @@ -21,19 +21,21 @@ import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import org.apache.cassandra.metrics.ThreadLocalMetrics; -import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.Task; +import org.apache.cassandra.service.accord.execution.TaskRunner; import io.netty.util.concurrent.FastThreadLocalThread; -public class CassandraThread extends FastThreadLocalThread implements AccordExecutor.AccordTaskRunner +public class CassandraThread extends FastThreadLocalThread implements TaskRunner { private ThreadLocalMetrics threadLocalMetrics; private ExecutorLocals executorLocals; private AccordExecutor accordActiveExecutor; private AccordExecutor accordLockedExecutor; private int accordLockedExecutorDepth; - private volatile AccordExecutor.Task accordActiveTask; - private static final AtomicReferenceFieldUpdater accordActiveTaskUpdater = AtomicReferenceFieldUpdater.newUpdater(CassandraThread.class, AccordExecutor.Task.class, "accordActiveTask"); + private volatile Task accordActiveTask; + private static final AtomicReferenceFieldUpdater accordActiveTaskUpdater = AtomicReferenceFieldUpdater.newUpdater(CassandraThread.class, Task.class, "accordActiveTask"); private final ImmediateTaskHolder immediateTaskHolder; @@ -129,19 +131,19 @@ public final void exitAccordLockedExecutor() accordLockedExecutor = null; } - public final AccordExecutor.Task accordActiveTask() + public final Task accordActiveTask() { return accordActiveTask; } // to be called only by the thread itself, so can (eventually) avoid any memory barriers - public final AccordExecutor.Task accordActiveSelfTask() + public final Task accordActiveSelfTask() { // TODO (expected): with newer JDK use accordActiveTaskUpdater.getPlain return accordActiveTask; } - public final void setAccordActiveTask(AccordExecutor.Task newActiveTask) + public final void setAccordActiveTask(Task newActiveTask) { accordActiveTaskUpdater.lazySet(this, newActiveTask); } diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 3a8c6e32bd8b..6e2fad78b428 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -132,11 +132,8 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.accord.AccordCache; -import org.apache.cassandra.service.accord.AccordCacheEntry; import org.apache.cassandra.service.accord.AccordCommandStore; import org.apache.cassandra.service.accord.AccordCommandStores; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.AccordKeyspace; import org.apache.cassandra.service.accord.AccordOperations; import org.apache.cassandra.service.accord.AccordService; @@ -154,6 +151,10 @@ import org.apache.cassandra.service.accord.debug.DebugTxnDepsOrdered; import org.apache.cassandra.service.accord.debug.DebugTxnGraph; import org.apache.cassandra.service.accord.debug.TxnKindsAndDomains; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.TaskInfo; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.service.consensus.migration.TableMigrationState; @@ -331,8 +332,8 @@ public void collect(PartitionsCollector collector) int executorId = executor.executorId(); collector.partition(executorId).collect(rows -> { int uniquePos = 0; - AccordExecutor.TaskInfo prev = null; - for (AccordExecutor.TaskInfo info : executor.taskSnapshot()) + TaskInfo prev = null; + for (TaskInfo info : executor.taskSnapshot()) { if (prev != null && info.status() == prev.status() && info.position() == prev.position()) ++uniquePos; else uniquePos = 0; diff --git a/src/java/org/apache/cassandra/db/virtual/QueriesTable.java b/src/java/org/apache/cassandra/db/virtual/QueriesTable.java index e58e874c14d0..673d7335adbd 100644 --- a/src/java/org/apache/cassandra/db/virtual/QueriesTable.java +++ b/src/java/org/apache/cassandra/db/virtual/QueriesTable.java @@ -25,8 +25,8 @@ import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.LocalPartitioner; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import static java.lang.Long.max; import static java.util.concurrent.TimeUnit.NANOSECONDS; diff --git a/src/java/org/apache/cassandra/io/util/PathUtils.java b/src/java/org/apache/cassandra/io/util/PathUtils.java index 334763f27113..780be3faafdb 100644 --- a/src/java/org/apache/cassandra/io/util/PathUtils.java +++ b/src/java/org/apache/cassandra/io/util/PathUtils.java @@ -662,7 +662,6 @@ static public void clearOnExitThreads() DeleteOnExit.clearOnExitThreads(); } - private static final class DeleteOnExit implements Runnable { private boolean isRegistered; diff --git a/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java b/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java index 59efb88044cf..00627e019a9c 100644 --- a/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java @@ -24,8 +24,8 @@ import com.codahale.metrics.Gauge; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.IAccordService; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import static org.apache.cassandra.metrics.AccordMetricUtils.fromAccordService; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; diff --git a/src/java/org/apache/cassandra/metrics/AccordExecutorMetrics.java b/src/java/org/apache/cassandra/metrics/AccordExecutorMetrics.java index 3d5132131a8a..2c48fe7d3d5d 100644 --- a/src/java/org/apache/cassandra/metrics/AccordExecutorMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordExecutorMetrics.java @@ -23,10 +23,10 @@ import com.codahale.metrics.Gauge; import org.apache.cassandra.metrics.ShardedDecayingHistograms.ShardedDecayingHistogram; -import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; -import static org.apache.cassandra.service.accord.AccordExecutor.HISTOGRAMS; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.HISTOGRAMS; public class AccordExecutorMetrics { diff --git a/src/java/org/apache/cassandra/metrics/AccordReplicaMetrics.java b/src/java/org/apache/cassandra/metrics/AccordReplicaMetrics.java index 6bc46f89c71b..556ffb574dfe 100644 --- a/src/java/org/apache/cassandra/metrics/AccordReplicaMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordReplicaMetrics.java @@ -33,11 +33,11 @@ import org.apache.cassandra.metrics.ShardedDecayingHistograms.DecayingHistogramsShard; import org.apache.cassandra.metrics.ShardedDecayingHistograms.ShardedDecayingHistogram; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.tracing.Tracing; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; -import static org.apache.cassandra.service.accord.AccordExecutor.HISTOGRAMS; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.HISTOGRAMS; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; public class AccordReplicaMetrics @@ -168,7 +168,7 @@ private static long elapsed(long unixNanos, TxnId txnId) private static LogLinearDecayingHistograms.Buffer buffer(SafeCommandStore safeStore) { - return ((AccordSafeCommandStore) safeStore).histogramBuffer(); + return ((SaferCommandStore) safeStore).histogramBuffer(); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index aeae9844e581..2dbbc07e0061 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory; import accord.api.Agent; -import accord.api.AsyncExecutor; import accord.api.DataStore; import accord.api.Journal; import accord.api.LocalListeners; @@ -72,7 +71,6 @@ import accord.primitives.PartialTxn; import accord.primitives.Range; import accord.primitives.Ranges; -import accord.primitives.RoutableKey; import accord.primitives.Route; import accord.primitives.SaveStatus; import accord.primitives.Status; @@ -98,10 +96,17 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable; -import org.apache.cassandra.service.accord.AccordExecutor.AccordTaskRunner; -import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.IAccordService.AccordCompactionInfo; -import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.ExclusiveExecutor; +import org.apache.cassandra.service.accord.execution.SafeTask; +import org.apache.cassandra.service.accord.execution.SaferCommand; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandsForKey; +import org.apache.cassandra.service.accord.execution.TaskRunner; +import org.apache.cassandra.service.accord.execution.Unterminatable; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.accord.journal.JournalRangeIndex; import org.apache.cassandra.service.accord.txn.TxnRead; @@ -138,10 +143,10 @@ public class AccordCommandStore extends CommandStore public static class Caches { private final AccordCache global; - private final AccordCache.Type.Instance commands; - private final AccordCache.Type.Instance commandsForKeys; + private final AccordCache.Type.Instance commands; + private final AccordCache.Type.Instance commandsForKeys; - Caches(AccordCache global, AccordCache.Type.Instance commandCache, AccordCache.Type.Instance commandsForKeyCache) + Caches(AccordCache global, AccordCache.Type.Instance commandCache, AccordCache.Type.Instance commandsForKeyCache) { this.global = global; this.commands = commandCache; @@ -153,29 +158,29 @@ public final AccordCache global() return global; } - public final AccordCache.Type.Instance commands() + public final AccordCache.Type.Instance commands() { return commands; } - public final AccordCache.Type.Instance commandsForKeys() + public final AccordCache.Type.Instance commandsForKeys() { return commandsForKeys; } } - public static final class ExclusiveCaches extends Caches implements CommandStoreCaches + public static final class ExclusiveCaches extends Caches implements CommandStoreCaches { private final AccordExecutor owner; - public ExclusiveCaches(AccordExecutor owner, AccordCache global, AccordCache.Type.Instance commands, AccordCache.Type.Instance commandsForKeys) + public ExclusiveCaches(AccordExecutor owner, AccordCache global, AccordCache.Type.Instance commands, AccordCache.Type.Instance commandsForKeys) { super(global, commands, commandsForKeys); this.owner = owner; } @Override - public AccordSafeCommand acquireIfLoaded(TxnId txnId) + public SaferCommand acquireIfLoaded(TxnId txnId) { // note: we must return false if the entry is locked to enforce ordering. // note importantly that this is also coupled to the safety of synchronously releasing ExclusiveExecutor.owner, @@ -184,7 +189,7 @@ public AccordSafeCommand acquireIfLoaded(TxnId txnId) } @Override - public AccordSafeCommandsForKey acquireIfLoaded(RoutingKey key) + public SaferCommandsForKey acquireIfLoaded(RoutingKey key) { return commandsForKeys().acquireIfLoadedAndPermitted(key); } @@ -193,7 +198,7 @@ public AccordSafeCommandsForKey acquireIfLoaded(RoutingKey key) public void close() { global().tryShrinkOrEvict(owner.unsafeLock()); - owner.unlock(AccordTaskRunner.get()); + owner.unlock(TaskRunner.get()); } } @@ -213,12 +218,10 @@ private boolean isReadyToTerminate() = AtomicReferenceFieldUpdater.newUpdater(AccordCommandStore.class, Termination.class, "terminated"); static final AtomicLong nextSafeRedundantBeforeTicket = new AtomicLong(); - private static final AtomicLong lastSystemTimestampMicros = new AtomicLong(); - public final String loggingId; public final Journal journal; private final AccordExecutor sharedExecutor; - final AccordExecutor.ExclusiveExecutor exclusiveExecutor; + private final ExclusiveExecutor exclusiveExecutor; private final ExclusiveCaches caches; private final RangeIndex rangeIndex; private final TableId tableId; @@ -227,8 +230,8 @@ private boolean isReadyToTerminate() volatile SafeRedundantBefore safeRedundantBefore; volatile Termination terminated; - private AccordSafeCommandStore current; - LogLinearDecayingHistograms.Buffer metricsBuffer; + private SaferCommandStore current; + public LogLinearDecayingHistograms.Buffer metricsBuffer; public AccordCommandStore(int id, NodeCommandStoreService node, @@ -257,15 +260,15 @@ public AccordCommandStore(int id, return a; }).orElseThrow(() -> Invariants.illegalState("CommandStore %d created with no ranges", id)); - final AccordCache.Type.Instance commands; - final AccordCache.Type.Instance commandsForKey; + final AccordCache.Type.Instance commands; + final AccordCache.Type.Instance commandsForKey; try (AccordExecutor.ExclusiveGlobalCaches exclusive = sharedExecutor.lockCaches()) { commands = exclusive.commands.newInstance(this); commandsForKey = exclusive.commandsForKey.newInstance(this); this.caches = new ExclusiveCaches(sharedExecutor, exclusive.global, commands, commandsForKey); } - this.exclusiveExecutor = sharedExecutor.executor(id); + this.exclusiveExecutor = sharedExecutor.newExclusiveExecutor(id); { AccordConfig.RangeIndexMode mode = getAccord().range_index_mode; @@ -298,12 +301,6 @@ public boolean inStore() return exclusiveExecutor.inExecutor(); } - void tryPreSetup(AccordTask task) - { - if (inStore() && current != null) - task.preSetup(current.task); - } - public final TableId tableId() { return tableId; @@ -317,7 +314,7 @@ public AccordExecutor executor() // TODO (desired): we use this for executing callbacks with mutual exclusivity, // but we don't need to block the actual CommandStore - could quite easily // inflate a separate queue dynamically in AccordExecutor - public AsyncExecutor taskExecutor() + public ExclusiveExecutor exclusiveExecutor() { return exclusiveExecutor; } @@ -325,13 +322,13 @@ public AsyncExecutor taskExecutor() public ExclusiveCaches lockCaches() { //noinspection LockAcquiredButNotSafelyReleased - caches.owner.lock(AccordTaskRunner.get()); + caches.owner.lock(TaskRunner.get()); return caches; } public ExclusiveCaches tryLockCaches() { - if (caches.owner.tryLock(AccordTaskRunner.get())) + if (caches.owner.tryLock(TaskRunner.get())) return caches; return null; } @@ -359,89 +356,50 @@ public void appendToLog(Command before, Command after, Runnable onFlush) journal.saveCommand(id, new CommandUpdate(before, after), onFlush); } - boolean validateCommand(TxnId txnId, Command evicting) - { - if (!Invariants.isParanoid()) - return true; - - Command reloaded = loadCommand(txnId); - return Objects.equals(evicting, reloaded); - } - @VisibleForTesting public void sanityCheckCommand(RedundantBefore redundantBefore, Command command) { ((AccordJournal) journal).sanityCheck(id, redundantBefore, command); } - CommandsForKey loadCommandsForKey(RoutableKey key) - { - CommandsForKey cfk = CommandsForKeyAccessor.load(id, (TokenKey) key); - if (cfk == null) - return null; - RedundantBefore.QuickBounds bounds = safeGetRedundantBefore().get(key); - if (!Invariants.expect(bounds != null, "No RedundantBefore information found when loading key %s", key)) - return cfk; - return cfk.withCleanCfkBeforeAtLeast(bounds.cleanCfkBefore(), false); - } - - boolean validateCommandsForKey(RoutableKey key, CommandsForKey evicting) - { - if (!Invariants.isParanoid()) - return true; - - CommandsForKey reloaded = CommandsForKeyAccessor.load(id, (TokenKey) key); - return Objects.equals(evicting, reloaded); - } - - @Nullable - Runnable saveCommandsForKey(RoutingKey key, CommandsForKey after, Object serialized) - { - return CommandsForKeyAccessor.systemTableUpdater(id, (TokenKey) key, after, serialized, nextSystemTimestampMicros()); - } - - public long nextSystemTimestampMicros() - { - return lastSystemTimestampMicros.accumulateAndGet(node.now(), (a, b) -> Math.max(a + 1, b)); - } @Override public AsyncChain chain(ExecutionContext context, Function function) { - return AccordTask.create(this, context, function).chain(); + return SafeTask.create(this, context, function).chain(); } @Override public AsyncChain chain(ExecutionContext context, Consumer consumer) { - return AccordTask.create(this, context, consumer).chain(); + return SafeTask.create(this, context, consumer).chain(); } @Override public AsyncChain chain(Callable call) { - return taskExecutor().chain(call); + return exclusiveExecutor().chain(call); } @Override public void execute(Runnable run) { - taskExecutor().execute(run); + exclusiveExecutor().execute(run); } @Override public boolean tryExecuteImmediately(Runnable run) { - return taskExecutor().tryExecuteImmediately(run); + return exclusiveExecutor().tryExecuteImmediately(run); } - public AccordSafeCommandStore begin(AccordSafeCommandStore safeStore) + public SaferCommandStore begin(SaferCommandStore safeStore) { require(current == null); current = safeStore; return current; } - public void complete(AccordSafeCommandStore store) + public void complete(SaferCommandStore store) { require(current == store); current = null; @@ -452,12 +410,12 @@ public boolean hasSafeStore() return current != null; } - DataStore dataStore() + public DataStore dataStore() { return dataStore; } - ProgressLog progressLog() + public ProgressLog progressLog() { return progressLog; } @@ -759,7 +717,7 @@ void maybeLoadRangesForEpoch(RangesForEpoch rangesForEpoch) AsyncChain saveState(Descriptor descriptor) { - return chain((AccordExecutor.Unterminatable)() -> "Save State", safeStore -> { + return chain((Unterminatable)() -> "Save State", safeStore -> { File storeDir = storeSaveDir(); { File[] tmpDirs = listTmpSaveDirs(storeDir); @@ -897,7 +855,7 @@ AsyncChain> restoreState() // TODO (expected): handle journal failures, and consider how we handle partial failures. // Very likely we will not be able to safely or cleanly handle partial failures of this logic, but decide and document. // TODO (desired): consider merging with PersistentField? This version is cheaper to manage which may be preferable at the CommandStore level. - static class SafeRedundantBefore + public static class SafeRedundantBefore { final long ticket; final RedundantBefore redundantBefore; @@ -912,6 +870,15 @@ static SafeRedundantBefore max(SafeRedundantBefore a, SafeRedundantBefore b) { return a.ticket >= b.ticket ? a : b; } + + public static Runnable updater(AccordCommandStore commandStore, RedundantBefore newRedundantBefore) + { + long ticket = nextSafeRedundantBeforeTicket.incrementAndGet(); + SafeRedundantBefore update = new SafeRedundantBefore(ticket, newRedundantBefore); + return () -> { + safeRedundantBeforeUpdater.accumulateAndGet(commandStore, update, SafeRedundantBefore::max); + }; + } } private @Nullable TableMetadata tableMetadata() diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index 14accb0fb885..228030ce3957 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -54,13 +54,19 @@ import org.apache.cassandra.journal.Descriptor; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordCommandStore.DurablyAppliedTo; -import org.apache.cassandra.service.accord.AccordExecutor.AccordExecutorFactory; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor.AccordExecutorFactory; +import org.apache.cassandra.service.accord.execution.AccordExecutorAsyncSubmit; +import org.apache.cassandra.service.accord.execution.AccordExecutorSemiSyncSubmit; +import org.apache.cassandra.service.accord.execution.AccordExecutorSignalLoop; +import org.apache.cassandra.service.accord.execution.AccordExecutorSimple; +import org.apache.cassandra.service.accord.execution.AccordExecutorSyncSubmit; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_PER_SHARD; import static org.apache.cassandra.config.DatabaseDescriptor.getAccord; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; -import static org.apache.cassandra.service.accord.AccordExecutor.constant; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.constant; import static org.apache.cassandra.service.accord.journal.ReplayMarkers.saveDirectory; import static org.apache.cassandra.utils.Clock.Global.nanoTime; diff --git a/src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java b/src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java index 6905b10bf99e..554ea61cd2b8 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java +++ b/src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java @@ -34,8 +34,9 @@ import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.accord.execution.Unstoppable; -class AccordDurableOnFlush implements BiConsumer +public class AccordDurableOnFlush implements BiConsumer { private static final Logger logger = LoggerFactory.getLogger(AccordDurableOnFlush.class); @@ -98,6 +99,11 @@ public String toString() { return redundantBefore.toString(); } + + public static void reportMaybeTerminate(AccordCommandStore commandStore, int flags) + { + commandStore.maybeTerminated(ReportDurable.isCommandStoreFlush(flags), ReportDurable.isDataStoreFlush(flags)); + } } private Int2ObjectHashMap commandStores = new Int2ObjectHashMap<>(); @@ -185,7 +191,7 @@ static void notifyInOrder(long memtableId, TableMetadata metadata, CommandStore static void notifyNow(CommandStore commandStore, ReportDurable report) { logger.debug("{} reporting flush with {}", commandStore, report); - commandStore.execute((AccordExecutor.Unstoppable) () -> "Report Durable", safeStore -> { + commandStore.execute((Unstoppable) () -> "Report Durable", safeStore -> { safeStore.reportDurable(report.redundantBefore, report.flags); }, commandStore.agent()); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java deleted file mode 100644 index 67a4d815c9dc..000000000000 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java +++ /dev/null @@ -1,3563 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.service.accord; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.CancellationException; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.LockSupport; -import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Function; -import java.util.function.IntFunction; -import java.util.stream.Stream; - -import javax.annotation.Nullable; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import accord.api.Agent; -import accord.api.ExclusiveAsyncExecutor; -import accord.api.RoutingKey; -import accord.impl.AbstractAsyncExecutor; -import accord.local.Command; -import accord.local.ExecutionContext; -import accord.local.ExecutionContext.ExecutionSequence; -import accord.local.cfk.CommandsForKey; -import accord.messages.Accept; -import accord.messages.Commit; -import accord.messages.MessageType; -import accord.messages.MessageType.StandardMessage; -import accord.messages.Request; -import accord.primitives.Ballot; -import accord.primitives.SaveStatus; -import accord.primitives.TxnId; -import accord.utils.ArrayBuffers; -import accord.utils.IntrusiveHeapNode; -import accord.utils.IntrusivePriorityHeap; -import accord.utils.Invariants; -import accord.utils.QuadConsumer; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; -import accord.utils.TinyEnumSet; -import accord.utils.TriConsumer; -import accord.utils.TriFunction; -import accord.utils.UnhandledEnum; -import accord.utils.async.AsyncCallbacks.CallAndCallback; -import accord.utils.async.AsyncCallbacks.RunOrFail; -import accord.utils.async.AsyncChain; -import accord.utils.async.AsyncChains; -import accord.utils.async.Cancellable; - -import org.apache.cassandra.cache.CacheSize; -import org.apache.cassandra.concurrent.CassandraThread; -import org.apache.cassandra.concurrent.DebuggableTask; -import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; -import org.apache.cassandra.concurrent.ExecutorLocals; -import org.apache.cassandra.concurrent.Shutdownable; -import org.apache.cassandra.config.AccordConfig; -import org.apache.cassandra.config.AccordConfig.QueueBalancingModel; -import org.apache.cassandra.config.AccordConfig.QueuePriorityModel; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.metrics.AccordCacheMetrics; -import org.apache.cassandra.metrics.AccordExecutorMetrics; -import org.apache.cassandra.metrics.AccordReplicaMetrics; -import org.apache.cassandra.metrics.AccordSystemMetrics; -import org.apache.cassandra.metrics.LogLinearDecayingHistograms; -import org.apache.cassandra.metrics.LogLinearDecayingHistograms.LogLinearDecayingHistogram; -import org.apache.cassandra.metrics.ShardedDecayingHistograms; -import org.apache.cassandra.metrics.ShardedDecayingHistograms.DecayingHistogramsShard; -import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; -import org.apache.cassandra.service.accord.AccordCacheEntry.SaveExecutor; -import org.apache.cassandra.service.accord.AccordCacheEntry.UniqueSave; -import org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup; -import org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup; -import org.apache.cassandra.service.accord.AccordExecutor.Task.GroupKind; -import org.apache.cassandra.service.accord.AccordExecutor.Task.State; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExclusiveExecutor; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutor; -import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; -import org.apache.cassandra.utils.Clock; -import org.apache.cassandra.utils.Closeable; -import org.apache.cassandra.utils.WithResources; -import org.apache.cassandra.utils.concurrent.AsyncPromise; -import org.apache.cassandra.utils.concurrent.Condition; -import org.apache.cassandra.utils.concurrent.Future; - -import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY; -import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY_ATOMIC; -import static accord.primitives.Routable.Domain.Range; -import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.ORIG_HLC_FIFO; -import static org.apache.cassandra.service.accord.AccordCache.CommandAdapter.COMMAND_ADAPTER; -import static org.apache.cassandra.service.accord.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER; -import static org.apache.cassandra.service.accord.AccordCache.registerJfrListener; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; -import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.COUNTER_LOWBITS; -import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.COUNTER_MASKS; -import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.minCounterValue; -import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.selectByOverflowBits; -import static org.apache.cassandra.service.accord.AccordExecutor.MultiTaskQueue.setOverflowWhenLessEqual; -import static org.apache.cassandra.service.accord.AccordExecutor.RunnableTaskQueue.RUNNABLE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.ACCEPT; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.APPLY; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.COMMIT; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.PREACCEPT; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.RANGE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.ExclusiveGroup.STABLE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.COMMAND_STORE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.LOAD; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.OTHER; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_LOAD; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_SCAN; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.SAVE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.MAX_TRANCHE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.CANCELLED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.EXECUTED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.FAILED_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_OPTIONAL; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_REQUIRED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING_OR_EXECUTED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_OPTIONAL; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_REQUIRED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; - -/** - * NOTE: We assume that NO BLOCKING TASKS are submitted to this executor AND WAITED ON by another task executing on this executor. - * (as we do not immediately schedule additional threads for submitted tasks, but schedule new threads only if necessary when the submitting execution completes) - */ -public abstract class AccordExecutor implements CacheSize, LoadExecutor, Boolean>, SaveExecutor, Shutdownable, AbstractAsyncExecutor -{ - private static final Logger logger = LoggerFactory.getLogger(AccordExecutor.class); - - private static final QueuePriorityModel PRIORITY_MODEL; - private static final QueueBalancingModel BALANCING_MODEL; - private static final long AGE_TO_FIFO; - // PRIORITY_FAIR blends two strategies (flow: least fairly serviced; age: earliest-queued work) by deficit - // round-robin; weights of BLEND_TOTAL come from a single imbalance ramp (onset..onset+width) trading age->flow. - private static final int BLEND_SHIFT = 6, BLEND_TOTAL = 1 << BLEND_SHIFT; - private static final int FLOW_ONSET, FLOW_WIDTH_SHIFT; - private static final boolean BALANCE_BY_POSITION; - private static final long GLOBAL_QUEUE_LIMITS, EXCLUSIVE_QUEUE_LIMITS; - static final int NONSYNC_MIN_BATCH_SIZE, NONSYNC_MAX_BATCH_SIZE, NONSYNC_BLOCKED_LIMIT; - static final boolean NONSYNC_ENABLED; - - static - { - AccordConfig config = DatabaseDescriptor.getAccord(); - AGE_TO_FIFO = config.queue_priority_age_to_fifo.to(TimeUnit.MICROSECONDS); - PRIORITY_MODEL = config.queue_priority_model != null ? config.queue_priority_model : QueuePriorityModel.HLC_FIFO; - BALANCING_MODEL = config.queue_balancing_model != null ? config.queue_balancing_model : QueueBalancingModel.BLENDED_PRIORITY_PHASE_FAIR; - FLOW_ONSET = config.queue_flow_imbalance_onset == null ? 4 : config.queue_flow_imbalance_onset; - FLOW_WIDTH_SHIFT = config.queue_flow_imbalance_width_shift == null ? 5 : config.queue_flow_imbalance_width_shift; - NONSYNC_MIN_BATCH_SIZE = config.queue_nonsync_min_batch_size == null ? 16 : config.queue_nonsync_min_batch_size; - NONSYNC_MAX_BATCH_SIZE = config.queue_nonsync_max_batch_size == null ? 64 : config.queue_nonsync_max_batch_size; - NONSYNC_ENABLED = config.queue_nonsync_enabled == null || config.queue_nonsync_enabled; - NONSYNC_BLOCKED_LIMIT = config.queue_nonsync_blocked_limit == null ? 8 : config.queue_nonsync_blocked_limit; - Invariants.require(FLOW_ONSET >= 0 && FLOW_WIDTH_SHIFT >= 0); - switch (BALANCING_MODEL) - { - default: throw new UnhandledEnum(BALANCING_MODEL); - case PRIORITY_ONLY: - case BLENDED_PRIORITY_PHASE_FAIR: - BALANCE_BY_POSITION = true; - break; - case PHASE_ONLY: - case PHASE_FAIR: - BALANCE_BY_POSITION = false; - } - - { - // TODO (required): pick default max loads/saves/range loads based on number of threads - long global = COUNTER_MASKS, exclusive = COUNTER_MASKS; - global ^= (0x7fL ^ 1) << (RANGE_SCAN.ordinal() * 8); - if (config.queue_active_limits != null) - { - long[] limits = parseEnumParams(config.queue_active_limits, "queue_active_limits"); - global = selectByOverflowBits(setOverflowWhenLessEqual(limits[0], 0), global, limits[0]); - exclusive = selectByOverflowBits(setOverflowWhenLessEqual(limits[1], 0), global, limits[1]); - } - GLOBAL_QUEUE_LIMITS = global; - EXCLUSIVE_QUEUE_LIMITS = exclusive; - } - - } - - public static final ShardedDecayingHistograms HISTOGRAMS = new ShardedDecayingHistograms(); - - public interface AccordExecutorFactory - { - AccordExecutor get(int executorId, Mode mode, int threads, IntFunction name, Agent agent); - } - - public enum Mode { RUN_WITH_LOCK, RUN_WITHOUT_LOCK } - - public interface AccordTaskRunner - { - AccordExecutor accordActiveExecutor(); - void setAccordActiveExecutor(AccordExecutor newExecutor); - - AccordExecutor accordLockedExecutor(); - boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor); - void exitAccordLockedExecutor(); - - AccordExecutor.Task accordActiveTask(); - // to be called only by the thread itself, so can (eventually) avoid any memory barriers - AccordExecutor.Task accordActiveSelfTask(); - void setAccordActiveTask(AccordExecutor.Task newActiveTask); - - static AccordTaskRunner get() - { - return get(Thread.currentThread()); - } - - static AccordTaskRunner get(Thread thread) - { - return thread instanceof CassandraThread ? (CassandraThread)thread : ThreadLocalAccordTaskRunner.threadLocal.get(); - } - } - - public static final class ThreadLocalAccordTaskRunner implements AccordTaskRunner - { - private AccordExecutor lockedExecutor; - private AccordExecutor activeExecutor; - volatile Task activeTask; - - private static final ThreadLocal threadLocal = ThreadLocal.withInitial(ThreadLocalAccordTaskRunner::new); - - @Override - public AccordExecutor accordActiveExecutor() - { - return activeExecutor; - } - - @Override - public void setAccordActiveExecutor(AccordExecutor newExecutor) - { - activeExecutor = newExecutor; - } - - @Override - public AccordExecutor accordLockedExecutor() - { - return lockedExecutor; - } - - @Override - public boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor) - { - if (lockedExecutor != null) - return false; - lockedExecutor = newLockedExecutor; - return true; - } - - @Override - public void exitAccordLockedExecutor() - { - lockedExecutor = null; - } - - @Override - public void setAccordActiveTask(Task newActiveTask) - { - activeTask = newActiveTask; - } - - @Override - public Task accordActiveTask() - { - return activeTask; - } - - @Override - public Task accordActiveSelfTask() - { - return activeTask; - } - } - - // WARNING: this is a shared object, so close is NOT idempotent - public static final class ExclusiveGlobalCaches extends GlobalCaches implements AutoCloseable - { - final AccordExecutor executor; - - public ExclusiveGlobalCaches(AccordExecutor executor, AccordCache global, AccordCache.Type commands, AccordCache.Type commandsForKey) - { - super(global, commands, commandsForKey); - this.executor = executor; - } - - @Override - public void close() - { - executor.beforeUnlockExternal(); - global.tryShrinkOrEvict(executor.lock); - executor.unlock(AccordTaskRunner.get()); - } - } - - public static class GlobalCaches - { - public final AccordCache global; - public final AccordCache.Type commands; - public final AccordCache.Type commandsForKey; - - public GlobalCaches(AccordCache global, AccordCache.Type commands, AccordCache.Type commandsForKey) - { - this.global = global; - this.commands = commands; - this.commandsForKey = commandsForKey; - } - } - - /** - * Tasks are separated into tranches based on their position. - * - * When we want to wait for all submitted tasks and their consequences to complete, we create a new tranche - * and track the number of tasks still extant for all prior tranches - once these reach zero we can signal - * the completion of the work. - */ - private static final class Tranches - { - class WithDeferred implements Runnable - { - final Runnable run; - final ArrayDeque deferred; - - WithDeferred(Runnable run, Runnable deferred) - { - this.run = run; - this.deferred = new ArrayDeque<>(1); - this.deferred.add(deferred); - } - - WithDeferred(Runnable run, ArrayDeque deferred) - { - this.run = run; - this.deferred = deferred; - } - - @Override - public void run() - { - Runnable register = deferred.poll(); - if (!deferred.isEmpty()) - { - Invariants.require(runs[firstIndex] != null); - Invariants.require(!(runs[firstIndex] instanceof WithDeferred)); - runs[firstIndex] = new WithDeferred(runs[firstIndex], deferred); - } - registerWait(register); - try { run.run(); } - catch (Throwable t) { owner.agent.onException(t); } - } - } - - final AccordExecutor owner; - - int firstTranche; - int firstIndex; - long[] mins = new long[8]; - int[] counts = new int[8]; - Runnable[] runs = new Runnable[8]; - - // the next tranche, that all new work is being collected against - // (and will move to the array accounting once a new wait is registered) - long nextMin; - int nextTranche; - int nextCount; - - private Tranches(AccordExecutor owner) - { - this.owner = owner; - } - - private int trancheToIndex(int tranche) - { - return firstIndex + trancheToIndexOffset(tranche); - } - - private int trancheToIndexOffset(int tranche) - { - int offset = tranche - firstTranche; - if (offset < 0) - offset += MAX_TRANCHE + 1; - return offset; - } - - int size() - { - return trancheToIndexOffset(nextTranche); - } - - int capacity() - { - return counts.length; - } - - int addNew(long position) - { - Invariants.require(position >= nextMin); - ++nextCount; - return nextTranche; - } - - void addInherited(int tranche, long position) - { - if (tranche == nextTranche) - { - Invariants.require(position >= nextMin); - ++nextCount; - } - else - { - int index = trancheToIndex(tranche); - Invariants.require(counts[index] > 0); - Invariants.require(mins[index] <= position); - ++counts[index]; - } - } - - void complete(int tranche) - { - if (tranche == nextTranche) - { - --nextCount; - } - else - { - - if (counts[trancheToIndex(tranche)] == 1) - owner.drainUnqueuedNewWorkExclusive(); // make sure we don't have any pending - - if (--counts[trancheToIndex(tranche)] == 0 && tranche == firstTranche) - { - do advance(); - while (firstTranche != nextTranche && counts[firstIndex] == 0); - } - - if (firstIndex >= counts.length / 2) - compact(); - } - } - - public void finishAll(long nextPosition) - { - while (firstTranche != nextTranche) - { - logger.warn("{} processed all pending tasks (<{}) but found {} waiting for {}", this, - nextPosition, counts[firstIndex], size() == 1 ? nextMin : mins[firstIndex + 1]); - advance(); - } - } - - private void advance() - { - Runnable run = runs[firstIndex]; - runs[firstIndex] = null; - ++firstIndex; - if (firstTranche == MAX_TRANCHE) firstTranche = 0; - else ++firstTranche; - try { run.run(); } - catch (Throwable t) { owner.agent.onException(t); } - } - - public void registerWait(Runnable run) - { - int newNextTranche = (nextTranche + 1) % (MAX_TRANCHE + 1); - if (newNextTranche == firstTranche) - { - Runnable cur = runs[firstIndex]; - if (cur instanceof WithDeferred) ((WithDeferred) cur).deferred.add(run); - else runs[firstIndex] = new WithDeferred(runs[firstIndex], run); - return; - } - - if ((firstIndex + size()) == capacity()) - growOrCompact(); - - int index = firstIndex + size(); - mins[index] = nextMin; - counts[index] = nextCount; - runs[index] = run; - nextMin = owner.minPosition = owner.nextPosition; - nextCount = 0; - nextTranche = newNextTranche; - } - - private void compact() - { - if (size() <= capacity()/4 && capacity() > 8) resize(capacity()/2); - else compact(mins, counts, runs); - } - - private void growOrCompact() - { - if (size() > capacity()/2) resize(capacity() * 2); - else compact(); - } - - private void resize(int newSize) - { - Invariants.require(newSize > 0); - long[] newMins = new long[newSize]; - int[] newCounts = new int[newSize]; - Runnable[] newRuns = new Runnable[newSize]; - compact(newMins, newCounts, newRuns); - mins = newMins; - counts = newCounts; - runs = newRuns; - } - - private void compact(long[] newMins, int[] newCounts, Runnable[] newRuns) - { - int size = size(); - System.arraycopy(mins, firstIndex, newMins, 0, size); - System.arraycopy(counts, firstIndex, newCounts, 0, size); - System.arraycopy(runs, firstIndex, newRuns, 0, size); - firstIndex = 0; - } - } - - final LogLinearDecayingHistograms histograms; - final LogLinearDecayingHistogram elapsedPreparingToRun; - final LogLinearDecayingHistogram elapsedWaitingToRun; - final LogLinearDecayingHistogram elapsedRunning; - final LogLinearDecayingHistogram elapsed; - final LogLinearDecayingHistogram keys; - public final AccordReplicaMetrics.Shard replicaMetrics; - - private final Lock lock; - final Agent agent; - final int executorId; - private final AccordCache cache; - private final ExclusiveGlobalCaches caches; - final AtomicLong uniqueCreatedAt = new AtomicLong(); - final DebugExecutor debug = DebugExecutor.maybeDebug(); - - private long maxWorkingSetSizeInBytes; - private long maxWorkingCapacityInBytes; - - final StandaloneTaskQueue> scanningRanges = new StandaloneTaskQueue<>(TinyEnumSet.encode(SCANNING_RANGES)); // never queried, just parked here while scanning - final StandaloneTaskQueue> loading = new StandaloneTaskQueue<>(TinyEnumSet.encode(LOADING_REQUIRED, LOADING_OPTIONAL)); - final StandaloneTaskQueue> waitingOnCacheQueues = new StandaloneTaskQueue<>(TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL)); - final RunnableTaskQueue runnable = new RunnableTaskQueue<>(); - - private final Tranches tranches = new Tranches(this); - - /** - * Newly submitted work must take a position >= minPosition, but this condition does not apply to consequences of - * previously submitted work; this inherits the originating operation's position and tranche. - * This is to ensure afterSubmittedAndConsequences functions correctly. - */ - private long minPosition = 1; - private long nextPosition = 1; - int tasks; - - private boolean hasPausedLoading; - - private List waitingForQuiescence; - - AccordExecutor(Lock lock, int executorId, Agent agent) - { - this.lock = lock; - this.executorId = executorId; - this.cache = new AccordCache(this, 0); - this.agent = agent; - - final AccordCache.Type commands; - final AccordCache.Type commandsForKey; - commands = cache.newType(TxnId.class, COMMAND_ADAPTER, AccordCacheMetrics.CommandsCacheMetrics.newShard(lock)); - registerJfrListener(executorId, commands, "Command"); - - commandsForKey = cache.newType(RoutingKey.class, CFK_ADAPTER, AccordCacheMetrics.CommandsForKeyCacheMetrics.newShard(lock)); - registerJfrListener(executorId, commandsForKey, "CommandsForKey"); - - this.caches = new ExclusiveGlobalCaches(this, cache, commands, commandsForKey); - - DecayingHistogramsShard histogramsShard = HISTOGRAMS.newShard(lock); - this.histograms = histogramsShard.unsafeGetInternal(); - this.elapsedPreparingToRun = AccordExecutorMetrics.INSTANCE.elapsedPreparingToRun.forShard(histogramsShard); - this.elapsedWaitingToRun = AccordExecutorMetrics.INSTANCE.elapsedWaitingToRun.forShard(histogramsShard); - this.elapsedRunning = AccordExecutorMetrics.INSTANCE.elapsedRunning.forShard(histogramsShard); - this.elapsed = AccordExecutorMetrics.INSTANCE.elapsed.forShard(histogramsShard); - this.keys = AccordExecutorMetrics.INSTANCE.keys.forShard(histogramsShard); - this.replicaMetrics = new AccordReplicaMetrics.Shard(histogramsShard); - } - - public int executorId() - { - return executorId; - } - - public ExclusiveGlobalCaches lockCaches() - { - lock(AccordTaskRunner.get(Thread.currentThread())); - return caches; - } - - abstract boolean isInLoop(); - - public final Lock unsafeLock() - { - return lock; - } - - final void lock(AccordTaskRunner self) - { - long startAt = DEBUG_EXECUTION ? 0 : Clock.Global.nanoTime(); - if (!self.tryEnterAccordLockedExecutor(this)) - throw new UnsupportedOperationException("To ensure system performance, it is not permitted to lock multiple AccordExecutor simultaneously with the same thread"); - //noinspection LockAcquiredButNotSafelyReleased - lock.lock(); - if (DEBUG_EXECUTION) debug.onEnterLock(startAt); - } - - final void unlock(AccordTaskRunner self) - { - self.exitAccordLockedExecutor(); - if (DEBUG_EXECUTION) debug.onExitLock(); - lock.unlock(); - } - - final boolean tryLock(AccordTaskRunner self) - { - AccordExecutor locked = self.accordLockedExecutor(); - if (locked != null) - return false; - - return onTryLock(self, lock.tryLock()); - } - - final boolean onTryLock(AccordTaskRunner self, boolean result) - { - if (result && !self.tryEnterAccordLockedExecutor(this)) - { - // shouldn't be possible, we should have checked this already - lock.unlock(); - throw new IllegalStateException(); - } - return result; - } - - public AccordCache cacheExclusive() - { - Invariants.require(isOwningThread()); - return cache; - } - - public AccordCache cacheUnsafe() - { - return cache; - } - - final boolean hasAlreadyWaitingToRun() - { - return runnable.hasWaitingToRun(); - } - - void updateWaitingToRunExclusive() - { - // TODO (expected): this should not be invoked on every update of waiting to run - maybeUnpauseLoading(); - } - - // drain only new work; specifically leave anything that would call completeTask queued. - // this is to maintain invariants in Tranches.complete, where we may have some consequence of some earlier task - // pending but unqueued, so that we have not incremented its tranche count, and in the interim we set the tranche - // count to zero - void drainUnqueuedNewWorkExclusive() - { - } - - final Task pollAlreadyWaitingToRunExclusive() - { - Task next = runnable.poll(); - if (DEBUG_EXECUTION && next != null) DebugTask.get(next).onPolled(); - return next; - } - - public Stream active() - { - return Stream.of(); - } - - public void waitForQuiescence() - { - AccordTaskRunner self = AccordTaskRunner.get(); - Condition condition; - lock(self); - try - { - if (tasks == 0) - return; - - if (waitingForQuiescence == null) - waitingForQuiescence = new ArrayList<>(); - condition = Condition.newOneTimeCondition(); - waitingForQuiescence.add(condition); - } - finally - { - unlock(self); - } - condition.awaitThrowUncheckedOnInterrupt(); - } - - protected void notifyQuiescentExclusive() - { - if (waitingForQuiescence != null) - { - waitingForQuiescence.forEach(Condition::signalAll); - waitingForQuiescence = null; - } - tranches.finishAll(nextPosition); - } - - public void afterSubmittedAndConsequences(Runnable run) - { - AccordTaskRunner self = AccordTaskRunner.get(); - - lock(self); - try - { - drainUnqueuedNewWorkExclusive(); - if (tasks == 0) - { - run.run(); - return; - } - - tranches.registerWait(run); - } - finally - { - unlock(self); - } - } - - final void maybeUnpauseLoading() - { - if (!hasPausedLoading) - return; - - if (cache.weightedSize() < maxWorkingCapacityInBytes && !runnable.hasWaitingToRun()) - { - hasPausedLoading = false; - runnable.restart(RANGE_SCAN.ordinal()); - runnable.restart(LOAD.ordinal()); - runnable.restart(RANGE_LOAD.ordinal()); - } - } - - final void maybePauseLoading() - { - if (hasPausedLoading) - return; - - if (cache.weightedSize() >= maxWorkingCapacityInBytes || !runnable.hasWaitingToRun()) - { - AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); - hasPausedLoading = true; - runnable.stop(RANGE_SCAN.ordinal()); - runnable.stop(LOAD.ordinal()); - runnable.stop(RANGE_LOAD.ordinal()); - } - } - - public abstract boolean hasTasks(); - abstract void beforeUnlockExternal(); - abstract boolean isOwningThread(); - - public ExclusiveExecutor executor() - { - return new ExclusiveExecutor(this); - } - - public ExclusiveExecutor executor(int commandStoreId) - { - return new ExclusiveExecutor(this, commandStoreId); - } - - public ExclusiveAsyncExecutor newExclusiveExecutor() - { - return new ExclusiveExecutor(this); - } - - public void cancel(AccordTask task) - { - Invariants.require(task.commandStore.executor() == this, - "%s is a wrong command store for %s, should be %s", - this, task, task); - submit(AccordExecutor::cancelExclusive, CancelTask::new, task); - } - - @Override - public LoadRunnable load(AccordTask parent, Boolean isForRange, AccordCacheEntry entry) - { - LoadRunnable load = newLoad(entry, isForRange); - return submitPlainExclusive(parent, load); - } - - @Override - public Cancellable save(AccordCacheEntry entry, UniqueSave identity, Runnable save) - { - return submitPlainExclusive(null, new SaveRunnable(entry, identity, save)); - } - - private void submit(BiConsumer sync, Function async, P1 p1) - { - submit((e, c, p1a, p2a, p3) -> c.accept(e, p1a), (f, p1a, p2a, p3) -> f.apply(p1a), sync, async, p1, null, null); - } - - private void submit(TriConsumer sync, BiFunction async, P1 p1, P2 p2) - { - submit((e, c, p1a, p2a, p3) -> c.accept(e, p1a, p2a), (f, p1a, p2a, p3) -> f.apply(p1a, p2a), sync, async, p1, p2, null); - } - - private void submit(QuadConsumer sync, TriFunction async, P1 p1, P2 p2, P3 p3) - { - submit((e, c, p1a, p2a, p3a) -> c.accept(e, p1a, p2a, p3a), TriFunction::apply, sync, async, p1, p2, p3); - } - - private void submit(QuintConsumer sync, QuadFunction async, P1 p1, P2 p2, P3 p3, P4 p4) - { - submit(sync, async, p1, p1, p2, p3, p4); - } - - abstract void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4); - - void submit(AccordTask operation) - { - submit((self, task) -> task.submitExclusive(self), i -> i, operation); - } - - public void submitExclusive(Runnable runnable) - { - submitPlainExclusive(new PlainRunnable(null, runnable, OTHER)); - } - - private Cancellable submitPlain(Plain task) - { - submit(AccordExecutor::submitPlainExclusive, i -> i, task); - return task; - } - - private void submitPlainExclusive(Plain task) - { - registerExclusive(task); - task.onLoaded(); - ExclusiveExecutor executor = task.executor(); - if (executor == null) runnable.enqueue(task, true); - else executor.enqueue(task, true); - } - - Cancellable submitPlainExclusive(Task parent, GlobalGroup group, AbstractIOTask task) - { - return submitPlainExclusive(parent, new WrappedIOTask(task, group, parent.position, parent.tranche())); - } - - final T submitPlainExclusive(Task parent, T task) - { - Invariants.require(isOwningThread()); - task.setStateExclusive(WAITING_TO_RUN); - if (parent == null) registerExclusive(task); - else registerConsequenceExclusive(parent, task); - task.onLoaded(); - runnable.enqueue(task, true); - return task; - } - - private void registerConsequenceExclusive(Task parent, Task task) - { - ++tasks; - int tranche = parent.tranche(); - tranches.addInherited(tranche, parent.position); - task.position = parent.position; - task.setInheritedWithTranche(tranche); - } - - void registerExclusive(Task task) - { - ++tasks; - if (task.hasInherited()) - { - tranches.addInherited(task.tranche(), task.position); - } - else - { - long position = task.position; - if (position == 0) - { - task.position = position = nextPosition++; - } - else - { - long delta = nextPosition - position; - if (delta >= AGE_TO_FIFO) - task.position = position = nextPosition++; - else if (delta <= 0) - nextPosition = position + 1; - else if (position < minPosition) - task.position = position = minPosition; - } - task.setTranche(tranches.addNew(position)); - } - } - - final void cleanupTaskExclusive(Task task, boolean executed) - { - runnable.cleanup(task); - try { task.cleanupExclusive(this, executed); } - finally - { - cache.tryShrinkOrEvict(lock); - maybeUnpauseLoading(); - } - } - - final void unregisterExclusive(Task task) - { - int tranch = task.tranche(); - --tasks; - tranches.complete(tranch); - } - - final void cancelExclusive(AccordTask task) - { - State state = task.state(); - switch (state) - { - default: throw new UnhandledEnum(state); - case SCANNING_RANGES: - case LOADING_REQUIRED: - case LOADING_OPTIONAL: - case WAITING_ON_REQUIRED: - case WAITING_ON_OPTIONAL: - case WAITING_TO_RUN: - if (!task.hasIncrementalStarted()) - { - task.unqueueIfQueued(); - try { task.cancelExclusive(); } - finally { cleanupTaskExclusive(task, false); } - break; - } - case UNINITIALIZED: // TODO (expected): preferable to be able to cancel at this stage, even if unlikely to trigger at this phase - case RUNNING: - case INCOMPLETE: - case EXECUTED: - case CANCELLED: - case FAILED_TO_LOAD: - // cannot safely cancel - } - } - - void onScannedRangesExclusive(AccordTask task, Throwable fail) - { - // the task may have already been cancelled, in which case we don't need to fail it - if (task.state().isExecuted()) - return; - - if (fail != null) failExclusive(task, fail, FAILED_TO_LOAD); - else task.rangeScanner().scannedExclusive(); - } - - private void failExclusive(AccordTask task, Throwable fail, State newState) - { - if (task.state().isExecuted()) - return; - - try { task.failExclusive(fail, newState); } - catch (Throwable t) { agent.onException(t); } - finally - { - task.unqueueIfQueued(); - cleanupTaskExclusive(task, false); - } - } - - private void onSavedExclusive(AccordCacheEntry state, Object identity, Throwable fail) - { - cache.saved(state, identity, fail); - } - - private void onLoadedExclusive(AccordCacheEntry loaded, V value, Throwable fail) - { - if (loaded.status() == EVICTED) - return; - - try (ArrayBuffers.BufferList> tasks = loaded.drainWaitingToLoad()) - { - if (fail != null) - { - for (AccordTask task : tasks) - failExclusive(task, fail, FAILED_TO_LOAD); - cache.failedToLoad(loaded); - } - else - { - cache.loaded(loaded, value); - for (AccordTask task : tasks) - task.onLoadOneExclusive(loaded); - } - } - - maybePauseLoading(); - } - - private Task inherit() - { - Thread thread = Thread.currentThread(); - AccordTaskRunner self = AccordTaskRunner.get(thread); - - if (self.accordActiveExecutor() != this) - return null; - - Task task = self.accordActiveSelfTask(); - if (task instanceof ExclusiveExecutorTask) - task = ((ExclusiveExecutorTask)task).queue.task; - return task; - } - - public Future submit(Runnable run) - { - Task inherit = inherit(); - PlainRunnable task = inherit == null ? new PlainRunnable(new AsyncPromise<>(), run, OTHER) - : new PlainRunnable(new AsyncPromise<>(), run, OTHER, inherit.position, inherit.tranche()); - submitPlain(task); - return task.result; - } - - public void execute(Runnable run) - { - Task inherit = inherit(); - PlainRunnable task = inherit == null ? new PlainRunnable(null, run, OTHER) - : new PlainRunnable(null, run, OTHER, inherit.position, inherit.tranche()); - submitPlain(task); - } - - @Override - public Cancellable execute(RunOrFail runOrFail) - { - Task inherit = inherit(); - PlainChain submit = inherit == null ? new PlainChain(runOrFail, null, ExclusiveGroup.OTHER) - : new PlainChain(runOrFail, null, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); - return submitPlain(submit); - } - - public AsyncChain buildDebuggable(Callable call, Object describe) - { - return new AsyncChains.Head<>() - { - @Override - protected Cancellable start(BiConsumer callback) - { - Task inherit = inherit(); - return submitPlain(inherit == null ? new DebuggableChain(new CallAndCallback<>(call, callback), null, describe) - : new DebuggableChain(new CallAndCallback<>(call, callback), null, inherit.position, inherit.tranche(), describe)); - } - }; - } - - public void executeDirectlyWithLock(Runnable command) - { - AccordTaskRunner self = AccordTaskRunner.get(); - lock(self); - try - { - self.setAccordActiveExecutor(this); - command.run(); - } - finally - { - beforeUnlockExternal(); - self.exitAccordLockedExecutor(); - unlock(self); - } - } - - @Override - public void setCapacity(long bytes) - { - Invariants.require(isOwningThread()); - cache.setCapacity(bytes); - maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; - } - - public void setWorkingSetSize(long bytes) - { - Invariants.require(isOwningThread()); - maxWorkingSetSizeInBytes = bytes; - maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; - if (maxWorkingCapacityInBytes < maxWorkingSetSizeInBytes) - maxWorkingCapacityInBytes = Long.MAX_VALUE; - } - - @Override - public long capacity() - { - return cache.capacity(); - } - - @Override - public int size() - { - return cache.size(); - } - - @Override - public long weightedSize() - { - return cache.weightedSize(); - } - - protected static abstract class TaskRunner implements DebuggableTaskRunner - { - private volatile AccordTaskRunner wrapped; - - protected void setWrapped(AccordTaskRunner wrapped) - { - this.wrapped = wrapped; - } - - @Override - public DebuggableTask running() - { - Task running = wrapped.accordActiveTask(); - return running == null ? null : running.debuggable(); - } - } - - public static abstract class Task extends IntrusiveHeapNode - { - private static final int WAITING_ON_OPTIONAL_BIT = 1 << 5; - private static final int WAITING_TO_RUN_BIT = 1 << 6; - private static final int INCOMPLETE_BIT = 1 << 9; - - enum State - { - UNINITIALIZED(), - SCANNING_RANGES(UNINITIALIZED), - LOADING_REQUIRED(UNINITIALIZED, SCANNING_RANGES), - LOADING_OPTIONAL(UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED), - WAITING_ON_REQUIRED(WAITING_ON_OPTIONAL_BIT | WAITING_TO_RUN_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), - WAITING_ON_OPTIONAL(WAITING_TO_RUN_BIT | INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED), - WAITING_TO_RUN(INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL), - RUNNING(WAITING_TO_RUN), - EXECUTED(RUNNING), - INCOMPLETE(RUNNING), - FAILED_TO_LOAD(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), - FAILED_OTHER(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN), - CANCELLED(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN, RUNNING), - ; - - private final int permittedFrom; - public static final int WAITING = TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN); - public static final int WAITING_OR_RUNNING = WAITING | TinyEnumSet.encode(RUNNING); - public static final int RUNNING_OR_EXECUTED = WAITING | TinyEnumSet.encode(RUNNING, EXECUTED); - static final State[] VALUES = values(); - - static - { - // hack to allow us to create loops in our enum transition declarations - Invariants.require(INCOMPLETE_BIT == 1 << INCOMPLETE.ordinal()); - } - - State() - { - this.permittedFrom = 0; - } - - State(State ... permittedFroms) - { - this(0, permittedFroms); - } - - State(int additional, State ... permittedFroms) - { - int permittedFrom = additional; - for (State state : permittedFroms) - permittedFrom |= 1 << state.ordinal(); - this.permittedFrom = permittedFrom; - } - - boolean isPermittedFrom(int prevOrdinal) - { - return (permittedFrom & (1 << prevOrdinal)) != 0; - } - - boolean isExecuted() - { - return this.compareTo(EXECUTED) >= 0; - } - - boolean hasStarted() - { - return this.compareTo(RUNNING) >= 0; - } - - static State forOrdinal(int ordinal) - { - return VALUES[ordinal]; - } - } - - enum RunState - { - NONE, PERSISTING, SUCCESS, FAILED; - - private static final RunState[] VALUES = values(); - static RunState forOrdinal(int ordinal) - { - return VALUES[ordinal]; - } - } - - enum GlobalGroup - { - COMMAND_STORE, - LOAD, - SAVE, - OTHER, - RANGE_LOAD, - RANGE_SCAN, - } - - enum ExclusiveGroup - { - APPLY, - STABLE, - COMMIT, - ACCEPT, - OTHER, - RECOVER, - PREACCEPT, - RANGE, - } - - public enum GroupKind - { - EXCLUSIVE(ExclusiveGroup.values().length, EXCLUSIVE_GROUP_SHIFT), - GLOBAL(GlobalGroup.values().length, GLOBAL_GROUP_SHIFT), - NONE(0, 0); - - final int count; - final byte shift; - GroupKind(int count, int shift) - { - this.count = count; - this.shift = (byte) shift; - } - } - - private static final int STATE_MASK = 0xf; - private static final int GROUP_MASK = 0x7; - private static final int EXCLUSIVE_GROUP_SHIFT = 4; - private static final int GLOBAL_GROUP_SHIFT = 7; - - private static final int NONSYNC_BIT = 1 << 10; - private static final int CACHE_QUEUED_BIT = 1 << 11; - - private static final int INCREMENTAL_MASK = 0x3 << 12; - private static final int INCREMENTAL = 0x1 << 12; - private static final int INCREMENTAL_STARTED = 0x2 << 12; - private static final int INCREMENTAL_FINISHING = 0x3 << 12; - - private static final int SEQUENCED_SHIFT = 14; - private static final int SEQUENCED_MASK = 0x3 << SEQUENCED_SHIFT; - private static final int SEQUENCED_PRIORITY = 0x1 << SEQUENCED_SHIFT; - private static final int SEQUENCED_ATOMIC = 0x2 << SEQUENCED_SHIFT; - private static final int SEQUENCED_ATOMIC_AND_QUEUED = 0x3 << SEQUENCED_SHIFT; - - // spare two bits - - private static final int HAS_TRANCHE_BIT = 1 << 18; - private static final int HAS_INHERITED_BIT = 1 << 19; - private static final int HAS_INHERITED_RANGE_SCAN_BIT = 1 << 20; - - private static final int TRANCHE_SHIFT = 22; - static final int MAX_TRANCHE = 0x3ff; - - static - { - Invariants.require(SEQUENCED_PRIORITY == BY_PRIORITY.ordinal() << SEQUENCED_SHIFT); - Invariants.require(SEQUENCED_ATOMIC == BY_PRIORITY_ATOMIC.ordinal() << SEQUENCED_SHIFT); - Invariants.require(ExecutionSequence.values().length <= 3); - } - - public final WithResources resources; - Task next; - - long position; - private int info; - - // TODO (expected): do we need this? we should be able to determine the queue from state() if needed for e.g. cancellation - private TaskQueue queued; - - public final long createdAt; - // TODO (expected): expose via executors vtable - // TODO (expected): use just one long and some flag bits to indicate which point it represents, and report incrementally - public long loadedAt, runningAt, completeAt; - private byte runState; - - Task(GlobalGroup group) - { - resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); - info = init(group, ExclusiveGroup.OTHER); - createdAt = nanoTime(); - } - - Task(ExclusiveGroup group) - { - resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); - info = init(GlobalGroup.OTHER, group); - createdAt = nanoTime(); - } - - Task(GlobalGroup group, long position, int tranche) - { - this(group); - this.position = position; - setInheritedWithTranche(tranche); - } - - Task(ExclusiveGroup group, long position, int tranche) - { - this(group); - this.position = position; - setInheritedWithTranche(tranche); - } - - protected Task(ExecutionContext context, AtomicLong lastCreatedAt) - { - resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); - createdAt = lastCreatedAt.accumulateAndGet(nanoTime(), (prev, next) -> next < prev ? prev + 1 : next); - ExclusiveGroup group = ExclusiveGroup.OTHER; - TxnId txnId = context.primaryTxnId(); - if (txnId != null) - { - if (txnId.is(Range)) group = ExclusiveGroup.RANGE; - else - { - switch (PRIORITY_MODEL) - { - case HLC_FIFO: - case ORIG_HLC_FIFO: - { - // TODO (expected): port to ExecutionKind; also we aren't consistent about using Ballot - if (context instanceof Request) - { - MessageType type = ((Request) context).type(); - if (type instanceof StandardMessage) - { - switch ((StandardMessage)type) - { - case APPLY_REQ: - { - group = APPLY; - break; - } - case READ_EPHEMERAL_REQ: - case READ_REQ: - case STABLE_THEN_READ_REQ: - { - group = STABLE; - break; - } - case COMMIT_REQ: - { - Commit commit = (Commit) context; - if (PRIORITY_MODEL == ORIG_HLC_FIFO && !commit.ballot.equals(Ballot.ZERO)) - txnId = null; - if (commit.kind.saveStatus == SaveStatus.Stable) group = STABLE; - else group = COMMIT; - break; - } - case ACCEPT_REQ: - { - Accept accept = (Accept) context; - if (PRIORITY_MODEL == ORIG_HLC_FIFO && !accept.ballot.equals(Ballot.ZERO)) - txnId = null; - group = ExclusiveGroup.ACCEPT; - break; - } - case GET_EPHEMERAL_READ_DEPS_REQ: - case PRE_ACCEPT_REQ: - { - group = ExclusiveGroup.PREACCEPT; - break; - } - default: - { - txnId = null; - } - } - } - } - else - { - txnId = null; - } - break; - } - case FIFO: - { - txnId = null; - break; - } - } - } - } - - this.info = init(GlobalGroup.OTHER, group); - if (txnId != null) - this.position = txnId.hlc(); - } - - public final Task unwrap() - { - if (this instanceof ExclusiveExecutorTask) - return ((ExclusiveExecutorTask) this).queue.task; - return this; - } - - static Task unwrap(Task task) - { - return task == null ? null : task.unwrap(); - } - - public DebuggableTask debuggable() { return null; } - - abstract String toDescription(); - - abstract void submitExclusive(AccordExecutor owner); - - /** - * Prepare to run while holding the state cache lock - */ - void preRunExclusive() { setStateExclusive(RUNNING); } - - /** - * Run the command; the state cache lock may or may not be held depending on the executor implementation - */ - abstract void run(); - - /** - * Fail the command; the state cache lock may or may not be held depending on the executor implementation - */ - abstract void reportFailure(Throwable fail); - - final void failExclusive(Throwable fail, State newState) - { - try { setStateExclusive(newState); } - finally { reportFailure(fail); } - - } - - final void failExecution(Throwable fail) - { - Invariants.require(is(RUNNING)); - try { setRunState(RunState.FAILED); } - finally { reportFailure(fail); } - - } - - abstract boolean isNewWork(); - - /** - * Cleanup the command while holding the state cache lock - */ - void cleanupExclusive(AccordExecutor executor, boolean executed) - { - if (executed) setStateExclusive(EXECUTED); - else Invariants.require(state().isExecuted()); - executor.unregisterExclusive(this); - completeAt = nanoTime(); - if (runningAt != 0) - { - if (loadedAt == 0) - loadedAt = runningAt; - executor.elapsedWaitingToRun.increment(runningAt - loadedAt, runningAt); - executor.elapsedPreparingToRun.increment(loadedAt - createdAt, runningAt); - executor.elapsedRunning.increment(completeAt - runningAt, completeAt); - executor.elapsed.increment(completeAt - createdAt, completeAt); - } - if (DEBUG_EXECUTION) DebugTask.get(this).onCompleted(executor.debug); - } - - void cancelExclusive(AccordExecutor owner) {} - - @Nullable - final TaskQueue queued() - { - return queued; - } - - final void unqueueIfQueued() - { - if (queued != null) - { - queued.unqueue(this); - queued = null; - } - } - - final void unqueue(TaskQueue expected) - { - Invariants.require(queued == expected, "%s != %s", queued, expected); - queued.unqueue(this); - queued = null; - } - - final void unsetQueue(TaskQueue expected) - { - Invariants.require(queued == expected, "%s != %s", queued, expected); - queued = null; - } - - final void setQueue(TaskQueue queue) - { - Invariants.require(queued == null); - Invariants.require(isCompatible(queue)); - queued = queue; - } - - final void onRunning() - { - runningAt = nanoTime(); - if (DEBUG_EXECUTION) ((DebugTask)resources).onRunning(); - } - - final void onRunComplete() - { - if (DEBUG_EXECUTION) ((DebugTask)resources).onRunComplete(); - } - - final void onLoaded() - { - loadedAt = nanoTime(); - } - - final State state() - { - return State.forOrdinal(stateOrdinal()); - } - - final RunState runState() - { - return RunState.forOrdinal(runState); - } - - final Enum describeState() - { - State state = state(); - if (state == RUNNING || state == EXECUTED) - { - RunState runState = runState(); - if (runState == RunState.NONE) - return state; - return runState; - } - return State.forOrdinal(stateOrdinal()); - } - - private int stateOrdinal() - { - return info & STATE_MASK; - } - - final boolean is(State state) - { - return stateOrdinal() == state.ordinal(); - } - - final boolean isState(int stateBitSet) - { - return TinyEnumSet.contains(stateBitSet, stateOrdinal()); - } - - final boolean is(GlobalGroup group) - { - return globalGroupOrdinal() == group.ordinal(); - } - - final boolean is(ExclusiveGroup group) - { - return exclusiveGroupOrdinal() == group.ordinal(); - } - - final void override(GlobalGroup group) - { - info = (info & ~(GROUP_MASK << GLOBAL_GROUP_SHIFT)) | (group.ordinal() << GLOBAL_GROUP_SHIFT); - } - - final int compareTo(State state) - { - return stateOrdinal() - state.ordinal(); - } - - final void setStateExclusive(State state) - { - Invariants.require(state.isPermittedFrom(stateOrdinal()), "%s forbidden from %s", state, this, Task::reportBadStateTransition); - setStateUnsafe(state); - } - - final void setRunState(RunState state) - { - Invariants.require(isState(RUNNING_OR_EXECUTED)); - runState = (byte) state.ordinal(); - } - - private static String reportBadStateTransition(Task task) - { - return task.state() + " for " + task.toDescription(); - } - - final void setStateUnsafe(State state) - { - info = (info & ~STATE_MASK) | state.ordinal(); - } - - final int globalGroupOrdinal() - { - return (info >>> GLOBAL_GROUP_SHIFT) & GROUP_MASK; - } - - final int exclusiveGroupOrdinal() - { - return (info >>> EXCLUSIVE_GROUP_SHIFT) & GROUP_MASK; - } - - private boolean isCompatible(TaskQueue queue) - { - int self = stateOrdinal(); - return TinyEnumSet.contains(queue.states, self); - } - - final boolean isSync() - { - return 0 == (info & NONSYNC_BIT); - } - - final boolean isNonSync() - { - return !isSync(); - } - - final void setNonSyncExclusive() - { - info |= NONSYNC_BIT; - } - - final boolean isIncremental() - { - return 0 != (info & INCREMENTAL_MASK); - } - - final void setIncrementalExclusive() - { - info |= INCREMENTAL | NONSYNC_BIT; - } - - final boolean hasIncrementalStarted() - { - return (info & INCREMENTAL_MASK) >= INCREMENTAL_STARTED; - } - - final void setIncrementalStartedExclusive() - { - Invariants.require(isIncremental()); - if (!isIncrementalFinishing()) - info = (info & ~INCREMENTAL_MASK) | INCREMENTAL_STARTED; - } - - final boolean isIncrementalFinishing() - { - return (info & INCREMENTAL_MASK) >= INCREMENTAL_FINISHING; - } - - final void setIncrementalFinishingExclusive() - { - Invariants.require(isIncremental()); - info |= INCREMENTAL_FINISHING; - } - - final void setSequencedExclusive(ExecutionSequence sequence) - { - Invariants.require(isUnsequenced()); - info |= sequence.ordinal() << SEQUENCED_SHIFT; - } - - final boolean isUnsequenced() - { - return (info & SEQUENCED_MASK) == 0; - } - - final boolean isSequencedByPriority() - { - return (info & SEQUENCED_MASK) == SEQUENCED_PRIORITY; - } - - final boolean isSequencedByPriorityAtomic() - { - return (info & SEQUENCED_MASK) >= SEQUENCED_ATOMIC; - } - - final boolean isCacheQueuedFifo() - { - return (info & SEQUENCED_MASK) == SEQUENCED_ATOMIC_AND_QUEUED; - } - - final boolean isCacheQueued() - { - return 0 != (info & CACHE_QUEUED_BIT); - } - - // supersedes priority, in whichever order they're called - final void setCacheQueuedFifoExclusive() - { - Invariants.require(isSequencedByPriorityAtomic()); - info |= SEQUENCED_ATOMIC_AND_QUEUED | CACHE_QUEUED_BIT; - } - - final void setCacheQueuedExclusive() - { - info |= CACHE_QUEUED_BIT; - } - - final int tranche() - { - Invariants.require((info & HAS_TRANCHE_BIT) != 0); - return info >>> TRANCHE_SHIFT; - } - - final void setTranche(int tranche) - { - Invariants.require(tranche <= MAX_TRANCHE); - info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; - } - - final void setInheritedWithTranche(int tranche) - { - Invariants.require(tranche <= MAX_TRANCHE); - info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT | HAS_INHERITED_BIT; - } - - final boolean hasInherited() - { - return (info & HAS_INHERITED_BIT) != 0; - } - - final void setInheritedRangeScan() - { - info = info | HAS_INHERITED_RANGE_SCAN_BIT; - } - - final boolean hasInheritedRangeScan() - { - return (info & HAS_INHERITED_RANGE_SCAN_BIT) != 0; - } - - static int init(GlobalGroup global, ExclusiveGroup exclusive) - { - return (global.ordinal() << GLOBAL_GROUP_SHIFT) | (exclusive.ordinal() << EXCLUSIVE_GROUP_SHIFT); - } - - static Task reverse(Task unqueued) - { - Task prev = null; - Task cur = unqueued; - while (cur != null) - { - Task next = cur.next; - cur.next = prev; - prev = cur; - cur = next; - } - return prev; - } - } - - // run the task even on a stopped commandStore - public interface Unstoppable extends ExecutionContext.Empty - { - } - - // run the task even on a terminated commandStore - public interface Unterminatable extends Unstoppable - { - } - - static final class ExclusiveExecutorTask extends Task - { - private final ExclusiveExecutor queue; - - ExclusiveExecutorTask(ExclusiveExecutor queue) - { - super(COMMAND_STORE); - this.queue = queue; - } - - @Override - String toDescription() - { - return queue.task.toDescription(); - } - - @Override void submitExclusive(AccordExecutor owner) { throw new UnsupportedOperationException(); } - - @Override - void preRunExclusive() - { - queue.preRunTask(); - } - - @Override - void run() - { - queue.runTask(); - } - - @Override - void reportFailure(Throwable t) - { - queue.task.reportFailure(t); - } - - @Override - boolean isNewWork() - { - throw new UnsupportedOperationException(); - } - - @Override - void cleanupExclusive(AccordExecutor executor, boolean executed) - { - queue.cleanupTask(executed); - } - - protected boolean isInHeap() - { - return super.isInHeap(); - } - } - - private static final AtomicReferenceFieldUpdater ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(ExclusiveExecutor.class, Thread.class, "owner"); - public final class ExclusiveExecutor extends MultiTaskQueue implements ExclusiveAsyncExecutor - { - final int commandStoreId; - final ExclusiveExecutorTask selfTask; - private Task task; - volatile Thread owner, waiting; - private boolean stopped; - private volatile boolean visibleStopped; - private boolean terminated; - - final DebugExclusiveExecutor debug; - - ExclusiveExecutor(AccordExecutor executor) - { - this(executor, -1); - } - - ExclusiveExecutor(AccordExecutor executor, int commandStoreId) - { - super(RUNNABLE, commandStoreId < 0 ? GroupKind.NONE : GroupKind.EXCLUSIVE, EXCLUSIVE_QUEUE_LIMITS); - this.commandStoreId = commandStoreId; - this.selfTask = new ExclusiveExecutorTask(this); - this.debug = DebugExclusiveExecutor.maybeDebug(executor.debug, commandStoreId); - } - - void preRunTask() - { - task.preRunExclusive(); - } - - void runTask() - { - Thread self = Thread.currentThread(); - if (!ownerUpdater.compareAndSet(this, null, self)) - { - if (DEBUG_EXECUTION) debug.onWaiting(); - Invariants.require(self == Thread.currentThread()); - waiting = self; - outer: do - { - while (true) - { - Thread owner = this.owner; - if (owner == self) break outer; - if (owner == null) continue outer; - LockSupport.park(); - } - } - while (!ownerUpdater.compareAndSet(this, null, self)); - } - waiting = null; - - if (stopped && reject(task)) - task.reportFailure(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((AccordTask) task).executionContext())); - else - task.run(); - - // NOTE: we can ONLY safely release owner here due to AccordCacheEntry locking, which remains in place until AccordTask.releaseResourcesExclusive - // this also relies on AccordSafeCommandStore$ExclusiveCaches.acquireIfLoaded returning false when the entry is locked - owner = null; - } - - private boolean reject(Task task) - { - if (!(task instanceof AccordTask)) - return true; - - ExecutionContext context = ((AccordTask) task).executionContext(); - return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); - } - - void cleanupTask(boolean executed) - { - try - { - task.unsetQueue(this); - task.cleanupExclusive(AccordExecutor.this, executed); - } - finally - { - active = 0; - task = super.pollMulti(); - if (DEBUG_EXECUTION) debug.onSetTask(task); - if (task != null) - { - selfTask.position = task.position; - selfTask.setStateUnsafe(WAITING_TO_RUN); - runnable.enqueue(selfTask, false); - } - } - } - - void enqueue(Task newTask, boolean incrementArrivals) - { - - if (task != null) - { - if (incrementArrivals) - runnable.incrementArrivals(selfTask); - // TODO (expected): restore some invariant here -// Invariants.require(selfTask.isInHeap() || selfTask.is(RUNNING)); - super.enqueueMulti(newTask, incrementArrivals); - } - else - { - Invariants.require(isEmptySingle()); - if (incrementArrivals) - incrementArrivals(newTask); - incrementDispatches(newTask); - task = newTask; - task.setQueue(this); - selfTask.position = newTask.position; - selfTask.setStateUnsafe(WAITING_TO_RUN); - runnable.enqueue(selfTask, incrementArrivals); - if (DEBUG_EXECUTION) debug.onSetTask(newTask); - } - } - - @Override - void unqueue(Task remove) - { - if (remove == task) removeCurrentTask(remove); - else super.unqueueMulti(remove); - } - - boolean tryUnqueueWaiting(Task remove) - { - if (remove == task) return tryRemoveCurrentTask(remove); - else return super.tryUnqueueWaiting(remove); - } - - private boolean tryRemoveCurrentTask(IntrusiveHeapNode remove) - { - if (runnable.isAssigned(selfTask)) - return false; - - removeCurrentTask(remove); - return true; - } - - private void removeCurrentTask(IntrusiveHeapNode remove) - { - Invariants.require(remove == task); - // cannot overwrite task while it is being executed - this cannot happen for AccordTask - // but can for other tasks that don't track their own state - - decrementDispatches(task); - task.unsetQueue(this); - task = pollMulti(); - if (DEBUG_EXECUTION) debug.onSetTask(task); - if (runnable.isWaiting(selfTask)) - { - if (task == null) runnable.unqueue(selfTask); - else - { - selfTask.position = task.position; - runnable.requeue(selfTask); - } - } - else - { - Invariants.expect(false, "%s should have been queued to run as it had the task %s pending, that has now been cancelled", this, remove); - if (task != null) - { - selfTask.position = task.position; - selfTask.setStateUnsafe(WAITING_TO_RUN); - runnable.enqueue(selfTask, false); - } - } - Invariants.require(task == null || runnable.isWaiting(selfTask)); - } - - public boolean inExecutor() - { - return owner == Thread.currentThread(); - } - - public boolean stopped() - { - return visibleStopped; - } - - void stop() - { - Invariants.require(inExecutor()); - this.stopped = true; - this.visibleStopped = true; - } - - void terminate() - { - Invariants.require(inExecutor()); - this.visibleStopped = this.terminated = this.stopped = true; - } - - @Override - public AsyncChain chain(Runnable run) - { - return AsyncChains.chain(this, run); - } - - @Override - public AsyncChain chain(Callable call) - { - return AsyncChains.chain(this, call); - } - - @Override - public AsyncChain flatChain(Callable> call) - { - return AsyncChains.flatChain(this, call); - } - - Task inherit() - { - Thread thread = Thread.currentThread(); - if (thread == owner) - return Task.unwrap(task); - return Task.unwrap(AccordTaskRunner.get(thread).accordActiveSelfTask()); - } - - @Override - public void execute(Runnable run) - { - Task inherit = inherit(); - PlainRunnable submit = inherit == null ? new PlainRunnable(null, run, this, ExclusiveGroup.OTHER) - : new PlainRunnable(null, run, this, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); - submitPlain(submit); - } - - @Override - public Cancellable execute(RunOrFail runOrFail) - { - Task inherit = inherit(); - PlainChain submit = inherit == null ? new PlainChain(runOrFail, ExclusiveExecutor.this, ExclusiveGroup.OTHER) - : new PlainChain(runOrFail, ExclusiveExecutor.this, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); - return submitPlain(submit); - } - - @Override - public boolean tryExecuteImmediately(Runnable run) - { - Thread thread = Thread.currentThread(); - Thread owner = this.owner; - if (owner != null && owner != thread) - return false; - - AccordTaskRunner self = AccordTaskRunner.get(thread); - AccordExecutor active = self.accordActiveExecutor(); - if (active != null && active != AccordExecutor.this) - return false; // prevent cross-executor locking/execution - - if (owner == null && !ownerUpdater.compareAndSet(this, null, thread)) - return false; - - if (active == null) - self.setAccordActiveExecutor(AccordExecutor.this); - - try { run.run(); } - catch (Throwable t) { agent.onException(t); } - finally - { - if (active == null) - self.setAccordActiveExecutor(null); - - if (owner == null) - { - Thread waiting = this.waiting; - Invariants.require(waiting != self); - this.owner = waiting; - if (waiting == null) // recheck, to ensure happens-before relation with a new waiter that expects any non-null owner to notify it - waiting = this.waiting; - if (waiting != null) - LockSupport.unpark(waiting); - } - } - return true; - } - } - - // has xSingle methods to distinguish from within MultiTaskQueue whether we're invoking the multi or single variation - static class TaskQueue extends IntrusivePriorityHeap - { - final int states; - public TaskQueue(int states) {this.states = states; } - - void unqueue(T task) - { - throw new UnsupportedOperationException(); - } - - @Override - public final int compare(T o1, T o2) - { - return Long.compare(o1.position, o2.position); - } - - @Override - protected final void ensureHeapified() - { - super.ensureHeapified(); - } - - protected final boolean requeueSingle(T requeue) - { - int oldIndex = updateNode(requeue); - if (oldIndex < 0) - return false; - - int newIndex = heapIndex(requeue); - return Math.min(oldIndex, newIndex) == 0 && heapifiedSize() > 0; - } - - final T peekSingle() - { - ensureHeapified(); - return peekNode(); - } - - final T pollSingle() - { - ensureHeapified(); - return pollNode(); - } - - final boolean isEmptySingle() - { - return isEmptyInternal(); - } - - final int enqueueSingle(T enqueue) - { - return insertNode(enqueue); - } - - final boolean unqueueSingle(T unqueue) - { - int heapIndex = heapIndex(unqueue); - removeNode(unqueue); - return heapIndex == 0; - } - - final boolean tryUnqueueSingle(T remove) - { - return removeNodeIfContains(remove); - } - - final T getSingle(int index) - { - return super.getNode(index); - } - - final boolean isQueuedSingle(T test) - { - return containsNode(test); - } - - @Override - public String toString() - { - return TinyEnumSet.toString(states, State::forOrdinal); - } - } - - static final class StandaloneTaskQueue extends TaskQueue - { - StandaloneTaskQueue(int states) - { - super(states); - } - - StandaloneTaskQueue(State state) - { - super(TinyEnumSet.encode(state)); - } - - void enqueue(T enqueue) - { - enqueue.setQueue(this); - enqueueSingle(enqueue); - } - - void unqueue(T unqueue) - { - unqueue.unsetQueue(this); - removeNode(unqueue); - } - - T peek() - { - return peekSingle(); - } - - T poll() - { - return pollSingle(); - } - - boolean isEmpty() - { - return isEmptySingle(); - } - } - - /** - * A {@link TaskQueue} sub-divided into up to eight per-group sub-queues (groups are phases for the exclusive - * executor, or work classes globally) plus a policy for choosing which sub-queue to serve next. The policy balances - * three competing concerns: ordering/priority (run the oldest / highest-priority work), fairness (do not let one - * group monopolise the executor), and throughput (do not let an even split leave a busy group's backlog growing - * without bound). - * - *

Packed counters

- * Per-group state is held in {@code long}s, one 7-bit lane per group; bit 7 of each byte is a spare "guard" bit used - * to run branch-free min/compare across all eight lanes at once (see {@code minCounters}). The lanes are: - *
    - *
  • {@code positions[]} - the head task's position (HLC, or submission order) per group; drives FIFO/priority.
    • - *
  • {@code recent} - a windowed count of recently served tasks per group (service received).
    • - *
  • {@code arrivals} - a windowed, size-biased count of recent arrivals per group (demand offered).
    • - *
  • {@code current} - tasks currently in flight per group, used to enforce {@code queue_active_limits}.
    • - *
  • {@code hasWork}/{@code dirty} - guard-bit masks: which groups have queued work / need a position refresh.
    • - *
- * - *

Windowing: a bounded memory of the past

- * {@code recent} and {@code arrivals} are not running totals. Whenever incrementing {@code recent} tips any lane to - * its maximum (0x80), both {@code recent} and {@code arrivals} are halved (see {@code incrementRecents}), - * and each lane also saturates at 0x7f. They are therefore an exponentially-decaying window keyed on service/time. - * This is what stops a one-off burst of arrivals granting a group a permanent boost: the burst saturates the lane - * briefly and then decays away as other work is served. - * - *

The fair selection counter

- * The fair models pick the group with the smallest {@code effective} counter, where per lane - * 
effective = min(0x7f, max(0, recent - arrivals) + bias)
- *
    - *
  • {@code max(0, recent - arrivals)} - a group whose arrivals have outpaced its recent service clamps to 0 and - * is therefore preferred; in steady state each group's service tracks its arrival rate, which bounds backlog. - * The clamp is symmetric, so a group cannot bank "credit" during a lull and later use it to monopolise service.
    • - *
- * - *

Choosing a group ({@code minGroup} / {@code pollByBlend})

- *
    - *
  • {@code PRIORITY_ONLY} - always the lowest {@code positions} (oldest / highest priority), ignoring fairness.
    • - *
  • {@code PHASE_OVERRIDE} - strict phase order: the lowest-index group with work, always.
    • - *
  • {@code PHASE_FAIR} - pure fairness: the smallest {@code effective}; ties broken by index, within a group by - * position. It has no priority fallback, so it is deliberately completing-first under contention.
    • - *
  • {@code PRIORITY_FAIR} (default) - a deficit round-robin blend of two strategies, chosen per poll: flow - * (smallest {@code effective}, i.e. least fairly serviced) and age (oldest {@code positions}, i.e. FIFO). - * The flow weight ramps up with the flow imbalance ({@code max-min} of {@code effective}) over - * {@code queue_flow_imbalance_onset..+width}, trading against age zero-sum; when balanced it is pure age. The - * ramp is smooth (not a hard mode switch), so there is no boundary to oscillate around and no hysteresis is - * needed. Age also drains stale/standing backlogs for free: a backlog's items are the oldest work, so age - * clears them without a separate backlog-aware strategy (the {@code effective} flow counter tracks arrival - * rate and is deliberately blind to standing stock).
    • - *
- * - *

Concurrency limits and eligibility

- * A group whose in-flight {@code current} count has reached its {@code queue_active_limits} limit is - * {@code saturated} and excluded from selection ({@code disabled}), as is any group with no queued work. - * - *

Note on {@code positions} freshness

- * {@code positions} is refreshed lazily from the sub-queue heads via the {@code dirty} mask; {@code dirty} must use - * the same guard-bit layout as {@code hasWork} so the refresh loop in {@code minGroupByPriority} actually runs - - * otherwise it silently degenerates to lowest-index-with-work and starves high-index / new-work groups. - * - *

NB it extends {@link TaskQueue} to keep the type hierarchy simple for method dispatch, and for efficiency for - * anonymous ExclusiveExecutors which do not use multiple queues, while letting ExclusiveExecutor share a parent - * class for both use cases. - */ - static abstract class MultiTaskQueue extends TaskQueue - { - private static final TaskQueue[] NO_QUEUES = new TaskQueue[0]; - private static final long[] NO_POSITIONS = new long[0]; - static final long COUNTER_OVERFLOWS = 0x8080808080808080L; - static final long COUNTER_MASKS = 0x7f7f7f7f7f7f7f7fL; - static final long COUNTER_LOWBITS = 0x0101010101010101L; - - final TaskQueue[] queues; - final long[] positions; - final byte groupShift; - final long limits; - - /** sets overflow bits for a queue that has been stopped */ - long stopped; - /** sets overflow bits for each counter when it needs its position updated */ - long dirty; - /** sets overflow bits for each counter when there's associated work */ - long hasWork; - /** Stores recent dequeue counts for up to 8 sub queues. */ - long dispatches; - // TODO (required): increment arrivals based on internal queue for ExclusiveExecutors - // also: experiment with decaying on arrival schedule rather than poll schedule, since this should respond to work growth more accurately - /** Stores recent enqueue counts for up to 8 sub queues. */ - long arrivals; - /** Stores currently-active counts for up to 8 sub queues. We can use this to impose limits on specific queues. */ - long active; - /** deficit-round-robin credits for the two PRIORITY_FAIR strategies (flow/age). */ - int creditFlow, creditAge; - - int waitingCount; - - MultiTaskQueue(int waitingStates, GroupKind groups, long limits) - { - super(waitingStates); - this.limits = limits; - int queueCount = groups.count; - Invariants.require(queueCount <= 8); - queues = queueCount == 0 ? NO_QUEUES : new TaskQueue[queueCount]; - positions = queueCount > 0 && BALANCE_BY_POSITION ? new long[queueCount] : NO_POSITIONS; - groupShift = groups.shift; - } - - final int group(Task task) - { - if (groupShift == 0) - return -1; - - return (task.info >>> groupShift) & Task.GROUP_MASK; - } - - void stop(int group) - { - stopped |= overflowBit(group); - } - - void restart(int group) - { - stopped &= ~overflowBit(group); - } - - final TaskQueue queue(Task task) - { - int group = group(task); - if (group < 0) - return this; - - return queue(group); - } - - final TaskQueue queue(int group) - { - TaskQueue queue = queues[group]; - if (queue == null) - queues[group] = queue = new TaskQueue<>(0); - - return queue; - } - - private int pollGroup() - { - if (hasWork == 0) - return -1; - - switch (BALANCING_MODEL) - { - default: throw new UnhandledEnum(PRIORITY_MODEL); - case PRIORITY_ONLY: return pollGroupByPriority(); - case PHASE_ONLY: return pollGroupByIndex(); - case PHASE_FAIR: return pollGroupByPhaseFair(); - case BLENDED_PRIORITY_PHASE_FAIR: return pollGroupByBlended(); - } - } - - private int pollGroupByPriority() - { - return pollGroupByPriority(unsaturatedWithWork()); - } - - private int pollGroupByPriority(long enabled) - { - long refresh = dirty & hasWork; - while (refresh != 0) - { - int bitIndex = Long.numberOfTrailingZeros(refresh); - int group = bitIndex / 8; - positions[group] = queues[group].peekSingle().position; - refresh ^= 1L << bitIndex; - } - dirty = 0; - - long minPosition = Long.MAX_VALUE; - int minGroup = -1; - long visit = enabled >>> 7; - while (visit != 0) - { - int bitIndex = Long.numberOfTrailingZeros(visit); - int group = bitIndex / 8; - long position = positions[group]; - if (position < minPosition) - { - minGroup = group; - minPosition = position; - } - visit ^= 1L << bitIndex; - } - - return minGroup; - } - - private int pollGroupByIndex() - { - long visit = (hasWork & unsaturated()) >>> 7; - if (visit == 0) - return -1; - - int bitIndex = Long.numberOfTrailingZeros(visit); - return bitIndex / 8; - } - - private int pollGroupByPhaseFair() - { - return minCounterIndex(recentFlowImbalances()); - } - - // PRIORITY_FAIR selection: a deficit round-robin blend of two strategies, chosen per poll: - // flow -> minCounterIndex(recent - arrivals + bias) (least fairly serviced) - // age -> minGroupByPriority() (earliest-queued work) - // wFlow = ramp(flow imbalance F = max-min of the selection counters); wAge = BLEND_TOTAL - wFlow. As flow gets - // uneven, polls trade from age to flow zero-sum; when balanced it is pure age (FIFO). A ramp (not a hard - // threshold) means there is no mode cliff to oscillate around, so no anti-oscillation penalty is needed. Age is - // also what drains a stale/standing backlog -- its items are the oldest -- so no separate stock strategy is needed. - private int pollGroupByBlended() - { - return pollGroupByBlended(saturatedOrWithoutWork()); - } - - private int pollGroupByBlended(long disabled) - { - long withoutWork = hasWork ^ COUNTER_OVERFLOWS; - long counters = recentFlowImbalances(); - long minMax = minMaxCounterValue(counters, withoutWork); - long min = minMax & 0x7f; - long max = minMax >>> 8; - int flowImbalance = (int) (max - min); - - int flowWeight = flowWeight(flowImbalance); - int priorityWeight = BLEND_TOTAL - flowWeight; - - creditFlow += flowWeight; - creditAge += priorityWeight; - - if (creditFlow >= creditAge) - { - creditFlow -= BLEND_TOTAL; - if (disabled != withoutWork) - min = minCounterValue(counters, disabled); - - return minCounterIndex(counters, min, disabled); - } - else - { - creditAge -= BLEND_TOTAL; - return pollGroupByPriority(disabled ^ COUNTER_OVERFLOWS); - } - } - - private long saturated() - { - return ((active | COUNTER_OVERFLOWS) - limits) & COUNTER_OVERFLOWS; - } - - private long unsaturated() - { - return saturated() ^ COUNTER_OVERFLOWS; - } - - private long unsaturatedWithWork() - { - return hasWork & unsaturated() & ~stopped; - } - - private long saturatedOrWithoutWork() - { - return (hasWork ^ COUNTER_OVERFLOWS) | saturated() | stopped; - } - - // arrivals is a windowed measure of ARRIVAL (incremented on enqueue, size-biased; decays on recent's overflow). - // Combined with recent (service) as effective = max(0, recent - arrivals): a queue whose arrivals outpace its - // service clamps to 0 and is preferred, so service converges to arrival rate (bounding the busy queue's backlog). - private long recentFlowImbalances() - { - return clampedSubtract(dispatches, arrivals); - } - - static long minCounterValue(long counters, long disabled) - { - long mins = counters; - mins |= overflowsToLowMasks(disabled); - mins = minCounters(mins, mins >>> 8); // each slot is min of slots [i..i+1] - mins = minCounters(mins, mins >>> 16); // each slot is min of slots [i..i+3] - mins = minCounters(mins, mins >>> 32); // each slot is min of slots [i..i+7] - return mins & 0x7f; - } - - static long minMaxCounterValue(long counters, long disabled) - { - long mins = counters; - long maxs = counters ^ COUNTER_MASKS; - long overflowMasks = overflowsToLowMasks(disabled); - mins |= overflowMasks; - maxs |= overflowMasks; - mins = minCounters(mins, mins >>> 8) & 0x007f007f007f007fL; // each slot is min of slots [i..i+1] - maxs = (minCounters(maxs, maxs << 8) & 0x7f007f007f007f00L); // each slot is min of slots ~[i..i+1] - long minmaxs = mins | maxs; - minmaxs = minCounters(minmaxs, minmaxs >>> 16); // each slot is min of slots [i..i+3] - minmaxs = minCounters(minmaxs, minmaxs >>> 32); // each slot is min of slots [i..i+7] - return (minmaxs ^ 0x7f00) & 0x7f7f; - } - - /** - * If provided two counters (containing 8 7 bit counters each), - * returns the minimum of each matching counter - */ - private static long minCounters(long a, long b) - { - // set overflow bits where a <= b - long selecta = setOverflowWhenLessEqual (a, b); - return selectByOverflowBits(selecta, a, b); - } - - static long setOverflowWhenLessEqual(long a, long b) - { - return ((b | COUNTER_OVERFLOWS) - a) & COUNTER_OVERFLOWS; - } - - // select a if overflow bit is set; b if it is unset - static long selectByOverflowBits(long selecta, long a, long b) - { - selecta = overflowsToLowMasks(selecta); - a &= selecta; - b &= ~selecta; - return a | b; - } - - static long overflowsToLowMasks(long v) - { - return v - (v >>> 7); - } - - private static int flowWeight(int flowImbalance) - { - if (flowImbalance <= FLOW_ONSET) return 0; - return Math.min(BLEND_TOTAL, ((flowImbalance - FLOW_ONSET) << BLEND_SHIFT) >>> FLOW_WIDTH_SHIFT); - } - - // per-lane max(0, a - b), carry-free: zero both a and b in lanes where a <= b, then subtract - private static long clampedSubtract(long a, long b) - { - long keep = ~overflowsToLowMasks(setOverflowWhenLessEqual(a, b)); - return (a & keep) - (b & keep); - } - - private int minCounterIndex(long counters) - { - return minCounterIndex(counters, saturatedOrWithoutWork()); - } - - private int minCounterIndex(long counters, long disabled) - { - return minCounterIndex(counters, minCounterValue(counters, disabled), disabled); - } - - private int minCounterIndex(long counters, long minCounterValue, long disabled) - { - long mins = minCounterValue * COUNTER_LOWBITS; - long select = ((mins | COUNTER_OVERFLOWS) - counters) & COUNTER_OVERFLOWS; - // now unset those overflow bits associated with disabled queues - select &= ~disabled; - if (select == 0) - return -1; - return (Long.numberOfTrailingZeros(select) - 7) / 8; - } - - final T pollMulti() - { - int group = pollGroup(); - if (group < 0) - { - // group < 0 can mean EITHER we don't have any nested queues OR those queues are either empty or DISABLED - T result = pollSingle(); - if (result != null) - --waitingCount; - return result; - } - - --waitingCount; - incrementActive(group); - incrementDispatches(group); - - TaskQueue queue = queues[group]; - T head = queue.pollSingle(); - // NOTE: must clear dirty when emptied, symmetrically with unqueue(): the fair selection paths - // never consume the dirty bit, so a group drained during a fairness episode would otherwise - // retain a stale dirty bit and NPE in minGroupByPriority.peekSingle() when balance is restored. - if (queue.isEmptySingle()) { unsetHasWork(group); unsetDirty(group); } - else setDirty(group); - return head; - } - - final void enqueueMulti(T task, boolean incrementArrivals) - { - task.setQueue(this); - int group = group(task); - if (group < 0) - { - enqueueSingle(task); - } - else - { - TaskQueue queue = queue(group); - int result = queue.enqueueSingle(task); - if (incrementArrivals) - incrementArrivals(group); - if (result < 0) setHasWork(group); - if (result != 0) setDirty(group); - } - ++waitingCount; - } - - final void requeue(T task) - { - int group = group(task); - if (group < 0) requeueSingle(task); - else - { - TaskQueue queue = queue(group); - Invariants.require(queue != null && queue.isQueuedSingle(task)); - if (queue.requeueSingle(task)) - setDirty(group); - } - } - - final void unqueueMulti(T task) - { - int group = group(task); - TaskQueue queue = group < 0 ? this : queue(task); - Invariants.require(queue.isQueuedSingle(task)); - unqueue(task, group, queue); - } - - // if there is an active collection, we return false and do not remove ourselves from it - boolean tryUnqueueWaiting(T task) - { - int group = group(task); - TaskQueue queue = group < 0 ? this : queue(task); - if (!queue.isQueuedSingle(task)) - return false; - - unqueue(task, group, queue); - return true; - } - - private void unqueue(T task, int group, TaskQueue queue) - { - task.unsetQueue(this); - boolean dirty = queue.unqueueSingle(task); - --waitingCount; - if (group >= 0) - { - if (queue.isEmptySingle()) - { - unsetHasWork(group); - unsetDirty(group); - } - else if (dirty) setDirty(group); - } - } - - final void incrementActive(int group) - { - active += lowBit(group); - } - - final void decrementActive(int group) - { - active -= lowBit(group); - } - - final void incrementDispatches(Task task) - { - int group = group(task); - if (group >= 0) - incrementDispatches(group); - } - - final void incrementDispatches(int group) - { - dispatches += lowBit(group); - if ((dispatches & COUNTER_OVERFLOWS) != 0) - { - dispatches = (dispatches >>> 1) & COUNTER_MASKS; - arrivals = (arrivals >>> 1) & COUNTER_MASKS; // arrivals (arrival) decays on the service/time clock - } - } - - final void decrementDispatches(Task task) - { - int group = group(task); - if (group >= 0) - decrementDispatches(group); - } - - final void decrementDispatches(int group) - { - long lowBit = lowBit(group); - dispatches -= lowBit; - dispatches += (dispatches >>> 7) & lowBit; - } - - final void incrementArrivals(Task task) - { - int group = group(task); - if (group >= 0) - incrementArrivals(group); - } - - final void incrementArrivals(int group) - { - int shift = group * 8; - long overflowBit = 0x80L << shift; - arrivals += 1L << shift; - // if we overflow, unset the overflow bit and set all other bits for the counter - long overflow = arrivals & overflowBit; - arrivals ^= overflow; - arrivals |= overflow - (overflow >>> 7); - } - - final void setHasWork(int group) - { - hasWork |= overflowBit(group); - } - - final void unsetHasWork(int group) - { - hasWork &= ~overflowBit(group); - } - - final void setDirty(int group) - { - dirty |= overflowBit(group); - } - - final void unsetDirty(int group) - { - dirty &= ~overflowBit(group); - } - - final boolean hasWaitingToRun() - { - return unsaturatedWithWork() != 0; - } - - final boolean isWaiting(T task) - { - return queue(task).isQueuedSingle(task); - } - - final int waitingCount() - { - return waitingCount; - } - - final long lowBit(int group) - { - return 1L << (group * 8); - } - - final long overflowBit(int group) - { - return 0x80L << (group * 8); - } - } - - static final class RunnableTaskQueue extends MultiTaskQueue - { - static final int RUNNABLE = TinyEnumSet.encode(WAITING_TO_RUN, RUNNING); - - final TaskQueue assigned; - - RunnableTaskQueue() - { - super(RUNNABLE, GroupKind.GLOBAL, GLOBAL_QUEUE_LIMITS); - this.assigned = new TaskQueue<>(0); - } - - T poll() - { - T next = pollMulti(); - if (next == null) - return null; - - assigned.enqueueSingle(next); - return next; - } - - void enqueue(T enqueue, boolean incrementArrivals) - { - enqueueMulti(enqueue, incrementArrivals); - } - - void unqueue(T unqueue) - { - if (assigned.isQueuedSingle(unqueue)) - { - int group = group(unqueue); - if (group >= 0) - decrementActive(group); - - unqueue.unsetQueue(this); - assigned.unqueueSingle(unqueue); - } - else - { - super.unqueueMulti(unqueue); - } - } - - int waitingOrAssignedCount() - { - return waitingCount + assigned.size(); - } - - boolean hasAssignedOrWaiting() - { - return waitingCount > 0 || !assigned.isEmptySingle(); - } - - boolean hasAssigned() - { - return !assigned.isEmptySingle(); - } - - boolean isAssigned(T task) - { - return assigned.isQueuedSingle(task); - } - - void cleanup(T task) - { - if (assigned.tryUnqueueSingle(task)) - { - int group = group(task); - if (group >= 0) - decrementActive(group); - task.unsetQueue(this); - } - } - } - - static class CancelTask extends Task - { - final Task cancel; - private CancelTask(Task cancel) - { - super(GlobalGroup.OTHER); - this.cancel = cancel; - } - - @Override void submitExclusive(AccordExecutor owner) { cancel.cancelExclusive(owner); } - @Override void preRunExclusive() { throw new UnsupportedOperationException(); } - @Override void run() { throw new UnsupportedOperationException(); } - @Override void reportFailure(Throwable fail) { throw new UnsupportedOperationException(); } - @Override boolean isNewWork() { return false; } - @Override String toDescription() { return "Cancel " + cancel.toDescription(); } - } - - static IntFunction constant(O out) - { - return ignore -> out; - } - - abstract class Plain extends Task implements Cancellable - { - Plain(GlobalGroup group, long position, int tranche) - { - super(group, position, tranche); - } - - Plain(ExclusiveGroup group, long position, int tranche) - { - super(group, position, tranche); - } - - Plain(GlobalGroup group) - { - super(group); - } - - Plain(ExclusiveGroup group) - { - super(group); - } - - abstract ExclusiveExecutor executor(); - - @Override - public void cancel() - { - submit((e, c) -> c.cancelExclusive(e), CancelTask::new, this); - } - - void cancelExclusive(AccordExecutor owner) - { - ExclusiveExecutor executor = executor(); - if ((executor == null ? runnable : executor).tryUnqueueWaiting(this)) - { - try { failExclusive(new CancellationException(), CANCELLED); } - catch (Throwable t) { agent.onException(t); } - finally { cleanupTaskExclusive(this, false); } - } - } - - @Override - final void submitExclusive(AccordExecutor owner) - { - setStateExclusive(WAITING_TO_RUN); - owner.submitPlainExclusive(this); - } - - @Override - protected boolean isNewWork() - { - return true; - } - } - - class PlainRunnable extends Plain implements Cancellable - { - final @Nullable AsyncPromise result; - final Runnable run; - final @Nullable ExclusiveExecutor executor; - - PlainRunnable(AsyncPromise result, Runnable run, GlobalGroup group, long position, int tranche) - { - super(group, position, tranche); - this.result = result; - this.run = run; - this.executor = null; - } - - PlainRunnable(AsyncPromise result, Runnable run, GlobalGroup group) - { - super(group); - this.result = result; - this.run = run; - this.executor = null; - } - - PlainRunnable(AsyncPromise result, Runnable run, ExclusiveExecutor executor, ExclusiveGroup group, long position, int tranche) - { - super(group, position, tranche); - this.result = result; - this.run = run; - this.executor = executor; - } - - PlainRunnable(AsyncPromise result, Runnable run, ExclusiveExecutor executor, ExclusiveGroup group) - { - super(group); - this.result = result; - this.run = run; - this.executor = executor; - } - - @Override - String toDescription() - { - // TODO (expected): ensure this is usefully descriptive, or accept a separate description - return run.toString(); - } - - @Override - protected void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - run.run(); - } - if (result != null) - result.trySuccess(null); - onRunComplete(); - } - - @Override - protected void reportFailure(Throwable t) - { - if (result != null) - result.tryFailure(t); - agent.onException(t); - } - - @Override - ExclusiveExecutor executor() - { - return executor; - } - } - - // a task that may be submitted to this executor or another - public abstract class IOTask extends Plain implements Cancellable, DebuggableTask - { - IOTask(GlobalGroup group, long position, int tranche) - { - super(group, position, tranche); - } - - IOTask(GlobalGroup group) - { - super(group); - } - - abstract void postRunExclusive(); - - @Override - void cleanupExclusive(AccordExecutor executor, boolean executed) - { - super.cleanupExclusive(executor, executed); - postRunExclusive(); - } - - @Override - ExclusiveExecutor executor() - { - return null; - } - - @Override - public long creationTimeNanos() - { - return createdAt; - } - - @Override - public long startTimeNanos() - { - return runningAt; - } - } - - static class FailureHolder - { - static final FailureHolder NOT_STARTED = new FailureHolder(new RuntimeException("Not started")); - - final Throwable fail; - - FailureHolder(Throwable fail) - { - this.fail = fail; - } - } - - LoadRunnable newLoad(AccordCacheEntry entry, boolean isForRange) - { - return new LoadRunnable<>(entry, isForRange ? RANGE_LOAD : LOAD); - } - - public class LoadRunnable extends IOTask - { - final AccordCacheEntry entry; - Object result = FailureHolder.NOT_STARTED; - - LoadRunnable(AccordCacheEntry entry, GlobalGroup group) - { - super(group); - Invariants.require(group == LOAD || group == RANGE_LOAD); - this.entry = entry; - } - - void postRunExclusive() - { - if (!(result instanceof FailureHolder)) onLoadedExclusive(entry, (V)result, null); - else onLoadedExclusive(entry, null, ((FailureHolder)result).fail); - } - - @Override - String toDescription() - { - return "Load " + entry.key(); - } - - @Override - public void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - result = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); - } - onRunComplete(); - } - - @Override - void reportFailure(Throwable t) - { - result = new FailureHolder(t); - } - - @Override - public String description() - { - return "Loading " + entry; - } - } - - static abstract class AbstractIOTask - { - abstract void runInternal(); - abstract void postRunExclusive(); - abstract void fail(Throwable t); - abstract String description(); - } - - class WrappedIOTask extends IOTask - { - final AbstractIOTask wrapped; - - WrappedIOTask(AbstractIOTask wrap, GlobalGroup group, long position, int tranche) - { - super(group, position, tranche); - this.wrapped = wrap; - } - - @Override - String toDescription() - { - return wrapped.description(); - } - - @Override - protected void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - wrapped.runInternal(); - } - onRunComplete(); - } - - @Override - void postRunExclusive() - { - wrapped.postRunExclusive(); - } - - @Override - public String description() - { - return wrapped.description(); - } - - @Override - protected void reportFailure(Throwable fail) - { - wrapped.fail(fail); - } - } - - private static final Throwable NOT_STARTED = new Throwable(); - class SaveRunnable extends IOTask - { - final AccordCacheEntry entry; - final UniqueSave identity; - final Runnable run; - Throwable failure = NOT_STARTED; - - SaveRunnable(AccordCacheEntry entry, UniqueSave identity, Runnable run) - { - super(SAVE); - this.entry = entry; - this.identity = identity; - this.run = run; - } - - @Override - void postRunExclusive() - { - onSavedExclusive(entry, identity, failure); - } - - @Override - String toDescription() - { - return "Save " + entry.key(); - } - - @Override - public void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - run.run(); - } - onRunComplete(); - failure = null; - } - - @Override - protected void reportFailure(Throwable t) - { - failure = t; - } - - @Override - public String description() - { - return "Save " + entry; - } - } - - class PlainChain extends Plain - { - final RunOrFail runOrFail; - final @Nullable ExclusiveExecutor executor; - - PlainChain(RunOrFail runOrFail, ExclusiveExecutor executor, ExclusiveGroup group) - { - super(group); - this.runOrFail = runOrFail; - this.executor = executor; - } - - PlainChain(RunOrFail runOrFail, ExclusiveExecutor executor, ExclusiveGroup group, long position, int tranche) - { - super(group, position, tranche); - this.runOrFail = runOrFail; - this.executor = executor; - } - - @Override - ExclusiveExecutor executor() - { - return executor; - } - - @Override - String toDescription() - { - return runOrFail.toString(); - } - - @Override - protected void run() - { - onRunning(); - try (Closeable close = resources.get()) - { - runOrFail.run(); - } - catch (Throwable t) - { - // shouldn't throw exceptions - agent.onException(t); - } - onRunComplete(); - } - - @Override - protected void reportFailure(Throwable fail) - { - try - { - runOrFail.fail(fail); - } - catch (Throwable t) - { - fail.addSuppressed(t); - agent.onException(fail); - } - } - } - - class DebuggableChain extends PlainChain implements DebuggableTask - { - final Object describe; - - DebuggableChain(RunOrFail runOrFail, @Nullable ExclusiveExecutor executor, Object describe) - { - super(runOrFail, executor, ExclusiveGroup.OTHER); - this.describe = Invariants.nonNull(describe); - } - - DebuggableChain(RunOrFail runOrFail, @Nullable ExclusiveExecutor executor, long position, int tranche, Object describe) - { - super(runOrFail, executor, ExclusiveGroup.OTHER, position, tranche); - this.describe = Invariants.nonNull(describe); - } - - @Override - public long creationTimeNanos() - { - return createdAt; - } - - @Override - public long startTimeNanos() - { - return runningAt; - } - - @Override - public String description() - { - return describe.toString(); - } - - @Override - public DebuggableTask debuggable() - { - return this; - } - } - - - public static class TaskInfo implements Comparable - { - // sorted in name order for reporting to virtual tables - public enum Status { LOADING, RUNNING, SCANNING_RANGES, WAITING_TO_LOAD, WAITING_TO_RUN } - - final Status status; - final int commandStoreId; - - final Task task; - - public TaskInfo(Status status, int commandStoreId, Task task) - { - this.status = status; - this.commandStoreId = commandStoreId; - this.task = task; - } - - public Status status() - { - return status; - } - - public Integer commandStoreId() - { - return commandStoreId >= 0 ? commandStoreId : null; - } - - public long position() - { - return task.position; - } - - public @Nullable String describe() - { - if (task instanceof AccordTask) - return ((AccordTask) task).executionContext().reason(); - - if (task instanceof DebuggableTask) - return ((DebuggableTask) task).description(); - - return null; - } - - public @Nullable ExecutionContext preLoadContext() - { - if (task instanceof AccordTask) - return ((AccordTask) task).executionContext(); - if (task instanceof WrappedIOTask && ((WrappedIOTask) task).wrapped instanceof AccordTask.RangeTxnScanner) - return ((AccordTask.RangeTxnScanner) ((WrappedIOTask) task).wrapped).preLoadContext(); - return null; - } - - @Override - public int compareTo(TaskInfo that) - { - int c = this.status.compareTo(that.status); - if (c == 0) c = Long.compare(this.position(), that.position()); - return c; - } - } - - public List taskSnapshot() - { - List result = new ArrayList<>(); - AccordTaskRunner self = AccordTaskRunner.get(); - lock(self); - try - { - addToSnapshot(result, scanningRanges, TaskInfo.Status.SCANNING_RANGES, TaskInfo.Status.SCANNING_RANGES); - addToSnapshot(result, loading, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); - for (TaskQueue queue : runnable.queues) - { - if (queue != null) - addToSnapshot(result, queue, TaskInfo.Status.WAITING_TO_RUN, TaskInfo.Status.WAITING_TO_RUN); - } - addToSnapshot(result, runnable.assigned, TaskInfo.Status.RUNNING, TaskInfo.Status.WAITING_TO_RUN); - } - finally - { - unlock(self); - } - result.sort(TaskInfo::compareTo); - return result; - } - - private static void addToSnapshot(List snapshot, TaskQueue queue, TaskInfo.Status ifCurrent, TaskInfo.Status ifQueued) - { - for (int i = 0 ; i < queue.size() ; ++i) - { - Task t = queue.getSingle(i); - if (t instanceof ExclusiveExecutorTask) - { - ExclusiveExecutor q = ((ExclusiveExecutorTask) t).queue; - snapshot.add(new TaskInfo(ifCurrent, q.commandStoreId, q.task)); - for (TaskQueue q0 : q.queues) - { - if (q0 != null) - { - for (int j = 0 ; j < q0.size() ; ++j) - snapshot.add(new TaskInfo(ifQueued, q.commandStoreId, q0.getSingle(j))); - } - } - } - else - { - int commmandStoreId = t instanceof AccordTask ? ((AccordTask) t).commandStore.id() : -1; - snapshot.add(new TaskInfo(ifCurrent, commmandStoreId, t)); - } - } - } - - public int unsafePreparingToRunCount() - { - return loading.size() + scanningRanges.size(); - } - - public int unsafeWaitingToRunCount() - { - return runnable.waitingCount(); - } - - public int unsafeRunningCount() - { - return runnable.assigned.size(); - } - - private static long[] parseEnumParams(String input, String describe) - { - String[] specs = input.split(";"); - if (specs.length != 2) - throw new IllegalArgumentException("Invalid specifiers in " + describe + "; expect [GlobalGroup];[ExclusiveGroup] but got: " + input); - long[] result = new long[2]; - result[0] = parseEnumParams(GlobalGroup::valueOf, specs[0], describe + " for GlobalGroup"); - result[1] = parseEnumParams(ExclusiveGroup::valueOf, specs[1], describe + " for ExclusiveGroup"); - return result; - } - - private static long parseEnumParams(Function> get, String input, String describe) - { - long result = 0; - for (String spec : input.split(",")) - { - if (spec.trim().isEmpty()) continue; - - String[] split = spec.split(":"); - if (split.length != 2) - throw new IllegalArgumentException("Invalid specifier " + spec + " in " + describe + ": " + input); - - try - { - Enum queue = get.apply(split[0]); - long value = Long.parseLong(split[1]); - if (value <= 0 || value >= 128) - throw new IllegalArgumentException("Invalid limit " + value + " in queue_active_limits: " + input); - - result |= value << (queue.ordinal() * 8); - } - catch (Throwable t) - { - throw new IllegalArgumentException("Invalid queue identifier " + split[0] + " in " + describe + ": " + input); - } - } - - return result; - } - - private static long encodeCounters(Map, Long> counters) - { - long result = 0; - for (Map.Entry, Long> e : counters.entrySet()) - result |= e.getValue() << (e.getKey().ordinal() * 8); - return result; - } - -} diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java index e268113b8604..a14d9b3cd468 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java @@ -245,7 +245,7 @@ private void maybeListen() logger.info("Reporting failure of plan {} for bootstrap of {} from {}", planId, range, from, fail); fail(from, Ranges.of(range), fail); } - }, ((AccordCommandStore) commandStore()).taskExecutor()); + }, ((AccordCommandStore) commandStore()).exclusiveExecutor()); } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java index 5123471acdeb..b0efd33b1bc0 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java +++ b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import com.google.common.annotations.VisibleForTesting; @@ -122,6 +123,7 @@ import static org.apache.cassandra.db.partitions.PartitionUpdate.singleRowUpdate; import static org.apache.cassandra.db.rows.BTreeRow.singleCellRow; import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME; +import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; public class AccordKeyspace { @@ -353,9 +355,15 @@ public static PartitionUpdate makeUpdate(int storeId, TokenKey key, long timesta BufferCell.live(CFKAccessor.data, timestampMicros, bytes))); } - public static Runnable systemTableUpdater(int storeId, TokenKey key, CommandsForKey update, Object serialized, long timestampMicros) + private static final AtomicLong lastSystemTimestampMicros = new AtomicLong(); + private static long nextSystemTimestampMicros() { - PartitionUpdate upd = makeUpdate(storeId, key, update, serialized, timestampMicros); + return lastSystemTimestampMicros.accumulateAndGet(currentTimeMillis(), (a, b) -> Math.max(a + 1, b)); + } + + public static Runnable systemTableUpdater(int storeId, TokenKey key, CommandsForKey update, Object serialized) + { + PartitionUpdate upd = makeUpdate(storeId, key, update, serialized, nextSystemTimestampMicros()); return () -> { ColumnFamilyStore cfs = AccordColumnFamilyStores.commandsForKey; try (OpOrder.Group group = Keyspace.writeOrder.start()) diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeState.java b/src/java/org/apache/cassandra/service/accord/AccordSafeState.java deleted file mode 100644 index ca8baafbedbe..000000000000 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeState.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.service.accord; - -import accord.local.SafeState; - -import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; - -public interface AccordSafeState & AccordSafeState> -{ - AccordCacheEntry global(); - void postExecute(AccordTask owner); - void preExecute(AccordTask owner, LockMode lockMode); - - static AccordCacheEntry global(SafeState safeState) - { - return safeState.getClass() == AccordSafeCommand.class ? ((AccordSafeCommand) safeState).global() - : ((AccordSafeCommandsForKey) safeState).global(); - } - - static void postExecute(SafeState safeState, AccordTask owner) - { - if (safeState.getClass() == AccordSafeCommand.class) ((AccordSafeCommand) safeState).postExecute(owner); - else ((AccordSafeCommandsForKey) safeState).postExecute(owner); - } - - static void preExecute(SafeState safeState, AccordTask owner, LockMode lockMode) - { - if (safeState.getClass() == AccordSafeCommand.class) ((AccordSafeCommand) safeState).preExecute(owner, lockMode); - else ((AccordSafeCommandsForKey) safeState).preExecute(owner, lockMode); - } -} diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index 98e1b239f4d9..994a553e8e67 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -126,6 +126,7 @@ import org.apache.cassandra.service.accord.api.AccordViolationHandler; import org.apache.cassandra.service.accord.api.CompositeTopologySorter; import org.apache.cassandra.service.accord.api.TokenKey.KeyspaceSplitter; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.interop.AccordInteropAdapter.AccordInteropFactory; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.accord.journal.ReplayMarkers; diff --git a/src/java/org/apache/cassandra/service/accord/IAccordService.java b/src/java/org/apache/cassandra/service/accord/IAccordService.java index 77296264e02b..abf7336b2241 100644 --- a/src/java/org/apache/cassandra/service/accord/IAccordService.java +++ b/src/java/org/apache/cassandra/service/accord/IAccordService.java @@ -63,6 +63,7 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.api.AccordScheduler; import org.apache.cassandra.service.accord.api.AccordTopologySorter; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.topology.AccordEndpointMapper; import org.apache.cassandra.service.accord.topology.AccordSyncPropagator; import org.apache.cassandra.service.accord.topology.AccordSyncPropagator.Notification; diff --git a/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java b/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java index 1af4dc96a00c..27da1e7324a6 100644 --- a/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java @@ -44,6 +44,7 @@ import accord.utils.async.Cancellable; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; import org.apache.cassandra.service.accord.serializers.CommandStoreSerializers; import static org.apache.cassandra.io.util.CompressedFrameDataInputPlus.readList; diff --git a/src/java/org/apache/cassandra/service/accord/RangeIndex.java b/src/java/org/apache/cassandra/service/accord/RangeIndex.java index 228aad3781bc..55f39c5bab3d 100644 --- a/src/java/org/apache/cassandra/service/accord/RangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/RangeIndex.java @@ -41,6 +41,7 @@ import org.apache.cassandra.exceptions.UnknownTableException; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; import org.apache.cassandra.service.accord.journal.CommandChanges; import org.apache.cassandra.service.accord.serializers.Version; @@ -59,10 +60,10 @@ public Loader(RedundantBefore redundantBefore, MaxDecidedRX maxDecidedRX, TxnId protected abstract AccordCommandStore commandStore(); - protected abstract void loadExclusive(Map into, AccordCommandStore.Caches caches); - protected abstract void load(Map into, BooleanSupplier abort); - protected abstract void finish(Map into); - protected abstract void cleanupExclusive(AccordCommandStore.Caches caches); + public abstract void loadExclusive(Map into, AccordCommandStore.Caches caches); + public abstract void load(Map into, BooleanSupplier abort); + public abstract void finish(Map into); + public abstract void cleanupExclusive(AccordCommandStore.Caches caches); protected CommandSummaries.Summary loadFromDisk(TxnId txnId) { diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java b/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java index e5ff023c80a2..15b34ca23fd5 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugBlockedTxns.java @@ -53,8 +53,6 @@ import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.utils.concurrent.Future; -import static accord.local.LoadKeys.SYNC; -import static accord.local.LoadKeysFor.READ_WRITE; import static java.util.Collections.emptyList; public class DebugBlockedTxns diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java index 037a09f1c4be..99aa8fe0ce41 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java @@ -29,7 +29,7 @@ import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.metrics.LogLinearHistogram; -import org.apache.cassandra.service.accord.AccordExecutor.Task; +import org.apache.cassandra.service.accord.execution.Task; import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.WithResources; diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java similarity index 88% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java rename to src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java index d7594025cd18..ac7e4390a07d 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java @@ -16,26 +16,26 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.locks.Lock; +import java.util.function.Consumer; +import java.util.function.Function; import accord.api.Agent; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; import org.apache.cassandra.concurrent.CassandraThread; -import org.apache.cassandra.service.accord.AccordExecutorLoops.LoopTask; +import org.apache.cassandra.service.accord.execution.Loops.LoopTask; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; -abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop +abstract class AbstractLockLoop extends AbstractLoop { int runningThreads; boolean shutdown; - AccordExecutorAbstractLockLoop(Lock lock, int executorId, Agent agent) + AbstractLockLoop(Lock lock, int executorId, Agent agent) { super(lock, executorId, agent); } @@ -43,17 +43,17 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutorAbstractLoop abstract void notifyWork(); abstract void notifyWorkExclusive(); abstract void awaitExclusive() throws InterruptedException; - abstract void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4); + abstract void submitExternal(Consumer sync, Function async, P1 p1); - final void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + final void submit(Consumer sync, Function async, P1 p1) { // if we're a loop thread, we will poll the waitingToRun queue when we come around // NOTE: this assumes no synchronous blocking tasks are submitted to this executor - if (isInLoop() || isOwningThread()) push(async.apply(p1a, p2, p3, p4)); - else submitExternal(sync, async, p1s, p1a, p2, p3, p4); + if (isInLoop() || isOwningThread()) push(async.apply(p1)); + else submitExternal(sync, async, p1); } - final void submitExternalExclusive(QuintConsumer sync, P1s p1s, P2 p2, P3 p3, P4 p4) + final void submitExternalExclusive(Consumer sync, Function async, P1 p1) { try { @@ -63,11 +63,11 @@ final void submitExternalExclusive(QuintConsumer unqueuedUpdater = AtomicReferenceFieldUpdater.newUpdater(AccordExecutorAbstractLoop.class, Task.class, "unqueued"); + static final AtomicReferenceFieldUpdater unqueuedUpdater = AtomicReferenceFieldUpdater.newUpdater(AbstractLoop.class, Task.class, "unqueued"); - AccordExecutorAbstractLoop(Lock lock, int executorId, Agent agent) + AbstractLoop(Lock lock, int executorId, Agent agent) { super(lock, executorId, agent); } - abstract AccordExecutorLoops loops(); + abstract Loops loops(); boolean hasUnqueued() { @@ -62,7 +62,7 @@ public boolean hasTasks() if (hasUnqueued() || tasks > 0) return true; - AccordTaskRunner self = AccordTaskRunner.get(); + TaskRunner self = TaskRunner.get(); lock(self); try { @@ -84,7 +84,7 @@ final Task enqueueOneExclusive(Task cur) Invariants.require(cur != null); Task next = cur.next; cur.next = null; - if (cur.is(Task.State.UNINITIALIZED)) cur.submitExclusive(this); + if (cur.is(Task.State.UNINITIALIZED)) cur.submitExclusive(); else cleanupTaskExclusive(cur, true); return next; } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractSemiSyncSubmit.java similarity index 65% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java rename to src/java/org/apache/cassandra/service/accord/execution/AbstractSemiSyncSubmit.java index dce92f103d67..79f6736ffa26 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractSemiSyncSubmit.java @@ -16,26 +16,26 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.locks.Lock; +import java.util.function.Consumer; +import java.util.function.Function; import accord.api.Agent; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; -abstract class AccordExecutorAbstractSemiSyncSubmit extends AccordExecutorAbstractLockLoop +abstract class AbstractSemiSyncSubmit extends AbstractLockLoop { - AccordExecutorAbstractSemiSyncSubmit(Lock lock, int executorId, Agent agent) + AbstractSemiSyncSubmit(Lock lock, int executorId, Agent agent) { super(lock, executorId, agent); } abstract void awaitExclusive() throws InterruptedException; - void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + void submitExternal(Consumer sync, Function async, P1 p1) { - if (push(async.apply(p1a, p2, p3, p4)) == null && !isInLoop()) + if (push(async.apply(p1)) == null && !isInLoop()) notifyWork(); } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCache.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java similarity index 90% rename from src/java/org/apache/cassandra/service/accord/AccordCache.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordCache.java index a75e6a6e27f8..8a5c2f699149 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCache.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.io.IOException; import java.nio.ByteBuffer; @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; @@ -44,6 +45,7 @@ import accord.api.RoutingKey; import accord.local.Command; +import accord.local.RedundantBefore; import accord.local.SafeState; import accord.local.cfk.CommandsForKey; import accord.local.cfk.Serialize; @@ -65,12 +67,17 @@ import org.apache.cassandra.metrics.AccordCacheMetrics; import org.apache.cassandra.metrics.LogLinearHistogram; import org.apache.cassandra.metrics.ShardedHitRate; -import org.apache.cassandra.service.accord.AccordCache.Adapter.Shrink; -import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; -import org.apache.cassandra.service.accord.AccordCacheEntry.Loading; -import org.apache.cassandra.service.accord.AccordCacheEntry.Status; -import org.apache.cassandra.service.accord.AccordSafeCommandsForKey.CommandsForKeyCacheEntry; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.AccordKeyspace; +import org.apache.cassandra.service.accord.AccordObjectSizes; +import org.apache.cassandra.service.accord.OrderedKeys; +import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.accord.events.CacheEvents; +import org.apache.cassandra.service.accord.execution.AccordCache.Adapter.Shrink; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LoadExecutor; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Loading; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; +import org.apache.cassandra.service.accord.execution.SaferCommandsForKey.CommandsForKeyCacheEntry; import org.apache.cassandra.service.accord.journal.CommandChangeWriter; import org.apache.cassandra.service.accord.journal.CommandChanges; import org.apache.cassandra.service.accord.serializers.CommandSerializers; @@ -81,11 +88,11 @@ import static accord.utils.Invariants.illegalState; import static accord.utils.Invariants.require; -import static org.apache.cassandra.service.accord.AccordCacheEntry.AGE_MASK; -import static org.apache.cassandra.service.accord.AccordCacheEntry.GENERATION_MASK; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.MODIFIED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.AGE_MASK; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.GENERATION_MASK; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.EVICTED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.LOADED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.MODIFIED; /** * Cache for AccordCommand and AccordCommandsForKey, available memory is shared between the two object types. @@ -102,15 +109,15 @@ public class AccordCache implements CacheSize // Debug mode to verify that loading from journal + system tables results in // functionally identical (or superceding) command to the one we've just evicted. - private static boolean VALIDATE_LOAD_ON_EVICT = false; + private static boolean DEBUG_VALIDATE_LOAD_ON_EVICT = false; @VisibleForTesting public static void validateLoadOnEvict(boolean value) { - VALIDATE_LOAD_ON_EVICT = value; + DEBUG_VALIDATE_LOAD_ON_EVICT = value; } - public interface Adapter & AccordSafeState> + public interface Adapter & SaferState> { enum Shrink { EVICT, DONE, PERFORM_WITHOUT_LOCK } @@ -194,7 +201,7 @@ public long capacity() /** * Make sure we don't have any items lingering too long in the no evict queue, to avoid cache memory leaks */ - void processNoEvictQueue() + public void processNoEvictQueue() { noEvictGeneration = (noEvictGeneration + 1) & GENERATION_MASK; if (noEvictQueue.isEmpty()) @@ -223,7 +230,7 @@ void processNoEvictQueue() * Roughly respects LRU semantics when evicting. Might consider prioritising keeping MODIFIED nodes around * for longer to maximise the chances of hitting system tables fewer times (or not at all). */ - void tryShrinkOrEvict(Lock lock) + public void tryShrinkOrEvict(Lock lock) { if (!tryShrinkOrEvict) return; @@ -336,7 +343,7 @@ private void evict(AccordCacheEntry node, boolean updateUnreference --parent.size; // TODO (expected): use listeners - if (node.status() == LOADED && VALIDATE_LOAD_ON_EVICT) + if (node.status() == LOADED && DEBUG_VALIDATE_LOAD_ON_EVICT) owner.validateLoadEvicted(node); AccordCacheEntry self = node.owner.remove(node.key()); @@ -376,19 +383,19 @@ void saved(AccordCacheEntry node, Object identity, Throwable fai evictQueue.addFirst(node); // add to front since we have just saved, so we were eligible for eviction } - public & AccordSafeState> void release(S safeRef, AccordTask owner) + public & SaferState> void release(S safeRef, SafeTask owner) { safeRef.global().owner.release(safeRef, owner); } - public & AccordSafeState> Type newType(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) + public & SaferState> Type newType(Class keyClass, Adapter adapter, AccordCacheMetrics.Shard metrics) { Type instance = new Type<>(keyClass, adapter, metrics); types.add(instance); return instance; } - public & AccordSafeState> Type newType( + public & SaferState> Type newType( Class keyClass, BiFunction loadFunction, QuadFunction saveFunction, @@ -401,7 +408,7 @@ public & AccordSafeState> Type n return newType(keyClass, loadFunction, saveFunction, quickShrink, (i, j) -> j, (c, i, j) -> (V)j, validateFunction, heapEstimator, i -> 0, safeRefFactory, metrics); } - public & AccordSafeState> Type newType( + public & SaferState> Type newType( Class keyClass, BiFunction loadFunction, QuadFunction saveFunction, @@ -433,7 +440,7 @@ default void onUpdate(AccordCacheEntry state) {} default void onEvict(AccordCacheEntry state) {} } - public class Type & AccordSafeState> implements CacheSize + public class Type & SaferState> implements CacheSize { public class Instance implements Iterable> { @@ -534,7 +541,7 @@ private AccordCacheEntry acquireExisting(AccordCacheEntry node return node; } - public final void release(S safeRef, AccordTask owner) + public final void release(S safeRef, SafeTask owner) { K key = safeRef.global().key(); logger.trace("Releasing resources for {}: {}", key, safeRef); @@ -566,8 +573,7 @@ public final void release(S safeRef, AccordTask owner) { evict = node.is(LOADED) && node.isNull(); } - node.remove(owner, safeRef.isSafe()); - safeRef.setReleased(); + node.remove(owner, safeRef.setReleased()); if (node.decrement() == 0) { @@ -666,21 +672,21 @@ public final boolean isReferenced(K key) } @VisibleForTesting - final boolean keyIsReferenced(Object key, Class> valClass) + final boolean keyIsReferenced(Object key, Class> valClass) { AccordCacheEntry node = cache.get(key); return node != null && node.references() > 0; } @VisibleForTesting - final boolean keyIsCached(Object key, Class> valClass) + final boolean keyIsCached(Object key, Class> valClass) { AccordCacheEntry node = cache.get(key); return node != null; } @VisibleForTesting - final int references(Object key, Class> valClass) + final int references(Object key, Class> valClass) { AccordCacheEntry node = cache.get(key); return node != null ? node.references() : 0; @@ -769,7 +775,7 @@ void updateSize(long newSize, long delta, boolean isUnreferenced, boolean update } // can be safely garbage collected if empty - Instance newInstance(AccordCommandStore commandStore) + public Instance newInstance(AccordCommandStore commandStore) { return keyClass == RoutingKey.class ? new KeyInstance(commandStore) : new Instance(commandStore); } @@ -908,7 +914,7 @@ public boolean isEmpty() return size() == 0; } - Iterable> evictionQueue() + public Iterable> evictionQueue() { return evictQueue::iterator; } @@ -1007,7 +1013,7 @@ private static void updateMutable(AccordCache.Type type, AccordCacheEnt event.update(); } - static class FunctionalAdapter & AccordSafeState> implements Adapter + static class FunctionalAdapter & SaferState> implements Adapter { final BiFunction load; final QuadFunction save; @@ -1120,7 +1126,7 @@ public Comparator keyComparator() } } - static class SettableWrapper & AccordSafeState> extends FunctionalAdapter + static class SettableWrapper & SaferState> extends FunctionalAdapter { volatile BiFunction load; @@ -1130,7 +1136,7 @@ static class SettableWrapper & AccordSafeState & AccordSafeState> Adapter loadOnly(BiFunction load) + public static & SaferState> Adapter loadOnly(BiFunction load) { SettableWrapper result = new SettableWrapper<>(new NoOpAdapter()); result.load = load; @@ -1144,7 +1150,7 @@ public V load(AccordCommandStore commandStore, K key) } } - static class NoOpAdapter & AccordSafeState> implements Adapter + static class NoOpAdapter & SaferState> implements Adapter { @Override public V load(AccordCommandStore commandStore, K key) { return null; } @Override public Runnable save(AccordCommandStore commandStore, K key, @Nullable V value, @Nullable Object shrunk) { return null; } @@ -1158,7 +1164,7 @@ static class NoOpAdapter & AccordSafeState @Override public S safeRef(AccordCacheEntry node) { return null; } } - public static class CommandsForKeyAdapter implements Adapter + public static class CommandsForKeyAdapter implements Adapter { public static final CommandsForKeyAdapter CFK_ADAPTER = new CommandsForKeyAdapter(); private static int SHRINK_WITHOUT_LOCK = -1; @@ -1168,7 +1174,13 @@ private CommandsForKeyAdapter() {} @Override public CommandsForKey load(AccordCommandStore commandStore, RoutingKey key) { - return commandStore.loadCommandsForKey(key); + CommandsForKey cfk = AccordKeyspace.CommandsForKeyAccessor.load(commandStore.id(), (TokenKey) key); + if (cfk == null) + return null; + RedundantBefore.QuickBounds bounds = commandStore.safeGetRedundantBefore().get(key); + if (!Invariants.expect(bounds != null, "No RedundantBefore information found when loading key %s", key)) + return cfk; + return cfk.withCleanCfkBeforeAtLeast(bounds.cleanCfkBefore(), false); } @Override @@ -1179,7 +1191,7 @@ public Runnable save(AccordCommandStore commandStore, RoutingKey key, @Nullable ByteBuffer bb = (ByteBuffer)serialized; serialized = bb.duplicate().position(prefixBytes(bb)); } - return commandStore.saveCommandsForKey(key, value, serialized); + return AccordKeyspace.CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey) key, value, serialized); } @Override @@ -1255,13 +1267,17 @@ public long estimateShrunkHeapSize(Object shrunk) @Override public boolean validate(AccordCommandStore commandStore, RoutingKey key, CommandsForKey value) { - return commandStore.validateCommandsForKey(key, value); + if (!Invariants.isParanoid()) + return true; + + CommandsForKey reloaded = AccordKeyspace.CommandsForKeyAccessor.load(commandStore.id(), (TokenKey) key); + return Objects.equals(value, reloaded); } @Override - public AccordSafeCommandsForKey safeRef(AccordCacheEntry node) + public SaferCommandsForKey safeRef(AccordCacheEntry node) { - return new AccordSafeCommandsForKey(node); + return new SaferCommandsForKey(node); } @Override @@ -1271,7 +1287,7 @@ public Comparator keyComparator() } @Override - public AccordCacheEntry newEntry(RoutingKey key, Type.Instance owner) + public AccordCacheEntry newEntry(RoutingKey key, Type.Instance owner) { CommandsForKeyCacheEntry entry = new CommandsForKeyCacheEntry(key, owner); entry.readyToLoad(); @@ -1279,7 +1295,7 @@ public AccordCacheEntry ne } } - public static class CommandAdapter implements Adapter + public static class CommandAdapter implements Adapter { private static final int SHRINK_WITHOUT_LOCK = -1; @@ -1392,19 +1408,23 @@ public long estimateShrunkHeapSize(Object shrunk) @Override public boolean validate(AccordCommandStore commandStore, TxnId key, Command value) { - return commandStore.validateCommand(key, value); + if (!Invariants.isParanoid()) + return true; + + Command reloaded = commandStore.loadCommand(key); + return Objects.equals(value, reloaded); } @Override - public AccordSafeCommand safeRef(AccordCacheEntry node) + public SaferCommand safeRef(AccordCacheEntry node) { - return new AccordSafeCommand(node); + return new SaferCommand(node); } @Override - public AccordCacheEntry newEntry(TxnId txnId, Type.Instance owner) + public AccordCacheEntry newEntry(TxnId txnId, Type.Instance owner) { - AccordCacheEntry node = new AccordCacheEntry<>(txnId, owner); + AccordCacheEntry node = new AccordCacheEntry<>(txnId, owner); if (txnId.is(Txn.Kind.EphemeralRead)) { node.initialize(null); diff --git a/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java similarity index 60% rename from src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java index 11d8a2b105cd..fefaad70a68c 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java @@ -15,13 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.function.BiConsumer; import java.util.function.BiPredicate; @@ -36,39 +33,38 @@ import accord.utils.IntrusiveLinkedList; import accord.utils.IntrusiveLinkedListNode; import accord.utils.Invariants; -import accord.utils.SortedArrays; -import accord.utils.TriConsumer; import accord.utils.UnhandledEnum; import accord.utils.async.Cancellable; -import org.apache.cassandra.service.accord.AccordCache.Adapter; -import org.apache.cassandra.service.accord.AccordCache.Adapter.Shrink; -import org.apache.cassandra.service.accord.AccordExecutor.IOTask; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.execution.AccordCache.Adapter; +import org.apache.cassandra.service.accord.execution.AccordCache.Adapter.Shrink; import org.apache.cassandra.utils.ObjectSizes; import static accord.utils.Invariants.nonNull; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.HOLD_QUEUE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.UNLOCKED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Queue.compare; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NEWLY_BLOCKING_RUNNABLE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NEWLY_RUNNABLE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_SAVE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADING; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.MODIFIED; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.SAVING; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.WAITING_TO_LOAD; -import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.WAITING_TO_SAVE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.HOLD_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.UNLOCKED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NEWLY_BLOCKING_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NEWLY_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.EVICTED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.FAILED_TO_LOAD; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.FAILED_TO_SAVE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.LOADED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.LOADING; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.MODIFIED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.SAVING; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.WAITING_TO_LOAD; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.WAITING_TO_SAVE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntryQueue.PRIORITY_START_INDEX; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntryQueue.compare; /** * Global (per CommandStore) state of a cached entity (Command or CommandsForKey). */ -public class AccordCacheEntry & AccordSafeState> extends IntrusiveLinkedListNode +public class AccordCacheEntry & SaferState> extends IntrusiveLinkedListNode { public enum Status { @@ -162,492 +158,6 @@ public enum Status static final int AGE_SHIFT = 24; static final int AGE_MASK = 0xff; - static class Queue - { - private static final int DEFAULT_CAPACITY = 4; - private static final int LOCKED_INDEX = 0; - private static final int PRIORITY_START_INDEX = LOCKED_INDEX + 1; - /** - * [priorityHead..priorityTail) stores a priority-sorted list of tasks - * (fifoTail...fifoHead] stores a fifo queue that runs ahead of any priority tasks - * (fifoTail-unsequencedCount...fifoTail] stores unsequenced tasks that are waiting for a queued incremental task. - * This only happens for TxnId cache entries, since they may lockAndHoldQueue. Once the lock is released, - * any pending unsequenced tasks are notified and immediately made (irrevocably) runnable for this entry. - */ - private AccordTask[] tasks; - // TODO (expected): use bytes/shorts for indexes to keep size down, and have an expanded version of the Queue - // with better algorithmic complexity (e.g. Hash -> IntrusivePriorityHeap) - private int priorityHead, priorityTail, fifoHead, fifoTail; - private int unsequencedSize; - - public Queue() - { - tasks = new AccordTask[DEFAULT_CAPACITY]; - priorityHead = priorityTail = PRIORITY_START_INDEX; - fifoHead = fifoTail = DEFAULT_CAPACITY - 1; - } - - private Queue(Queue copy) - { - tasks = copy.tasks.clone(); - priorityHead = copy.priorityHead; - priorityTail = copy.priorityTail; - fifoHead = copy.fifoHead; - fifoTail = copy.fifoTail; - } - - // returns true if no fifo tasks already queue (i.e. so we become head) - boolean addFifo(AccordTask task) - { - ensureCapacity(); - boolean isHead = fifoHead == fifoTail; - if (unsequencedSize > 0) // simply displace the unsequence task, as they're an unordered list - tasks[fifoTail - unsequencedSize] = tasks[fifoTail]; - tasks[fifoTail--] = task; - validate(); - return isHead; - } - - boolean addUnsequenced(AccordTask task) - { - Invariants.require(task.isUnsequenced()); - ensureCapacity(); - tasks[fifoTail - unsequencedSize++] = task; - return unsequencedSize == 1 && sequencedSize() == 1; - } - - boolean isLocked(AccordTask task) - { - return tasks[LOCKED_INDEX] == task; - } - - AccordTask lockedBy() - { - return tasks[LOCKED_INDEX]; - } - - boolean removeIfHead(AccordTask task) - { - return removeIfFifoHead(task) || removeIfPriorityHead(task); - } - - boolean removeIfFifoHead(AccordTask task) - { - if (!hasFifo()) - return false; - - if (task != tasks[fifoHead]) - return false; - tasks[fifoHead--] = null; - return true; - } - - boolean removeIfPriorityHead(AccordTask task) - { - if (!hasPriority()) - return false; - - if (task != tasks[priorityHead]) - return false; - tasks[priorityHead++] = null; - return true; - } - - void lock(AccordTask task) - { - tasks[LOCKED_INDEX] = task; - } - - void unlock(AccordTask task) - { - Invariants.require(tasks[LOCKED_INDEX] == task); - tasks[LOCKED_INDEX] = null; - } - - // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue - boolean addPrioritised(AccordTask task) - { - ensureCapacity(); - int insertPos = Arrays.binarySearch(tasks, priorityHead, priorityTail, task, Queue::compare); - if (insertPos < 0) - insertPos = -1 - insertPos; - - if (priorityHead == PRIORITY_START_INDEX || insertPos > (priorityTail + priorityHead)/2) - { - System.arraycopy(tasks, insertPos, tasks, insertPos + 1, priorityTail - insertPos); - tasks[insertPos] = task; - priorityTail++; - } - else - { - System.arraycopy(tasks, priorityHead, tasks, priorityHead - 1, insertPos - priorityHead); - tasks[insertPos - 1] = task; - priorityHead--; - } - - validate(); - return fifoHead == fifoTail && tasks[priorityHead] == task; - } - - // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue - void addWaitingToLoad(AccordTask task) - { - ensureCapacity(); - tasks[priorityTail++] = task; - } - - private boolean hasTailRoom() - { - if (priorityTail + unsequencedSize <= fifoTail) - return true; - Invariants.require(priorityTail + unsequencedSize == 1 + fifoTail); - return false; - } - - private void ensureCapacity() - { - if (!hasTailRoom()) - { - if (fifoHead == fifoTail && unsequencedSize == 0 && fifoTail < tasks.length - 1) fifoHead = fifoTail = tasks.length - 1; - else if (priorityHead == priorityTail && priorityHead > PRIORITY_START_INDEX) priorityHead = priorityTail = PRIORITY_START_INDEX; - else if (totalSize() >= (tasks.length - 1) / 2) compact(new AccordTask[tasks.length * 2]); - else compact(tasks); - Invariants.require(hasTailRoom()); - } - } - - private void compact(AccordTask[] into) - { - if (priorityHead == priorityTail) priorityHead = priorityTail = PRIORITY_START_INDEX; - else - { - int priorityLength = priorityTail - priorityHead; - System.arraycopy(tasks, priorityHead, into, PRIORITY_START_INDEX, priorityLength); - int newTail = PRIORITY_START_INDEX + priorityLength; - Invariants.require(newTail <= priorityTail); - if (into == tasks) - Arrays.fill(into, newTail, priorityTail, null); - priorityHead = PRIORITY_START_INDEX; - priorityTail = newTail; - } - - if (fifoHead == fifoTail && unsequencedSize == 0) fifoHead = fifoTail = into.length - 1; - else - { - int fifoLength = fifoHead - fifoTail; - int copyLength = fifoLength + unsequencedSize; - int copyFrom = (fifoTail - unsequencedSize) + 1; - int copyTo = into.length - copyLength; - Invariants.require(copyTo >= copyFrom); - System.arraycopy(tasks, copyFrom, into, copyTo, copyLength); - if (into == tasks) - Arrays.fill(into, copyFrom, copyTo, null); - fifoHead = into.length - 1; - fifoTail = fifoHead - fifoLength; - } - - if (tasks != into) - { - into[LOCKED_INDEX] = tasks[LOCKED_INDEX]; - tasks = into; - } - validate(); - } - - private void validate() - { - for (int i = PRIORITY_START_INDEX ; i < priorityHead ; ++i) - Invariants.require(tasks[i] == null); - for (int i = priorityHead ; i < priorityTail ; ++i) - Invariants.require(tasks[i] != null); - for (int i = priorityTail; i <= fifoTail - unsequencedSize; ++i) - Invariants.require(tasks[i] == null); - for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoHead ; ++i) - Invariants.require(tasks[i] != null); - for (int i = fifoHead + 1 ; i < tasks.length ; ++i) - Invariants.require(tasks[i] == null); - } - - AccordTask peek() - { - if (hasFifo()) return tasks[fifoHead]; - if (hasPriority()) return tasks[priorityHead]; - return null; - } - - AccordTask peekFifo() - { - return hasFifo() ? tasks[fifoHead] : null; - } - - // second task - AccordTask peekBehind() - { - int fifoSize = fifoSize(); - if (fifoSize > 1) - return tasks[fifoHead - 1]; - int priorityIndex = priorityHead + (1 - fifoSize); - if (priorityIndex < priorityTail) - return tasks[priorityIndex]; - return null; - } - - boolean hasFifo() - { - return fifoHead != fifoTail; - } - - boolean hasPriority() - { - return priorityHead != priorityTail; - } - - boolean hasUnsequenced() - { - return unsequencedSize > 0; - } - - int sequencedSize() - { - return prioritySize() + fifoSize(); - } - - int unsequencedSize() - { - return unsequencedSize; - } - - int totalSize() - { - return sequencedSize() + unsequencedSize; - } - - int prioritySize() - { - return priorityTail - priorityHead; - } - - int fifoSize() - { - return fifoHead - fifoTail; - } - - // true iff was head - boolean removeFifoOrPriority(AccordTask task, boolean permitMissing) - { - int fifoIndex = fifoIndexOf(task); - if (fifoIndex >= 0) - { - if (fifoIndex == fifoHead) - { - tasks[fifoHead--] = null; - validate(); - return true; - } - else - { - if (remove(fifoIndex, fifoTail + 1, fifoHead + 1)) ++fifoTail; - else --fifoHead; - validate(); - return false; - } - } - - int priorityIndex = priorityIndexOf(task); - Invariants.require(priorityIndex >= 0 || permitMissing); - if (priorityIndex >= 0) - { - if (priorityIndex == priorityHead) - { - tasks[priorityHead++] = null; - return !hasFifo(); - } - - if (remove(priorityIndex, priorityHead, priorityTail)) ++priorityHead; - else --priorityTail; - return false; - } - - return false; - } - - boolean removeUnsequenced(AccordTask task) - { - int unsequencedIndex = unsequencedIndexOf(task); - if (unsequencedIndex < 0) - return false; - - --unsequencedSize; - tasks[unsequencedIndex] = tasks[fifoTail - unsequencedSize]; - tasks[fifoTail - unsequencedSize] = null; - return true; - } - - // return true IFF was head - private boolean removePriority(AccordTask task, boolean permitAbsent) - { - int i = priorityIndexOf(task); - if (i < 0) - { - Invariants.require(permitAbsent); - return false; - } - - boolean wasHead = i == priorityHead; - if (remove(i, priorityHead, priorityTail)) ++priorityHead; - else --priorityTail; - return wasHead; - } - - // return true if we move the start forwards, false if we moved the end back - private boolean remove(int i, int start, int end) - { - if (i < (start + end)/2) - { - System.arraycopy(tasks, start, tasks, start + 1, i - start); - tasks[start] = null; - return true; - } - else - { - System.arraycopy(tasks, i + 1, tasks, i, end - (i + 1)); - tasks[end - 1] = null; - return false; - } - } - - boolean contains(AccordTask task) - { - return indexOf(task) >= 0; - } - - private int indexOf(AccordTask task) - { - if (tasks[priorityHead] == task) - return priorityHead; - - if (tasks[fifoHead] == task) - return fifoHead; - - int i = priorityIndexOf(task); - if (i >= 0) - return i; - - return fifoIndexOf(task); - } - - private int priorityIndexOf(AccordTask task) - { - if (priorityTail - priorityHead > 16) - { - if (tasks[priorityHead] == task) - return priorityHead; - - int i = SortedArrays.binarySearch(tasks, priorityHead + 1, priorityTail, task, Queue::compare, SortedArrays.Search.CEIL); - if (i < 0) - return -1; - - while (i < priorityTail) - { - if (tasks[i] == task) - return i; - if (compare(task, tasks[i]) != 0) - break; - ++i; - } - } - - for (int i = priorityHead ; i < priorityTail ; ++i) - { - if (tasks[i] == task) - return i; - } - return -1; - } - - private int fifoIndexOf(AccordTask task) - { - for (int i = fifoHead ; i > fifoTail ; --i) - { - if (tasks[i] == task) - return i; - } - return -1; - } - - private int unsequencedIndexOf(AccordTask task) - { - for (int i = (fifoTail - unsequencedSize) + 1 ; i <= fifoTail ; ++i) - { - if (tasks[i] == task) - return i; - } - return -1; - } - - int drainUnsequenced(TriConsumer, P1, P2> forEach, P1 p1, P2 p2) - { - for (int i = (fifoTail - unsequencedSize) + 1 ; i <= fifoTail ; ++i) - { - AccordTask task = tasks[i]; - tasks[i] = null; - // should not be reentrant - forEach.accept(task, p1, p2); - } - int count = unsequencedSize; - unsequencedSize = 0; - return count; - } - - RunnableStatus ensureHeadFifo(AccordTask task) - { - if (hasFifo()) - { - Invariants.require(tasks[fifoHead] == task); - return NOT_RUNNABLE; - } - - if (tasks[priorityHead] == task) - { - tasks[priorityHead++] = null; - addFifo(task); - return STILL_RUNNABLE; - } - else - { - boolean wasPriorityHead = removePriority(task, false); - boolean isFifoHead = addFifo(task); - if (!isFifoHead) - return NOT_RUNNABLE; - if (wasPriorityHead) - return STILL_RUNNABLE; - if (hasPriority() || hasUnsequenced()) - return NEWLY_BLOCKING_RUNNABLE; - return NEWLY_RUNNABLE; - } - } - - static int compare(AccordTask a, AccordTask b) - { - Invariants.require(a != null && b != null); - int c = Long.compare(a.position, b.position); - if (c == 0) - c = a.executionContext().executionKind().compareTo(b.executionContext().executionKind()); - if (c == 0) - c = Long.compare(a.createdAt, b.createdAt); - if (c == 0) - c = Long.compare(a.loadedAt, b.loadedAt); - if (c == 0) - c = a.loggingId().compareTo(b.loggingId()); - return c; - } - - public boolean hasQueued() - { - return hasFifo() || hasPriority(); - } - } - static final long EMPTY_SIZE = ObjectSizes.measure(new AccordCacheEntry<>(null, null)); private final K key; @@ -676,13 +186,13 @@ public boolean hasQueued() private RunnableStatus validate(RunnableStatus status) { Invariants.require(queue != null); - AccordTask head = queue instanceof Queue ? ((Queue) queue).peek() : (AccordTask) queue; + SafeTask head = queue instanceof AccordCacheEntryQueue ? ((AccordCacheEntryQueue) queue).peek() : (SafeTask) queue; Invariants.require(isRunnable(head) || status == NOT_RUNNABLE); return status; } // TODO (expected): don't unwrap when only one entry, since this may cause us to flap when locking unsequenced tasks - private void maybeUnwrap(Queue q) + private void maybeUnwrap(AccordCacheEntryQueue q) { int size = q.sequencedSize(); switch (size) @@ -699,7 +209,7 @@ private void maybeUnwrap(Queue q) } } - private boolean maybeUnwrap(Queue q, AccordTask lockedBy) + private boolean maybeUnwrap(AccordCacheEntryQueue q, SafeTask lockedBy) { if (q.sequencedSize() == 0) { @@ -711,11 +221,11 @@ private boolean maybeUnwrap(Queue q, AccordTask lockedBy) } // assumes already queued with priority - final RunnableStatus moveToFifo(AccordTask task) + final RunnableStatus moveToFifo(SafeTask task) { if (queue != task) { - Queue q = (Queue) queue; + AccordCacheEntryQueue q = (AccordCacheEntryQueue) queue; RunnableStatus status = q.ensureHeadFifo(task); if (status == NEWLY_BLOCKING_RUNNABLE && isLoaded()) onChangedHead(q, null, q.peekBehind()); @@ -725,22 +235,22 @@ final RunnableStatus moveToFifo(AccordTask task) } // drains ONLY those queued with addWaitingToLoad; addFifo are included in the result but are not removed from the collection - public final BufferList> drainWaitingToLoad() + public final BufferList> drainWaitingToLoad() { Invariants.require(isLoading()); Invariants.require(!isLocked()); - BufferList> list = new BufferList<>(); + BufferList> list = new BufferList<>(); if (queue != null) { - if (queue instanceof Queue) + if (queue instanceof AccordCacheEntryQueue) { - Queue q = (Queue) queue; + AccordCacheEntryQueue q = (AccordCacheEntryQueue) queue; for (int i = q.priorityHead ; i < q.priorityTail ; ++i) { list.add(q.tasks[i]); q.tasks[i] = null; } - q.priorityHead = q.priorityTail = Queue.PRIORITY_START_INDEX; + q.priorityHead = q.priorityTail = PRIORITY_START_INDEX; for (int i = q.fifoHead ; i > q.fifoTail ; --i) list.add(q.tasks[i]); @@ -748,7 +258,7 @@ public final BufferList> drainWaitingToLoad() } else { - AccordTask task = (AccordTask) queue; + SafeTask task = (SafeTask) queue; list.add(task); Invariants.require(!isLocked()); if (!task.isCacheQueuedFifo()) @@ -758,14 +268,14 @@ public final BufferList> drainWaitingToLoad() return list; } - final void remove(AccordTask task, boolean ownsLock) + final void remove(SafeTask task, boolean ownsLock) { - if (queue instanceof Queue) + if (queue instanceof AccordCacheEntryQueue) { - Queue q = (Queue) queue; + AccordCacheEntryQueue q = (AccordCacheEntryQueue) queue; boolean remove; boolean isLocked = isLocked() && q.isLocked(task); - Invariants.require(isLocked == ownsLock); + Invariants.require(isLocked || !ownsLock); if (isLocked) { // if locked, we've already released unsequenced/pririty/fifo positions unless isLockedHoldingQueue @@ -789,7 +299,7 @@ else if (task.isUnsequenced(this)) boolean wasHead = remove && q.removeFifoOrPriority(task, false); if (isLoaded() && wasHead) { - unsequenced += q.drainUnsequenced(AccordTask::onChangeHeadStatus, this, NEWLY_RUNNABLE); + unsequenced += q.drainUnsequenced(SafeTask::onChangeHeadStatus, this, NEWLY_RUNNABLE); onChangedHead(q, q.peek(), null); } } @@ -800,7 +310,7 @@ else if (task.isUnsequenced(this)) else if (queue == task) { boolean isLocked = isLocked(); - Invariants.require(isLocked == ownsLock); + Invariants.require(isLocked || !ownsLock); if (isLocked) { status &= ~LOCKED_MASK; @@ -822,28 +332,28 @@ else if (task.isUnsequenced(this)) final boolean isCommandsForKey() { - return getClass() == AccordSafeCommandsForKey.CommandsForKeyCacheEntry.class; + return getClass() == SaferCommandsForKey.CommandsForKeyCacheEntry.class; } - final RunnableStatus headStatus(AccordTask task) + final RunnableStatus headStatus(SafeTask task) { if (queue == task) return validate(isRunnable(task) ? NEWLY_RUNNABLE : NOT_RUNNABLE); - Queue q = (Queue) queue; + AccordCacheEntryQueue q = (AccordCacheEntryQueue) queue; if (q.peek() != task || !isRunnable(task)) return NOT_RUNNABLE; return validate(q.totalSize() == 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); } - private Queue ensureQueue() + private AccordCacheEntryQueue ensureQueue() { - if (queue instanceof Queue) - return (Queue) queue; + if (queue instanceof AccordCacheEntryQueue) + return (AccordCacheEntryQueue) queue; - AccordTask head = (AccordTask) this.queue; - Queue q = new Queue(); + SafeTask head = (SafeTask) this.queue; + AccordCacheEntryQueue q = new AccordCacheEntryQueue(); if (isLocked()) { if (isLockedHoldingQueue()) @@ -856,32 +366,32 @@ private Queue ensureQueue() return q; } - final void addWaitingToLoad(AccordTask task) + final void addWaitingToLoad(SafeTask task) { Invariants.require(isLoading()); if (queue == null) queue = task; else ensureQueue().addWaitingToLoad(task); } - final AccordTask head() + final SafeTask head() { if (queue == null) return null; - if (queue instanceof Queue) - return ((Queue) queue).peek(); + if (queue instanceof AccordCacheEntryQueue) + return ((AccordCacheEntryQueue) queue).peek(); if (isLocked() && !isLockedHoldingQueue()) return null; - return (AccordTask) queue; + return (SafeTask) queue; } - final RunnableStatus addUnsequenced(AccordTask task) + final RunnableStatus addUnsequenced(SafeTask task) { Invariants.require(isLoaded()); - AccordTask head = head(); + SafeTask head = head(); if (head != null && head.holdsLocksBetweenRuns()) { boolean wait = compare(task, head) > 0 || (unsequenced == 0 && head.hasIncrementalStarted()); @@ -907,8 +417,8 @@ final RunnableStatus addUnsequenced(AccordTask task) final int waitingCount() { Invariants.require(isLoading()); - return queue == null ? 0 : queue instanceof Queue - ? ((Queue)queue).sequencedSize() + return queue == null ? 0 : queue instanceof AccordCacheEntryQueue + ? ((AccordCacheEntryQueue)queue).sequencedSize() : isLocked() == isLockedHoldingQueue() ? 1 : 0; } @@ -917,12 +427,12 @@ public enum RunnableStatus NOT_RUNNABLE, STILL_RUNNABLE, NEWLY_RUNNABLE, NEWLY_BLOCKING_RUNNABLE, STILL_RUNNABLE_NEWLY_BLOCKING } - private boolean isRunnable(AccordTask head) + private boolean isRunnable(SafeTask head) { return !head.holdsLocksBetweenRuns() || unsequenced == 0; } - private RunnableStatus add(AccordTask task, BiPredicate> add) + private RunnableStatus add(SafeTask task, BiPredicate> add) { Object prev = this.queue; if (prev == null) @@ -931,12 +441,12 @@ private RunnableStatus add(AccordTask task, BiPredicate> return validate(isRunnable(task) ? NEWLY_RUNNABLE : NOT_RUNNABLE); } - Queue q = ensureQueue(); + AccordCacheEntryQueue q = ensureQueue(); if (!add.test(q, task)) { if (isLoaded() && q.totalSize() == 2) { - AccordTask head = q.peek(); + SafeTask head = q.peek(); if (isRunnable(head)) head.onChangeHeadStatus(this, STILL_RUNNABLE_NEWLY_BLOCKING); } @@ -955,18 +465,18 @@ private RunnableStatus add(AccordTask task, BiPredicate> return validate(isRunnable ? NEWLY_BLOCKING_RUNNABLE : NOT_RUNNABLE); } - final RunnableStatus addPrioritised(AccordTask task) + final RunnableStatus addPrioritised(SafeTask task) { Invariants.require(!isLoading()); - return add(task, Queue::addPrioritised); + return add(task, AccordCacheEntryQueue::addPrioritised); } - final RunnableStatus addFifo(AccordTask task) + final RunnableStatus addFifo(SafeTask task) { - return add(task, Queue::addFifo); + return add(task, AccordCacheEntryQueue::addFifo); } - private void onChangedHead(Queue q, @Nullable AccordTask notifyNewHead, @Nullable AccordTask notifyPrevHead) + private void onChangedHead(AccordCacheEntryQueue q, @Nullable SafeTask notifyNewHead, @Nullable SafeTask notifyPrevHead) { if (notifyNewHead != null && isRunnable(notifyNewHead)) notifyNewHead.onChangeHeadStatus(this, q.totalSize() == 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); @@ -1003,7 +513,7 @@ public enum LockMode /** * On lock we remove ourselves from the priority/fifo queues and notify the new head */ - public final V lockExclusive(AccordTask owner, LockMode lockMode) + public final V lockExclusive(SafeTask owner, LockMode lockMode) { Invariants.require(!isLocked()); Invariants.require(isRunnable(owner) || owner.isUnsequenced(this)); @@ -1028,7 +538,7 @@ else if (queue == null) } else { - Queue q = ensureQueue(); + AccordCacheEntryQueue q = ensureQueue(); switch (lockMode) { default: throw UnhandledEnum.unknown(lockMode); @@ -1052,7 +562,7 @@ else if (queue == null) Invariants.require(wasHead); if (isLoaded()) { - unsequenced += q.drainUnsequenced(AccordTask::onChangeHeadStatus, this, NEWLY_RUNNABLE); + unsequenced += q.drainUnsequenced(SafeTask::onChangeHeadStatus, this, NEWLY_RUNNABLE); onChangedHead(q, q.peek(), null); } if (maybeUnwrap(q, owner)) @@ -1067,12 +577,12 @@ else if (queue == null) return getExclusive(); } - private void releaseUnsequenced(Queue q, AccordTask release) + private void releaseUnsequenced(AccordCacheEntryQueue q, SafeTask release) { Invariants.require(release.isCacheQueued()); if (--unsequenced == 0) { - AccordTask head = q.peek(); + SafeTask head = q.peek(); if (head != null && head.holdsLocksBetweenRuns()) onChangedHead(q, head, null); } @@ -1086,10 +596,10 @@ final boolean hasFifoOrLocked() if (queue == null) return false; - if (queue instanceof AccordTask) - return ((AccordTask) queue).isCacheQueuedFifo(); + if (queue instanceof SafeTask) + return ((SafeTask) queue).isCacheQueuedFifo(); - return ((Queue)queue).hasFifo(); + return ((AccordCacheEntryQueue)queue).hasFifo(); } final void unlink() @@ -1137,7 +647,7 @@ final boolean isLoaded() return (status & IS_LOADED) != 0; } - final boolean isModified() + public final boolean isModified() { return (status & IS_NOT_EVICTED) >= MODIFIED.ordinal(); } @@ -1342,7 +852,7 @@ public final V getExclusive() return (V) maybeUnwrap(); } - public final void releaseExclusive(S safeState, AccordTask task) + public final void releaseExclusive(S safeState, SafeTask task) { owner.release(safeState, task); } @@ -1644,7 +1154,7 @@ public Throwable failure() } } - public static & AccordSafeState> AccordCacheEntry createReadyToLoad(K key, AccordCache.Type.Instance owner) + public static & SaferState> AccordCacheEntry createReadyToLoad(K key, AccordCache.Type.Instance owner) { AccordCacheEntry node = new AccordCacheEntry<>(key, owner); node.readyToLoad(); diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java new file mode 100644 index 000000000000..9e4b60df7d81 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java @@ -0,0 +1,518 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.Arrays; + +import accord.utils.Invariants; +import accord.utils.SortedArrays; +import accord.utils.TriConsumer; + +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NEWLY_BLOCKING_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NEWLY_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; + +class AccordCacheEntryQueue +{ + private static final int DEFAULT_CAPACITY = 4; + static final int LOCKED_INDEX = 0; + static final int PRIORITY_START_INDEX = LOCKED_INDEX + 1; + /** + * [priorityHead..priorityTail) stores a priority-sorted list of tasks + * (fifoTail...fifoHead] stores a fifo queue that runs ahead of any priority tasks + * (fifoTail-unsequencedCount...fifoTail] stores unsequenced tasks that are waiting for a queued incremental task. + * This only happens for TxnId cache entries, since they may lockAndHoldQueue. Once the lock is released, + * any pending unsequenced tasks are notified and immediately made (irrevocably) runnable for this entry. + */ + SafeTask[] tasks; + // TODO (expected): use bytes/shorts for indexes to keep size down, and have an expanded version of the Queue + // with better algorithmic complexity (e.g. Hash -> IntrusivePriorityHeap) + int priorityHead, priorityTail, fifoHead, fifoTail; + int unsequencedSize; + + public AccordCacheEntryQueue() + { + tasks = new SafeTask[DEFAULT_CAPACITY]; + priorityHead = priorityTail = PRIORITY_START_INDEX; + fifoHead = fifoTail = DEFAULT_CAPACITY - 1; + } + + AccordCacheEntryQueue(AccordCacheEntryQueue copy) + { + tasks = copy.tasks.clone(); + priorityHead = copy.priorityHead; + priorityTail = copy.priorityTail; + fifoHead = copy.fifoHead; + fifoTail = copy.fifoTail; + } + + // returns true if no fifo tasks already queue (i.e. so we become head) + boolean addFifo(SafeTask task) + { + ensureCapacity(); + boolean isHead = fifoHead == fifoTail; + if (unsequencedSize > 0) // simply displace the unsequence task, as they're an unordered list + tasks[fifoTail - unsequencedSize] = tasks[fifoTail]; + tasks[fifoTail--] = task; + validate(); + return isHead; + } + + boolean addUnsequenced(SafeTask task) + { + Invariants.require(task.isUnsequenced()); + ensureCapacity(); + tasks[fifoTail - unsequencedSize++] = task; + return unsequencedSize == 1 && sequencedSize() == 1; + } + + boolean isLocked(SafeTask task) + { + return tasks[LOCKED_INDEX] == task; + } + + SafeTask lockedBy() + { + return tasks[LOCKED_INDEX]; + } + + boolean removeIfHead(SafeTask task) + { + return removeIfFifoHead(task) || removeIfPriorityHead(task); + } + + boolean removeIfFifoHead(SafeTask task) + { + if (!hasFifo()) + return false; + + if (task != tasks[fifoHead]) + return false; + tasks[fifoHead--] = null; + return true; + } + + boolean removeIfPriorityHead(SafeTask task) + { + if (!hasPriority()) + return false; + + if (task != tasks[priorityHead]) + return false; + tasks[priorityHead++] = null; + return true; + } + + void lock(SafeTask task) + { + tasks[LOCKED_INDEX] = task; + } + + void unlock(SafeTask task) + { + Invariants.require(tasks[LOCKED_INDEX] == task); + tasks[LOCKED_INDEX] = null; + } + + // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue + boolean addPrioritised(SafeTask task) + { + ensureCapacity(); + int insertPos = Arrays.binarySearch(tasks, priorityHead, priorityTail, task, AccordCacheEntryQueue::compare); + if (insertPos < 0) + insertPos = -1 - insertPos; + + if (priorityHead == PRIORITY_START_INDEX || insertPos > (priorityTail + priorityHead) / 2) + { + System.arraycopy(tasks, insertPos, tasks, insertPos + 1, priorityTail - insertPos); + tasks[insertPos] = task; + priorityTail++; + } + else + { + System.arraycopy(tasks, priorityHead, tasks, priorityHead - 1, insertPos - priorityHead); + tasks[insertPos - 1] = task; + priorityHead--; + } + + validate(); + return fifoHead == fifoTail && tasks[priorityHead] == task; + } + + // should always return false, as should never be invoked on an empty queue, and returns true only if we're head of the queue + void addWaitingToLoad(SafeTask task) + { + ensureCapacity(); + tasks[priorityTail++] = task; + } + + private boolean hasTailRoom() + { + if (priorityTail + unsequencedSize <= fifoTail) + return true; + Invariants.require(priorityTail + unsequencedSize == 1 + fifoTail); + return false; + } + + private void ensureCapacity() + { + if (!hasTailRoom()) + { + if (fifoHead == fifoTail && unsequencedSize == 0 && fifoTail < tasks.length - 1) + fifoHead = fifoTail = tasks.length - 1; + else if (priorityHead == priorityTail && priorityHead > PRIORITY_START_INDEX) + priorityHead = priorityTail = PRIORITY_START_INDEX; + else if (totalSize() >= (tasks.length - 1) / 2) compact(new SafeTask[tasks.length * 2]); + else compact(tasks); + Invariants.require(hasTailRoom()); + } + } + + private void compact(SafeTask[] into) + { + if (priorityHead == priorityTail) priorityHead = priorityTail = PRIORITY_START_INDEX; + else + { + int priorityLength = priorityTail - priorityHead; + System.arraycopy(tasks, priorityHead, into, PRIORITY_START_INDEX, priorityLength); + int newTail = PRIORITY_START_INDEX + priorityLength; + Invariants.require(newTail <= priorityTail); + if (into == tasks) + Arrays.fill(into, newTail, priorityTail, null); + priorityHead = PRIORITY_START_INDEX; + priorityTail = newTail; + } + + if (fifoHead == fifoTail && unsequencedSize == 0) fifoHead = fifoTail = into.length - 1; + else + { + int fifoLength = fifoHead - fifoTail; + int copyLength = fifoLength + unsequencedSize; + int copyFrom = (fifoTail - unsequencedSize) + 1; + int copyTo = into.length - copyLength; + Invariants.require(copyTo >= copyFrom); + System.arraycopy(tasks, copyFrom, into, copyTo, copyLength); + if (into == tasks) + Arrays.fill(into, copyFrom, copyTo, null); + fifoHead = into.length - 1; + fifoTail = fifoHead - fifoLength; + } + + if (tasks != into) + { + into[LOCKED_INDEX] = tasks[LOCKED_INDEX]; + tasks = into; + } + validate(); + } + + private void validate() + { + for (int i = PRIORITY_START_INDEX; i < priorityHead; ++i) + Invariants.require(tasks[i] == null); + for (int i = priorityHead; i < priorityTail; ++i) + Invariants.require(tasks[i] != null); + for (int i = priorityTail; i <= fifoTail - unsequencedSize; ++i) + Invariants.require(tasks[i] == null); + for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoHead; ++i) + Invariants.require(tasks[i] != null); + for (int i = fifoHead + 1; i < tasks.length; ++i) + Invariants.require(tasks[i] == null); + } + + SafeTask peek() + { + if (hasFifo()) return tasks[fifoHead]; + if (hasPriority()) return tasks[priorityHead]; + return null; + } + + SafeTask peekFifo() + { + return hasFifo() ? tasks[fifoHead] : null; + } + + // second task + SafeTask peekBehind() + { + int fifoSize = fifoSize(); + if (fifoSize > 1) + return tasks[fifoHead - 1]; + int priorityIndex = priorityHead + (1 - fifoSize); + if (priorityIndex < priorityTail) + return tasks[priorityIndex]; + return null; + } + + boolean hasFifo() + { + return fifoHead != fifoTail; + } + + boolean hasPriority() + { + return priorityHead != priorityTail; + } + + boolean hasUnsequenced() + { + return unsequencedSize > 0; + } + + int sequencedSize() + { + return prioritySize() + fifoSize(); + } + + int unsequencedSize() + { + return unsequencedSize; + } + + int totalSize() + { + return sequencedSize() + unsequencedSize; + } + + int prioritySize() + { + return priorityTail - priorityHead; + } + + int fifoSize() + { + return fifoHead - fifoTail; + } + + // true iff was head + boolean removeFifoOrPriority(SafeTask task, boolean permitMissing) + { + int fifoIndex = fifoIndexOf(task); + if (fifoIndex >= 0) + { + if (fifoIndex == fifoHead) + { + tasks[fifoHead--] = null; + validate(); + return true; + } + else + { + if (remove(fifoIndex, fifoTail + 1, fifoHead + 1)) ++fifoTail; + else --fifoHead; + validate(); + return false; + } + } + + int priorityIndex = priorityIndexOf(task); + Invariants.require(priorityIndex >= 0 || permitMissing); + if (priorityIndex >= 0) + { + if (priorityIndex == priorityHead) + { + tasks[priorityHead++] = null; + return !hasFifo(); + } + + if (remove(priorityIndex, priorityHead, priorityTail)) ++priorityHead; + else --priorityTail; + return false; + } + + return false; + } + + boolean removeUnsequenced(SafeTask task) + { + int unsequencedIndex = unsequencedIndexOf(task); + if (unsequencedIndex < 0) + return false; + + --unsequencedSize; + tasks[unsequencedIndex] = tasks[fifoTail - unsequencedSize]; + tasks[fifoTail - unsequencedSize] = null; + return true; + } + + // return true IFF was head + private boolean removePriority(SafeTask task, boolean permitAbsent) + { + int i = priorityIndexOf(task); + if (i < 0) + { + Invariants.require(permitAbsent); + return false; + } + + boolean wasHead = i == priorityHead; + if (remove(i, priorityHead, priorityTail)) ++priorityHead; + else --priorityTail; + return wasHead; + } + + // return true if we move the start forwards, false if we moved the end back + private boolean remove(int i, int start, int end) + { + if (i < (start + end) / 2) + { + System.arraycopy(tasks, start, tasks, start + 1, i - start); + tasks[start] = null; + return true; + } + else + { + System.arraycopy(tasks, i + 1, tasks, i, end - (i + 1)); + tasks[end - 1] = null; + return false; + } + } + + boolean contains(SafeTask task) + { + return indexOf(task) >= 0; + } + + private int indexOf(SafeTask task) + { + if (tasks[priorityHead] == task) + return priorityHead; + + if (tasks[fifoHead] == task) + return fifoHead; + + int i = priorityIndexOf(task); + if (i >= 0) + return i; + + return fifoIndexOf(task); + } + + private int priorityIndexOf(SafeTask task) + { + if (priorityTail - priorityHead > 16) + { + if (tasks[priorityHead] == task) + return priorityHead; + + int i = SortedArrays.binarySearch(tasks, priorityHead + 1, priorityTail, task, AccordCacheEntryQueue::compare, SortedArrays.Search.CEIL); + if (i < 0) + return -1; + + while (i < priorityTail) + { + if (tasks[i] == task) + return i; + if (compare(task, tasks[i]) != 0) + break; + ++i; + } + } + + for (int i = priorityHead; i < priorityTail; ++i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + private int fifoIndexOf(SafeTask task) + { + for (int i = fifoHead; i > fifoTail; --i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + private int unsequencedIndexOf(SafeTask task) + { + for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoTail; ++i) + { + if (tasks[i] == task) + return i; + } + return -1; + } + + int drainUnsequenced(TriConsumer, P1, P2> forEach, P1 p1, P2 p2) + { + for (int i = (fifoTail - unsequencedSize) + 1; i <= fifoTail; ++i) + { + SafeTask task = tasks[i]; + tasks[i] = null; + // should not be reentrant + forEach.accept(task, p1, p2); + } + int count = unsequencedSize; + unsequencedSize = 0; + return count; + } + + AccordCacheEntry.RunnableStatus ensureHeadFifo(SafeTask task) + { + if (hasFifo()) + { + Invariants.require(tasks[fifoHead] == task); + return NOT_RUNNABLE; + } + + if (tasks[priorityHead] == task) + { + tasks[priorityHead++] = null; + addFifo(task); + return STILL_RUNNABLE; + } + else + { + boolean wasPriorityHead = removePriority(task, false); + boolean isFifoHead = addFifo(task); + if (!isFifoHead) + return NOT_RUNNABLE; + if (wasPriorityHead) + return STILL_RUNNABLE; + if (hasPriority() || hasUnsequenced()) + return NEWLY_BLOCKING_RUNNABLE; + return NEWLY_RUNNABLE; + } + } + + static int compare(SafeTask a, SafeTask b) + { + Invariants.require(a != null && b != null); + int c = Long.compare(a.position, b.position); + if (c == 0) + c = a.executionContext().executionKind().compareTo(b.executionContext().executionKind()); + if (c == 0) + c = Long.compare(a.createdAt, b.createdAt); + if (c == 0) + c = Long.compare(a.loadedAt, b.loadedAt); + if (c == 0) + c = a.loggingId().compareTo(b.loggingId()); + return c; + } + + public boolean hasQueued() + { + return hasFifo() || hasPriority(); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java new file mode 100644 index 000000000000..a87e405dd257 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java @@ -0,0 +1,823 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.IntFunction; +import java.util.stream.Stream; + +import accord.api.Agent; +import accord.api.ExclusiveAsyncExecutor; +import accord.api.RoutingKey; +import accord.impl.AbstractAsyncExecutor; +import accord.local.Command; +import accord.local.cfk.CommandsForKey; +import accord.primitives.TxnId; +import accord.utils.ArrayBuffers; +import accord.utils.Invariants; +import accord.utils.TinyEnumSet; +import accord.utils.UnhandledEnum; +import accord.utils.async.AsyncCallbacks.CallAndCallback; +import accord.utils.async.AsyncCallbacks.RunOrFail; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.cache.CacheSize; +import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; +import org.apache.cassandra.concurrent.Shutdownable; +import org.apache.cassandra.config.AccordConfig; +import org.apache.cassandra.config.AccordConfig.QueueBalancingModel; +import org.apache.cassandra.config.AccordConfig.QueuePriorityModel; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.metrics.AccordCacheMetrics; +import org.apache.cassandra.metrics.AccordExecutorMetrics; +import org.apache.cassandra.metrics.AccordReplicaMetrics; +import org.apache.cassandra.metrics.AccordSystemMetrics; +import org.apache.cassandra.metrics.LogLinearDecayingHistograms; +import org.apache.cassandra.metrics.LogLinearDecayingHistograms.LogLinearDecayingHistogram; +import org.apache.cassandra.metrics.ShardedDecayingHistograms; +import org.apache.cassandra.metrics.ShardedDecayingHistograms.DecayingHistogramsShard; +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugExecutor; +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LoadExecutor; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.SaveExecutor; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.UniqueSave; +import org.apache.cassandra.service.accord.execution.ExclusiveExecutor.ExclusiveExecutorTask; +import org.apache.cassandra.service.accord.execution.IOTaskWrapper.WrappableIOTask; +import org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup; +import org.apache.cassandra.service.accord.execution.Task.GlobalGroup; +import org.apache.cassandra.service.accord.execution.Task.State; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.Condition; +import org.apache.cassandra.utils.concurrent.Future; + +import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.AccordCache.CommandAdapter.COMMAND_ADAPTER; +import static org.apache.cassandra.service.accord.execution.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER; +import static org.apache.cassandra.service.accord.execution.AccordCache.registerJfrListener; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status.EVICTED; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.OTHER; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_SCAN; +import static org.apache.cassandra.service.accord.execution.Task.State.FAILED_TO_LOAD; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.SCANNING_RANGES; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_REQUIRED; +import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.COUNTER_MASKS; +import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.selectByOverflowBits; +import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.setOverflowWhenLessEqual; + +/** + * NOTE: We assume that NO BLOCKING TASKS are submitted to this executor AND WAITED ON by another task executing on this executor. + * (as we do not immediately schedule additional threads for submitted tasks, but schedule new threads only if necessary when the submitting execution completes) + */ +public abstract class AccordExecutor implements CacheSize, LoadExecutor, Boolean>, SaveExecutor, Shutdownable, AbstractAsyncExecutor +{ + static final QueuePriorityModel PRIORITY_MODEL; + static final QueueBalancingModel BALANCING_MODEL; + static final long AGE_TO_FIFO; + // PRIORITY_FAIR blends two strategies (flow: least fairly serviced; age: earliest-queued work) by deficit + // round-robin; weights of BLEND_TOTAL come from a single imbalance ramp (onset..onset+width) trading age->flow. + static final int BLEND_SHIFT = 6, BLEND_TOTAL = 1 << BLEND_SHIFT; + static final int FLOW_ONSET, FLOW_WIDTH_SHIFT; + static final boolean BALANCE_BY_POSITION; + static final long GLOBAL_QUEUE_LIMITS, EXCLUSIVE_QUEUE_LIMITS; + static final int NONSYNC_MIN_BATCH_SIZE, NONSYNC_MAX_BATCH_SIZE, NONSYNC_BLOCKED_LIMIT; + static final boolean NONSYNC_ENABLED; + + static + { + AccordConfig config = DatabaseDescriptor.getAccord(); + AGE_TO_FIFO = config.queue_priority_age_to_fifo.to(TimeUnit.MICROSECONDS); + PRIORITY_MODEL = config.queue_priority_model != null ? config.queue_priority_model : QueuePriorityModel.HLC_FIFO; + BALANCING_MODEL = config.queue_balancing_model != null ? config.queue_balancing_model : QueueBalancingModel.BLENDED_PRIORITY_PHASE_FAIR; + FLOW_ONSET = config.queue_flow_imbalance_onset == null ? 4 : config.queue_flow_imbalance_onset; + FLOW_WIDTH_SHIFT = config.queue_flow_imbalance_width_shift == null ? 5 : config.queue_flow_imbalance_width_shift; + NONSYNC_MIN_BATCH_SIZE = config.queue_nonsync_min_batch_size == null ? 16 : config.queue_nonsync_min_batch_size; + NONSYNC_MAX_BATCH_SIZE = config.queue_nonsync_max_batch_size == null ? 64 : config.queue_nonsync_max_batch_size; + NONSYNC_ENABLED = config.queue_nonsync_enabled == null || config.queue_nonsync_enabled; + NONSYNC_BLOCKED_LIMIT = config.queue_nonsync_blocked_limit == null ? 8 : config.queue_nonsync_blocked_limit; + Invariants.require(FLOW_ONSET >= 0 && FLOW_WIDTH_SHIFT >= 0); + switch (BALANCING_MODEL) + { + default: throw new UnhandledEnum(BALANCING_MODEL); + case PRIORITY_ONLY: + case BLENDED_PRIORITY_PHASE_FAIR: + BALANCE_BY_POSITION = true; + break; + case PHASE_ONLY: + case PHASE_FAIR: + BALANCE_BY_POSITION = false; + } + + { + // TODO (required): pick default max loads/saves/range loads based on number of threads + long global = COUNTER_MASKS, exclusive = COUNTER_MASKS; + global ^= (0x7fL ^ 1) << (RANGE_SCAN.ordinal() * 8); + if (config.queue_active_limits != null) + { + long[] limits = parseEnumParams(config.queue_active_limits, "queue_active_limits"); + global = selectByOverflowBits(setOverflowWhenLessEqual(limits[0], 0), global, limits[0]); + exclusive = selectByOverflowBits(setOverflowWhenLessEqual(limits[1], 0), global, limits[1]); + } + GLOBAL_QUEUE_LIMITS = global; + EXCLUSIVE_QUEUE_LIMITS = exclusive; + } + + } + + public static final ShardedDecayingHistograms HISTOGRAMS = new ShardedDecayingHistograms(); + + public interface AccordExecutorFactory + { + AccordExecutor get(int executorId, Mode mode, int threads, IntFunction name, Agent agent); + } + + public enum Mode { RUN_WITH_LOCK, RUN_WITHOUT_LOCK } + + // WARNING: this is a shared object, so close is NOT idempotent + public static final class ExclusiveGlobalCaches extends GlobalCaches implements AutoCloseable + { + final AccordExecutor executor; + + public ExclusiveGlobalCaches(AccordExecutor executor, AccordCache global, AccordCache.Type commands, AccordCache.Type commandsForKey) + { + super(global, commands, commandsForKey); + this.executor = executor; + } + + @Override + public void close() + { + executor.beforeUnlockExternal(); + global.tryShrinkOrEvict(executor.lock); + executor.unlock(TaskRunner.get()); + } + } + + public static class GlobalCaches + { + public final AccordCache global; + public final AccordCache.Type commands; + public final AccordCache.Type commandsForKey; + + public GlobalCaches(AccordCache global, AccordCache.Type commands, AccordCache.Type commandsForKey) + { + this.global = global; + this.commands = commands; + this.commandsForKey = commandsForKey; + } + } + + final LogLinearDecayingHistograms histograms; + final LogLinearDecayingHistogram elapsedPreparingToRun; + final LogLinearDecayingHistogram elapsedWaitingToRun; + final LogLinearDecayingHistogram elapsedRunning; + final LogLinearDecayingHistogram elapsed; + final LogLinearDecayingHistogram keys; + public final AccordReplicaMetrics.Shard replicaMetrics; + + private final Lock lock; + final Agent agent; + public final int executorId; + private final AccordCache cache; + private final ExclusiveGlobalCaches caches; + final AtomicLong uniqueCreatedAt = new AtomicLong(); + final DebugExecutor debug = DebugExecutor.maybeDebug(); + + private long maxWorkingSetSizeInBytes; + private long maxWorkingCapacityInBytes; + + final TaskQueueStandalone> scanningRanges = new TaskQueueStandalone<>(TinyEnumSet.encode(SCANNING_RANGES)); // never queried, just parked here while scanning + final TaskQueueStandalone> loading = new TaskQueueStandalone<>(TinyEnumSet.encode(LOADING_REQUIRED, LOADING_OPTIONAL)); + final TaskQueueStandalone> waitingOnCacheQueues = new TaskQueueStandalone<>(TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL)); + final TaskQueueRunnable runnable = new TaskQueueRunnable<>(); + + private final Tranches tranches = new Tranches(this); + + /** + * Newly submitted work must take a position >= minPosition, but this condition does not apply to consequences of + * previously submitted work; this inherits the originating task's position and tranche. + * This is to ensure afterSubmittedAndConsequences functions correctly. + */ + long minPosition = 1; + long nextPosition = 1; + int tasks; + + private boolean hasPausedLoading; + + private List waitingForQuiescence; + + AccordExecutor(Lock lock, int executorId, Agent agent) + { + this.lock = lock; + this.executorId = executorId; + this.cache = new AccordCache(this, 0); + this.agent = agent; + + final AccordCache.Type commands; + final AccordCache.Type commandsForKey; + commands = cache.newType(TxnId.class, COMMAND_ADAPTER, AccordCacheMetrics.CommandsCacheMetrics.newShard(lock)); + registerJfrListener(executorId, commands, "Command"); + + commandsForKey = cache.newType(RoutingKey.class, CFK_ADAPTER, AccordCacheMetrics.CommandsForKeyCacheMetrics.newShard(lock)); + registerJfrListener(executorId, commandsForKey, "CommandsForKey"); + + this.caches = new ExclusiveGlobalCaches(this, cache, commands, commandsForKey); + + DecayingHistogramsShard histogramsShard = HISTOGRAMS.newShard(lock); + this.histograms = histogramsShard.unsafeGetInternal(); + this.elapsedPreparingToRun = AccordExecutorMetrics.INSTANCE.elapsedPreparingToRun.forShard(histogramsShard); + this.elapsedWaitingToRun = AccordExecutorMetrics.INSTANCE.elapsedWaitingToRun.forShard(histogramsShard); + this.elapsedRunning = AccordExecutorMetrics.INSTANCE.elapsedRunning.forShard(histogramsShard); + this.elapsed = AccordExecutorMetrics.INSTANCE.elapsed.forShard(histogramsShard); + this.keys = AccordExecutorMetrics.INSTANCE.keys.forShard(histogramsShard); + this.replicaMetrics = new AccordReplicaMetrics.Shard(histogramsShard); + } + + abstract boolean isInLoop(); + abstract void beforeUnlockExternal(); + abstract void submit(Consumer sync, Function async, P1 p1); + public abstract boolean hasTasks(); + public abstract boolean isOwningThread(); + + public final Lock unsafeLock() + { + return lock; + } + + public final void lock(TaskRunner self) + { + long startAt = DEBUG_EXECUTION ? Clock.Global.nanoTime() : 0; + if (!self.tryEnterAccordLockedExecutor(this)) + throw new UnsupportedOperationException("To ensure system performance, it is not permitted to utilise multiple AccordExecutor simultaneously with the same thread"); + //noinspection LockAcquiredButNotSafelyReleased + lock.lock(); + if (DEBUG_EXECUTION) debug.onEnterLock(startAt); + } + + public final void unlock(TaskRunner self) + { + self.exitAccordLockedExecutor(); + if (DEBUG_EXECUTION) debug.onExitLock(); + lock.unlock(); + } + + public final boolean tryLock(TaskRunner self) + { + AccordExecutor active = self.accordActiveExecutor(); + if (active != null && active != this) + return false; + + return onTryLock(self, lock.tryLock()); + } + + final boolean onTryLock(TaskRunner self, boolean result) + { + if (result && !self.tryEnterAccordLockedExecutor(this)) + { + // shouldn't be possible, we should have checked this already + lock.unlock(); + throw new IllegalStateException(); + } + return result; + } + + public ExclusiveGlobalCaches lockCaches() + { + lock(TaskRunner.get(Thread.currentThread())); + return caches; + } + + public AccordCache cacheExclusive() + { + Invariants.require(isOwningThread()); + return cache; + } + + public AccordCache cacheUnsafe() + { + return cache; + } + + final boolean hasAlreadyWaitingToRun() + { + return runnable.hasWaitingToRun(); + } + + void updateWaitingToRunExclusive() + { + // TODO (expected): this should not be invoked on every update of waiting to run + maybeUnpauseLoading(); + } + + // drain only new work; specifically leave anything that would call completeTask queued. + // this is to maintain invariants in Tranches.complete, where we may have some consequence of some earlier task + // pending but unqueued, so that we have not incremented its tranche count, and in the interim we set the tranche + // count to zero + void drainUnqueuedNewWorkExclusive() + { + } + + final Task pollAlreadyWaitingToRunExclusive() + { + Task next = runnable.poll(); + if (DEBUG_EXECUTION && next != null) DebugTask.get(next).onPolled(); + return next; + } + + public void waitForQuiescence() + { + TaskRunner self = TaskRunner.get(); + Condition condition; + lock(self); + try + { + if (tasks == 0) + return; + + if (waitingForQuiescence == null) + waitingForQuiescence = new ArrayList<>(); + condition = Condition.newOneTimeCondition(); + waitingForQuiescence.add(condition); + } + finally + { + unlock(self); + } + condition.awaitThrowUncheckedOnInterrupt(); + } + + protected void notifyQuiescentExclusive() + { + if (waitingForQuiescence != null) + { + waitingForQuiescence.forEach(Condition::signalAll); + waitingForQuiescence = null; + } + tranches.finishAll(nextPosition); + } + + public void afterSubmittedAndConsequences(Runnable run) + { + TaskRunner self = TaskRunner.get(); + + lock(self); + try + { + drainUnqueuedNewWorkExclusive(); + if (tasks == 0) + { + run.run(); + return; + } + + tranches.registerWait(run); + } + finally + { + unlock(self); + } + } + + final void maybeUnpauseLoading() + { + if (!hasPausedLoading) + return; + + if (cache.weightedSize() < maxWorkingCapacityInBytes && !runnable.hasWaitingToRun()) + { + hasPausedLoading = false; + runnable.restart(RANGE_SCAN.ordinal()); + runnable.restart(LOAD.ordinal()); + runnable.restart(RANGE_LOAD.ordinal()); + } + } + + final void maybePauseLoading() + { + if (hasPausedLoading) + return; + + if (cache.weightedSize() >= maxWorkingCapacityInBytes || !runnable.hasWaitingToRun()) + { + AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); + hasPausedLoading = true; + runnable.stop(RANGE_SCAN.ordinal()); + runnable.stop(LOAD.ordinal()); + runnable.stop(RANGE_LOAD.ordinal()); + } + } + + public ExclusiveExecutor newExclusiveExecutor(int commandStoreId) + { + return new ExclusiveExecutor(this, commandStoreId); + } + + public ExclusiveAsyncExecutor newExclusiveExecutor() + { + return new ExclusiveExecutor(this); + } + + @Override + public IOTaskLoad load(SafeTask parent, Boolean isForRange, AccordCacheEntry entry) + { + IOTaskLoad result = newLoad(entry, isForRange); + result.submitExclusive(null, parent); + return result; + } + + @Override + public Cancellable save(AccordCacheEntry entry, UniqueSave identity, Runnable save) + { + return new IOTaskSave(this, entry, identity, save).submitExclusive(null, null); + } + + Cancellable submit(Task task) + { + submit(Task::submitExclusive, i -> i, task); + return task; + } + + public Future submit(Runnable run) + { + Task inherit = inherit(); + PlainRunnable task = inherit == null ? new PlainRunnable(this, new AsyncPromise<>(), run, OTHER) + : new PlainRunnable(this, new AsyncPromise<>(), run, OTHER, inherit.position, inherit.tranche()); + submit(task); + return task.result; + } + + public void execute(Runnable run) + { + Task inherit = inherit(); + PlainRunnable task = inherit == null ? new PlainRunnable(this, null, run, OTHER) + : new PlainRunnable(this, null, run, OTHER, inherit.position, inherit.tranche()); + submit(task); + } + + @Override + public Cancellable execute(RunOrFail runOrFail) + { + Task inherit = inherit(); + PlainChain submit = inherit == null ? new PlainChain(this, runOrFail, null, ExclusiveGroup.OTHER) + : new PlainChain(this, runOrFail, null, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); + return submit(submit); + } + + public AsyncChain buildDebuggable(Callable call, Object describe) + { + return new AsyncChains.Head<>() + { + @Override + protected Cancellable start(BiConsumer callback) + { + Task inherit = inherit(); + return submit(inherit == null ? new PlainChainDebuggable(AccordExecutor.this, new CallAndCallback<>(call, callback), null, describe) + : new PlainChainDebuggable(AccordExecutor.this, new CallAndCallback<>(call, callback), null, inherit.position, inherit.tranche(), describe)); + } + }; + } + + public void submitExclusive(Runnable runnable) + { + new PlainRunnable(this, null, runnable, OTHER).submitExclusive(null, null); + } + + Cancellable submitExclusive(Task parent, GlobalGroup group, WrappableIOTask task) + { + return new IOTaskWrapper(this, task, group, parent.position, parent.tranche()).submitExclusive(null, parent); + } + + Task inherit() + { + return inherit(Thread.currentThread()); + } + + Task inherit(Thread thread) + { + TaskRunner self = TaskRunner.get(thread); + + if (self.accordActiveExecutor() != this) + return null; + + Task task = self.accordActiveSelfTask(); + if (task instanceof ExclusiveExecutorTask) + task = ((ExclusiveExecutorTask)task).queue.task; + return task; + } + + void registerConsequenceExclusive(Task parent, Task task) + { + ++tasks; + int tranche = parent.tranche(); + tranches.addInherited(tranche, parent.position); + task.position = parent.position; + task.setInheritedWithTranche(tranche); + } + + void registerExclusive(Task task) + { + ++tasks; + if (task.hasInherited()) + { + tranches.addInherited(task.tranche(), task.position); + } + else + { + long position = task.position; + if (position == 0) + { + task.position = position = nextPosition++; + } + else + { + long delta = nextPosition - position; + if (delta >= AGE_TO_FIFO) + task.position = position = nextPosition++; + else if (delta <= 0) + nextPosition = position + 1; + else if (position < minPosition) + task.position = position = minPosition; + } + task.setTranche(tranches.addNew(position)); + } + } + + final void cleanupTaskExclusive(Task task, boolean executed) + { + runnable.cleanup(task); + try { task.cleanupExclusive(this, executed); } + finally + { + cache.tryShrinkOrEvict(lock); + maybeUnpauseLoading(); + } + } + + final void unregisterExclusive(Task task) + { + int tranch = task.tranche(); + --tasks; + tranches.complete(tranch); + } + + void onScannedRangesExclusive(SafeTask task, Throwable fail) + { + // the task may have already been cancelled, in which case we don't need to fail it + if (task.state().isExecuted()) + return; + + if (fail != null) failExclusive(task, fail, FAILED_TO_LOAD); + else task.rangeScanner().scannedExclusive(); + } + + private void failExclusive(SafeTask task, Throwable fail, State newState) + { + if (task.state().isExecuted()) + return; + + try { task.failExclusive(fail, newState); } + catch (Throwable t) { agent.onException(t); } + finally + { + task.unqueueIfQueued(); + cleanupTaskExclusive(task, false); + } + } + + void onSavedExclusive(AccordCacheEntry state, Object identity, Throwable fail) + { + cache.saved(state, identity, fail); + } + + void onLoadedExclusive(AccordCacheEntry loaded, V value, Throwable fail) + { + if (loaded.status() == EVICTED) + return; + + try (ArrayBuffers.BufferList> tasks = loaded.drainWaitingToLoad()) + { + if (fail != null) + { + for (SafeTask task : tasks) + failExclusive(task, fail, FAILED_TO_LOAD); + cache.failedToLoad(loaded); + } + else + { + cache.loaded(loaded, value); + for (SafeTask task : tasks) + task.onLoadOneExclusive(loaded); + } + } + + maybePauseLoading(); + } + + public void executeDirectlyWithLock(Runnable command) + { + TaskRunner self = TaskRunner.get(); + lock(self); + try + { + self.setAccordActiveExecutor(this); + command.run(); + } + finally + { + beforeUnlockExternal(); + self.setAccordActiveExecutor(null); + unlock(self); + } + } + + @Override + public void setCapacity(long bytes) + { + Invariants.require(isOwningThread()); + cache.setCapacity(bytes); + maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; + } + + public void setWorkingSetSize(long bytes) + { + Invariants.require(isOwningThread()); + maxWorkingSetSizeInBytes = bytes; + maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; + if (maxWorkingCapacityInBytes < maxWorkingSetSizeInBytes) + maxWorkingCapacityInBytes = Long.MAX_VALUE; + } + + @Override + public long capacity() + { + return cache.capacity(); + } + + @Override + public int size() + { + return cache.size(); + } + + @Override + public long weightedSize() + { + return cache.weightedSize(); + } + + public int unsafePreparingToRunCount() + { + return loading.size() + scanningRanges.size(); + } + + public int unsafeWaitingToRunCount() + { + return runnable.waitingCount(); + } + + public int unsafeRunningCount() + { + return runnable.assigned.size(); + } + + public Stream active() + { + return Stream.of(); + } + + public int executorId() + { + return executorId; + } + + public static IntFunction constant(O out) + { + return ignore -> out; + } + + IOTaskLoad newLoad(AccordCacheEntry entry, boolean isForRange) + { + return new IOTaskLoad<>(this, entry, isForRange ? RANGE_LOAD : LOAD); + } + + public List taskSnapshot() + { + List result = new ArrayList<>(); + TaskRunner self = TaskRunner.get(); + lock(self); + try + { + addToSnapshot(result, scanningRanges, TaskInfo.Status.SCANNING_RANGES, TaskInfo.Status.SCANNING_RANGES); + addToSnapshot(result, loading, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); + for (TaskQueue queue : runnable.queues) + { + if (queue != null) + addToSnapshot(result, queue, TaskInfo.Status.WAITING_TO_RUN, TaskInfo.Status.WAITING_TO_RUN); + } + addToSnapshot(result, runnable.assigned, TaskInfo.Status.RUNNING, TaskInfo.Status.WAITING_TO_RUN); + } + finally + { + unlock(self); + } + result.sort(TaskInfo::compareTo); + return result; + } + + private static void addToSnapshot(List snapshot, TaskQueue queue, TaskInfo.Status ifCurrent, TaskInfo.Status ifQueued) + { + for (int i = 0 ; i < queue.size() ; ++i) + { + Task t = queue.getSingle(i); + if (t instanceof ExclusiveExecutorTask) + { + ExclusiveExecutor q = ((ExclusiveExecutorTask) t).queue; + snapshot.add(new TaskInfo(ifCurrent, q.commandStoreId, q.task)); + for (TaskQueue q0 : q.queues) + { + if (q0 != null) + { + for (int j = 0 ; j < q0.size() ; ++j) + snapshot.add(new TaskInfo(ifQueued, q.commandStoreId, q0.getSingle(j))); + } + } + } + else + { + int commmandStoreId = t instanceof SafeTask ? ((SafeTask) t).commandStore.id() : -1; + snapshot.add(new TaskInfo(ifCurrent, commmandStoreId, t)); + } + } + } + + private static long[] parseEnumParams(String input, String describe) + { + String[] specs = input.split(";"); + if (specs.length != 2) + throw new IllegalArgumentException("Invalid specifiers in " + describe + "; expect [GlobalGroup];[ExclusiveGroup] but got: " + input); + long[] result = new long[2]; + result[0] = parseEnumParams(GlobalGroup::valueOf, specs[0], describe + " for GlobalGroup"); + result[1] = parseEnumParams(ExclusiveGroup::valueOf, specs[1], describe + " for ExclusiveGroup"); + return result; + } + + private static long parseEnumParams(Function> get, String input, String describe) + { + long result = 0; + for (String spec : input.split(",")) + { + if (spec.trim().isEmpty()) continue; + + String[] split = spec.split(":"); + if (split.length != 2) + throw new IllegalArgumentException("Invalid specifier " + spec + " in " + describe + ": " + input); + + try + { + Enum queue = get.apply(split[0]); + long value = Long.parseLong(split[1]); + if (value <= 0 || value >= 128) + throw new IllegalArgumentException("Invalid limit " + value + " in queue_active_limits: " + input); + + result |= value << (queue.ordinal() * 8); + } + catch (Throwable t) + { + throw new IllegalArgumentException("Invalid queue identifier " + split[0] + " in " + describe + ": " + input); + } + } + + return result; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorAsyncSubmit.java similarity index 88% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorAsyncSubmit.java index a2c57a4556f1..55a0086cf1db 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorAsyncSubmit.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.function.IntFunction; @@ -26,9 +26,9 @@ import org.apache.cassandra.utils.concurrent.LockWithAsyncSignal; // WARNING: experimental - needs more testing -public class AccordExecutorAsyncSubmit extends AccordExecutorAbstractSemiSyncSubmit +public class AccordExecutorAsyncSubmit extends AbstractSemiSyncSubmit { - private final AccordExecutorLoops loops; + private final Loops loops; private final LockWithAsyncSignal lock; public AccordExecutorAsyncSubmit(int executorId, Mode mode, int threads, IntFunction name, Agent agent) @@ -40,7 +40,7 @@ private AccordExecutorAsyncSubmit(LockWithAsyncSignal lock, int executorId, Mode { super(lock, executorId, agent); this.lock = lock; - this.loops = new AccordExecutorLoops(mode, threads, name, this::task); + this.loops = new Loops(mode, threads, name, this::task); } @Override @@ -52,7 +52,7 @@ void awaitExclusive() throws InterruptedException } @Override - AccordExecutorLoops loops() + Loops loops() { return loops; } @@ -76,7 +76,7 @@ void notifyWorkExclusive() } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isOwner(Thread.currentThread()); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java similarity index 89% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java index 67e897d47964..2bafc5b4b625 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; @@ -26,9 +26,9 @@ import accord.api.Agent; // WARNING: experimental - needs more testing -public class AccordExecutorSemiSyncSubmit extends AccordExecutorAbstractSemiSyncSubmit +public class AccordExecutorSemiSyncSubmit extends AbstractSemiSyncSubmit { - private final AccordExecutorLoops loops; + private final Loops loops; private final ReentrantLock lock; private final Condition hasWork; private int waiting; @@ -43,7 +43,7 @@ private AccordExecutorSemiSyncSubmit(ReentrantLock lock, int executorId, Mode mo super(lock, executorId, agent); this.lock = lock; this.hasWork = lock.newCondition(); - this.loops = new AccordExecutorLoops(mode, threads, name, this::task); + this.loops = new Loops(mode, threads, name, this::task); } @Override @@ -61,7 +61,7 @@ private void awaitWork() throws InterruptedException } @Override - AccordExecutorLoops loops() + Loops loops() { return loops; } @@ -87,7 +87,7 @@ void notifyWorkExclusive() } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isHeldByCurrentThread(); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java similarity index 92% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java index 392ce2a0154b..acb5eb7202d6 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSignalLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java @@ -16,33 +16,33 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.IntFunction; import javax.annotation.Nullable; import accord.api.Agent; import accord.utils.Invariants; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; import org.apache.cassandra.utils.concurrent.SignalLock; import static accord.utils.Invariants.nonNull; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.UNINITIALIZED; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.Task.State.UNINITIALIZED; -public class AccordExecutorSignalLoop extends AccordExecutorAbstractLoop +public class AccordExecutorSignalLoop extends AbstractLoop { private static class ShutdownException extends RuntimeException {} private static final int MAX_LOOPS = 1000; // limit the amount of time we hold the lock for private final SignalLock lock; - private final AccordExecutorLoops loops; + private final Loops loops; private int readyToRunTarget = 1; private final int readyToRunLimit; // TODO (desired): intrusive queue using Task.next, but a little challenging because we reuse SequentialQueueTask so have ABA problem @@ -64,13 +64,13 @@ public AccordExecutorSignalLoop(SignalLock lock, int executorId, Mode mode, int super(lock, executorId, agent); Invariants.require(threads <= SignalLock.MAX_THREADS); this.lock = lock; - this.loops = new AccordExecutorLoops(mode, threads, name, this::task); + this.loops = new Loops(mode, threads, name, this::task); this.readyToRunLimit = Math.min(threads * 4, SignalLock.MAX_SIGNAL_COUNT); } - void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + void submit(Consumer sync, Function async, P1 p1) { - Task next = async.apply(p1a, p2, p3, p4); + Task next = async.apply(p1); Task prev = push(next); if (prev == null) lock.signalLockWork(); @@ -134,7 +134,7 @@ final void drainUnqueuedNewWorkExclusive() { Task next = submit.next; submit.next = null; - submit.submitExclusive(this); + submit.submitExclusive(); submit = next; } } @@ -159,7 +159,7 @@ private boolean enqueueOnePending() pendingNewHead = destructiveNext(submit); if (pendingNewHead == null) pendingNewTail = null; - submit.submitExclusive(this); + submit.submitExclusive(); } return true; } @@ -315,7 +315,7 @@ private boolean updateAndEnqueuePendingUntilHasWaitingToRun(int processAtLeast) return hasAlreadyWaitingToRun(); } - class LoopTask extends AccordExecutorLoops.LoopTask + class LoopTask extends Loops.LoopTask { final int index; @@ -325,7 +325,7 @@ class LoopTask extends AccordExecutorLoops.LoopTask this.index = index; } - private Task awaitWork(AccordTaskRunner self) + private Task awaitWork(TaskRunner self) { while (true) { @@ -353,7 +353,7 @@ private Task awaitWork(AccordTaskRunner self) } } - private Task executedAndMaybeGetWork(AccordTaskRunner self, @Nullable Task executed) + private Task executedAndMaybeGetWork(TaskRunner self, @Nullable Task executed) { if (lock.tryAcquireAsyncWork()) { @@ -394,7 +394,7 @@ private Task pushExecutedAndReturn(Task complete, Task result) return result; } - final boolean tryLock(AccordTaskRunner self) + final boolean tryLock(TaskRunner self) { if (self.accordLockedExecutor() != null) return false; @@ -402,7 +402,7 @@ final boolean tryLock(AccordTaskRunner self) return onTryLock(self, lock.tryLock(index)); } - final boolean unlockAndAcquire(AccordTaskRunner self) + final boolean unlockAndAcquire(TaskRunner self) { self.exitAccordLockedExecutor(); if (DEBUG_EXECUTION) debug.onExitLock(); @@ -413,7 +413,7 @@ final boolean unlockAndAcquire(AccordTaskRunner self) public void run() { Thread thread = Thread.currentThread(); - AccordTaskRunner self = AccordTaskRunner.get(thread); + TaskRunner self = TaskRunner.get(thread); self.setAccordActiveExecutor(AccordExecutorSignalLoop.this); setWrapped(self); @@ -482,7 +482,7 @@ public void shutdown() } @Override - AccordExecutorLoops loops() + Loops loops() { return loops; } @@ -494,7 +494,7 @@ boolean isInLoop() } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isOwner(); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java similarity index 90% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java index cadd7a766eb3..4d2508b87ff6 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java @@ -16,22 +16,22 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.IntFunction; import accord.api.Agent; import accord.utils.Invariants; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; import org.apache.cassandra.concurrent.ExecutorPlus; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; -class AccordExecutorSimple extends AccordExecutor +public class AccordExecutorSimple extends AccordExecutor { final ExecutorPlus executor; final ReentrantLock lock; @@ -82,7 +82,7 @@ void beforeUnlockExternal() protected void run() { - AccordTaskRunner self = AccordTaskRunner.get(); + TaskRunner self = TaskRunner.get(); self.setAccordActiveExecutor(AccordExecutorSimple.this); lock.lock(); try @@ -126,12 +126,12 @@ protected void run() } @Override - void submit(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + void submit(Consumer sync, Function async, P1 p1) { lock.lock(); try { - sync.accept(this, p1s, p2, p3, p4); + sync.accept(p1); } finally { @@ -155,7 +155,7 @@ final Task pollWaitingToRunExclusive() } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isHeldByCurrentThread(); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSyncSubmit.java similarity index 79% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java rename to src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSyncSubmit.java index e4efac3ad46d..e6d47f8910de 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSyncSubmit.java @@ -16,20 +16,20 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.IntFunction; import accord.api.Agent; -import accord.utils.QuadFunction; -import accord.utils.QuintConsumer; -public class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop +public class AccordExecutorSyncSubmit extends AbstractLockLoop { - private final AccordExecutorLoops loops; + private final Loops loops; private final ReentrantLock lock; private final Condition hasWork; @@ -48,7 +48,7 @@ private AccordExecutorSyncSubmit(ReentrantLock lock, int executorId, Mode mode, super(lock, executorId, agent); this.lock = lock; this.hasWork = lock.newCondition(); - this.loops = new AccordExecutorLoops(mode, threads, name, this::task); + this.loops = new Loops(mode, threads, name, this::task); } @Override @@ -58,7 +58,7 @@ void awaitExclusive() throws InterruptedException } @Override - AccordExecutorLoops loops() + Loops loops() { return loops; } @@ -70,7 +70,7 @@ boolean isInLoop() } @Override - boolean isOwningThread() + public boolean isOwningThread() { return lock.isHeldByCurrentThread(); } @@ -89,13 +89,13 @@ void notifyWorkExclusive() hasWork.signal(); } - void submitExternal(QuintConsumer sync, QuadFunction async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4) + void submitExternal(Consumer sync, Function async, P1 p1) { - AccordTaskRunner self = AccordTaskRunner.get(); + TaskRunner self = TaskRunner.get(); lock(self); try { - submitExternalExclusive(sync, p1s, p2, p3, p4); + submitExternalExclusive(sync, async, p1); } finally { diff --git a/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java b/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java new file mode 100644 index 000000000000..c171ed08de9b --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +final class CancelTask extends Task +{ + final Task cancel; + + CancelTask(Task cancel) + { + super(GlobalGroup.OTHER); + this.cancel = cancel; + } + + @Override + void submitExclusive() + { + cancel.cancelExclusive(); + } + + @Override + boolean isNewWork() + { + return false; + } + + @Override + String toDescription() + { + return "Cancel " + cancel.toDescription(); + } + + @Override void preRunExclusive() { throw new UnsupportedOperationException(); } + @Override void run() { throw new UnsupportedOperationException(); } + @Override void reportFailure(Throwable fail) { throw new UnsupportedOperationException(); } + @Override public void cancel() { throw new UnsupportedOperationException(); } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java b/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java new file mode 100644 index 000000000000..b0bd8d37e6fc --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java @@ -0,0 +1,390 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.concurrent.Callable; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.concurrent.locks.LockSupport; + +import accord.api.ExclusiveAsyncExecutor; +import accord.local.ExecutionContext; +import accord.utils.IntrusiveHeapNode; +import accord.utils.Invariants; +import accord.utils.async.AsyncCallbacks; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.service.accord.debug.DebugExecution; + +import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.COMMAND_STORE; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.service.accord.execution.TaskQueueRunnable.RUNNABLE; + +public final class ExclusiveExecutor extends TaskQueueMulti implements ExclusiveAsyncExecutor +{ + private static final AtomicReferenceFieldUpdater ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(ExclusiveExecutor.class, Thread.class, "owner"); + + static final class ExclusiveExecutorTask extends Task + { + final ExclusiveExecutor queue; + + ExclusiveExecutorTask(ExclusiveExecutor queue) + { + super(COMMAND_STORE); + this.queue = queue; + } + + @Override void submitExclusive() { throw new UnsupportedOperationException(); } + @Override boolean isNewWork() { throw new UnsupportedOperationException(); } + @Override public void cancel() { throw new UnsupportedOperationException(); } + + @Override + void preRunExclusive() + { + super.preRunExclusive(); + queue.preRunTask(); + } + + @Override + void run() + { + queue.runTask(); + } + + @Override + void cleanupExclusive(AccordExecutor executor, boolean executed) + { + unsafeSetStateExclusive(WAITING_TO_RUN); + queue.cleanupTask(executed); + } + + @Override + void reportFailure(Throwable t) + { + queue.task.reportFailure(t); + } + + @Override + String toDescription() + { + return queue.task.toDescription(); + } + + protected boolean isInHeap() + { + return super.isInHeap(); + } + } + + private final AccordExecutor executor; + final int commandStoreId; + final ExclusiveExecutorTask selfTask; + Task task; + volatile Thread owner, waiting; + private boolean stopped; + private volatile boolean visibleStopped; + private boolean terminated; + + final DebugExecution.DebugExclusiveExecutor debug; + + ExclusiveExecutor(AccordExecutor executor) + { + this(executor, -1); + } + + ExclusiveExecutor(AccordExecutor executor, int commandStoreId) + { + super(RUNNABLE, commandStoreId < 0 ? Task.GroupKind.NONE : Task.GroupKind.EXCLUSIVE, AccordExecutor.EXCLUSIVE_QUEUE_LIMITS); + this.executor = executor; + this.commandStoreId = commandStoreId; + this.selfTask = new ExclusiveExecutorTask(this); + this.debug = DebugExecution.DebugExclusiveExecutor.maybeDebug(executor.debug, commandStoreId); + } + + void preRunTask() + { + task.preRunExclusive(); + } + + void runTask() + { + Thread self = Thread.currentThread(); + if (!ownerUpdater.compareAndSet(this, null, self)) + { + if (DEBUG_EXECUTION) debug.onWaiting(); + Invariants.require(waiting == null); + waiting = self; + outer: + do + { + while (true) + { + Thread owner = this.owner; + if (owner == self) break outer; + if (owner == null) continue outer; + LockSupport.park(); + } + } + while (!ownerUpdater.compareAndSet(this, null, self)); + Invariants.require(waiting == self); + waiting = null; + } + + try + { + if (stopped && reject(task)) + task.reportFailure(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((SafeTask) task).executionContext())); + else + task.run(); + } + finally + { + // NOTE: we can ONLY safely release owner here due to AccordCacheEntry locking, which remains in place until AccordTask.releaseResourcesExclusive + // this also relies on AccordSafeCommandStore$ExclusiveCaches.acquireIfLoaded returning false when the entry is locked + owner = null; + } + } + + private boolean reject(Task task) + { + if (!(task instanceof SafeTask)) + return true; + + ExecutionContext context = ((SafeTask) task).executionContext(); + return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); + } + + void cleanupTask(boolean executed) + { + try + { + task.unsetQueue(this); + task.cleanupExclusive(executor, executed); + } + finally + { + active = 0; + task = super.pollMulti(); + if (DEBUG_EXECUTION) debug.onSetTask(task); + if (task != null) + { + selfTask.position = task.position; + selfTask.unsafeSetStateExclusive(WAITING_TO_RUN); + executor.runnable.enqueue(selfTask, false); + } + } + } + + void enqueue(Task newTask, boolean incrementArrivals) + { + + if (task != null) + { + if (incrementArrivals) + executor.runnable.incrementArrivals(selfTask); + // TODO (expected): restore some invariant here +// Invariants.require(selfTask.isInHeap() || selfTask.is(RUNNING)); + super.enqueueMulti(newTask, incrementArrivals); + } + else + { + Invariants.require(isEmptySingle()); + if (incrementArrivals) + incrementArrivals(newTask); + incrementDispatches(newTask); + task = newTask; + task.setQueue(this); + selfTask.position = newTask.position; + selfTask.unsafeSetStateExclusive(WAITING_TO_RUN); + executor.runnable.enqueue(selfTask, incrementArrivals); + if (DEBUG_EXECUTION) debug.onSetTask(newTask); + } + } + + @Override + void unqueue(Task remove) + { + if (remove == task) removeCurrentTask(remove); + else super.unqueueMulti(remove); + } + + boolean tryUnqueueWaiting(Task remove) + { + if (remove == task) return tryRemoveCurrentTask(remove); + else return super.tryUnqueueWaiting(remove); + } + + private boolean tryRemoveCurrentTask(IntrusiveHeapNode remove) + { + if (executor.runnable.isAssigned(selfTask)) + return false; + + removeCurrentTask(remove); + return true; + } + + private void removeCurrentTask(IntrusiveHeapNode remove) + { + Invariants.require(remove == task); + // cannot overwrite task while it is being executed - this cannot happen for AccordTask + // but can for other tasks that don't track their own state + + decrementDispatches(task); + task.unsetQueue(this); + task = pollMulti(); + if (DEBUG_EXECUTION) debug.onSetTask(task); + if (executor.runnable.isWaiting(selfTask)) + { + if (task == null) executor.runnable.unqueue(selfTask); + else + { + selfTask.position = task.position; + executor.runnable.requeue(selfTask); + } + } + else + { + Invariants.expect(false, "%s should have been queued to run as it had the task %s pending, that has now been cancelled", this, remove); + if (task != null) + { + selfTask.position = task.position; + selfTask.unsafeSetStateExclusive(WAITING_TO_RUN); + executor.runnable.enqueue(selfTask, false); + } + } + Invariants.require(task == null || executor.runnable.isWaiting(selfTask)); + } + + public boolean inExecutor() + { + return owner == Thread.currentThread(); + } + + public boolean stopped() + { + return visibleStopped; + } + + public void stop() + { + Invariants.require(inExecutor()); + this.stopped = true; + this.visibleStopped = true; + } + + public void terminate() + { + Invariants.require(inExecutor()); + this.visibleStopped = this.terminated = this.stopped = true; + } + + @Override + public AsyncChain chain(Runnable run) + { + return AsyncChains.chain(this, run); + } + + @Override + public AsyncChain chain(Callable call) + { + return AsyncChains.chain(this, call); + } + + @Override + public AsyncChain flatChain(Callable> call) + { + return AsyncChains.flatChain(this, call); + } + + Task inherit() + { + Thread thread = Thread.currentThread(); + if (thread == owner) + return Task.unwrap(task); + + return executor.inherit(thread); + } + + @Override + public void execute(Runnable run) + { + Task inherit = inherit(); + PlainRunnable submit = inherit == null ? new PlainRunnable(executor, null, run, this, Task.ExclusiveGroup.OTHER) + : new PlainRunnable(executor, null, run, this, Task.ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); + executor.submit(submit); + } + + @Override + public Cancellable execute(AsyncCallbacks.RunOrFail runOrFail) + { + Task inherit = inherit(); + PlainChain submit = inherit == null ? new PlainChain(executor, runOrFail, ExclusiveExecutor.this, Task.ExclusiveGroup.OTHER) + : new PlainChain(executor, runOrFail, ExclusiveExecutor.this, Task.ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); + return executor.submit(submit); + } + + @Override + public boolean tryExecuteImmediately(Runnable run) + { + Thread thread = Thread.currentThread(); + Thread owner = this.owner; + if (owner != null && owner != thread) + return false; + + TaskRunner self = TaskRunner.get(thread); + AccordExecutor active = self.accordActiveExecutor(); + if (active != null && active != executor) + return false; // prevent cross-executor locking/execution + + if (owner == null && !ownerUpdater.compareAndSet(this, null, thread)) + return false; + + try + { + if (active == null) + self.setAccordActiveExecutor(executor); + + run.run(); + } + catch (Throwable t) + { + executor.agent.onException(t); + } + finally + { + if (owner == null) + { + Thread waiting = this.waiting; + Invariants.require(waiting != self); + this.owner = waiting; + if (waiting == null) // recheck, to ensure happens-before relation with a new waiter that expects any non-null owner to notify it + waiting = this.waiting; + if (waiting != null) + LockSupport.unpark(waiting); + } + + if (active == null) + self.setAccordActiveExecutor(null); + } + return true; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTask.java b/src/java/org/apache/cassandra/service/accord/execution/IOTask.java new file mode 100644 index 000000000000..68c976eb7bac --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTask.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.async.Cancellable; + +import org.apache.cassandra.concurrent.DebuggableTask; + +// a task that may be submitted to this executor or another +public abstract class IOTask extends Plain implements Cancellable, DebuggableTask +{ + IOTask(AccordExecutor executor, GlobalGroup group, long position, int tranche) + { + super(executor, group, position, tranche); + } + + IOTask(AccordExecutor executor, GlobalGroup group) + { + super(executor, group); + } + + abstract void postRunExclusive(); + + @Override + void cleanupExclusive(AccordExecutor executor, boolean executed) + { + super.cleanupExclusive(executor, executed); + postRunExclusive(); + } + + @Override + ExclusiveExecutor exclusiveExecutor() + { + return null; + } + + @Override + public long creationTimeNanos() + { + return createdAt; + } + + @Override + public long startTimeNanos() + { + return runningAt; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java new file mode 100644 index 000000000000..aa54cfcf2a2b --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.Invariants; + +import org.apache.cassandra.utils.Closeable; + +import static org.apache.cassandra.service.accord.execution.IOTaskLoad.FailureHolder.NOT_STARTED; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; + +public class IOTaskLoad extends IOTask +{ + static class FailureHolder + { + static final FailureHolder NOT_STARTED = new FailureHolder(new RuntimeException("Not started")); + + final Throwable fail; + + FailureHolder(Throwable fail) + { + this.fail = fail; + } + } + + final AccordCacheEntry entry; + Object result = NOT_STARTED; + + IOTaskLoad(AccordExecutor executor, AccordCacheEntry entry, GlobalGroup group) + { + super(executor, group); + Invariants.require(group == LOAD || group == RANGE_LOAD); + this.entry = entry; + } + + void postRunExclusive() + { + if (!(result instanceof FailureHolder)) + executor.onLoadedExclusive(entry, (V) result, null); + else + executor.onLoadedExclusive(entry, null, ((FailureHolder) result).fail); + } + + @Override + String toDescription() + { + return "Load " + entry.key(); + } + + @Override + public void run() + { + onRunning(); + try (Closeable close = resources.get()) + { + result = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); + } + onRunComplete(); + } + + @Override + void reportFailure(Throwable t) + { + result = new FailureHolder(t); + } + + @Override + public String description() + { + return "Loading " + entry; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java new file mode 100644 index 000000000000..d010b6e1f102 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import org.apache.cassandra.utils.Closeable; + +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.SAVE; + +class IOTaskSave extends IOTask +{ + private static final Throwable NOT_STARTED = new Throwable(); + + final AccordCacheEntry entry; + final AccordCacheEntry.UniqueSave identity; + final Runnable run; + Throwable failure = NOT_STARTED; + + IOTaskSave(AccordExecutor executor, AccordCacheEntry entry, AccordCacheEntry.UniqueSave identity, Runnable run) + { + super(executor, SAVE); + this.entry = entry; + this.identity = identity; + this.run = run; + } + + @Override + void postRunExclusive() + { + executor.onSavedExclusive(entry, identity, failure); + } + + @Override + String toDescription() + { + return "Save " + entry.key(); + } + + @Override + public void run() + { + onRunning(); + try (Closeable close = resources.get()) + { + run.run(); + } + onRunComplete(); + failure = null; + } + + @Override + protected void reportFailure(Throwable t) + { + failure = t; + } + + @Override + public String description() + { + return "Save " + entry; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java new file mode 100644 index 000000000000..ce6d60d5308d --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import org.apache.cassandra.utils.Closeable; + +class IOTaskWrapper extends IOTask +{ + static abstract class WrappableIOTask + { + abstract void runInternal(); + abstract void postRunExclusive(); + abstract void fail(Throwable t); + abstract String description(); + } + + final WrappableIOTask wrapped; + + IOTaskWrapper(AccordExecutor executor, WrappableIOTask wrap, GlobalGroup group, long position, int tranche) + { + super(executor, group, position, tranche); + this.wrapped = wrap; + } + + @Override + String toDescription() + { + return wrapped.description(); + } + + @Override + protected void run() + { + onRunning(); + try (Closeable close = resources.get()) + { + wrapped.runInternal(); + } + onRunComplete(); + } + + @Override + void postRunExclusive() + { + wrapped.postRunExclusive(); + } + + @Override + public String description() + { + return wrapped.description(); + } + + @Override + protected void reportFailure(Throwable fail) + { + wrapped.fail(fail); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorLoops.java b/src/java/org/apache/cassandra/service/accord/execution/Loops.java similarity index 87% rename from src/java/org/apache/cassandra/service/accord/AccordExecutorLoops.java rename to src/java/org/apache/cassandra/service/accord/execution/Loops.java index 3eddaaf9e28a..d15afc51c69d 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordExecutorLoops.java +++ b/src/java/org/apache/cassandra/service/accord/execution/Loops.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -29,19 +29,19 @@ import accord.utils.TriFunction; import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; -import org.apache.cassandra.service.accord.AccordExecutor.Mode; -import org.apache.cassandra.service.accord.AccordExecutor.TaskRunner; +import org.apache.cassandra.service.accord.execution.AccordExecutor.Mode; +import org.apache.cassandra.service.accord.execution.TaskRunner.DebuggableWrapper; import org.apache.cassandra.utils.concurrent.Condition; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorThreadTag.INFINITE_LOOP; import static org.apache.cassandra.concurrent.ExecutorFactory.SystemThreadTag.NON_DAEMON; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; import static org.apache.cassandra.utils.Clock.Global.nanoTime; -class AccordExecutorLoops +class Loops { - static abstract class LoopTask extends TaskRunner implements Runnable + static abstract class LoopTask extends DebuggableWrapper implements Runnable { final String id; LoopTask(String id) { this.id = id; } @@ -54,7 +54,7 @@ static abstract class LoopTask extends TaskRunner implements Runnable private final AtomicInteger running = new AtomicInteger(); private final Condition terminated = Condition.newOneTimeCondition(); - public AccordExecutorLoops(Mode mode, int threads, IntFunction loopName, TriFunction loopFactory) + public Loops(Mode mode, int threads, IntFunction loopName, TriFunction loopFactory) { Invariants.require(mode == RUN_WITH_LOCK ? threads == 1 : threads >= 1); running.addAndGet(threads); diff --git a/src/java/org/apache/cassandra/service/accord/execution/Plain.java b/src/java/org/apache/cassandra/service/accord/execution/Plain.java new file mode 100644 index 000000000000..aff55550e28a --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Plain.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.concurrent.CancellationException; + +import accord.utils.Invariants; +import accord.utils.async.Cancellable; + +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; + +abstract class Plain extends Task implements Cancellable +{ + final AccordExecutor executor; + + Plain(AccordExecutor executor, GlobalGroup group, long position, int tranche) + { + super(group, position, tranche); + this.executor = executor; + } + + Plain(AccordExecutor executor, ExclusiveGroup group, long position, int tranche) + { + super(group, position, tranche); + this.executor = executor; + } + + Plain(AccordExecutor executor, GlobalGroup group) + { + super(group); + this.executor = executor; + } + + Plain(AccordExecutor executor, ExclusiveGroup group) + { + super(group); + this.executor = executor; + } + + abstract ExclusiveExecutor exclusiveExecutor(); + + @Override + public void cancel() + { + executor.submit(Task::cancelExclusive, CancelTask::new, this); + } + + void cancelExclusive() + { + ExclusiveExecutor exclusiveExecutor = exclusiveExecutor(); + if ((exclusiveExecutor == null ? executor.runnable : exclusiveExecutor).tryUnqueueWaiting(this)) + { + try + { + failExclusive(new CancellationException(), CANCELLED); + } + catch (Throwable t) + { + executor.agent.onException(t); + } + finally + { + executor.cleanupTaskExclusive(this, false); + } + } + } + + @Override + final void submitExclusive() + { + submitExclusive(exclusiveExecutor(), null); + } + + final Cancellable submitExclusive(ExclusiveExecutor exclusiveExecutor, Task parent) + { + Invariants.require(executor.isOwningThread()); + setStateExclusive(WAITING_TO_RUN); + + if (parent == null) executor.registerExclusive(this); + else executor.registerConsequenceExclusive(parent, this); + onLoaded(); + + if (exclusiveExecutor == null) executor.runnable.enqueue(this, true); + else exclusiveExecutor.enqueue(this, true); + return this; + } + + @Override + protected boolean isNewWork() + { + return true; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java b/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java new file mode 100644 index 000000000000..c729f58e841f --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import javax.annotation.Nullable; + +import accord.utils.async.AsyncCallbacks; + +import org.apache.cassandra.utils.Closeable; + +class PlainChain extends Plain +{ + final AsyncCallbacks.RunOrFail runOrFail; + final @Nullable ExclusiveExecutor exclusiveExecutor; + + PlainChain(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group) + { + super(executor, group); + this.runOrFail = runOrFail; + this.exclusiveExecutor = exclusiveExecutor; + } + + PlainChain(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group, long position, int tranche) + { + super(executor, group, position, tranche); + this.runOrFail = runOrFail; + this.exclusiveExecutor = exclusiveExecutor; + } + + @Override + ExclusiveExecutor exclusiveExecutor() + { + return exclusiveExecutor; + } + + @Override + String toDescription() + { + return runOrFail.toString(); + } + + @Override + protected void run() + { + onRunning(); + try (Closeable close = resources.get()) + { + runOrFail.run(); + } + catch (Throwable t) + { + // shouldn't throw exceptions + executor.agent.onException(t); + } + onRunComplete(); + } + + @Override + protected void reportFailure(Throwable fail) + { + try + { + runOrFail.fail(fail); + } + catch (Throwable t) + { + fail.addSuppressed(t); + executor.agent.onException(fail); + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java b/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java new file mode 100644 index 000000000000..6a2ab7950b46 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import javax.annotation.Nullable; + +import accord.utils.Invariants; +import accord.utils.async.AsyncCallbacks; + +import org.apache.cassandra.concurrent.DebuggableTask; + +class PlainChainDebuggable extends PlainChain implements DebuggableTask +{ + final Object describe; + + PlainChainDebuggable(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, @Nullable ExclusiveExecutor exclusiveExecutor, Object describe) + { + super(executor, runOrFail, exclusiveExecutor, ExclusiveGroup.OTHER); + this.describe = Invariants.nonNull(describe); + } + + PlainChainDebuggable(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, @Nullable ExclusiveExecutor exclusiveExecutor, long position, int tranche, Object describe) + { + super(executor, runOrFail, exclusiveExecutor, ExclusiveGroup.OTHER, position, tranche); + this.describe = Invariants.nonNull(describe); + } + + @Override + public long creationTimeNanos() + { + return createdAt; + } + + @Override + public long startTimeNanos() + { + return runningAt; + } + + @Override + public String description() + { + return describe.toString(); + } + + @Override + public DebuggableTask debuggable() + { + return this; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java b/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java new file mode 100644 index 000000000000..594ff107bf64 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import javax.annotation.Nullable; + +import accord.utils.async.Cancellable; + +import org.apache.cassandra.utils.Closeable; +import org.apache.cassandra.utils.concurrent.AsyncPromise; + +class PlainRunnable extends Plain implements Cancellable +{ + final @Nullable AsyncPromise result; + final Runnable run; + final @Nullable ExclusiveExecutor exclusiveExecutor; + + PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, GlobalGroup group, long position, int tranche) + { + super(executor, group, position, tranche); + this.result = result; + this.run = run; + this.exclusiveExecutor = null; + } + + PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, GlobalGroup group) + { + super(executor, group); + this.result = result; + this.run = run; + this.exclusiveExecutor = null; + } + + PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group, long position, int tranche) + { + super(executor, group, position, tranche); + this.result = result; + this.run = run; + this.exclusiveExecutor = exclusiveExecutor; + } + + PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group) + { + super(executor, group); + this.result = result; + this.run = run; + this.exclusiveExecutor = exclusiveExecutor; + } + + @Override + String toDescription() + { + // TODO (expected): ensure this is usefully descriptive, or accept a separate description + return run.toString(); + } + + @Override + protected void run() + { + onRunning(); + try (Closeable close = resources.get()) + { + run.run(); + } + if (result != null) + result.trySuccess(null); + onRunComplete(); + } + + @Override + protected void reportFailure(Throwable t) + { + if (result != null) + result.tryFailure(t); + executor.agent.onException(t); + } + + @Override + ExclusiveExecutor exclusiveExecutor() + { + return exclusiveExecutor; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordTask.java b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java similarity index 84% rename from src/java/org/apache/cassandra/service/accord/AccordTask.java rename to src/java/org/apache/cassandra/service/accord/execution/SafeTask.java index 30e6998396f2..77c61ecf4047 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTask.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -69,15 +69,16 @@ import org.apache.cassandra.concurrent.DebuggableTask; import org.apache.cassandra.metrics.LogLinearDecayingHistograms; -import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; -import org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus; -import org.apache.cassandra.service.accord.AccordCacheEntry.Loading; -import org.apache.cassandra.service.accord.AccordCacheEntry.Status; +import org.apache.cassandra.service.accord.AccordCommandStore; import org.apache.cassandra.service.accord.AccordCommandStore.Caches; -import org.apache.cassandra.service.accord.AccordExecutor.Task; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; +import org.apache.cassandra.service.accord.RangeIndex; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Loading; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; import org.apache.cassandra.service.accord.serializers.CommandSerializers; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.Closeable; @@ -90,50 +91,50 @@ import static accord.local.LoadKeysFor.RECOVERY; import static accord.local.LoadKeysFor.WRITE; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.HOLD_QUEUE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; -import static org.apache.cassandra.service.accord.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; -import static org.apache.cassandra.service.accord.AccordExecutor.NONSYNC_BLOCKED_LIMIT; -import static org.apache.cassandra.service.accord.AccordExecutor.NONSYNC_ENABLED; -import static org.apache.cassandra.service.accord.AccordExecutor.NONSYNC_MIN_BATCH_SIZE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.LOAD; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.GlobalGroup.RANGE_LOAD; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.RunState.PERSISTING; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.CANCELLED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.INCOMPLETE; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_OPTIONAL; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.LOADING_REQUIRED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.RUNNING; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_OPTIONAL; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_ON_REQUIRED; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_OR_RUNNING; -import static org.apache.cassandra.service.accord.AccordExecutor.Task.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.AccordSafeState.global; -import static org.apache.cassandra.service.accord.AccordSafeState.postExecute; -import static org.apache.cassandra.service.accord.AccordSafeState.preExecute; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; import static org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask.SANITY_CHECK; - -public final class AccordTask extends Task implements Cancellable, DebuggableTask +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.HOLD_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.RELEASE_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.NOT_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.RunnableStatus.STILL_RUNNABLE_NEWLY_BLOCKING; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.NONSYNC_BLOCKED_LIMIT; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.NONSYNC_ENABLED; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.NONSYNC_MIN_BATCH_SIZE; +import static org.apache.cassandra.service.accord.execution.SaferState.global; +import static org.apache.cassandra.service.accord.execution.SaferState.postExecute; +import static org.apache.cassandra.service.accord.execution.SaferState.preExecute; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; +import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; +import static org.apache.cassandra.service.accord.execution.Task.RunState.PERSISTING; +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.execution.Task.State.INCOMPLETE; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.SCANNING_RANGES; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_OR_RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; + +public final class SafeTask extends Task implements Cancellable, DebuggableTask { - private static final Logger logger = LoggerFactory.getLogger(AccordTask.class); + private static final Logger logger = LoggerFactory.getLogger(SafeTask.class); private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES); - public static AccordTask create(CommandStore commandStore, ExecutionContext context, Consumer consumer) + public static SafeTask create(CommandStore commandStore, ExecutionContext context, Consumer consumer) { - return new AccordTask<>((AccordCommandStore) commandStore, context, safeStore -> { + return new SafeTask<>((AccordCommandStore) commandStore, context, safeStore -> { consumer.accept(safeStore); return null; }); } - public static AccordTask create(CommandStore commandStore, ExecutionContext context, Function function) + public static SafeTask create(CommandStore commandStore, ExecutionContext context, Function function) { - return new AccordTask<>((AccordCommandStore) commandStore, context, function); + return new SafeTask<>((AccordCommandStore) commandStore, context, function); } final class NonSyncState extends ExecutionContext.Wrapped implements ExecutionContext @@ -221,7 +222,7 @@ void preRunExclusive() if ((blocking == null || !populate(keys, blocking)) && notBlocking != null) populate(keys, notBlocking); - keys.forEach(key -> preExecute(refs.get(key), AccordTask.this, RELEASE_QUEUE)); + keys.forEach(key -> preExecute(refs.get(key), SafeTask.this, RELEASE_QUEUE)); keys.sort(RoutingKey::compareTo); active = RoutingKeys.of(keys); } @@ -258,7 +259,7 @@ void postRunExclusive() if (active != null) { for (RoutingKey key : active) - postExecute(refs.remove(key), AccordTask.this); + postExecute(refs.remove(key), SafeTask.this); active = null; } ready = readyCount(); @@ -301,7 +302,7 @@ void postRunExclusive() private BiConsumer callback; - public AccordTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext, Function function) + public SafeTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext, Function function) { super(executionContext, commandStore.executor().uniqueCreatedAt); this.commandStore = commandStore; @@ -332,7 +333,7 @@ public String toDescription() return toBriefString() + ": " + (queued() == null ? "unqueued" : describeState()) + ", primaryTxnId: " + executionContext.primaryTxnId() - + ", state: " + summarise(refs, AccordSafeState::global); + + ", state: " + summarise(refs, SaferState::global); } @@ -383,8 +384,8 @@ public AsyncChain chain() protected Cancellable start(BiConsumer callback) { preSetup(callback); - executor().submit(AccordTask.this); - return AccordTask.this; + executor().submit(SafeTask.this); + return SafeTask.this; } }; } @@ -393,11 +394,18 @@ private void preSetup(BiConsumer callback) { Invariants.require(this.callback == null); this.callback = callback; - commandStore.tryPreSetup(this); + TaskRunner self = TaskRunner.get(); + Task task = self.accordActiveSelfTask(); + if (task instanceof SafeTask) + { + SafeTask parent = (SafeTask) task; + if (parent.commandStore == commandStore) + preSetup(parent); + } } // to be invoked only by the CommandStore owning thread, to take references to objects already in use by the current execution - public void preSetup(AccordTask parent) + public void preSetup(SafeTask parent) { this.position = parent.position; setInheritedWithTranche(parent.tranche()); @@ -441,7 +449,7 @@ public void preSetup(AccordTask parent) { // TODO (desired): custom map we can more cheaply fork/copy parent.refs.forEach((key, val) -> { - if (val instanceof AccordSafeCommandsForKey) + if (val instanceof SaferCommandsForKey) preSetup((RoutingKey) key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); }); } @@ -457,7 +465,7 @@ public void preSetup(AccordTask parent) case Range: AbstractRanges ranges = (AbstractRanges) keysOrRanges; parent.refs.forEach((key, val) -> { - if (val instanceof AccordSafeCommandsForKey && ranges.contains((RoutingKey) key)) + if (val instanceof SaferCommandsForKey && ranges.contains((RoutingKey) key)) preSetup((RoutingKey) key, parent.refs, commandStore.cachesUnsafe().commandsForKeys()); }); break; @@ -470,14 +478,10 @@ public void preSetup(AccordTask parent) } @Override - void submitExclusive(AccordExecutor owner) - { - owner.registerExclusive(this); - setupInternal(commandStore.cachesExclusive()); - } - - private void setupInternal(Caches caches) + void submitExclusive() { + Caches caches = commandStore.cachesExclusive(); + executor().registerExclusive(this); boolean hasPreSetup = hasInherited(); LoadKeys loadKeys = loadKeys(executionContext); if (loadKeys != NONE) @@ -496,8 +500,8 @@ private void setupInternal(Caches caches) case Range: if (!hasInheritedRangeScan()) setupRangeLoadsExclusive(caches); else refs.forEach((k, v) -> { - if (v instanceof AccordSafeCommandsForKey) - completePresetupExclusive((AccordSafeCommandsForKey)v); + if (v instanceof SaferCommandsForKey) + completePresetupExclusive((SaferCommandsForKey)v); }); break; case Key: @@ -551,7 +555,7 @@ private void setupRangeLoadsExclusive(Caches caches) } // expects mutual exclusivity only on the command store - private & AccordSafeState> void preSetup(K k, Map> parentMap, AccordCache.Type.Instance cache) + private & SaferState> void preSetup(K k, Map> parentMap, AccordCache.Type.Instance cache) { S ref = (S) parentMap.get(k); if (ref == null) @@ -566,7 +570,7 @@ private & AccordSafeState> void preSetup( keys++; } - private & AccordSafeState> boolean completePresetupExclusive(K k) + private & SaferState> boolean completePresetupExclusive(K k) { S preacquired = (S) refs.get(k); if (preacquired != null) @@ -577,7 +581,7 @@ private & AccordSafeState> boolean comple return false; } - private & AccordSafeState> void completePresetupExclusive(S preacquired) + private & SaferState> void completePresetupExclusive(S preacquired) { AccordCacheEntry entry = preacquired.global(); if (entry.isLoaded()) completeSetupOfLoaded(entry); @@ -585,7 +589,7 @@ private & AccordSafeState> void completeP } // expects to hold lock - private & AccordSafeState> void setupExclusive(K k, AccordCache.Type.Instance cache, int waitForIncrement) + private & SaferState> void setupExclusive(K k, AccordCache.Type.Instance cache, int waitForIncrement) { S safeRef = cache.acquire(k); AccordCacheEntry entry = safeRef.global(); @@ -706,7 +710,7 @@ private void waitOnCacheQueuesExclusive() Invariants.require(waitingForState == 0); onLoaded(); executor().runnable.incrementArrivals(this); - commandStore.exclusiveExecutor.incrementArrivals(this); + commandStore.exclusiveExecutor().incrementArrivals(this); this.refs.forEach((key, safeState) -> { AccordCacheEntry entry = global(safeState); @@ -838,7 +842,7 @@ private void incrementWaitingWhileAlreadyWaiting() else { // TODO (expected): this is potentially costly; maybe we don't want to swap these in and out (but harder to maintain invariants) - unqueue(commandStore.exclusiveExecutor); + unqueue(commandStore.exclusiveExecutor()); setStateExclusive(WAITING_ON_REQUIRED); executor().waitingOnCacheQueues.enqueue(this); } @@ -875,7 +879,7 @@ void onChangeOptionalHeadStatus(AccordCacheEntry entry, RunnableStatus nonSync.onNotHead(entry); if (is(WAITING_TO_RUN) && !nonSync.isWaitReady()) { - unqueue(commandStore.exclusiveExecutor); + unqueue(commandStore.exclusiveExecutor()); setStateExclusive(WAITING_ON_OPTIONAL); executor().waitingOnCacheQueues.enqueue(this); } @@ -904,7 +908,7 @@ void onChangeOptionalHeadStatus(AccordCacheEntry entry, RunnableStatus void waitToRunExclusive() { setStateExclusive(WAITING_TO_RUN); - commandStore.exclusiveExecutor.enqueue(this, false); + commandStore.exclusiveExecutor().enqueue(this, false); } public ExecutionContext executionContext() @@ -979,13 +983,13 @@ protected void preRunExclusive() public void run() { onRunning(); - AccordSafeCommandStore safeStore = null; + SaferCommandStore safeStore = null; try (Closeable close = resources.get()) { if (Tracing.isTracing()) Tracing.trace(executionContext.describe()); - commandStore.begin(safeStore = new AccordSafeCommandStore(this, isSync() ? executionContext : nonSync)); + commandStore.begin(safeStore = new SaferCommandStore(this, isSync() ? executionContext : nonSync)); R result = function.apply(safeStore); @@ -995,9 +999,9 @@ public void run() List changes = new ArrayList<>(); // TODO (expected): save any TxnId we add so that we don't need to iterate all of refs refs.forEach((key, value) -> { - if (value instanceof AccordSafeCommand) + if (value instanceof SaferCommand) { - AccordSafeCommand safeCommand = (AccordSafeCommand) value; + SaferCommand safeCommand = (SaferCommand) value; Journal.CommandUpdate diff = safeCommand.update(); if (diff != null) { @@ -1060,7 +1064,7 @@ private void save(List diffs, Runnable onFlush) } } - private void maybeSanityCheck(AccordSafeCommand safeCommand) + private void maybeSanityCheck(SaferCommand safeCommand) { if (SANITY_CHECK) { @@ -1112,27 +1116,52 @@ protected void cleanupExclusive(AccordExecutor executor, boolean executed) public void cancel() { if (!state().hasStarted()) - executor().cancel(this); + executor().submit(Task::cancelExclusive, CancelTask::new, this); } void cancelExclusive() { - logger.info("Cancelling {}", executionContext); - setStateExclusive(CANCELLED); - if (rangeScanner != null) - rangeScanner.cancelled = true; - if (callback != null) - { - if (executor().isInLoop()) callback.accept(null, new CancellationException()); - else executor().submit(() -> callback.accept(null, new CancellationException())); + State state = state(); + switch (state) + { + default: throw new UnhandledEnum(state); + case SCANNING_RANGES: + case LOADING_REQUIRED: + case LOADING_OPTIONAL: + case WAITING_ON_REQUIRED: + case WAITING_ON_OPTIONAL: + case WAITING_TO_RUN: + if (!hasIncrementalStarted()) + { + unqueueIfQueued(); + setStateExclusive(CANCELLED); + if (rangeScanner != null) + rangeScanner.cancelled = true; + try + { + logger.info("Cancelling {}", executionContext); + if (callback != null) + { + if (executor().isInLoop()) callback.accept(null, new CancellationException()); + else executor().submit(() -> callback.accept(null, new CancellationException())); + } + } + finally + { + executor().cleanupTaskExclusive(this, false); + } + break; + } + case UNINITIALIZED: // TODO (expected): preferable to be able to cancel at this stage, even if unlikely to trigger at this phase + case RUNNING: + case INCOMPLETE: + case EXECUTED: + case CANCELLED: + case FAILED_TO_LOAD: + // cannot safely cancel } } - void cancelExclusive(AccordExecutor owner) - { - owner.cancelExclusive(this); - } - private void finish(R result, Throwable failure) { setRunState(failure == null ? RunState.SUCCESS : RunState.FAILED); @@ -1155,7 +1184,7 @@ void releaseResourcesExclusive(Caches caches) } refs.forEach((key, safeState) -> { - AccordSafeState.postExecute(safeState, this); + SaferState.postExecute(safeState, this); }); if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedState(); } @@ -1178,7 +1207,7 @@ private void releaseResourcesSlowExclusive(Throwable suppressedBy) refs.forEach((k, safeState) -> { if (!safeState.isReleased()) { - try { AccordSafeState.postExecute(safeState, this); } + try { SaferState.postExecute(safeState, this); } catch (Throwable t) { suppressedBy.addSuppressed(t); } } }); @@ -1193,16 +1222,16 @@ class KeyWatcher implements AccordCache.Listener public void onUpdate(AccordCacheEntry state) { if (ranges.contains(state.key())) - reference((AccordCacheEntry) state); + reference((AccordCacheEntry) state); } } final Set intersectingKeys = new ObjectHashSet<>(); final Ranges ranges = ((AbstractRanges) executionContext.keys()).toRanges(); - final AccordCache.Type.Instance commandsForKeyCache; + final AccordCache.Type.Instance commandsForKeyCache; KeyWatcher keyWatcher = new KeyWatcher(); - public RangeTxnAndKeyScanner(AccordCache.Type.Instance commandsForKeyCache) + public RangeTxnAndKeyScanner(AccordCache.Type.Instance commandsForKeyCache) { this.commandsForKeyCache = commandsForKeyCache; } @@ -1223,7 +1252,7 @@ protected void runInternal() super.runInternal(); } - private void reference(AccordCacheEntry entry) + private void reference(AccordCacheEntry entry) { switch (entry.status()) { @@ -1306,7 +1335,7 @@ CommandSummaries finish(Caches caches) } } - public class RangeTxnScanner extends AccordExecutor.AbstractIOTask + public class RangeTxnScanner extends IOTaskWrapper.WrappableIOTask { final Map summaries = new HashMap<>(); final Map guardedSummaries = Collections.synchronizedMap(summaries); @@ -1330,7 +1359,7 @@ ExecutionContext preLoadContext() @Override protected void postRunExclusive() { - executor().onScannedRangesExclusive(AccordTask.this, failure); + executor().onScannedRangesExclusive(SafeTask.this, failure); } @Override @@ -1343,9 +1372,9 @@ public void start() { Caches caches = commandStore.cachesExclusive(); setStateExclusive(SCANNING_RANGES); - executor().scanningRanges.enqueue(AccordTask.this); + executor().scanningRanges.enqueue(SafeTask.this); startInternal(caches); - executor().submitPlainExclusive(AccordTask.this, GlobalGroup.RANGE_SCAN, this); + executor().submitExclusive(SafeTask.this, GlobalGroup.RANGE_SCAN, this); } void startInternal(Caches caches) @@ -1356,7 +1385,7 @@ void startInternal(Caches caches) public void scannedExclusive() { - Invariants.require(is(SCANNING_RANGES), "Expected SCANNING_RANGES; found %s", AccordTask.this, AccordTask::toDescription); + Invariants.require(is(SCANNING_RANGES), "Expected SCANNING_RANGES; found %s", SafeTask.this, SafeTask::toDescription); scanned = true; scannedInternal(); unqueue(executor().scanningRanges); diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java b/src/java/org/apache/cassandra/service/accord/execution/SaferCommand.java similarity index 77% rename from src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java rename to src/java/org/apache/cassandra/service/accord/execution/SaferCommand.java index 1263a3e1ec2d..3d066aa425db 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferCommand.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.Objects; @@ -25,14 +25,14 @@ import accord.local.SafeCommand; import accord.primitives.TxnId; -import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; -public class AccordSafeCommand extends SafeCommand implements AccordSafeState +public class SaferCommand extends SafeCommand implements SaferState { - private final AccordCacheEntry global; + private final AccordCacheEntry global; private Command original; - public AccordSafeCommand(AccordCacheEntry global) + public SaferCommand(AccordCacheEntry global) { super(global.key()); this.global = global; @@ -43,7 +43,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - AccordSafeCommand that = (AccordSafeCommand) o; + SaferCommand that = (SaferCommand) o; return Objects.equals(this.original, that.original) && Objects.equals(this.current(), that.current()); } @@ -64,13 +64,13 @@ public String toString() '}'; } - public AccordCacheEntry global() + public AccordCacheEntry global() { return global; } @Override - public void postExecute(AccordTask owner) + public void postExecute(SafeTask owner) { global.releaseExclusive(this, owner); } @@ -80,7 +80,7 @@ public Journal.CommandUpdate update() return new Journal.CommandUpdate(original, current()); } - public void preExecute(AccordTask owner, LockMode lockMode) + public void preExecute(SafeTask owner, LockMode lockMode) { requireUninitialised(); original = global.lockExclusive(owner, lockMode); diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java similarity index 85% rename from src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java rename to src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java index 6b02b5ab9df6..e1b73cf15e89 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.function.Predicate; @@ -46,19 +46,20 @@ import accord.utils.UnhandledEnum; import org.apache.cassandra.metrics.LogLinearDecayingHistograms; +import org.apache.cassandra.service.accord.AccordCommandStore; import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches; import org.apache.cassandra.service.accord.AccordCommandStore.SafeRedundantBefore; import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable; import org.apache.cassandra.service.paxos.PaxosState; import static accord.utils.Invariants.illegalState; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.UNQUEUED; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.UNQUEUED; -public final class AccordSafeCommandStore extends AbstractSafeCommandStore +public final class SaferCommandStore extends AbstractSafeCommandStore { - final AccordTask task; + final SafeTask task; - AccordSafeCommandStore(AccordTask task, ExecutionContext context) + SaferCommandStore(SafeTask task, ExecutionContext context) { super(context); this.task = task; @@ -73,9 +74,9 @@ public Iterable safeCommandsForKeys() } @Override - protected AccordSafeCommand getInternal(TxnId txnId) + protected SaferCommand getInternal(TxnId txnId) { - return (AccordSafeCommand) task.refs.get(txnId); + return (SaferCommand) task.refs.get(txnId); } @Override @@ -92,7 +93,7 @@ protected ExclusiveCaches tryGetCaches() return commandStore().tryLockCaches(); } - protected AccordSafeCommand add(AccordSafeCommand safeCommand, ExclusiveCaches caches) + protected SaferCommand add(SaferCommand safeCommand, ExclusiveCaches caches) { Object check = task.refs.putIfAbsent(safeCommand.txnId(), safeCommand); if (check == null) @@ -122,11 +123,7 @@ void persistFieldUpdatesInternal(Runnable onDone) if (updates.newRedundantBefore != null) { - long ticket = AccordCommandStore.nextSafeRedundantBeforeTicket.incrementAndGet(); - SafeRedundantBefore update = new SafeRedundantBefore(ticket, updates.newRedundantBefore); - Runnable reportRedundantBefore = () -> { - AccordCommandStore.safeRedundantBeforeUpdater.accumulateAndGet(commandStore(), update, SafeRedundantBefore::max); - }; + Runnable reportRedundantBefore = SafeRedundantBefore.updater(commandStore(), updates.newRedundantBefore); Runnable prevOnDone = onDone; onDone = prevOnDone == null ? reportRedundantBefore : () -> { try { reportRedundantBefore.run(); } @@ -136,7 +133,7 @@ void persistFieldUpdatesInternal(Runnable onDone) commandStore().persistFieldUpdates(updates, onDone); } - protected AccordSafeCommandsForKey add(AccordSafeCommandsForKey safeCfk, ExclusiveCaches caches) + protected SaferCommandsForKey add(SaferCommandsForKey safeCfk, ExclusiveCaches caches) { Object check = task.refs.putIfAbsent(safeCfk.key(), safeCfk); if (check == null) @@ -152,9 +149,9 @@ protected AccordSafeCommandsForKey add(AccordSafeCommandsForKey safeCfk, Exclusi } @Override - protected AccordSafeCommandsForKey getInternal(RoutingKey key) + protected SaferCommandsForKey getInternal(RoutingKey key) { - AccordSafeCommandsForKey safeCfk = (AccordSafeCommandsForKey) task.refs.get(key); + SaferCommandsForKey safeCfk = (SaferCommandsForKey) task.refs.get(key); if (safeCfk == null || safeCfk.isUninitialised()) return null; return safeCfk; @@ -177,7 +174,7 @@ public void updateCommandsForRanges(Command prev, Command updated, boolean force public void reportDurable(RedundantBefore addRedundantBefore, int flags) { upsertRedundantBefore(addRedundantBefore); - commandStore().maybeTerminated(ReportDurable.isCommandStoreFlush(flags), ReportDurable.isDataStoreFlush(flags)); + ReportDurable.reportMaybeTerminate(commandStore(), flags); } @Override @@ -239,7 +236,7 @@ private boolean visitForKey(Unseekables keysOrRanges, Predicate skip = context.keys().without(keysOrRanges); for (SafeState safeState : task.refs.values()) { - if (!(safeState instanceof AccordSafeCommandsForKey)) + if (!(safeState instanceof SaferCommandsForKey)) continue; SafeCommandsForKey safeCfk = (SafeCommandsForKey) safeState; diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandsForKey.java similarity index 72% rename from src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java rename to src/java/org/apache/cassandra/service/accord/execution/SaferCommandsForKey.java index fc38874caae4..ea188159beaf 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandsForKey.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.Objects; @@ -24,25 +24,24 @@ import accord.local.cfk.CommandsForKey; import accord.local.cfk.NotifySink; import accord.local.cfk.SafeCommandsForKey; -import accord.utils.Invariants; -import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; -public class AccordSafeCommandsForKey extends SafeCommandsForKey implements AccordSafeState +public class SaferCommandsForKey extends SafeCommandsForKey implements SaferState { - public static class CommandsForKeyCacheEntry extends AccordCacheEntry + public static class CommandsForKeyCacheEntry extends AccordCacheEntry { private NotifySink overrideSink; - CommandsForKeyCacheEntry(RoutingKey key, AccordCache.Type.Instance owner) + CommandsForKeyCacheEntry(RoutingKey key, AccordCache.Type.Instance owner) { super(key, owner); } } - private final AccordCacheEntry global; + private final AccordCacheEntry global; - public AccordSafeCommandsForKey(AccordCacheEntry global) + public SaferCommandsForKey(AccordCacheEntry global) { super(global.key()); this.global = global; @@ -53,7 +52,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - AccordSafeCommandsForKey that = (AccordSafeCommandsForKey) o; + SaferCommandsForKey that = (SaferCommandsForKey) o; return Objects.equals(current, that.current); } @@ -73,13 +72,13 @@ public String toString() '}'; } - public final AccordCacheEntry global() + public final AccordCacheEntry global() { return global; } @Override - public void postExecute(AccordTask owner) + public void postExecute(SafeTask owner) { global.releaseExclusive(this, owner); } @@ -96,7 +95,7 @@ public NotifySink overrideSink() return ((CommandsForKeyCacheEntry)global).overrideSink; } - public void preExecute(AccordTask owner, LockMode lockMode) + public void preExecute(SafeTask owner, LockMode lockMode) { requireUninitialised(); current = global.lockExclusive(owner, lockMode); diff --git a/src/java/org/apache/cassandra/service/accord/execution/SaferState.java b/src/java/org/apache/cassandra/service/accord/execution/SaferState.java new file mode 100644 index 000000000000..6708bdaa05e3 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferState.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service.accord.execution; + +import accord.local.SafeState; + +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; + +public interface SaferState & SaferState> +{ + AccordCacheEntry global(); + void postExecute(SafeTask owner); + void preExecute(SafeTask owner, LockMode lockMode); + + static AccordCacheEntry global(SafeState safeState) + { + return safeState.getClass() == SaferCommand.class ? ((SaferCommand) safeState).global() + : ((SaferCommandsForKey) safeState).global(); + } + + static void postExecute(SafeState safeState, SafeTask owner) + { + if (safeState.getClass() == SaferCommand.class) ((SaferCommand) safeState).postExecute(owner); + else ((SaferCommandsForKey) safeState).postExecute(owner); + } + + static void preExecute(SafeState safeState, SafeTask owner, LockMode lockMode) + { + if (safeState.getClass() == SaferCommand.class) ((SaferCommand) safeState).preExecute(owner, lockMode); + else ((SaferCommandsForKey) safeState).preExecute(owner, lockMode); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/Task.java b/src/java/org/apache/cassandra/service/accord/execution/Task.java new file mode 100644 index 000000000000..0d2395f8deca --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Task.java @@ -0,0 +1,724 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.concurrent.atomic.AtomicLong; + +import javax.annotation.Nullable; + +import accord.local.ExecutionContext; +import accord.messages.Accept; +import accord.messages.Commit; +import accord.messages.MessageType; +import accord.messages.Request; +import accord.primitives.Ballot; +import accord.primitives.SaveStatus; +import accord.primitives.TxnId; +import accord.utils.IntrusiveHeapNode; +import accord.utils.Invariants; +import accord.utils.TinyEnumSet; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.concurrent.DebuggableTask; +import org.apache.cassandra.concurrent.ExecutorLocals; +import org.apache.cassandra.service.accord.debug.DebugExecution; +import org.apache.cassandra.utils.WithResources; + +import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY; +import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY_ATOMIC; +import static accord.primitives.Routable.Domain.Range; +import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.ORIG_HLC_FIFO; +import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.APPLY; +import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.COMMIT; +import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.STABLE; +import static org.apache.cassandra.service.accord.execution.Task.State.EXECUTED; +import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING_OR_EXECUTED; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +public abstract class Task extends IntrusiveHeapNode implements Cancellable +{ + private static final int WAITING_ON_OPTIONAL_BIT = 1 << 5; + private static final int WAITING_TO_RUN_BIT = 1 << 6; + private static final int INCOMPLETE_BIT = 1 << 9; + + enum State + { + UNINITIALIZED(), + SCANNING_RANGES(UNINITIALIZED), + LOADING_REQUIRED(UNINITIALIZED, SCANNING_RANGES), + LOADING_OPTIONAL(UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED), + WAITING_ON_REQUIRED(WAITING_ON_OPTIONAL_BIT | WAITING_TO_RUN_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), + WAITING_ON_OPTIONAL(WAITING_TO_RUN_BIT | INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED), + WAITING_TO_RUN(INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL), + RUNNING(WAITING_TO_RUN), + EXECUTED(RUNNING), + INCOMPLETE(RUNNING), + FAILED_TO_LOAD(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), + FAILED_OTHER(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN), + CANCELLED(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN, RUNNING), + ; + + private final int permittedFrom; + public static final int WAITING = TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN); + public static final int WAITING_OR_RUNNING = WAITING | TinyEnumSet.encode(RUNNING); + public static final int RUNNING_OR_EXECUTED = WAITING | TinyEnumSet.encode(RUNNING, EXECUTED); + static final State[] VALUES = values(); + + static + { + // hack to allow us to create loops in our enum transition declarations + Invariants.require(INCOMPLETE_BIT == 1 << INCOMPLETE.ordinal()); + } + + State() + { + this.permittedFrom = 0; + } + + State(State... permittedFroms) + { + this(0, permittedFroms); + } + + State(int additional, State... permittedFroms) + { + int permittedFrom = additional; + for (State state : permittedFroms) + permittedFrom |= 1 << state.ordinal(); + this.permittedFrom = permittedFrom; + } + + boolean isPermittedFrom(int prevOrdinal) + { + return (permittedFrom & (1 << prevOrdinal)) != 0; + } + + boolean isExecuted() + { + return this.compareTo(EXECUTED) >= 0; + } + + boolean hasStarted() + { + return this.compareTo(RUNNING) >= 0; + } + + static State forOrdinal(int ordinal) + { + return VALUES[ordinal]; + } + } + + enum RunState + { + NONE, PERSISTING, SUCCESS, FAILED; + + private static final RunState[] VALUES = values(); + + static RunState forOrdinal(int ordinal) + { + return VALUES[ordinal]; + } + } + + enum GlobalGroup + { + COMMAND_STORE, + LOAD, + SAVE, + OTHER, + RANGE_LOAD, + RANGE_SCAN, + } + + enum ExclusiveGroup + { + APPLY, + STABLE, + COMMIT, + ACCEPT, + OTHER, + RECOVER, + PREACCEPT, + RANGE, + } + + public enum GroupKind + { + EXCLUSIVE(ExclusiveGroup.values().length, EXCLUSIVE_GROUP_SHIFT), + GLOBAL(GlobalGroup.values().length, GLOBAL_GROUP_SHIFT), + NONE(0, 0); + + final int count; + final byte shift; + + GroupKind(int count, int shift) + { + this.count = count; + this.shift = (byte) shift; + } + } + + private static final int STATE_MASK = 0xf; + static final int GROUP_MASK = 0x7; + private static final int EXCLUSIVE_GROUP_SHIFT = 4; + private static final int GLOBAL_GROUP_SHIFT = 7; + + private static final int NONSYNC_BIT = 1 << 10; + private static final int CACHE_QUEUED_BIT = 1 << 11; + + private static final int INCREMENTAL_MASK = 0x3 << 12; + private static final int INCREMENTAL = 0x1 << 12; + private static final int INCREMENTAL_STARTED = 0x2 << 12; + private static final int INCREMENTAL_FINISHING = 0x3 << 12; + + private static final int SEQUENCED_SHIFT = 14; + private static final int SEQUENCED_MASK = 0x3 << SEQUENCED_SHIFT; + private static final int SEQUENCED_PRIORITY = 0x1 << SEQUENCED_SHIFT; + private static final int SEQUENCED_ATOMIC = 0x2 << SEQUENCED_SHIFT; + private static final int SEQUENCED_ATOMIC_AND_QUEUED = 0x3 << SEQUENCED_SHIFT; + + // spare two bits + + private static final int HAS_TRANCHE_BIT = 1 << 18; + private static final int HAS_INHERITED_BIT = 1 << 19; + private static final int HAS_INHERITED_RANGE_SCAN_BIT = 1 << 20; + + private static final int TRANCHE_SHIFT = 22; + static final int MAX_TRANCHE = 0x3ff; + + static + { + Invariants.require(SEQUENCED_PRIORITY == BY_PRIORITY.ordinal() << SEQUENCED_SHIFT); + Invariants.require(SEQUENCED_ATOMIC == BY_PRIORITY_ATOMIC.ordinal() << SEQUENCED_SHIFT); + Invariants.require(ExecutionContext.ExecutionSequence.values().length <= 3); + } + + public final WithResources resources; + Task next; + + long position; + int info; + + // TODO (expected): do we need this? we should be able to determine the queue from state() if needed for e.g. cancellation + private TaskQueue queued; + + public final long createdAt; + // TODO (expected): expose via executors vtable + // TODO (expected): use just one long and some flag bits to indicate which point it represents, and report incrementally + public long loadedAt, runningAt, completeAt; + private byte runState; + + Task(GlobalGroup group) + { + resources = DebugExecution.DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + info = init(group, ExclusiveGroup.OTHER); + createdAt = nanoTime(); + } + + Task(ExclusiveGroup group) + { + resources = DebugExecution.DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + info = init(GlobalGroup.OTHER, group); + createdAt = nanoTime(); + } + + Task(GlobalGroup group, long position, int tranche) + { + this(group); + this.position = position; + setInheritedWithTranche(tranche); + } + + Task(ExclusiveGroup group, long position, int tranche) + { + this(group); + this.position = position; + setInheritedWithTranche(tranche); + } + + protected Task(ExecutionContext context, AtomicLong lastCreatedAt) + { + resources = DebugExecution.DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + createdAt = lastCreatedAt.accumulateAndGet(nanoTime(), (prev, next) -> next < prev ? prev + 1 : next); + ExclusiveGroup group = ExclusiveGroup.OTHER; + TxnId txnId = context.primaryTxnId(); + if (txnId != null) + { + if (txnId.is(Range)) group = ExclusiveGroup.RANGE; + else + { + switch (AccordExecutor.PRIORITY_MODEL) + { + case HLC_FIFO: + case ORIG_HLC_FIFO: + { + // TODO (expected): port to ExecutionKind; also we aren't consistent about using Ballot + if (context instanceof Request) + { + MessageType type = ((Request) context).type(); + if (type instanceof MessageType.StandardMessage) + { + switch ((MessageType.StandardMessage) type) + { + case APPLY_REQ: + { + group = APPLY; + break; + } + case READ_EPHEMERAL_REQ: + case READ_REQ: + case STABLE_THEN_READ_REQ: + { + group = STABLE; + break; + } + case COMMIT_REQ: + { + Commit commit = (Commit) context; + if (AccordExecutor.PRIORITY_MODEL == ORIG_HLC_FIFO && !commit.ballot.equals(Ballot.ZERO)) + txnId = null; + if (commit.kind.saveStatus == SaveStatus.Stable) group = STABLE; + else group = COMMIT; + break; + } + case ACCEPT_REQ: + { + Accept accept = (Accept) context; + if (AccordExecutor.PRIORITY_MODEL == ORIG_HLC_FIFO && !accept.ballot.equals(Ballot.ZERO)) + txnId = null; + group = ExclusiveGroup.ACCEPT; + break; + } + case GET_EPHEMERAL_READ_DEPS_REQ: + case PRE_ACCEPT_REQ: + { + group = ExclusiveGroup.PREACCEPT; + break; + } + default: + { + txnId = null; + } + } + } + } + else + { + txnId = null; + } + break; + } + case FIFO: + { + txnId = null; + break; + } + } + } + } + + this.info = init(GlobalGroup.OTHER, group); + if (txnId != null) + this.position = txnId.hlc(); + } + + public final Task unwrap() + { + if (this instanceof ExclusiveExecutor.ExclusiveExecutorTask) + return ((ExclusiveExecutor.ExclusiveExecutorTask) this).queue.task; + return this; + } + + static Task unwrap(Task task) + { + return task == null ? null : task.unwrap(); + } + + public DebuggableTask debuggable() + { + return null; + } + + abstract String toDescription(); + + abstract void submitExclusive(); + + /** + * Prepare to run while holding the state cache lock + */ + void preRunExclusive() + { + setStateExclusive(RUNNING); + } + + /** + * Run the command; the state cache lock may or may not be held depending on the executor implementation + */ + abstract void run(); + + /** + * Fail the command; the state cache lock may or may not be held depending on the executor implementation + */ + abstract void reportFailure(Throwable fail); + + final void failExclusive(Throwable fail, State newState) + { + try + { + setStateExclusive(newState); + } + finally + { + reportFailure(fail); + } + } + + final void failExecution(Throwable fail) + { + Invariants.require(is(RUNNING)); + try + { + setRunState(RunState.FAILED); + } + finally + { + reportFailure(fail); + } + } + + abstract boolean isNewWork(); + + /** + * Cleanup the command while holding the state cache lock + */ + void cleanupExclusive(AccordExecutor executor, boolean executed) + { + if (executed) setStateExclusive(EXECUTED); + else Invariants.require(state().isExecuted()); + executor.unregisterExclusive(this); + completeAt = nanoTime(); + if (runningAt != 0) + { + if (loadedAt == 0) + loadedAt = runningAt; + executor.elapsedWaitingToRun.increment(runningAt - loadedAt, runningAt); + executor.elapsedPreparingToRun.increment(loadedAt - createdAt, runningAt); + executor.elapsedRunning.increment(completeAt - runningAt, completeAt); + executor.elapsed.increment(completeAt - createdAt, completeAt); + } + if (DEBUG_EXECUTION) DebugExecution.DebugTask.get(this).onCompleted(executor.debug); + } + + void cancelExclusive() + { + } + + @Nullable + final TaskQueue queued() + { + return queued; + } + + final void unqueueIfQueued() + { + if (queued != null) + { + queued.unqueue(this); + queued = null; + } + } + + final void unqueue(TaskQueue expected) + { + Invariants.require(queued == expected, "%s != %s", queued, expected); + queued.unqueue(this); + queued = null; + } + + final void unsetQueue(TaskQueue expected) + { + Invariants.require(queued == expected, "%s != %s", queued, expected); + queued = null; + } + + final void setQueue(TaskQueue queue) + { + Invariants.require(queued == null); + Invariants.require(isCompatible(queue)); + queued = queue; + } + + final void onRunning() + { + runningAt = nanoTime(); + if (DEBUG_EXECUTION) ((DebugExecution.DebugTask) resources).onRunning(); + } + + final void onRunComplete() + { + if (DEBUG_EXECUTION) ((DebugExecution.DebugTask) resources).onRunComplete(); + } + + final void onLoaded() + { + loadedAt = nanoTime(); + } + + final State state() + { + return State.forOrdinal(stateOrdinal()); + } + + final RunState runState() + { + return RunState.forOrdinal(runState); + } + + final Enum describeState() + { + State state = state(); + if (state == RUNNING || state == EXECUTED) + { + RunState runState = runState(); + if (runState == RunState.NONE) + return state; + return runState; + } + return State.forOrdinal(stateOrdinal()); + } + + private int stateOrdinal() + { + return info & STATE_MASK; + } + + final boolean is(State state) + { + return stateOrdinal() == state.ordinal(); + } + + final boolean isState(int stateBitSet) + { + return TinyEnumSet.contains(stateBitSet, stateOrdinal()); + } + + final boolean is(GlobalGroup group) + { + return globalGroupOrdinal() == group.ordinal(); + } + + final boolean is(ExclusiveGroup group) + { + return exclusiveGroupOrdinal() == group.ordinal(); + } + + final void override(GlobalGroup group) + { + info = (info & ~(GROUP_MASK << GLOBAL_GROUP_SHIFT)) | (group.ordinal() << GLOBAL_GROUP_SHIFT); + } + + final int compareTo(State state) + { + return stateOrdinal() - state.ordinal(); + } + + final void setStateExclusive(State state) + { + Invariants.require(state.isPermittedFrom(stateOrdinal()), "%s forbidden from %s", state, this, Task::reportBadStateTransition); + unsafeSetStateExclusive(state); + } + + final void setRunState(RunState state) + { + Invariants.require(isState(RUNNING_OR_EXECUTED)); + runState = (byte) state.ordinal(); + } + + private static String reportBadStateTransition(Task task) + { + return task.state() + " for " + task.toDescription(); + } + + final void unsafeSetStateExclusive(State state) + { + info = (info & ~STATE_MASK) | state.ordinal(); + } + + final int globalGroupOrdinal() + { + return (info >>> GLOBAL_GROUP_SHIFT) & GROUP_MASK; + } + + final int exclusiveGroupOrdinal() + { + return (info >>> EXCLUSIVE_GROUP_SHIFT) & GROUP_MASK; + } + + private boolean isCompatible(TaskQueue queue) + { + int self = stateOrdinal(); + return TinyEnumSet.contains(queue.states, self); + } + + final boolean isSync() + { + return 0 == (info & NONSYNC_BIT); + } + + final boolean isNonSync() + { + return !isSync(); + } + + final void setNonSyncExclusive() + { + info |= NONSYNC_BIT; + } + + final boolean isIncremental() + { + return 0 != (info & INCREMENTAL_MASK); + } + + final void setIncrementalExclusive() + { + info |= INCREMENTAL | NONSYNC_BIT; + } + + final boolean hasIncrementalStarted() + { + return (info & INCREMENTAL_MASK) >= INCREMENTAL_STARTED; + } + + final void setIncrementalStartedExclusive() + { + Invariants.require(isIncremental()); + if (!isIncrementalFinishing()) + info = (info & ~INCREMENTAL_MASK) | INCREMENTAL_STARTED; + } + + final boolean isIncrementalFinishing() + { + return (info & INCREMENTAL_MASK) >= INCREMENTAL_FINISHING; + } + + final void setIncrementalFinishingExclusive() + { + Invariants.require(isIncremental()); + info |= INCREMENTAL_FINISHING; + } + + final void setSequencedExclusive(ExecutionContext.ExecutionSequence sequence) + { + Invariants.require(isUnsequenced()); + info |= sequence.ordinal() << SEQUENCED_SHIFT; + } + + final boolean isUnsequenced() + { + return (info & SEQUENCED_MASK) == 0; + } + + final boolean isSequencedByPriority() + { + return (info & SEQUENCED_MASK) == SEQUENCED_PRIORITY; + } + + final boolean isSequencedByPriorityAtomic() + { + return (info & SEQUENCED_MASK) >= SEQUENCED_ATOMIC; + } + + final boolean isCacheQueuedFifo() + { + return (info & SEQUENCED_MASK) == SEQUENCED_ATOMIC_AND_QUEUED; + } + + final boolean isCacheQueued() + { + return 0 != (info & CACHE_QUEUED_BIT); + } + + // supersedes priority, in whichever order they're called + final void setCacheQueuedFifoExclusive() + { + Invariants.require(isSequencedByPriorityAtomic()); + info |= SEQUENCED_ATOMIC_AND_QUEUED | CACHE_QUEUED_BIT; + } + + final void setCacheQueuedExclusive() + { + info |= CACHE_QUEUED_BIT; + } + + final int tranche() + { + Invariants.require((info & HAS_TRANCHE_BIT) != 0); + return info >>> TRANCHE_SHIFT; + } + + final void setTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; + } + + final void setInheritedWithTranche(int tranche) + { + Invariants.require(tranche <= MAX_TRANCHE); + info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT | HAS_INHERITED_BIT; + } + + final boolean hasInherited() + { + return (info & HAS_INHERITED_BIT) != 0; + } + + final void setInheritedRangeScan() + { + info = info | HAS_INHERITED_RANGE_SCAN_BIT; + } + + final boolean hasInheritedRangeScan() + { + return (info & HAS_INHERITED_RANGE_SCAN_BIT) != 0; + } + + static int init(GlobalGroup global, ExclusiveGroup exclusive) + { + return (global.ordinal() << GLOBAL_GROUP_SHIFT) | (exclusive.ordinal() << EXCLUSIVE_GROUP_SHIFT); + } + + static Task reverse(Task unqueued) + { + Task prev = null; + Task cur = unqueued; + while (cur != null) + { + Task next = cur.next; + cur.next = prev; + prev = cur; + cur = next; + } + return prev; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java b/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java new file mode 100644 index 000000000000..d9163d5a587c --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import javax.annotation.Nullable; + +import accord.local.ExecutionContext; + +import org.apache.cassandra.concurrent.DebuggableTask; + +public class TaskInfo implements Comparable +{ + // sorted in name order for reporting to virtual tables + public enum Status + { + LOADING, RUNNING, SCANNING_RANGES, WAITING_TO_LOAD, WAITING_TO_RUN + } + + final Status status; + final int commandStoreId; + + final Task task; + + public TaskInfo(Status status, int commandStoreId, Task task) + { + this.status = status; + this.commandStoreId = commandStoreId; + this.task = task; + } + + public Status status() + { + return status; + } + + public Integer commandStoreId() + { + return commandStoreId >= 0 ? commandStoreId : null; + } + + public long position() + { + return task.position; + } + + public @Nullable String describe() + { + if (task instanceof SafeTask) + return ((SafeTask) task).executionContext().reason(); + + if (task instanceof DebuggableTask) + return ((DebuggableTask) task).description(); + + return null; + } + + public @Nullable ExecutionContext preLoadContext() + { + if (task instanceof SafeTask) + return ((SafeTask) task).executionContext(); + if (task instanceof IOTaskWrapper && ((IOTaskWrapper) task).wrapped instanceof SafeTask.RangeTxnScanner) + return ((SafeTask.RangeTxnScanner) ((IOTaskWrapper) task).wrapped).preLoadContext(); + return null; + } + + @Override + public int compareTo(TaskInfo that) + { + int c = this.status.compareTo(that.status); + if (c == 0) c = Long.compare(this.position(), that.position()); + return c; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java new file mode 100644 index 000000000000..ec483dd6409c --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.IntrusivePriorityHeap; +import accord.utils.TinyEnumSet; + +// has xSingle methods to distinguish from within MultiTaskQueue whether we're invoking the multi or single variation +class TaskQueue extends IntrusivePriorityHeap +{ + final int states; + + public TaskQueue(int states) + { + this.states = states; + } + + void unqueue(T task) + { + throw new UnsupportedOperationException(); + } + + @Override + public final int compare(T o1, T o2) + { + return Long.compare(o1.position, o2.position); + } + + @Override + protected final void ensureHeapified() + { + super.ensureHeapified(); + } + + protected final boolean requeueSingle(T requeue) + { + int oldIndex = updateNode(requeue); + if (oldIndex < 0) + return false; + + int newIndex = heapIndex(requeue); + return Math.min(oldIndex, newIndex) == 0 && heapifiedSize() > 0; + } + + final T peekSingle() + { + ensureHeapified(); + return peekNode(); + } + + final T pollSingle() + { + ensureHeapified(); + return pollNode(); + } + + final boolean isEmptySingle() + { + return isEmptyInternal(); + } + + final int enqueueSingle(T enqueue) + { + return insertNode(enqueue); + } + + final boolean unqueueSingle(T unqueue) + { + int heapIndex = heapIndex(unqueue); + removeNode(unqueue); + return heapIndex == 0; + } + + final boolean tryUnqueueSingle(T remove) + { + return removeNodeIfContains(remove); + } + + final T getSingle(int index) + { + return super.getNode(index); + } + + final boolean isQueuedSingle(T test) + { + return containsNode(test); + } + + @Override + public String toString() + { + return TinyEnumSet.toString(states, Task.State::forOrdinal); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java new file mode 100644 index 000000000000..9784e51ad905 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java @@ -0,0 +1,612 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.Invariants; +import accord.utils.UnhandledEnum; + +/** + * A {@link TaskQueue} sub-divided into up to eight per-group sub-queues (groups are phases for the exclusive + * executor, or work classes globally) plus a policy for choosing which sub-queue to serve next. The policy balances + * three competing concerns: ordering/priority (run the oldest / highest-priority work), fairness (do not let one + * group monopolise the executor), and throughput (do not let an even split leave a busy group's backlog growing + * without bound). + * + *

Packed counters

+ * Per-group state is held in {@code long}s, one 7-bit lane per group; bit 7 of each byte is a spare "guard" bit used + * to run branch-free min/compare across all eight lanes at once (see {@code minCounters}). The lanes are: + *
    + *
  • {@code positions[]} - the head task's position (HLC, or submission order) per group; drives FIFO/priority.
    • + *
  • {@code recent} - a windowed count of recently served tasks per group (service received).
    • + *
  • {@code arrivals} - a windowed, size-biased count of recent arrivals per group (demand offered).
    • + *
  • {@code current} - tasks currently in flight per group, used to enforce {@code queue_active_limits}.
    • + *
  • {@code hasWork}/{@code dirty} - guard-bit masks: which groups have queued work / need a position refresh.
    • + *
+ * + *

Windowing: a bounded memory of the past

+ * {@code recent} and {@code arrivals} are not running totals. Whenever incrementing {@code recent} tips any lane to + * its maximum (0x80), both {@code recent} and {@code arrivals} are halved (see {@code incrementRecents}), + * and each lane also saturates at 0x7f. They are therefore an exponentially-decaying window keyed on service/time. + * This is what stops a one-off burst of arrivals granting a group a permanent boost: the burst saturates the lane + * briefly and then decays away as other work is served. + * + *

The fair selection counter

+ * The fair models pick the group with the smallest {@code effective} counter, where per lane + * 
effective = min(0x7f, max(0, recent - arrivals) + bias)
+ *
    + *
  • {@code max(0, recent - arrivals)} - a group whose arrivals have outpaced its recent service clamps to 0 and + * is therefore preferred; in steady state each group's service tracks its arrival rate, which bounds backlog. + * The clamp is symmetric, so a group cannot bank "credit" during a lull and later use it to monopolise service.
    • + *
+ * + *

Choosing a group ({@code minGroup} / {@code pollByBlend})

+ *
    + *
  • {@code PRIORITY_ONLY} - always the lowest {@code positions} (oldest / highest priority), ignoring fairness.
    • + *
  • {@code PHASE_OVERRIDE} - strict phase order: the lowest-index group with work, always.
    • + *
  • {@code PHASE_FAIR} - pure fairness: the smallest {@code effective}; ties broken by index, within a group by + * position. It has no priority fallback, so it is deliberately completing-first under contention.
    • + *
  • {@code PRIORITY_FAIR} (default) - a deficit round-robin blend of two strategies, chosen per poll: flow + * (smallest {@code effective}, i.e. least fairly serviced) and age (oldest {@code positions}, i.e. FIFO). + * The flow weight ramps up with the flow imbalance ({@code max-min} of {@code effective}) over + * {@code queue_flow_imbalance_onset..+width}, trading against age zero-sum; when balanced it is pure age. The + * ramp is smooth (not a hard mode switch), so there is no boundary to oscillate around and no hysteresis is + * needed. Age also drains stale/standing backlogs for free: a backlog's items are the oldest work, so age + * clears them without a separate backlog-aware strategy (the {@code effective} flow counter tracks arrival + * rate and is deliberately blind to standing stock).
    • + *
+ * + *

Concurrency limits and eligibility

+ * A group whose in-flight {@code current} count has reached its {@code queue_active_limits} limit is + * {@code saturated} and excluded from selection ({@code disabled}), as is any group with no queued work. + * + *

Note on {@code positions} freshness

+ * {@code positions} is refreshed lazily from the sub-queue heads via the {@code dirty} mask; {@code dirty} must use + * the same guard-bit layout as {@code hasWork} so the refresh loop in {@code minGroupByPriority} actually runs - + * otherwise it silently degenerates to lowest-index-with-work and starves high-index / new-work groups. + * + *

NB it extends {@link TaskQueue} to keep the type hierarchy simple for method dispatch, and for efficiency for + * anonymous ExclusiveExecutors which do not use multiple queues, while letting ExclusiveExecutor share a parent + * class for both use cases. + */ +abstract class TaskQueueMulti extends TaskQueue +{ + private static final TaskQueue[] NO_QUEUES = new TaskQueue[0]; + private static final long[] NO_POSITIONS = new long[0]; + static final long COUNTER_OVERFLOWS = 0x8080808080808080L; + static final long COUNTER_MASKS = 0x7f7f7f7f7f7f7f7fL; + static final long COUNTER_LOWBITS = 0x0101010101010101L; + + final TaskQueue[] queues; + final long[] positions; + final byte groupShift; + final long limits; + + /** + * sets overflow bits for a queue that has been stopped + */ + long stopped; + /** + * sets overflow bits for each counter when it needs its position updated + */ + long dirty; + /** + * sets overflow bits for each counter when there's associated work + */ + long hasWork; + /** + * Stores recent dequeue counts for up to 8 sub queues. + */ + long dispatches; + // TODO (required): increment arrivals based on internal queue for ExclusiveExecutors + // also: experiment with decaying on arrival schedule rather than poll schedule, since this should respond to work growth more accurately + /** + * Stores recent enqueue counts for up to 8 sub queues. + */ + long arrivals; + /** + * Stores currently-active counts for up to 8 sub queues. We can use this to impose limits on specific queues. + */ + long active; + /** + * deficit-round-robin credits for the two PRIORITY_FAIR strategies (flow/age). + */ + int creditFlow, creditAge; + + int waitingCount; + + TaskQueueMulti(int waitingStates, Task.GroupKind groups, long limits) + { + super(waitingStates); + this.limits = limits; + int queueCount = groups.count; + Invariants.require(queueCount <= 8); + queues = queueCount == 0 ? NO_QUEUES : new TaskQueue[queueCount]; + positions = queueCount > 0 && AccordExecutor.BALANCE_BY_POSITION ? new long[queueCount] : NO_POSITIONS; + groupShift = groups.shift; + } + + final int group(Task task) + { + if (groupShift == 0) + return -1; + + return (task.info >>> groupShift) & Task.GROUP_MASK; + } + + void stop(int group) + { + stopped |= overflowBit(group); + } + + void restart(int group) + { + stopped &= ~overflowBit(group); + } + + final TaskQueue queue(Task task) + { + int group = group(task); + if (group < 0) + return this; + + return queue(group); + } + + final TaskQueue queue(int group) + { + TaskQueue queue = queues[group]; + if (queue == null) + queues[group] = queue = new TaskQueue<>(0); + + return queue; + } + + private int pollGroup() + { + if (hasWork == 0) + return -1; + + switch (AccordExecutor.BALANCING_MODEL) + { + default: + throw new UnhandledEnum(AccordExecutor.PRIORITY_MODEL); + case PRIORITY_ONLY: + return pollGroupByPriority(); + case PHASE_ONLY: + return pollGroupByIndex(); + case PHASE_FAIR: + return pollGroupByPhaseFair(); + case BLENDED_PRIORITY_PHASE_FAIR: + return pollGroupByBlended(); + } + } + + private int pollGroupByPriority() + { + return pollGroupByPriority(unsaturatedWithWork()); + } + + private int pollGroupByPriority(long enabled) + { + long refresh = dirty & hasWork; + while (refresh != 0) + { + int bitIndex = Long.numberOfTrailingZeros(refresh); + int group = bitIndex / 8; + positions[group] = queues[group].peekSingle().position; + refresh ^= 1L << bitIndex; + } + dirty = 0; + + long minPosition = Long.MAX_VALUE; + int minGroup = -1; + long visit = enabled >>> 7; + while (visit != 0) + { + int bitIndex = Long.numberOfTrailingZeros(visit); + int group = bitIndex / 8; + long position = positions[group]; + if (position < minPosition) + { + minGroup = group; + minPosition = position; + } + visit ^= 1L << bitIndex; + } + + return minGroup; + } + + private int pollGroupByIndex() + { + long visit = (hasWork & unsaturated()) >>> 7; + if (visit == 0) + return -1; + + int bitIndex = Long.numberOfTrailingZeros(visit); + return bitIndex / 8; + } + + private int pollGroupByPhaseFair() + { + return minCounterIndex(recentFlowImbalances()); + } + + // PRIORITY_FAIR selection: a deficit round-robin blend of two strategies, chosen per poll: + // flow -> minCounterIndex(recent - arrivals + bias) (least fairly serviced) + // age -> minGroupByPriority() (earliest-queued work) + // wFlow = ramp(flow imbalance F = max-min of the selection counters); wAge = BLEND_TOTAL - wFlow. As flow gets + // uneven, polls trade from age to flow zero-sum; when balanced it is pure age (FIFO). A ramp (not a hard + // threshold) means there is no mode cliff to oscillate around, so no anti-oscillation penalty is needed. Age is + // also what drains a stale/standing backlog -- its items are the oldest -- so no separate stock strategy is needed. + private int pollGroupByBlended() + { + return pollGroupByBlended(saturatedOrWithoutWork()); + } + + private int pollGroupByBlended(long disabled) + { + long withoutWork = hasWork ^ COUNTER_OVERFLOWS; + long counters = recentFlowImbalances(); + long minMax = minMaxCounterValue(counters, withoutWork); + long min = minMax & 0x7f; + long max = minMax >>> 8; + int flowImbalance = (int) (max - min); + + int flowWeight = flowWeight(flowImbalance); + int priorityWeight = AccordExecutor.BLEND_TOTAL - flowWeight; + + creditFlow += flowWeight; + creditAge += priorityWeight; + + if (creditFlow >= creditAge) + { + creditFlow -= AccordExecutor.BLEND_TOTAL; + if (disabled != withoutWork) + min = minCounterValue(counters, disabled); + + return minCounterIndex(counters, min, disabled); + } + else + { + creditAge -= AccordExecutor.BLEND_TOTAL; + return pollGroupByPriority(disabled ^ COUNTER_OVERFLOWS); + } + } + + private long saturated() + { + return ((active | COUNTER_OVERFLOWS) - limits) & COUNTER_OVERFLOWS; + } + + private long unsaturated() + { + return saturated() ^ COUNTER_OVERFLOWS; + } + + private long unsaturatedWithWork() + { + return hasWork & unsaturated() & ~stopped; + } + + private long saturatedOrWithoutWork() + { + return (hasWork ^ COUNTER_OVERFLOWS) | saturated() | stopped; + } + + // arrivals is a windowed measure of ARRIVAL (incremented on enqueue, size-biased; decays on recent's overflow). + // Combined with recent (service) as effective = max(0, recent - arrivals): a queue whose arrivals outpace its + // service clamps to 0 and is preferred, so service converges to arrival rate (bounding the busy queue's backlog). + private long recentFlowImbalances() + { + return clampedSubtract(dispatches, arrivals); + } + + static long minCounterValue(long counters, long disabled) + { + long mins = counters; + mins |= overflowsToLowMasks(disabled); + mins = minCounters(mins, mins >>> 8); // each slot is min of slots [i..i+1] + mins = minCounters(mins, mins >>> 16); // each slot is min of slots [i..i+3] + mins = minCounters(mins, mins >>> 32); // each slot is min of slots [i..i+7] + return mins & 0x7f; + } + + static long minMaxCounterValue(long counters, long disabled) + { + long mins = counters; + long maxs = counters ^ COUNTER_MASKS; + long overflowMasks = overflowsToLowMasks(disabled); + mins |= overflowMasks; + maxs |= overflowMasks; + mins = minCounters(mins, mins >>> 8) & 0x007f007f007f007fL; // each slot is min of slots [i..i+1] + maxs = (minCounters(maxs, maxs << 8) & 0x7f007f007f007f00L); // each slot is min of slots ~[i..i+1] + long minmaxs = mins | maxs; + minmaxs = minCounters(minmaxs, minmaxs >>> 16); // each slot is min of slots [i..i+3] + minmaxs = minCounters(minmaxs, minmaxs >>> 32); // each slot is min of slots [i..i+7] + return (minmaxs ^ 0x7f00) & 0x7f7f; + } + + /** + * If provided two counters (containing 8 7 bit counters each), + * returns the minimum of each matching counter + */ + private static long minCounters(long a, long b) + { + // set overflow bits where a <= b + long selecta = setOverflowWhenLessEqual(a, b); + return selectByOverflowBits(selecta, a, b); + } + + static long setOverflowWhenLessEqual(long a, long b) + { + return ((b | COUNTER_OVERFLOWS) - a) & COUNTER_OVERFLOWS; + } + + // select a if overflow bit is set; b if it is unset + static long selectByOverflowBits(long selecta, long a, long b) + { + selecta = overflowsToLowMasks(selecta); + a &= selecta; + b &= ~selecta; + return a | b; + } + + static long overflowsToLowMasks(long v) + { + return v - (v >>> 7); + } + + private static int flowWeight(int flowImbalance) + { + if (flowImbalance <= AccordExecutor.FLOW_ONSET) return 0; + return Math.min(AccordExecutor.BLEND_TOTAL, ((flowImbalance - AccordExecutor.FLOW_ONSET) << AccordExecutor.BLEND_SHIFT) >>> AccordExecutor.FLOW_WIDTH_SHIFT); + } + + // per-lane max(0, a - b), carry-free: zero both a and b in lanes where a <= b, then subtract + private static long clampedSubtract(long a, long b) + { + long keep = ~overflowsToLowMasks(setOverflowWhenLessEqual(a, b)); + return (a & keep) - (b & keep); + } + + private int minCounterIndex(long counters) + { + return minCounterIndex(counters, saturatedOrWithoutWork()); + } + + private int minCounterIndex(long counters, long disabled) + { + return minCounterIndex(counters, minCounterValue(counters, disabled), disabled); + } + + private int minCounterIndex(long counters, long minCounterValue, long disabled) + { + long mins = minCounterValue * COUNTER_LOWBITS; + long select = ((mins | COUNTER_OVERFLOWS) - counters) & COUNTER_OVERFLOWS; + // now unset those overflow bits associated with disabled queues + select &= ~disabled; + if (select == 0) + return -1; + return (Long.numberOfTrailingZeros(select) - 7) / 8; + } + + final T pollMulti() + { + int group = pollGroup(); + if (group < 0) + { + // group < 0 can mean EITHER we don't have any nested queues OR those queues are either empty or DISABLED + T result = pollSingle(); + if (result != null) + --waitingCount; + return result; + } + + --waitingCount; + incrementActive(group); + incrementDispatches(group); + + TaskQueue queue = queues[group]; + T head = queue.pollSingle(); + // NOTE: must clear dirty when emptied, symmetrically with unqueue(): the fair selection paths + // never consume the dirty bit, so a group drained during a fairness episode would otherwise + // retain a stale dirty bit and NPE in minGroupByPriority.peekSingle() when balance is restored. + if (queue.isEmptySingle()) + { + unsetHasWork(group); + unsetDirty(group); + } + else setDirty(group); + return head; + } + + final void enqueueMulti(T task, boolean incrementArrivals) + { + task.setQueue(this); + int group = group(task); + if (group < 0) + { + enqueueSingle(task); + } + else + { + TaskQueue queue = queue(group); + int result = queue.enqueueSingle(task); + if (incrementArrivals) + incrementArrivals(group); + if (result < 0) setHasWork(group); + if (result != 0) setDirty(group); + } + ++waitingCount; + } + + final void requeue(T task) + { + int group = group(task); + if (group < 0) requeueSingle(task); + else + { + TaskQueue queue = queue(group); + Invariants.require(queue != null && queue.isQueuedSingle(task)); + if (queue.requeueSingle(task)) + setDirty(group); + } + } + + final void unqueueMulti(T task) + { + int group = group(task); + TaskQueue queue = group < 0 ? this : queue(task); + Invariants.require(queue.isQueuedSingle(task)); + unqueue(task, group, queue); + } + + // if there is an active collection, we return false and do not remove ourselves from it + boolean tryUnqueueWaiting(T task) + { + int group = group(task); + TaskQueue queue = group < 0 ? this : queue(task); + if (!queue.isQueuedSingle(task)) + return false; + + unqueue(task, group, queue); + return true; + } + + private void unqueue(T task, int group, TaskQueue queue) + { + task.unsetQueue(this); + boolean dirty = queue.unqueueSingle(task); + --waitingCount; + if (group >= 0) + { + if (queue.isEmptySingle()) + { + unsetHasWork(group); + unsetDirty(group); + } + else if (dirty) setDirty(group); + } + } + + final void incrementActive(int group) + { + active += lowBit(group); + } + + final void decrementActive(int group) + { + active -= lowBit(group); + } + + final void incrementDispatches(Task task) + { + int group = group(task); + if (group >= 0) + incrementDispatches(group); + } + + final void incrementDispatches(int group) + { + dispatches += lowBit(group); + if ((dispatches & COUNTER_OVERFLOWS) != 0) + { + dispatches = (dispatches >>> 1) & COUNTER_MASKS; + arrivals = (arrivals >>> 1) & COUNTER_MASKS; // arrivals (arrival) decays on the service/time clock + } + } + + final void decrementDispatches(Task task) + { + int group = group(task); + if (group >= 0) + decrementDispatches(group); + } + + final void decrementDispatches(int group) + { + long lowBit = lowBit(group); + dispatches -= lowBit; + dispatches += (dispatches >>> 7) & lowBit; + } + + final void incrementArrivals(Task task) + { + int group = group(task); + if (group >= 0) + incrementArrivals(group); + } + + final void incrementArrivals(int group) + { + int shift = group * 8; + long overflowBit = 0x80L << shift; + arrivals += 1L << shift; + // if we overflow, unset the overflow bit and set all other bits for the counter + long overflow = arrivals & overflowBit; + arrivals ^= overflow; + arrivals |= overflow - (overflow >>> 7); + } + + final void setHasWork(int group) + { + hasWork |= overflowBit(group); + } + + final void unsetHasWork(int group) + { + hasWork &= ~overflowBit(group); + } + + final void setDirty(int group) + { + dirty |= overflowBit(group); + } + + final void unsetDirty(int group) + { + dirty &= ~overflowBit(group); + } + + final boolean hasWaitingToRun() + { + return unsaturatedWithWork() != 0; + } + + final boolean isWaiting(T task) + { + return queue(task).isQueuedSingle(task); + } + + final int waitingCount() + { + return waitingCount; + } + + final long lowBit(int group) + { + return 1L << (group * 8); + } + + final long overflowBit(int group) + { + return 0x80L << (group * 8); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java new file mode 100644 index 000000000000..8aad433fb62a --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.TinyEnumSet; + +import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; + +final class TaskQueueRunnable extends TaskQueueMulti +{ + static final int RUNNABLE = TinyEnumSet.encode(WAITING_TO_RUN, RUNNING); + + final TaskQueue assigned; + + TaskQueueRunnable() + { + super(RUNNABLE, Task.GroupKind.GLOBAL, AccordExecutor.GLOBAL_QUEUE_LIMITS); + this.assigned = new TaskQueue<>(0); + } + + T poll() + { + T next = pollMulti(); + if (next == null) + return null; + + assigned.enqueueSingle(next); + return next; + } + + void enqueue(T enqueue, boolean incrementArrivals) + { + enqueueMulti(enqueue, incrementArrivals); + } + + void unqueue(T unqueue) + { + if (assigned.isQueuedSingle(unqueue)) + { + int group = group(unqueue); + if (group >= 0) + decrementActive(group); + + unqueue.unsetQueue(this); + assigned.unqueueSingle(unqueue); + } + else + { + super.unqueueMulti(unqueue); + } + } + + int waitingOrAssignedCount() + { + return waitingCount + assigned.size(); + } + + boolean hasAssignedOrWaiting() + { + return waitingCount > 0 || !assigned.isEmptySingle(); + } + + boolean hasAssigned() + { + return !assigned.isEmptySingle(); + } + + boolean isAssigned(T task) + { + return assigned.isQueuedSingle(task); + } + + void cleanup(T task) + { + if (assigned.tryUnqueueSingle(task)) + { + int group = group(task); + if (group >= 0) + decrementActive(group); + task.unsetQueue(this); + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java new file mode 100644 index 000000000000..7dd8b1def0be --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.utils.TinyEnumSet; + +final class TaskQueueStandalone extends TaskQueue +{ + TaskQueueStandalone(int states) + { + super(states); + } + + TaskQueueStandalone(Task.State state) + { + super(TinyEnumSet.encode(state)); + } + + void enqueue(T enqueue) + { + enqueue.setQueue(this); + enqueueSingle(enqueue); + } + + void unqueue(T unqueue) + { + unqueue.unsetQueue(this); + removeNode(unqueue); + } + + T peek() + { + return peekSingle(); + } + + T poll() + { + return pollSingle(); + } + + boolean isEmpty() + { + return isEmptySingle(); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java b/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java new file mode 100644 index 000000000000..2dffb11e1850 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import org.apache.cassandra.concurrent.CassandraThread; +import org.apache.cassandra.concurrent.DebuggableTask; + +public interface TaskRunner +{ + AccordExecutor accordActiveExecutor(); + void setAccordActiveExecutor(AccordExecutor newExecutor); + + AccordExecutor accordLockedExecutor(); + boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor); + void exitAccordLockedExecutor(); + + // to be called only by the thread itself, so can (eventually) avoid any memory barriers + Task accordActiveSelfTask(); + Task accordActiveTask(); + void setAccordActiveTask(Task newActiveTask); + + static TaskRunner get() + { + return get(Thread.currentThread()); + } + + static TaskRunner get(Thread thread) + { + return thread instanceof CassandraThread ? (CassandraThread) thread : ThreadLocalTaskRunner.threadLocal.get(); + } + + final class ThreadLocalTaskRunner implements TaskRunner + { + private AccordExecutor lockedExecutor; + private AccordExecutor activeExecutor; + volatile Task activeTask; + + private static final ThreadLocal threadLocal = ThreadLocal.withInitial(ThreadLocalTaskRunner::new); + + @Override + public AccordExecutor accordActiveExecutor() + { + return activeExecutor; + } + + @Override + public void setAccordActiveExecutor(AccordExecutor newExecutor) + { + activeExecutor = newExecutor; + } + + @Override + public AccordExecutor accordLockedExecutor() + { + return lockedExecutor; + } + + @Override + public boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor) + { + if (lockedExecutor != null) + return false; + lockedExecutor = newLockedExecutor; + return true; + } + + @Override + public void exitAccordLockedExecutor() + { + lockedExecutor = null; + } + + @Override + public void setAccordActiveTask(Task newActiveTask) + { + activeTask = newActiveTask; + } + + @Override + public Task accordActiveTask() + { + return activeTask; + } + + @Override + public Task accordActiveSelfTask() + { + return activeTask; + } + } + + abstract class DebuggableWrapper implements DebuggableTask.DebuggableTaskRunner + { + private volatile TaskRunner wrapped; + + protected void setWrapped(TaskRunner wrapped) + { + this.wrapped = wrapped; + } + + @Override + public DebuggableTask running() + { + Task running = wrapped.accordActiveTask(); + return running == null ? null : running.debuggable(); + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/Tranches.java b/src/java/org/apache/cassandra/service/accord/execution/Tranches.java new file mode 100644 index 000000000000..5b5b3f266854 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Tranches.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayDeque; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import accord.utils.Invariants; + +import static org.apache.cassandra.service.accord.execution.Task.MAX_TRANCHE; + +/** + * Tasks are separated into tranches based on their position. + *

+ * When we want to wait for all submitted tasks and their consequences to complete, we create a new tranche + * and track the number of tasks still extant for all prior tranches - once these reach zero we can signal + * the completion of the work. + */ +final class Tranches +{ + private static final Logger logger = LoggerFactory.getLogger(Tranches.class); + + class WithDeferred implements Runnable + { + final Runnable run; + final ArrayDeque deferred; + + WithDeferred(Runnable run, Runnable deferred) + { + this.run = run; + this.deferred = new ArrayDeque<>(1); + this.deferred.add(deferred); + } + + WithDeferred(Runnable run, ArrayDeque deferred) + { + this.run = run; + this.deferred = deferred; + } + + @Override + public void run() + { + Runnable register = deferred.poll(); + if (!deferred.isEmpty()) + { + Invariants.require(runs[firstIndex] != null); + Invariants.require(!(runs[firstIndex] instanceof WithDeferred)); + runs[firstIndex] = new WithDeferred(runs[firstIndex], deferred); + } + registerWait(register); + try + { + run.run(); + } + catch (Throwable t) + { + owner.agent.onException(t); + } + } + } + + final AccordExecutor owner; + + int firstTranche; + int firstIndex; + long[] mins = new long[8]; + int[] counts = new int[8]; + Runnable[] runs = new Runnable[8]; + + // the next tranche, that all new work is being collected against + // (and will move to the array accounting once a new wait is registered) + long nextMin; + int nextTranche; + int nextCount; + + Tranches(AccordExecutor owner) + { + this.owner = owner; + } + + private int trancheToIndex(int tranche) + { + return firstIndex + trancheToIndexOffset(tranche); + } + + private int trancheToIndexOffset(int tranche) + { + int offset = tranche - firstTranche; + if (offset < 0) + offset += MAX_TRANCHE + 1; + return offset; + } + + int size() + { + return trancheToIndexOffset(nextTranche); + } + + int capacity() + { + return counts.length; + } + + int addNew(long position) + { + Invariants.require(position >= nextMin); + ++nextCount; + return nextTranche; + } + + void addInherited(int tranche, long position) + { + if (tranche == nextTranche) + { + Invariants.require(position >= nextMin); + ++nextCount; + } + else + { + int index = trancheToIndex(tranche); + Invariants.require(counts[index] > 0); + Invariants.require(mins[index] <= position); + ++counts[index]; + } + } + + void complete(int tranche) + { + if (tranche == nextTranche) + { + --nextCount; + } + else + { + + if (counts[trancheToIndex(tranche)] == 1) + owner.drainUnqueuedNewWorkExclusive(); // make sure we don't have any pending + + if (--counts[trancheToIndex(tranche)] == 0 && tranche == firstTranche) + { + do + { + advance(); + } + while (firstTranche != nextTranche && counts[firstIndex] == 0); + } + + if (firstIndex >= counts.length / 2) + compact(); + } + } + + public void finishAll(long nextPosition) + { + while (firstTranche != nextTranche) + { + logger.warn("{} processed all pending tasks (<{}) but found {} waiting for {}", this, + nextPosition, counts[firstIndex], size() == 1 ? nextMin : mins[firstIndex + 1]); + advance(); + } + } + + private void advance() + { + Runnable run = runs[firstIndex]; + runs[firstIndex] = null; + ++firstIndex; + if (firstTranche == MAX_TRANCHE) firstTranche = 0; + else ++firstTranche; + try + { + run.run(); + } + catch (Throwable t) + { + owner.agent.onException(t); + } + } + + public void registerWait(Runnable run) + { + int newNextTranche = (nextTranche + 1) % (MAX_TRANCHE + 1); + if (newNextTranche == firstTranche) + { + Runnable cur = runs[firstIndex]; + if (cur instanceof WithDeferred) ((WithDeferred) cur).deferred.add(run); + else runs[firstIndex] = new WithDeferred(runs[firstIndex], run); + return; + } + + if ((firstIndex + size()) == capacity()) + growOrCompact(); + + int index = firstIndex + size(); + mins[index] = nextMin; + counts[index] = nextCount; + runs[index] = run; + nextMin = owner.minPosition = owner.nextPosition; + nextCount = 0; + nextTranche = newNextTranche; + } + + private void compact() + { + if (size() <= capacity() / 4 && capacity() > 8) resize(capacity() / 2); + else compact(mins, counts, runs); + } + + private void growOrCompact() + { + if (size() > capacity() / 2) resize(capacity() * 2); + else compact(); + } + + private void resize(int newSize) + { + Invariants.require(newSize > 0); + long[] newMins = new long[newSize]; + int[] newCounts = new int[newSize]; + Runnable[] newRuns = new Runnable[newSize]; + compact(newMins, newCounts, newRuns); + mins = newMins; + counts = newCounts; + runs = newRuns; + } + + private void compact(long[] newMins, int[] newCounts, Runnable[] newRuns) + { + int size = size(); + System.arraycopy(mins, firstIndex, newMins, 0, size); + System.arraycopy(counts, firstIndex, newCounts, 0, size); + System.arraycopy(runs, firstIndex, newRuns, 0, size); + firstIndex = 0; + } +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/Unstoppable.java b/src/java/org/apache/cassandra/service/accord/execution/Unstoppable.java new file mode 100644 index 000000000000..d4851481a647 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Unstoppable.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import accord.local.ExecutionContext; + +// run the task even on a stopped commandStore +public interface Unstoppable extends ExecutionContext.Empty +{ +} diff --git a/src/java/org/apache/cassandra/service/accord/execution/Unterminatable.java b/src/java/org/apache/cassandra/service/accord/execution/Unterminatable.java new file mode 100644 index 000000000000..20badc4edab6 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/execution/Unterminatable.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +// run the task even on a terminated commandStore +public interface Unterminatable extends Unstoppable +{ +} diff --git a/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java b/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java index c3be7c87a0ed..375575032458 100644 --- a/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java +++ b/src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java @@ -54,13 +54,13 @@ import accord.utils.btree.BTree; import accord.utils.btree.IntervalBTree; -import org.apache.cassandra.service.accord.AccordCache; -import org.apache.cassandra.service.accord.AccordCacheEntry; import org.apache.cassandra.service.accord.AccordCommandStore; import org.apache.cassandra.service.accord.AccordCommandStore.Caches; import org.apache.cassandra.service.accord.RangeIndex; import org.apache.cassandra.service.accord.TokenRange; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; import static accord.local.CommandSummaries.Relevance.IRRELEVANT; import static accord.local.LoadKeysFor.RECOVERY; @@ -199,7 +199,7 @@ public void load(Map into, BooleanSupplier abort) } @Override - protected void finish(Map into) + public void finish(Map into) { } @@ -233,7 +233,7 @@ boolean isMaybeRelevant(TxnIdInterval txnIdInterval) } @Override - protected void cleanupExclusive(Caches caches) + public void cleanupExclusive(Caches caches) { if (commandWatcher != null) { diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java b/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java index 29f02207909e..87ce5336389f 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java @@ -58,10 +58,10 @@ import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.TokenRange; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; import org.apache.cassandra.service.accord.serializers.Version; diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java b/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java index bddb2b6fba7a..3c72c34ba135 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java @@ -55,8 +55,8 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; import org.apache.cassandra.service.accord.serializers.Version; diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java b/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java index 719eb0f5900c..90cd9898c95e 100644 --- a/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java +++ b/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java @@ -64,8 +64,8 @@ import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; import org.apache.cassandra.service.accord.serializers.Version; diff --git a/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java b/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java index 88a785d9a92c..d82bd4ab83c7 100644 --- a/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java +++ b/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java @@ -24,6 +24,7 @@ import org.junit.Test; import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.utils.concurrent.Semaphore; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; diff --git a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java index 7e91f5a7fb61..4b573b255138 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java +++ b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java @@ -83,7 +83,7 @@ import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.Verb; -import org.apache.cassandra.service.accord.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCache; import static java.lang.System.currentTimeMillis; import static java.util.concurrent.TimeUnit.MILLISECONDS; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java index 506407cb13e8..107488f0d5c4 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java @@ -46,7 +46,7 @@ import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.streaming.StreamEvent; import org.apache.cassandra.streaming.StreamEventHandler; @@ -262,7 +262,7 @@ public void bootstrapTest(Consumer bootstrapAndJoinNode) throws Throwab getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> { if (safeStore.ranges().currentRanges().contains(partitionKey)) { - AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore; + SaferCommandStore ss = (SaferCommandStore) safeStore; Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); Assert.assertFalse(ss.safeToReadAt().isEmpty()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java index edff67507ac0..97f98f319be2 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java @@ -24,7 +24,6 @@ import accord.local.CommandStores; import accord.local.ExecutionContext; -import accord.local.LoadKeys; import accord.local.Node; import accord.local.cfk.CommandsForKey; import accord.local.cfk.SafeCommandsForKey; @@ -42,11 +41,10 @@ import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.TokenRange; -import static accord.local.LoadKeysFor.READ_WRITE; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; import static org.apache.cassandra.service.accord.AccordService.getBlocking; @@ -140,7 +138,7 @@ protected static void validateAccord(Cluster cluster, TableId id) { AccordCommandStore store = (AccordCommandStore) stores.forId(storeId); getBlocking(store.chain(ctx, input -> { - AccordSafeCommandStore safe = (AccordSafeCommandStore) input; + SaferCommandStore safe = (SaferCommandStore) input; for (SafeCommandsForKey safeCfk : safe.safeCommandsForKeys()) { CommandsForKey cfk = safeCfk.current(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java index 95d79b6fa79f..d3c1fadfc4e4 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java @@ -62,7 +62,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.accord.IAccordService.DelegatingAccordService; @@ -209,7 +209,7 @@ private static TxnId awaitLocalApplyOnKey(TokenKey key) Node node = accordService().node(); AtomicReference waitFor = new AtomicReference<>(null); getBlocking(node.commandStores().forEach("Test", RoutingKeys.of(key), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> { - AccordSafeCommandStore store = (AccordSafeCommandStore) safeStore; + SaferCommandStore store = (SaferCommandStore) safeStore; SafeCommandsForKey safeCfk = store.ifLoadedAndInitialised(key); if (safeCfk == null) return; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java index dff776cbf22b..6e6bb30ab74b 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java @@ -54,7 +54,7 @@ import org.apache.cassandra.metrics.RatioGaugeSet; import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.exceptions.AccordReadPreemptedException; import org.apache.cassandra.service.accord.exceptions.AccordWritePreemptedException; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java index 4d600f0c7105..881728882fec 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java @@ -38,7 +38,7 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.api.PartitionKey; import static com.google.common.collect.Iterables.getOnlyElement; @@ -112,7 +112,7 @@ public void moveTest() throws Throwable getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), moveMax, moveMax, safeStore -> { if (!safeStore.ranges().allAt(preMove).contains(partitionKey)) { - AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore; + SaferCommandStore ss = (SaferCommandStore) safeStore; Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); Assert.assertFalse(ss.safeToReadAt().isEmpty()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java index 79cf938c7ef1..4f4b7942f00b 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java @@ -47,7 +47,7 @@ import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordOperations; -import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.execution.SaferCommandStore; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; @@ -187,7 +187,7 @@ public void replaceWithAvailabilityLossTest() throws Throwable getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> { if (safeStore.ranges().currentRanges().contains(partitionKey)) { - AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore; + SaferCommandStore ss = (SaferCommandStore) safeStore; Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); Assert.assertFalse(ss.safeToReadAt().isEmpty()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java index 1b864aea1b04..67493f5a2c8b 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalConsistentExpungeTest.java @@ -36,9 +36,9 @@ import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.accord.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordSafeCommand; +import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.utils.ByteBufferUtil; @@ -87,7 +87,7 @@ public void loadCommandErasedTest() throws Throwable Node node = service.node(); AccordCommandStore commandStore = (AccordCommandStore) node.commandStores().unsafeForKey(key.toUnseekable()); - Iterator> iterator = commandStore.cachesUnsafe().commands().iterator(); + Iterator> iterator = commandStore.cachesUnsafe().commands().iterator(); TxnId txnId = TxnId.NONE; diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java index d5a8f79c63ae..640ce44740a1 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java @@ -74,7 +74,7 @@ public void rebootstrapFuzzTest() throws Throwable HashSet downInstances = new HashSet<>(); AtomicInteger nextId = new AtomicInteger(); withRandom(rng -> { - Generator schemaGen = SchemaGenerators.trivialSchema(KEYSPACE, () -> "bootstrap_fuzz" + (nextId.incrementAndGet()), POPULATION, + Generator schemaGen = SchemaGenerators.trivialSchema(KEYSPACE, () -> "bsfuzz" + (nextId.incrementAndGet()), POPULATION, SchemaSpec.optionsBuilder() .addWriteTimestamps(false) .withTransactionalMode(TransactionalMode.full) diff --git a/test/simulator/test/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulationTest.java b/test/simulator/test/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulationTest.java index fcdc7deb4181..f6262a7db2c4 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulationTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulationTest.java @@ -45,8 +45,8 @@ public void parseSuccess() "\tat accord.impl.RequestCallbacks$CallbackStripe$RegisteredCallback.lambda$safeInvoke$0(RequestCallbacks.java:140)\n" + "\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n" + "\tat accord.utils.async.AsyncChains.lambda$encapsulate$0(AsyncChains.java:498)\n" + - "\tat org.apache.cassandra.service.accord.AccordExecutor$PlainRunnable.run(AccordExecutor.java:989)\n" + - "\tat org.apache.cassandra.service.accord.AccordExecutor$CommandStoreQueue.run(AccordExecutor.java:729)\n" + + "\tat org.apache.cassandra.service.accord.execution.AccordExecutor$PlainRunnable.run(AccordExecutor.java:989)\n" + + "\tat org.apache.cassandra.service.accord.execution.AccordExecutor$CommandStoreQueue.run(AccordExecutor.java:729)\n" + "\tat org.apache.cassandra.service.accord.AccordExecutorSimple.run(AccordExecutorSimple.java:95)\n" + "\tat org.apache.cassandra.concurrent.FutureTask$2.call(FutureTask.java:124)\n" + "\tat org.apache.cassandra.concurrent.SyncFutureTask.run(SyncFutureTask.java:68)\n" + @@ -77,8 +77,8 @@ public void parseFailure() "\tat accord.impl.RequestCallbacks$CallbackStripe$RegisteredCallback.lambda$safeInvoke$0(RequestCallbacks.java:140)\n" + "\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n" + "\tat accord.utils.async.AsyncChains.lambda$encapsulate$0(AsyncChains.java:498)\n" + - "\tat org.apache.cassandra.service.accord.AccordExecutor$PlainRunnable.run(AccordExecutor.java:989)\n" + - "\tat org.apache.cassandra.service.accord.AccordExecutor$CommandStoreQueue.run(AccordExecutor.java:729)\n" + + "\tat org.apache.cassandra.service.accord.execution.AccordExecutor$PlainRunnable.run(AccordExecutor.java:989)\n" + + "\tat org.apache.cassandra.service.accord.execution.AccordExecutor$CommandStoreQueue.run(AccordExecutor.java:729)\n" + "\tat org.apache.cassandra.service.accord.AccordExecutorSimple.run(AccordExecutorSimple.java:95)\n" + "\tat org.apache.cassandra.concurrent.FutureTask$2.call(FutureTask.java:124)\n" + "\tat org.apache.cassandra.concurrent.SyncFutureTask.run(SyncFutureTask.java:68)\n" + diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java deleted file mode 100644 index 6bf1f46dfd97..000000000000 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorAndCacheTest.java +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.simulator.test; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executor; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.LockSupport; -import java.util.function.BooleanSupplier; -import java.util.function.Consumer; - -import javax.annotation.Nullable; - -import org.junit.Test; - -import accord.api.AsyncExecutor; -import accord.api.ExclusiveAsyncExecutor; -import accord.utils.async.Cancellable; - -import org.apache.cassandra.concurrent.ExecutorPlus; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; -import org.apache.cassandra.service.accord.AccordExecutor; -import org.apache.cassandra.service.accord.AccordExecutorAsyncSubmit; -import org.apache.cassandra.service.accord.AccordExecutorSignalLoop; -import org.apache.cassandra.service.accord.api.AccordAgent; -import org.apache.cassandra.utils.concurrent.AsyncPromise; -import org.apache.cassandra.utils.concurrent.SignalLock; - -import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; - -// TODO (required): have simulator intercept ReentantLock so we can test SyncSubmit and SemiSyncSubmit -public class AccordExecutorAndCacheTest extends SimulationTestBase -{ - @Test - public void signalLoopTest() - { - executorTest(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, SignalLock.MAX_THREADS, -1, -1, TimeUnit.MICROSECONDS, i -> "Loop" + i, new AccordAgent()), - 16); - } - - static class Submitted - { - final AtomicInteger nextId = new AtomicInteger(); - final AtomicInteger doneBefore = new AtomicInteger(); - final ConcurrentHashMap>> consequences = new ConcurrentHashMap<>(); - - boolean isDone() - { - return isDoneBetween(0, nextId.get()); - } - - boolean isDoneBefore(int before) - { - if (!isDoneBetween(doneBefore.get(), before)) - return false; - doneBefore.accumulateAndGet(before, Integer::max); - return true; - } - - boolean isDoneBetween(int from, int before) - { - for (int id = from ; id < before ; ++id) - { - for (Future future : consequences.get(id)) - { - if (!future.isDone()) - return false; - } - } - return true; - } - - Collection> start() - { - ConcurrentLinkedQueue> result = new ConcurrentLinkedQueue<>(); - - while (true) - { - int id = nextId.get(); - Object prev = consequences.putIfAbsent(id, result); - nextId.compareAndSet(id, id + 1); - if (prev == null) - return result; - } - } - } - - static class Control extends ConcurrentLinkedQueue - { - final Submitted submitted; - final AtomicInteger count = new AtomicInteger(); - final float cancelChance; - float processChance; - - Control(float cancelChance, Submitted submitted) - { - this(submitted, cancelChance, ThreadLocalRandom.current().nextFloat() * 0.5f); - } - - Control(Submitted submitted, float cancelChance, float processChance) - { - this.submitted = submitted; - this.cancelChance = cancelChance; - this.processChance = processChance; - } - - void submit(AsyncExecutor executor, Collection> consequences, Consumer>> run) - { - AsyncPromise future = new AsyncPromise<>(); - Cancellable cancel = executor.chain(() -> run.accept(consequences)).begin((success, fail) -> { - if (fail == null) future.trySuccess(null); - else - { - future.tryFailure(fail); - if (fail instanceof CancellationException) - run.accept(submitted.start()); - } - }); - consequences.add(future); - - if (cancel != null && ThreadLocalRandom.current().nextFloat() <= cancelChance) - { - add(cancel); - count.incrementAndGet(); - } - - if (count.get() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance) - { - int cancelCount = 0; - do - { - ++cancelCount; - - float delta = ThreadLocalRandom.current().nextFloat() - 0.5f; - if (delta < 0) processChance /= delta; - else processChance *= -delta; - if (processChance < 0.001f || processChance > 0.999f) - processChance = cancelChance; - } while (count.decrementAndGet() > 0 && ThreadLocalRandom.current().nextFloat() <= processChance); - - // do outside of loop to avoid reentry - while (cancelCount-- > 0) - remove().cancel(); - } - } - } - - public void executorTest(SerializableSupplier supplier, int submissionThreads) - { - simulate(arr(() -> { - try - { - DatabaseDescriptor.daemonInitialization(); - ExecutorPlus submit = executorFactory().pooled("submit-test", submissionThreads); - AccordExecutor executor = supplier.get(); - Lock lock = executor.unsafeLock(); - ExclusiveAsyncExecutor sequentialExecutor = executor.newExclusiveExecutor(); - Executor lockExecutor = executorFactory().sequential("lock"); - - for (float sleepChance : new float[] { 0f, 0.01f, 0.1f }) - { - for (float lockChance : new float[] { 0f, 0.01f, 0.1f }) - { - for (float cancelChance : new float[] { 0f, 0.01f, 0.1f }) - { - System.out.println(String.format("sleepChance %.2f, lockChance %.2f, cancelChance %.2f", sleepChance, lockChance, cancelChance)); - List> done = new ArrayList<>(); - Submitted submitted = new Submitted(); - for (int i = 0; i < submissionThreads; ++i) - { - int id = i; - done.add(submit.submit(() -> { - try - { - submitLoop(id, lock, executor, sequentialExecutor, lockExecutor, 20, 10, sleepChance, lockChance, new Control(cancelChance, submitted), submitted); - } - catch (ExecutionException | InterruptedException e) - { - throw new RuntimeException(e); - } - })); - } - for (Future f : done) - f.get(); - - if (!submitted.isDone()) - throw new AssertionError(); - } - } - } - } - catch (Throwable t) - { - throw new RuntimeException(t); - } - }), - () -> {}, 1L); - } - - private static void submitLoop(int id, Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, Executor lockExecutor, int outerLoop, int innerLoop, float sleepChance, float lockChance, Control control, Submitted submitted) throws ExecutionException, InterruptedException - { - ConcurrentLinkedQueue> awaitConsequences = new ConcurrentLinkedQueue<>(); - while (outerLoop-- > 0) - { - List>> allAwaitSubmitted = new ArrayList<>(); - for (int i = 0; i < innerLoop; ++i) - { - Collection> awaitSubmitted = submitted.start(); - allAwaitSubmitted.add(awaitSubmitted); - submitRecursive(lock, executor, sequentialExecutor, 1 + i, awaitSubmitted, awaitConsequences, submitted, sleepChance, lockChance, control); - } - - AtomicBoolean done = new AtomicBoolean(); - submitUntil(lock, lockExecutor, sleepChance, done::get); - for (Collection> awaitSubmitted : allAwaitSubmitted) - await(awaitSubmitted, CancellationException.class); - await(awaitConsequences, null); - done.set(true); - System.out.println("Loop " + id + '.' + (1 + outerLoop)); - } - } - - private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Collection> consequences, Collection> awaitConsequences, Submitted submitted, float sleepChance, float lockChance, Control control) - { - AsyncExecutor submitTo = ThreadLocalRandom.current().nextBoolean() ? executor : sequentialExecutor; - - control.submit(submitTo, consequences, nextConsequences -> { - ThreadLocalRandom rnd = ThreadLocalRandom.current(); - boolean locked = false; - if (rnd.nextFloat() < lockChance) - { - if (rnd.nextBoolean()) locked = lock.tryLock(); - else { locked = true; lock.lock(); } - } - if (ThreadLocalRandom.current().nextFloat() < 0.01f) - { - int expectDoneBefore = submitted.nextId.get(); - AsyncPromise afterConsequences = new AsyncPromise<>(); - executor.afterSubmittedAndConsequences(() -> { - if (!submitted.isDoneBefore(expectDoneBefore)) - throw new AssertionError(); - afterConsequences.setSuccess(null); - }); - awaitConsequences.add(afterConsequences); - } - try - { - if (count > 1) - submitRecursive(lock, executor, sequentialExecutor, count -1, nextConsequences, awaitConsequences, submitted, sleepChance, lockChance, control); - if (rnd.nextFloat() < sleepChance) - LockSupport.parkNanos(rnd.nextInt(10000, 100000)); - } - finally - { - if (locked) - lock.unlock(); - } - }); - } - - private static void submitUntil(Lock lock, Executor executor, float sleepChance, BooleanSupplier done) - { - if (done.getAsBoolean()) - return; - - executor.execute(() -> { - - ThreadLocalRandom rnd = ThreadLocalRandom.current(); - boolean tryLock = rnd.nextBoolean(); - boolean locked = !tryLock; - if (tryLock) locked = lock.tryLock(); - else lock.lock(); - try - { - if (rnd.nextFloat() < sleepChance) - LockSupport.parkNanos(rnd.nextInt(10000, 100000)); - - submitUntil(lock, executor, sleepChance, done); - } - finally - { - if (locked) - lock.unlock(); - } - }); - } - - private static void await(Collection> await, @Nullable Class ignore) throws InterruptedException, ExecutionException - { - for (Future future : await) - { - try { future.get(); } - catch (ExecutionException e) - { - if (ignore == null || !(ignore.isInstance(e.getCause()))) - throw e; - } - } - } -} diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java index 52a2b85409ee..3d441ae04e15 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java @@ -47,15 +47,16 @@ import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; -import org.apache.cassandra.service.accord.AccordExecutor; -import org.apache.cassandra.service.accord.AccordExecutorAsyncSubmit; -import org.apache.cassandra.service.accord.AccordExecutorSignalLoop; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutorAsyncSubmit; +import org.apache.cassandra.service.accord.execution.AccordExecutorSignalLoop; import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.simulator.test.AccordExecutorTest.Submitted.Consequences; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.SignalLock; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; // TODO (required): have simulator intercept ReentantLock so we can test SyncSubmit and SemiSyncSubmit public class AccordExecutorTest extends SimulationTestBase @@ -114,19 +115,38 @@ boolean isDoneBetween(int from, int before) return true; } - Collection> start() + class Consequences extends ConcurrentLinkedQueue> { - ConcurrentLinkedQueue> result = new ConcurrentLinkedQueue<>(); + volatile boolean started; - while (true) + void ensureStarted() { - int id = nextId.get(); - Object prev = consequences.putIfAbsent(id, result); - nextId.compareAndSet(id, id + 1); - if (prev == null) - return result; + if (started) + return; + + synchronized (this) + { + if (started) + return; + + while (true) + { + int id = nextId.get(); + Object prev = consequences.putIfAbsent(id, this); + nextId.compareAndSet(id, id + 1); + if (prev == null) + break; + } + + started = true; + } } } + + Consequences allocate() + { + return new Consequences(); + } } static class Control extends ConcurrentLinkedQueue @@ -148,19 +168,20 @@ static class Control extends ConcurrentLinkedQueue this.processChance = processChance; } - void submit(AsyncExecutor executor, Collection> consequences, Consumer>> run) + void submit(AsyncExecutor executor, Consequences consequences, Consumer run) { AsyncPromise future = new AsyncPromise<>(); + consequences.add(future); Cancellable cancel = executor.chain(() -> run.accept(consequences)).begin((success, fail) -> { if (fail == null) future.trySuccess(null); else { future.tryFailure(fail); if (fail instanceof CancellationException) - run.accept(submitted.start()); + run.accept(submitted.allocate()); } }); - consequences.add(future); + consequences.ensureStarted(); if (cancel != null && ThreadLocalRandom.current().nextFloat() <= cancelChance) { @@ -249,7 +270,7 @@ private static void submitLoop(int id, Lock lock, AccordExecutor executor, Exclu List>> allAwaitSubmitted = new ArrayList<>(); for (int i = 0; i < innerLoop; ++i) { - Collection> awaitSubmitted = submitted.start(); + Consequences awaitSubmitted = submitted.allocate(); allAwaitSubmitted.add(awaitSubmitted); submitRecursive(lock, executor, sequentialExecutor, 1 + i, awaitSubmitted, awaitConsequences, submitted, sleepChance, lockChance, control); } @@ -264,7 +285,7 @@ private static void submitLoop(int id, Lock lock, AccordExecutor executor, Exclu } } - private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Collection> consequences, Collection> awaitConsequences, Submitted submitted, float sleepChance, float lockChance, Control control) + private static void submitRecursive(Lock lock, AccordExecutor executor, ExclusiveAsyncExecutor sequentialExecutor, int count, Consequences consequences, Collection> awaitConsequences, Submitted submitted, float sleepChance, float lockChance, Control control) { AsyncExecutor submitTo = ThreadLocalRandom.current().nextBoolean() ? executor : sequentialExecutor; diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java index 6f836c0e8671..cca5292f81e3 100644 --- a/test/unit/org/apache/cassandra/cql3/CQLTester.java +++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java @@ -188,7 +188,7 @@ import org.apache.cassandra.service.ClientWarn; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.accord.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCache; import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.transport.Event; diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java index 69f3722bd0f2..97a9fb22fffa 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java @@ -83,17 +83,14 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.AccordTestUtils; import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.utils.FBUtilities; -import static accord.local.ExecutionContext.contextFor; import static accord.local.ExecutionContext.unsequencedReadWrite; -import static accord.local.LoadKeys.SYNC; -import static accord.local.LoadKeysFor.READ_WRITE; import static accord.local.RedundantStatus.SomeStatus.GC_BEFORE_AND_LOCALLY_DURABLE; import static accord.primitives.Routable.Domain.Range; import static accord.primitives.Timestamp.Flag.HLC_BOUND; diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java index a8bbec8fdf7e..92081f8918db 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java @@ -62,6 +62,9 @@ import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.SaferCommand; +import org.apache.cassandra.service.accord.execution.SaferCommandsForKey; +import org.apache.cassandra.service.accord.execution.SafeTask; import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializerTest.TestSafeCommandStore; import org.apache.cassandra.service.accord.txn.TxnDataResult; import org.apache.cassandra.service.accord.txn.TxnUpdate; @@ -71,15 +74,15 @@ import static accord.primitives.Status.Durability.AllQuorums; import static com.google.common.collect.Iterables.getOnlyElement; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.RELEASE_QUEUE; import static org.apache.cassandra.service.accord.AccordService.getBlocking; import static org.apache.cassandra.service.accord.AccordTestUtils.Commands.preaccepted; import static org.apache.cassandra.service.accord.AccordTestUtils.ballot; import static org.apache.cassandra.service.accord.AccordTestUtils.createAccordCommandStore; import static org.apache.cassandra.service.accord.AccordTestUtils.createPartialTxn; -import static org.apache.cassandra.service.accord.AccordTestUtils.loaded; import static org.apache.cassandra.service.accord.AccordTestUtils.timestamp; import static org.apache.cassandra.service.accord.AccordTestUtils.txnId; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; public class AccordCommandStoreTest { @@ -138,8 +141,8 @@ public void commandLoadSave() throws Throwable Command expected = Command.Executed.executed(txnId, SaveStatus.Applied, AllQuorums, StoreParticipants.all(route), promised, executeAt, txn, dependencies, accepted, waitingOn, result.left, TxnDataResult.PERSISTABLE); - AccordSafeCommand safeCommand = new AccordSafeCommand(loaded(txnId, null)); - safeCommand.preExecute(new AccordTask<>(null, ExecutionContext.unsequenced(txnId, "Test"), null), RELEASE_QUEUE); + SaferCommand safeCommand = new SaferCommand(loaded(txnId, null)); + safeCommand.preExecute(new SafeTask<>(null, ExecutionContext.unsequenced(txnId, "Test"), null), RELEASE_QUEUE); safeCommand.set(expected); // In practice we should never need to save it with the condition boolean set // Not sure why this test does that @@ -167,13 +170,13 @@ public void commandsForKeyLoadSave() Command command1 = preaccepted(txnId1, txn, timestamp(1, clock.incrementAndGet(), 1)); Command command2 = preaccepted(txnId2, txn, timestamp(1, clock.incrementAndGet(), 1)); - AccordSafeCommandsForKey cfk = new AccordSafeCommandsForKey(loaded(key, null)); - cfk.preExecute(new AccordTask<>(null, ExecutionContext.unsequenced(txnId1, "Test"), null), RELEASE_QUEUE); + SaferCommandsForKey cfk = new SaferCommandsForKey(loaded(key, null)); + cfk.preExecute(new SafeTask<>(null, ExecutionContext.unsequenced(txnId1, "Test"), null), RELEASE_QUEUE); cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.unsequenced(command1.txnId(), "Test")), command1).cfk()); cfk.set(cfk.current().update(new TestSafeCommandStore(ExecutionContext.unsequenced(command1.txnId(), "Test")), command2).cfk()); - CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null, commandStore.nextSystemTimestampMicros()).run(); + CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null).run(); logger.info("E: {}", cfk); CommandsForKey actual = CommandsForKeyAccessor.load(commandStore.id(), key); logger.info("A: {}", actual); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordExpungeTest.java b/test/unit/org/apache/cassandra/service/accord/AccordExpungeTest.java index 66514033a5fc..2353fd5ac2d4 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordExpungeTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordExpungeTest.java @@ -48,6 +48,7 @@ import org.apache.cassandra.journal.TestParams; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.accord.journal.CommandChanges; import org.apache.cassandra.utils.AccordGenerators; @@ -70,7 +71,7 @@ * *

Mechanism under test: when the FULL-input cleanup path decides EXPUNGE, * {@link accord.impl.CommandChange.Builder#construct} returns {@code null}. - * Downstream, {@link AccordSafeCommand#preExecute} maps {@code null} to a + * Downstream, {@link SaferCommand#preExecute} maps {@code null} to a * {@code Command.NotDefined.uninitialised(...)} — which is exactly the bogus * NotDefined the user observed. The journal load API itself should never collapse * "this txnId has been erased" into the same answer as "we have never heard of this diff --git a/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java b/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java index b467edce537e..d73fdaa62b17 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java @@ -70,6 +70,7 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.utils.CassandraGenerators; import static accord.local.Command.Committed.committed; @@ -82,6 +83,7 @@ import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor.findAllKeysBetween; import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor.makeSystemTableKeyBytes; import static org.apache.cassandra.service.accord.AccordTestUtils.createTxn; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport; import static org.apache.cassandra.utils.AccordGenerators.fromQT; @@ -120,7 +122,7 @@ public void serde() Command.Committed committed = committed(id, SaveStatus.Committed, Status.Durability.NotDurable, participants, Ballot.ZERO, id, partialTxn, deps.intersecting(scope), Ballot.ZERO, waitingOn); - AccordSafeCommand safeCommand = new AccordSafeCommand(AccordTestUtils.loaded(id, null)); + SaferCommand safeCommand = new SaferCommand(loaded(id, null)); safeCommand.set(committed); AccordTestUtils.appendCommandsBlocking(store, null, committed); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index 297d13a79d96..c8f9d7cb7b42 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -57,7 +57,6 @@ import accord.local.Node.Id; import accord.local.NodeCommandStoreService; import accord.local.SafeCommandStore; -import accord.local.SafeState; import accord.local.StoreParticipants; import accord.local.TimeService; import accord.local.durability.DurabilityService; @@ -85,8 +84,6 @@ import accord.utils.async.AsyncChains; import org.apache.cassandra.ServerTestUtils; -import org.apache.cassandra.concurrent.ExecutorPlus; -import org.apache.cassandra.concurrent.ManualExecutor; import org.apache.cassandra.config.AccordConfig; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; @@ -103,10 +100,12 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ClientState; -import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor; -import org.apache.cassandra.service.accord.AccordExecutor.IOTask; import org.apache.cassandra.service.accord.api.AccordAgent; import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutorSyncSubmit; +import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.service.accord.journal.AccordJournal; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; @@ -123,9 +122,9 @@ import static accord.primitives.Status.Durability.NotDurable; import static accord.primitives.Txn.Kind.Write; import static java.lang.String.format; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; -import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK; import static org.apache.cassandra.service.accord.AccordService.getBlocking; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; public class AccordTestUtils { @@ -163,17 +162,10 @@ private static FullRoute route(PartialTxn txn) } } - public static & AccordSafeState> AccordCacheEntry loaded(K key, V value) + public static SaferCommand safeCommand(Command command) { - AccordCacheEntry global = new AccordCacheEntry<>(key, null); - global.initialize(value); - return global; - } - - public static AccordSafeCommand safeCommand(Command command) - { - AccordCacheEntry global = loaded(command.txnId(), command); - return new AccordSafeCommand(global); + AccordCacheEntry global = loaded(command.txnId(), command); + return new SaferCommand(global); } public static Function testableLoad(K key, V val) @@ -184,39 +176,6 @@ public static Function testableLoad(K key, V val) }; } - private static LoadExecutor loadExecutor(ExecutorPlus executor) - { - return new LoadExecutor<>() - { - @Override - public IOTask load(P1 p1, P2 p2, AccordCacheEntry entry) - { - executor.submit(() -> { - V v; - try { v = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); } - catch (Throwable t) - { - entry.failedToLoad(); - throw t; - } - entry.loaded(v); - }); - return null; - } - }; - } - - public static & AccordSafeState> void testLoad(ManualExecutor executor, S safeState, V val) - { - Assert.assertEquals(AccordCacheEntry.Status.WAITING_TO_LOAD, safeState.global().status()); - safeState.global().load(loadExecutor(executor), null, null); - Assert.assertEquals(AccordCacheEntry.Status.LOADING, safeState.global().status()); - executor.runOne(); - Assert.assertEquals(AccordCacheEntry.Status.LOADED, safeState.global().status()); - safeState.preExecute(new AccordTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); - Assert.assertEquals(val, safeState.current()); - } - public static TxnId txnId(long epoch, long hlc, int node) { return txnId(epoch, hlc, node, Write); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java similarity index 95% rename from test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java rename to test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java index 9ca2b116785c..2da44ed37fee 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java @@ -77,10 +77,15 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches; -import org.apache.cassandra.service.accord.AccordExecutor.ExclusiveGlobalCaches; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutor.ExclusiveGlobalCaches; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.SaferCommand; +import org.apache.cassandra.service.accord.execution.SafeTask; import org.apache.cassandra.utils.AssertionUtils; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Condition; @@ -95,12 +100,12 @@ import static org.apache.cassandra.service.accord.AccordTestUtils.createAccordCommandStore; import static org.apache.cassandra.service.accord.AccordTestUtils.createPartialTxn; import static org.apache.cassandra.service.accord.AccordTestUtils.keys; -import static org.apache.cassandra.service.accord.AccordTestUtils.loaded; import static org.apache.cassandra.service.accord.AccordTestUtils.txnId; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; -public class AccordTaskTest +public class SafeTaskTest { - private static final Logger logger = LoggerFactory.getLogger(AccordTaskTest.class); + private static final Logger logger = LoggerFactory.getLogger(SafeTaskTest.class); private static final AtomicLong clock = new AtomicLong(0); @BeforeClass @@ -173,7 +178,7 @@ public void optionalCommandsForKeyTest() throws Throwable private static Command createStableAndPersist(AccordCommandStore commandStore, TxnId txnId, Timestamp executeAt) { Command command = AccordTestUtils.Commands.stable(txnId, createPartialTxn(0), executeAt); - AccordSafeCommand safeCommand = new AccordSafeCommand(loaded(txnId, null)); + SaferCommand safeCommand = new SaferCommand(loaded(txnId, null)); safeCommand.set(command); appendDiffToLog(commandStore).accept(null, command); @@ -313,7 +318,7 @@ public void loadFail() throw new NullPointerException("txn_id " + txnId); }); } - AccordTask o1 = AccordTask.create(commandStore, ctx, consumer); + SafeTask o1 = SafeTask.create(commandStore, ctx, consumer); AssertionUtils.assertThatThrownBy(() -> getBlocking(o1.chain())) .hasRootCause() .isInstanceOf(NullPointerException.class) @@ -334,7 +339,7 @@ public void loadFail() return cmd; }); } - AccordTask o2 = AccordTask.create(commandStore, ctx, store -> { + SafeTask o2 = SafeTask.create(commandStore, ctx, store -> { ids.forEach(id -> { store.ifInitialised(id).readyToExecute(store); }); @@ -370,7 +375,7 @@ public void consumerFails() String errorMsg = "txn_ids " + ids; Mockito.doThrow(new NullPointerException(errorMsg)).when(consumer).accept(Mockito.any()); - AccordTask operation = AccordTask.create(commandStore, ctx, consumer); + SafeTask operation = SafeTask.create(commandStore, ctx, consumer); AssertionUtils.assertThatThrownBy(() -> getBlocking(operation.chain())) .hasRootCause() diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java index 00705915dd5c..165115d8fdc0 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java @@ -100,6 +100,10 @@ import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.AccordCache; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry; +import org.apache.cassandra.service.accord.execution.AccordExecutor; +import org.apache.cassandra.service.accord.execution.AccordExecutorSimple; import org.apache.cassandra.service.accord.journal.RangeSearcher; import org.apache.cassandra.service.accord.journal.SegmentRangeSearcher; import org.apache.cassandra.service.accord.topology.AccordTopology; diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java index fcd7ccdde6cd..5da64a83c246 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java @@ -53,6 +53,7 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.accord.SimulatedAccordCommandStore.FunctionWrapper; import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.service.accord.execution.SafeTask; import org.apache.cassandra.utils.Pair; import static accord.utils.Property.qt; @@ -179,12 +180,12 @@ private static TokenRange range(TableId tableId, long start, long end) private enum Action { SUCCESS, FAILURE, LOAD_FAILURE } - private static AccordTask operation(SimulatedAccordCommandStore instance, ExecutionContext ctx, Action action, BooleanSupplier delay) + private static SafeTask operation(SimulatedAccordCommandStore instance, ExecutionContext ctx, Action action, BooleanSupplier delay) { Function function = action == Action.FAILURE ? safeStore -> { throw new SimulatedFault("Operation failed for keys " + ctx.keys()); } : safeStore -> null; - return AccordTask.create(instance.commandStore, ctx, function); + return SafeTask.create(instance.commandStore, ctx, function); } private static class Counter implements BiConsumer diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java b/test/unit/org/apache/cassandra/service/accord/execution/AccordCacheEntryTest.java similarity index 88% rename from test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java rename to test/unit/org/apache/cassandra/service/accord/execution/AccordCacheEntryTest.java index 4c174a37b725..4f61205a65f2 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java +++ b/test/unit/org/apache/cassandra/service/accord/execution/AccordCacheEntryTest.java @@ -15,24 +15,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import org.junit.Assert; import org.junit.Test; import accord.local.SafeState; -import org.apache.cassandra.service.accord.AccordCache.Type; -import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; -import org.apache.cassandra.service.accord.AccordCacheEntry.Status; +import org.apache.cassandra.service.accord.execution.AccordCache.Type; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; public class AccordCacheEntryTest { - static class TestSafeState extends SafeState implements AccordSafeState + static class TestSafeState extends SafeState implements SaferState { @Override public AccordCacheEntry global() { return null; } - @Override public void preExecute(AccordTask owner, LockMode lockMode) {} - @Override public void postExecute(AccordTask owner) {} + @Override public void preExecute(SafeTask owner, LockMode lockMode) {} + @Override public void postExecute(SafeTask owner) {} } static class CacheEntry extends AccordCacheEntry diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java b/test/unit/org/apache/cassandra/service/accord/execution/AccordCacheTest.java similarity index 95% rename from test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java rename to test/unit/org/apache/cassandra/service/accord/execution/AccordCacheTest.java index aedf100df066..669d25fe3008 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java +++ b/test/unit/org/apache/cassandra/service/accord/execution/AccordCacheTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.accord; +package org.apache.cassandra.service.accord.execution; import java.util.function.Function; @@ -30,12 +30,12 @@ import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.concurrent.ManualExecutor; import org.apache.cassandra.metrics.AccordCacheMetrics; -import org.apache.cassandra.service.accord.AccordCacheEntry.LockMode; -import org.apache.cassandra.service.accord.AccordCacheEntry.SaveExecutor; -import org.apache.cassandra.service.accord.AccordCacheEntry.Status; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.SaveExecutor; +import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; -import static org.apache.cassandra.service.accord.AccordCacheEntry.LockMode.RELEASE_QUEUE; -import static org.apache.cassandra.service.accord.AccordTestUtils.testLoad; +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.RELEASE_QUEUE; +import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.testLoad; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.mockito.Mockito.mock; @@ -45,7 +45,7 @@ public class AccordCacheTest { private static final long DEFAULT_NODE_SIZE = nodeSize(0); - private static abstract class TestSafeState & AccordSafeState> extends SafeState implements AccordSafeState + private static abstract class TestSafeState & SaferState> extends SafeState implements SaferState { protected final AccordCacheEntry global; @@ -61,7 +61,7 @@ public AccordCacheEntry global() public final T key() { return global.key(); } - public void preExecute(AccordTask owner, LockMode lockMode) + public void preExecute(SafeTask owner, LockMode lockMode) { requireUninitialised(); current = global.lockExclusive(owner, lockMode); @@ -77,7 +77,7 @@ public SafeString(AccordCacheEntry global) } @Override - public void postExecute(AccordTask owner) + public void postExecute(SafeTask owner) { global.releaseExclusive(this, owner); } @@ -91,7 +91,7 @@ public SafeInt(AccordCacheEntry global) } @Override - public void postExecute(AccordTask owner) + public void postExecute(SafeTask owner) { global.releaseExclusive(this, owner); } @@ -240,7 +240,7 @@ public void testRotation() assertCacheMetrics(cacheMetrics, 0, 3, 3, 3); SafeString safeString = instance.acquire("1"); - safeString.preExecute(new AccordTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); + safeString.preExecute(new SafeTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); Assert.assertEquals(Status.LOADED, safeString.global.status()); assertCacheState(cache, 1, 3, nodeSize(1) * 3); @@ -369,7 +369,7 @@ public void testMultiAcquireRelease() assertCacheState(cache, 1, 1, nodeSize(1)); SafeString safeString2 = instance.acquire("0"); - safeString2.preExecute(new AccordTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); + safeString2.preExecute(new SafeTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); Assert.assertEquals("0", safeString2.current()); Assert.assertEquals(Status.LOADED, safeString1.global.status()); Assert.assertEquals(2, instance.references("0", SafeString.class)); diff --git a/test/unit/org/apache/cassandra/service/accord/execution/AccordExecutionTestUtils.java b/test/unit/org/apache/cassandra/service/accord/execution/AccordExecutionTestUtils.java new file mode 100644 index 000000000000..1a92a205d965 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/accord/execution/AccordExecutionTestUtils.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import org.junit.Assert; + +import accord.local.ExecutionContext; +import accord.local.SafeState; + +import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.concurrent.ManualExecutor; + +import static org.apache.cassandra.service.accord.execution.AccordCacheEntry.LockMode.RELEASE_QUEUE; + +public class AccordExecutionTestUtils +{ + public static & SaferState> AccordCacheEntry loaded(K key, V value) + { + AccordCacheEntry global = new AccordCacheEntry<>(key, null); + global.initialize(value); + return global; + } + + private static AccordCacheEntry.LoadExecutor loadExecutor(ExecutorPlus executor) + { + return new AccordCacheEntry.LoadExecutor<>() + { + @Override + public IOTask load(P1 p1, P2 p2, AccordCacheEntry entry) + { + executor.submit(() -> { + V v; + try { v = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); } + catch (Throwable t) + { + entry.failedToLoad(); + throw t; + } + entry.loaded(v); + }); + return null; + } + }; + } + + public static & SaferState> void testLoad(ManualExecutor executor, S safeState, V val) + { + Assert.assertEquals(AccordCacheEntry.Status.WAITING_TO_LOAD, safeState.global().status()); + safeState.global().load(loadExecutor(executor), null, null); + Assert.assertEquals(AccordCacheEntry.Status.LOADING, safeState.global().status()); + executor.runOne(); + Assert.assertEquals(AccordCacheEntry.Status.LOADED, safeState.global().status()); + safeState.preExecute(new SafeTask<>(null, (ExecutionContext.Empty)() -> "Test", null), RELEASE_QUEUE); + Assert.assertEquals(val, safeState.current()); + } +} From c4b0257b3879f4010d31b3d05d438965fc74b107 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Tue, 28 Jul 2026 23:29:38 +0100 Subject: [PATCH 06/10] Refactor AccordExecutor/Task responsibilities, in particular: - fix race conditions when updating Task.info (between running and Exclusive methods) - do not run consequences of actions that fail and do not become durable (e.g. applying to data store/post-applying a transaction that we failed to complete the PreApplied/Applying step for) --- modules/accord | 2 +- .../db/virtual/AccordDebugKeyspace.java | 2 +- .../apache/cassandra/gms/FailureDetector.java | 4 + .../cassandra/metrics/AccordCacheMetrics.java | 2 +- .../metrics/LogLinearDecayingHistograms.java | 5 + .../service/accord/AccordCommandStores.java | 8 +- .../service/accord/AccordDataStore.java | 2 +- .../service/accord/AccordService.java | 10 +- .../service/accord/api/AccordAgent.java | 12 +- .../service/accord/debug/DebugExecution.java | 23 +- .../accord/execution/AbstractLockLoop.java | 76 +- .../accord/execution/AbstractLoop.java | 6 +- .../service/accord/execution/AccordCache.java | 13 +- .../accord/execution/AccordCacheEntry.java | 14 +- .../execution/AccordCacheEntryQueue.java | 28 +- .../accord/execution/AccordExecutor.java | 160 ++--- .../AccordExecutorSemiSyncSubmit.java | 5 +- .../execution/AccordExecutorSignalLoop.java | 48 +- .../execution/AccordExecutorSimple.java | 20 +- .../service/accord/execution/CancelTask.java | 24 +- .../accord/execution/ExclusiveExecutor.java | 122 ++-- .../service/accord/execution/IOTask.java | 26 - .../service/accord/execution/IOTaskLoad.java | 28 +- .../service/accord/execution/IOTaskSave.java | 29 +- .../accord/execution/IOTaskWrapper.java | 31 +- .../service/accord/execution/Plain.java | 81 +-- .../service/accord/execution/PlainChain.java | 44 +- .../execution/PlainChainDebuggable.java | 24 - .../accord/execution/PlainRunnable.java | 43 +- .../service/accord/execution/SafeTask.java | 653 ++++++++++-------- .../accord/execution/SaferCommandStore.java | 16 +- .../service/accord/execution/Task.java | 516 ++++++++++---- .../service/accord/execution/TaskInfo.java | 8 +- .../service/accord/execution/TaskQueue.java | 11 +- .../accord/execution/TaskQueueMulti.java | 37 +- .../accord/execution/TaskQueueRunnable.java | 13 +- .../accord/execution/TaskQueueStandalone.java | 22 +- .../apache/cassandra/utils/NoSpamLogger.java | 161 ++++- .../CompactionAccordIteratorsTest.java | 4 +- .../service/accord/AccordCommandTest.java | 4 +- .../service/accord/AccordTestUtils.java | 2 +- .../service/accord/SafeTaskTest.java | 15 +- .../accord/execution/TaskLifecycleTest.java | 529 ++++++++++++++ 43 files changed, 1858 insertions(+), 1025 deletions(-) create mode 100644 test/unit/org/apache/cassandra/service/accord/execution/TaskLifecycleTest.java diff --git a/modules/accord b/modules/accord index 8d2cdb35ae96..4d091037c7b7 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 8d2cdb35ae96205b44da16582797bf42aa6fddcc +Subproject commit 4d091037c7b751631d6ce204e0c192e2301162da diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 6e2fad78b428..5b3e9f79e5de 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -1795,7 +1795,7 @@ protected void applyRowUpdate(Object[] partitionKeys, Object[] clusteringKeys, C SafeCommand safeCommand = safeStore.unsafeGet(txnId); Command command = safeCommand.current(); if (command.saveStatus() == SaveStatus.Applying) - return Commands.applyChain(safeStore, command); + return Commands.applyChain(safeStore, command.asExecuted()); Commands.maybeExecute(safeStore, safeCommand, command, true, true, NotifyWaitingOnPlus.adapter(ignore -> {}, true, true)); return AsyncChains.success(null); }); diff --git a/src/java/org/apache/cassandra/gms/FailureDetector.java b/src/java/org/apache/cassandra/gms/FailureDetector.java index 13d6266ca039..eaa73d09cc9f 100644 --- a/src/java/org/apache/cassandra/gms/FailureDetector.java +++ b/src/java/org/apache/cassandra/gms/FailureDetector.java @@ -377,6 +377,10 @@ public void interpret(InetAddressAndPort ep) logger.debug("Still not marking nodes down due to local pause"); return; } + + if (!isAlive(ep)) + return; // don't convict nodes that are already down - this helps Accord on startup which doesn't report itself alive in Gossip until ready to serve traffic + double phi = hbWnd.phi(now); logger.trace("PHI for {} : {}", ep, phi); diff --git a/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java b/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java index 00627e019a9c..a9c5c4c28ebf 100644 --- a/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java +++ b/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java @@ -46,7 +46,7 @@ public static class AccordCacheGlobalMetrics public AccordCacheGlobalMetrics() { DefaultNameFactory factory = new DefaultNameFactory(ACCORD_CACHE); - this.usedBytes = Metrics.gauge(factory.createMetricName("UsedBytes"), fromAccordService(sumExecutors(executor -> executor.cacheUnsafe().weightedSize()), 0L)); + this.usedBytes = Metrics.gauge(factory.createMetricName("UsedBytes"), fromAccordService(sumExecutors(AccordExecutor::weightedSize), 0L)); this.unreferencedBytes = Metrics.gauge(factory.createMetricName("UnreferencedBytes"), fromAccordService(sumExecutors(executor -> executor.cacheUnsafe().unreferencedBytes()), 0L)); } diff --git a/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java b/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java index 823346917ae0..4a1779482785 100644 --- a/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java +++ b/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java @@ -73,6 +73,11 @@ private void add(long histogramIndex, long value) buffer[bufferCount++] = value; } + public void clear() + { + bufferCount = 0; + } + public void flush(long at) { for (int i = 0 ; i < bufferCount ; ++i) diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index 228030ce3957..576fdf482bef 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -38,6 +38,7 @@ import accord.local.CommandStores; import accord.local.NodeCommandStoreService; import accord.local.ShardDistributor; +import accord.utils.Invariants; import accord.utils.RandomSource; import accord.utils.Reduce; import accord.utils.async.AsyncChain; @@ -63,6 +64,7 @@ import org.apache.cassandra.service.accord.execution.AccordExecutorSyncSubmit; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_PER_SHARD; +import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_PER_SHARD_SYNC_QUEUE; import static org.apache.cassandra.config.DatabaseDescriptor.getAccord; import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; @@ -123,19 +125,21 @@ static Factory factory() break; case EXEC_ST: factory = AccordExecutorSimple::new; + Invariants.require(executors.length == 1, "Executors that hold their lock during execution cannot support multiple executor shards, as they cannot lock multiple executors at once"); maxThreads = 1; break; } + QueueShardModel shardModel = config.queue_shard_model; + Invariants.require(shardModel != THREAD_PER_SHARD_SYNC_QUEUE || executors.length == 1, "Executors that hold their lock during execution cannot support multiple executor shards, as they cannot lock multiple executors at once"); for (int id = 0; id < executors.length; id++) { - QueueShardModel shardModel = config.queue_shard_model; String baseName = AccordExecutor.class.getSimpleName() + '[' + id; switch (shardModel) { case THREAD_PER_SHARD: case THREAD_PER_SHARD_SYNC_QUEUE: - executors[id] = factory.get(id, shardModel == THREAD_PER_SHARD ? RUN_WITHOUT_LOCK : RUN_WITH_LOCK, 1, constant(baseName + ']'), agent); + executors[id] = factory.get(id, shardModel == THREAD_PER_SHARD_SYNC_QUEUE ? RUN_WITH_LOCK : RUN_WITHOUT_LOCK, 1, constant(baseName + ']'), agent); break; case THREAD_POOL_PER_SHARD: int threads = Math.min(maxThreads, Math.max(DatabaseDescriptor.getAccordConcurrentOps() / executors.length, 1)); diff --git a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java index 7eba765c5b63..0e757ba418d5 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java @@ -165,7 +165,7 @@ public void abort(Ranges ranges) break; case ABORT: case ERROR: - IllegalStateException ex = new IllegalStateException(String.format("Repair failed: %s", event)); + RuntimeException ex = new RuntimeException(String.format("Repair failed (%s):", event.getType(), event.getMessage())); callback.fail(ranges, ex); syncResult.tryFailure(ex); break; diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index 994a553e8e67..19a0f9da098f 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -385,6 +385,11 @@ public synchronized static void localStartup(NodeId tcmId) AccordService as = new AccordService(tcmIdToAccord(tcmId)); unsafeInstance = replyInstance = as; as.localStartup(); + + AccordReplicaMetrics.touch(); + AccordSystemMetrics.touch(); + AccordExecutorMetrics.touch(); + AccordViolationHandler.setup(); } } @@ -398,11 +403,6 @@ public synchronized static AccordService distributedStartup() return as; as.distributedStartupInternal(); - - AccordReplicaMetrics.touch(); - AccordSystemMetrics.touch(); - AccordExecutorMetrics.touch(); - AccordViolationHandler.setup(); return as; } diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java index 454fa9edbeb8..3188fe00294e 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java @@ -43,6 +43,7 @@ import accord.coordinate.Preempted; import accord.coordinate.Timeout; import accord.local.Command; +import accord.local.CommandStore; import accord.local.LogUnavailableException; import accord.local.Node; import accord.local.SafeCommand; @@ -84,6 +85,7 @@ import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.NoSpamLogger; +import org.apache.cassandra.utils.NoSpamLogger.NoDuplicateSpamLogStatement; import static accord.primitives.Routable.Domain.Key; import static accord.utils.SortedArrays.SortedArrayList.ofSorted; @@ -107,12 +109,14 @@ import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowRead; import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowTxnPreaccept; import static org.apache.cassandra.service.accord.txn.TxnResult.Kind.txn_data; +import static org.apache.cassandra.utils.NoSpamLogger.NoDuplicateSpamLogStatement.exceptionId; // TODO (expected): merge with AccordService public class AccordAgent implements Agent, OwnershipEventListener { private static final Logger logger = LoggerFactory.getLogger(AccordAgent.class); private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, MINUTES); + private static final NoDuplicateSpamLogStatement noSpamException = new NoDuplicateSpamLogStatement(logger, "", 1L, MINUTES); private static final ReplicaEventListener replicaEventListener = new AccordReplicaMetrics.Listener(); private static BiConsumer onFailedBarrier; @@ -155,6 +159,12 @@ public void setup(Node.Id id) config = DatabaseDescriptor.getAccord(); } + @Override + public void onSuccessfulBootstrap(CommandStore commandStore, int attempt, long epoch, Ranges ranges) + { + logger.info("{}: Completed bootstrap of {} on epoch {}", commandStore, ranges, epoch); + } + @Override public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Runnable fail, Throwable failure) { @@ -214,7 +224,7 @@ public static void handleException(Throwable t) AccordSystemMetrics.metrics.errors.inc(); if (t instanceof CancellationException || t instanceof TimeoutException || t instanceof Timeout || t instanceof Preempted || t instanceof Exhausted || t instanceof LogUnavailableException) // TODO (required): leaky logger, permitting multiple messages per time period and reporting how many were dropped - noSpamLogger.warn("", t); + noSpamException.warn(exceptionId(t), t); else JVMStabilityInspector.uncaughtException(Thread.currentThread(), t); } diff --git a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java index 99aa8fe0ce41..e01a8363d82f 100644 --- a/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java +++ b/src/java/org/apache/cassandra/service/accord/debug/DebugExecution.java @@ -132,15 +132,16 @@ public void onSetTask(Task next) public void onComplete(Task completed) { long readyAt = setTaskAt; + long runningAt = DebugTask.get(completed).runningAt; if (waitingAt > setTaskAt) { readyAt = waitingAt; - long waitingMicros = (completed.runningAt - waitingAt)/1000; + long waitingMicros = (runningAt - waitingAt)/1000; owner.sequentialExecutorWaitingToRunLatency.increment(waitingMicros); if (waitingMicros > REPORT_MAX_LATENCY_MICROS) report("{} spent {}us blocked by a direct execution on queue {}", completed, waitingMicros, commandStoreId); } - long atHeadMicros = (completed.runningAt - readyAt)/1000; + long atHeadMicros = (runningAt - readyAt)/1000; owner.sequentialExecutorSetHeadToRunLatency.increment(atHeadMicros); if (atHeadMicros > REPORT_MAX_LATENCY_MICROS) { @@ -172,7 +173,7 @@ public DebugTask(WithResources resources, Task task) } public List sanityCheck; // for AccordTask only - long polledAt, preRunAt, runCompleteAt, completedAt; + long polledAt, preRunAt, runningAt, runCompleteAt, completeAt, completedAt; long releasedRangeScannerAt, releasedStateAt; long runningAtCpu, runCompleteAtCpu; Thread thread; @@ -191,6 +192,7 @@ public void onRunning() { thread = Thread.currentThread(); runningAtCpu = nowCpu(); + runningAt = nanoTime(); } public void onRunComplete() @@ -209,22 +211,27 @@ public void onReleasedState() releasedStateAt = nanoTime(); } + public void onComplete() + { + completeAt = nanoTime(); + } + public void onCompleted(DebugExecutor owner) { completedAt = nanoTime(); - if (task.runningAt > 0 && polledAt > 0) + if (runningAt > 0 && polledAt > 0) { - long pollToRunMicros = (task.runningAt - polledAt)/1000; + long pollToRunMicros = (runningAt - polledAt)/1000; owner.pollToRun.increment(pollToRunMicros); long runningMicros = -1; if (runCompleteAt > 0) { - runningMicros = (runCompleteAt - task.runningAt) / 1000; + runningMicros = (runCompleteAt - runningAt) / 1000; owner.running.increment(runningMicros); } - long runToCleanMicros = (task.completeAt - runCompleteAt) / 1000; + long runToCleanMicros = (completeAt - runCompleteAt) / 1000; owner.runToCleanup.increment(runToCleanMicros); - long cleanupMicros = (completedAt - task.completeAt) / 1000; + long cleanupMicros = (completedAt - completeAt) / 1000; owner.cleanup.increment(cleanupMicros); long totalMicros = (completedAt - polledAt)/1000; owner.taskTotal.increment(totalMicros); diff --git a/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java index ac7e4390a07d..41738b8dd3f4 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java @@ -142,7 +142,7 @@ final void drainUnqueuedNewWorkExclusive() while (cur != null) { Task next = cur.next; - cur.submitExclusive(); + cur.submitExclusiveNoExcept(); cur.next = null; cur = next; } @@ -209,28 +209,7 @@ public void run() while (true) { task = pollWaitingToRunExclusive(); - - if (task != null) - { - self.setAccordActiveTask(task); - boolean executed = false; - try - { - task.preRunExclusive(); - executed = true; - task.run(); - } - catch (Throwable t) - { - executed = false; - task.failExecution(t); - } - finally - { - cleanupTaskExclusive(task, executed); - self.setAccordActiveTask(null); - } - } + if (task != null) prepareRunComplete(self, task); else { if (shutdown) @@ -250,7 +229,6 @@ public void run() catch (Throwable t) { exitLockLoop(); - try { agent.onException(t); } catch (Throwable t2) { } } @@ -285,18 +263,20 @@ public void run() { Task tmp = task; task = null; - cleanupTaskExclusive(tmp, true); - self.setAccordActiveTask(null); + tmp.completeExclusiveNoExcept(); } while (true) { task = pollWaitingToRunExclusive(); - if (task != null) { - self.setAccordActiveTask(task); - task.preRunExclusive(); + if (!task.prepareExclusiveNoExcept()) + { + task = null; + continue; + } + if (DEBUG_EXECUTION) debug.onExitLock(); exitLockLoop(); break; @@ -319,21 +299,8 @@ public void run() } catch (Throwable t) { - if (task != null) - { - try { task.failExecution(t); } - catch (Throwable t2) { t.addSuppressed(t2); } - try { cleanupTaskExclusive(task, false); } - catch (Throwable t2) { t.addSuppressed(t2); } - try { agent.onException(t); } - catch (Throwable t2) { /* nothing we can sensibly do after already reporting */ } - task = null; - } - else - { - try { agent.onException(t); } - catch (Throwable t2) { /* nothing we can sensibly do after already reporting */ } - } + try { agent.onException(t); } + catch (Throwable t2) { } exitLockLoop(); continue; } @@ -343,26 +310,7 @@ public void run() unlock(self); } - try - { - task.run(); - } - catch (Throwable t) - { - try { task.failExecution(t); } - catch (Throwable t2) - { - try - { - t2.addSuppressed(t); - agent.onException(t2); - } - catch (Throwable t3) - { - // empty to ensure we definitely loop so we cleanup the task - } - } - } + task.runNoExcept(self); } } }; diff --git a/src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java index 684160b8702c..de6d8020f79e 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractLoop.java @@ -27,6 +27,8 @@ import org.apache.cassandra.concurrent.DebuggableTask.DebuggableTaskRunner; +import static org.apache.cassandra.service.accord.execution.Task.State.REGISTERED; + abstract class AbstractLoop extends AccordExecutor { volatile Task unqueued; @@ -84,8 +86,8 @@ final Task enqueueOneExclusive(Task cur) Invariants.require(cur != null); Task next = cur.next; cur.next = null; - if (cur.is(Task.State.UNINITIALIZED)) cur.submitExclusive(); - else cleanupTaskExclusive(cur, true); + if (cur.compareTo(REGISTERED) < 0) cur.submitExclusiveNoExcept(); + else cur.completeExclusiveNoExcept(); return next; } diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java index 8a5c2f699149..21248920c69a 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCache.java @@ -102,7 +102,7 @@ * * TODO (required): we only iterate over unreferenced entries */ -public class AccordCache implements CacheSize +public class AccordCache { private static final Logger logger = LoggerFactory.getLogger(AccordCache.class); private static final NoSpamLogStatement evictNoEvict = NoSpamLogger.getStatement(logger, "Found and expired {} marked no evict, with age {}, exceeding its expected max age of {}", 1L, TimeUnit.MINUTES); @@ -180,8 +180,8 @@ public AccordCache(AccordCacheEntry.SaveExecutor saveExecutor, long maxSizeInByt } // note: only affects current contents after lock is released - @Override - public void setCapacity(long sizeInBytes) + // should NOT be called directly, should be called via AccordExecutor.setCapacity, so it may refresh its derived values + void setCapacity(long sizeInBytes) { maxSizeInBytes = sizeInBytes; tryShrinkOrEvict = true; @@ -192,7 +192,6 @@ public void setShrinkingOn(boolean shrinkingOn) this.shrinkingOn = shrinkingOn; } - @Override public long capacity() { return maxSizeInBytes; @@ -945,14 +944,12 @@ public long unreferencedBytes() return unreferencedBytes; } - @Override - public int size() + int size() { return cacheSize(); } - @Override - public long weightedSize() + long weightedSize() { return bytesCached; } diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java index fefaad70a68c..43ad91fedcc5 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntry.java @@ -283,6 +283,7 @@ final void remove(SafeTask task, boolean ownsLock) status &= ~LOCKED_MASK; q.unlock(task); } + else if (isLoading()) remove = true; else if (task.isUnsequenced(this)) { if (task.isCacheQueued()) @@ -326,7 +327,14 @@ else if (task.isUnsequenced(this)) { Invariants.require(!ownsLock); if (task.isUnsequenced(this) && task.isCacheQueued()) - --unsequenced; // nothing to release if we hit zero + { + if (--unsequenced == 0 && queue != null && isLoaded()) + { + SafeTask head = (SafeTask) queue; + if (head.holdsLocksBetweenRuns()) + head.onChangeOptionalHeadStatus(this, NEWLY_RUNNABLE); + } + } } } @@ -479,7 +487,7 @@ final RunnableStatus addFifo(SafeTask task) private void onChangedHead(AccordCacheEntryQueue q, @Nullable SafeTask notifyNewHead, @Nullable SafeTask notifyPrevHead) { if (notifyNewHead != null && isRunnable(notifyNewHead)) - notifyNewHead.onChangeHeadStatus(this, q.totalSize() == 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); + notifyNewHead.onChangeHeadStatus(this, q.totalSize() <= 1 ? NEWLY_RUNNABLE : NEWLY_BLOCKING_RUNNABLE); if (notifyPrevHead != null && isRunnable(notifyPrevHead)) notifyPrevHead.onChangeHeadStatus(this, NOT_RUNNABLE); } @@ -583,7 +591,7 @@ private void releaseUnsequenced(AccordCacheEntryQueue q, SafeTask release) if (--unsequenced == 0) { SafeTask head = q.peek(); - if (head != null && head.holdsLocksBetweenRuns()) + if (head != null && head.holdsLocksBetweenRuns() && isLoaded()) onChangedHead(q, head, null); } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java index 9e4b60df7d81..06025e49c77b 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordCacheEntryQueue.java @@ -31,6 +31,8 @@ class AccordCacheEntryQueue { + static final AccordCacheEntryQueue EMPTY = new AccordCacheEntryQueue(0); + private static final int DEFAULT_CAPACITY = 4; static final int LOCKED_INDEX = 0; static final int PRIORITY_START_INDEX = LOCKED_INDEX + 1; @@ -49,9 +51,14 @@ class AccordCacheEntryQueue public AccordCacheEntryQueue() { - tasks = new SafeTask[DEFAULT_CAPACITY]; + this(DEFAULT_CAPACITY); + } + + public AccordCacheEntryQueue(int capacity) + { + tasks = new SafeTask[capacity]; priorityHead = priorityTail = PRIORITY_START_INDEX; - fifoHead = fifoTail = DEFAULT_CAPACITY - 1; + fifoHead = fifoTail = capacity - 1; } AccordCacheEntryQueue(AccordCacheEntryQueue copy) @@ -412,18 +419,7 @@ private int priorityIndexOf(SafeTask task) if (tasks[priorityHead] == task) return priorityHead; - int i = SortedArrays.binarySearch(tasks, priorityHead + 1, priorityTail, task, AccordCacheEntryQueue::compare, SortedArrays.Search.CEIL); - if (i < 0) - return -1; - - while (i < priorityTail) - { - if (tasks[i] == task) - return i; - if (compare(task, tasks[i]) != 0) - break; - ++i; - } + return Arrays.binarySearch(tasks, priorityHead + 1, priorityTail, task, AccordCacheEntryQueue::compare); } for (int i = priorityHead; i < priorityTail; ++i) @@ -504,10 +500,6 @@ static int compare(SafeTask a, SafeTask b) c = a.executionContext().executionKind().compareTo(b.executionContext().executionKind()); if (c == 0) c = Long.compare(a.createdAt, b.createdAt); - if (c == 0) - c = Long.compare(a.loadedAt, b.loadedAt); - if (c == 0) - c = a.loggingId().compareTo(b.loggingId()); return c; } diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java index a87e405dd257..37ba493d3269 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutor.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; @@ -39,7 +40,6 @@ import accord.primitives.TxnId; import accord.utils.ArrayBuffers; import accord.utils.Invariants; -import accord.utils.TinyEnumSet; import accord.utils.UnhandledEnum; import accord.utils.async.AsyncCallbacks.CallAndCallback; import accord.utils.async.AsyncCallbacks.RunOrFail; @@ -70,8 +70,8 @@ import org.apache.cassandra.service.accord.execution.ExclusiveExecutor.ExclusiveExecutorTask; import org.apache.cassandra.service.accord.execution.IOTaskWrapper.WrappableIOTask; import org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup; +import org.apache.cassandra.service.accord.execution.Task.ExecutorQueue; import org.apache.cassandra.service.accord.execution.Task.GlobalGroup; -import org.apache.cassandra.service.accord.execution.Task.State; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Condition; @@ -86,13 +86,11 @@ import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.OTHER; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_SCAN; -import static org.apache.cassandra.service.accord.execution.Task.State.FAILED_TO_LOAD; -import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_OPTIONAL; -import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_REQUIRED; -import static org.apache.cassandra.service.accord.execution.Task.State.SCANNING_RANGES; -import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_OPTIONAL; -import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.EXECUTED; +import static org.apache.cassandra.service.accord.execution.Task.State.FAILED; +import static org.apache.cassandra.service.accord.execution.Task.State.UNREGISTERED; import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.COUNTER_MASKS; +import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.overflowBit; import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.selectByOverflowBits; import static org.apache.cassandra.service.accord.execution.TaskQueueMulti.setOverflowWhenLessEqual; @@ -113,6 +111,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor> scanningRanges = new TaskQueueStandalone<>(TinyEnumSet.encode(SCANNING_RANGES)); // never queried, just parked here while scanning - final TaskQueueStandalone> loading = new TaskQueueStandalone<>(TinyEnumSet.encode(LOADING_REQUIRED, LOADING_OPTIONAL)); - final TaskQueueStandalone> waitingOnCacheQueues = new TaskQueueStandalone<>(TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL)); + final TaskQueueStandalone> loading = new TaskQueueStandalone<>(ExecutorQueue.LOADING); + final TaskQueueStandalone> waiting = new TaskQueueStandalone<>(ExecutorQueue.WAITING); final TaskQueueRunnable runnable = new TaskQueueRunnable<>(); private final Tranches tranches = new Tranches(this); @@ -409,17 +407,17 @@ public void afterSubmittedAndConsequences(Runnable run) } } - final void maybeUnpauseLoading() + private boolean hasNonLoadingWaitingToRun() { - if (!hasPausedLoading) - return; + return runnable.hasWaitingToRunExcluding(LOADING_GROUPS); + } - if (cache.weightedSize() < maxWorkingCapacityInBytes && !runnable.hasWaitingToRun()) + final void maybeUnpauseLoading() + { + if (hasPausedLoading && (cache.weightedSize() < maxWorkingCapacityInBytes || !hasNonLoadingWaitingToRun())) { hasPausedLoading = false; - runnable.restart(RANGE_SCAN.ordinal()); - runnable.restart(LOAD.ordinal()); - runnable.restart(RANGE_LOAD.ordinal()); + runnable.restart(LOADING_GROUPS); } } @@ -428,13 +426,11 @@ final void maybePauseLoading() if (hasPausedLoading) return; - if (cache.weightedSize() >= maxWorkingCapacityInBytes || !runnable.hasWaitingToRun()) + if (!hasPausedLoading && cache.weightedSize() >= maxWorkingCapacityInBytes && hasNonLoadingWaitingToRun()) { AccordSystemMetrics.metrics.pausedExecutorLoading.inc(); hasPausedLoading = true; - runnable.stop(RANGE_SCAN.ordinal()); - runnable.stop(LOAD.ordinal()); - runnable.stop(RANGE_LOAD.ordinal()); + runnable.stop(LOADING_GROUPS); } } @@ -452,46 +448,48 @@ public ExclusiveAsyncExecutor newExclusiveExecutor() public IOTaskLoad load(SafeTask parent, Boolean isForRange, AccordCacheEntry entry) { IOTaskLoad result = newLoad(entry, isForRange); - result.submitExclusive(null, parent); + result.inherit(parent).submitExclusiveNoExcept(); return result; } @Override public Cancellable save(AccordCacheEntry entry, UniqueSave identity, Runnable save) { - return new IOTaskSave(this, entry, identity, save).submitExclusive(null, null); + IOTaskSave task = new IOTaskSave(this, entry, identity, save); + task.submitExclusiveNoExcept(); + return task; } - Cancellable submit(Task task) + void submitTask(Task task) { - submit(Task::submitExclusive, i -> i, task); - return task; + submit(Task::submitExclusiveNoExcept, i -> i, task); } public Future submit(Runnable run) { Task inherit = inherit(); - PlainRunnable task = inherit == null ? new PlainRunnable(this, new AsyncPromise<>(), run, OTHER) - : new PlainRunnable(this, new AsyncPromise<>(), run, OTHER, inherit.position, inherit.tranche()); - submit(task); + PlainRunnable task = new PlainRunnable(this, new AsyncPromise<>(), run, OTHER); + if (inherit != null) inherit.addConsequence(task); + else submitTask(task); return task.result; } public void execute(Runnable run) { Task inherit = inherit(); - PlainRunnable task = inherit == null ? new PlainRunnable(this, null, run, OTHER) - : new PlainRunnable(this, null, run, OTHER, inherit.position, inherit.tranche()); - submit(task); + PlainRunnable task = new PlainRunnable(this, null, run, OTHER); + if (inherit != null) inherit.addConsequence(task); + else submitTask(task); } @Override public Cancellable execute(RunOrFail runOrFail) { Task inherit = inherit(); - PlainChain submit = inherit == null ? new PlainChain(this, runOrFail, null, ExclusiveGroup.OTHER) - : new PlainChain(this, runOrFail, null, ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); - return submit(submit); + PlainChain task = new PlainChain(this, runOrFail, null, ExclusiveGroup.OTHER); + if (inherit != null) inherit.addConsequence(task); + else submitTask(task); + return task; } public AsyncChain buildDebuggable(Callable call, Object describe) @@ -502,30 +500,29 @@ public AsyncChain buildDebuggable(Callable call, Object describe) protected Cancellable start(BiConsumer callback) { Task inherit = inherit(); - return submit(inherit == null ? new PlainChainDebuggable(AccordExecutor.this, new CallAndCallback<>(call, callback), null, describe) - : new PlainChainDebuggable(AccordExecutor.this, new CallAndCallback<>(call, callback), null, inherit.position, inherit.tranche(), describe)); + PlainChainDebuggable task = new PlainChainDebuggable(AccordExecutor.this, new CallAndCallback<>(call, callback), null, describe); + if (inherit != null) inherit.addConsequence(task); + else submitTask(task); + return task; } }; } public void submitExclusive(Runnable runnable) { - new PlainRunnable(this, null, runnable, OTHER).submitExclusive(null, null); + new PlainRunnable(this, null, runnable, OTHER).submitExclusiveNoExcept(); } - Cancellable submitExclusive(Task parent, GlobalGroup group, WrappableIOTask task) + Cancellable submitExclusive(Task parent, GlobalGroup group, WrappableIOTask wrap) { - return new IOTaskWrapper(this, task, group, parent.position, parent.tranche()).submitExclusive(null, parent); + IOTaskWrapper task = new IOTaskWrapper(this, wrap, group); + task.inherit(parent).submitExclusiveNoExcept(); + return task; } Task inherit() { - return inherit(Thread.currentThread()); - } - - Task inherit(Thread thread) - { - TaskRunner self = TaskRunner.get(thread); + TaskRunner self = TaskRunner.get(); if (self.accordActiveExecutor() != this) return null; @@ -536,17 +533,10 @@ Task inherit(Thread thread) return task; } - void registerConsequenceExclusive(Task parent, Task task) - { - ++tasks; - int tranche = parent.tranche(); - tranches.addInherited(tranche, parent.position); - task.position = parent.position; - task.setInheritedWithTranche(tranche); - } - void registerExclusive(Task task) { + Invariants.require(isOwningThread()); + Invariants.require(task.is(UNREGISTERED)); ++tasks; if (task.hasInherited()) { @@ -573,15 +563,14 @@ else if (position < minPosition) } } - final void cleanupTaskExclusive(Task task, boolean executed) + final void completedTaskExclusive(Task task) { + Invariants.require(task.compareTo(EXECUTED) >= 0); + if (DEBUG_EXECUTION) DebugTask.get(task).onCompleted(debug); + unregisterExclusive(task); runnable.cleanup(task); - try { task.cleanupExclusive(this, executed); } - finally - { - cache.tryShrinkOrEvict(lock); - maybeUnpauseLoading(); - } + cache.tryShrinkOrEvict(lock); + maybeUnpauseLoading(); } final void unregisterExclusive(Task task) @@ -594,25 +583,13 @@ final void unregisterExclusive(Task task) void onScannedRangesExclusive(SafeTask task, Throwable fail) { // the task may have already been cancelled, in which case we don't need to fail it - if (task.state().isExecuted()) - return; - - if (fail != null) failExclusive(task, fail, FAILED_TO_LOAD); - else task.rangeScanner().scannedExclusive(); - } - - private void failExclusive(SafeTask task, Throwable fail, State newState) - { - if (task.state().isExecuted()) + if (task.state().isDone()) return; - try { task.failExclusive(fail, newState); } - catch (Throwable t) { agent.onException(t); } - finally - { - task.unqueueIfQueued(); - cleanupTaskExclusive(task, false); - } + SafeTask.RangeTxnScanner scanner = task.rangeScanner(); + if (scanner == null && fail == null) fail = new CancellationException(); + if (fail != null) task.tryFailAndCompleteExclusive(fail, FAILED); + else scanner.scannedExclusive(); } void onSavedExclusive(AccordCacheEntry state, Object identity, Throwable fail) @@ -630,7 +607,7 @@ void onLoadedExclusive(AccordCacheEntry loaded, V value, Throwab if (fail != null) { for (SafeTask task : tasks) - failExclusive(task, fail, FAILED_TO_LOAD); + task.tryFailAndCompleteExclusive(fail, FAILED); cache.failedToLoad(loaded); } else @@ -666,13 +643,18 @@ public void setCapacity(long bytes) { Invariants.require(isOwningThread()); cache.setCapacity(bytes); - maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; + refreshCapacity(); } public void setWorkingSetSize(long bytes) { Invariants.require(isOwningThread()); maxWorkingSetSizeInBytes = bytes; + refreshCapacity(); + } + + private void refreshCapacity() + { maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes; if (maxWorkingCapacityInBytes < maxWorkingSetSizeInBytes) maxWorkingCapacityInBytes = Long.MAX_VALUE; @@ -698,7 +680,7 @@ public long weightedSize() public int unsafePreparingToRunCount() { - return loading.size() + scanningRanges.size(); + return loading.size(); } public int unsafeWaitingToRunCount() @@ -738,8 +720,7 @@ public List taskSnapshot() lock(self); try { - addToSnapshot(result, scanningRanges, TaskInfo.Status.SCANNING_RANGES, TaskInfo.Status.SCANNING_RANGES); - addToSnapshot(result, loading, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD); + addToSnapshot(result, loading, TaskInfo.Status.LOADING, TaskInfo.Status.LOADING); for (TaskQueue queue : runnable.queues) { if (queue != null) @@ -820,4 +801,13 @@ private static long parseEnumParams(Function> get, Str return result; } + + protected static void prepareRunComplete(TaskRunner self, Task task) + { + if (task.prepareExclusiveNoExcept()) + { + try { task.runNoExcept(self); } + finally { task.completeExclusiveNoExcept(); } + } + } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java index 2bafc5b4b625..ac17576f2692 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSemiSyncSubmit.java @@ -31,7 +31,6 @@ public class AccordExecutorSemiSyncSubmit extends AbstractSemiSyncSubmit private final Loops loops; private final ReentrantLock lock; private final Condition hasWork; - private int waiting; public AccordExecutorSemiSyncSubmit(int executorId, Mode mode, int threads, IntFunction name, Agent agent) { @@ -55,9 +54,7 @@ void awaitExclusive() throws InterruptedException private void awaitWork() throws InterruptedException { - waiting++; - try { hasWork.await(); } - finally { waiting--; } + hasWork.await(); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java index acb5eb7202d6..51275ee4fe25 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSignalLoop.java @@ -34,7 +34,7 @@ import static accord.utils.Invariants.nonNull; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; -import static org.apache.cassandra.service.accord.execution.Task.State.UNINITIALIZED; +import static org.apache.cassandra.service.accord.execution.Task.State.REGISTERED; public class AccordExecutorSignalLoop extends AbstractLoop { @@ -134,7 +134,7 @@ final void drainUnqueuedNewWorkExclusive() { Task next = submit.next; submit.next = null; - submit.submitExclusive(); + submit.submitExclusiveNoExcept(); submit = next; } } @@ -151,7 +151,7 @@ private boolean enqueueOnePending() pendingExecutedHead = destructiveNext(executed); if (pendingExecutedHead == null) pendingExecutedTail = null; - cleanupTaskExclusive(executed, true); + executed.completeExclusiveNoExcept(); } else { @@ -159,7 +159,7 @@ private boolean enqueueOnePending() pendingNewHead = destructiveNext(submit); if (pendingNewHead == null) pendingNewTail = null; - submit.submitExclusive(); + submit.submitExclusiveNoExcept(); } return true; } @@ -227,19 +227,9 @@ else if (waiters > 0 && signals > 1 && SignalLock.activeEnabledThreadCount(state return; } } - else + else if (task.prepareExclusiveNoExcept()) { - try { task.preRunExclusive(); } - catch (Throwable t) - { - try { task.failExclusive(t, Task.State.FAILED_OTHER); } - catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } - try { cleanupTaskExclusive(task, false); } - catch (Throwable t2) { try { t.addSuppressed(t2); } catch (Throwable t3) {} } - continue; - } if (DEBUG_EXECUTION) DebugTask.get(task).onPreRun(); - addReadyToRun(task); boolean incremented = lock.incrementAsyncWork(false); Invariants.require(incremented); @@ -260,7 +250,7 @@ private boolean updatePendingUnqueued() while (cur != null) { Task next = cur.next; - if (cur.is(UNINITIALIZED)) + if (cur.compareTo(REGISTERED) < 0) { if (addNewHead == null) addNewHead = addNewTail = setNextNull(cur); else addNewHead = reverseOne(addNewHead, cur); @@ -368,7 +358,8 @@ private Task executedAndMaybeGetWork(TaskRunner self, @Nullable Task executed) try { - cleanupTaskExclusive(executed, true); + if (executed != null) + executed.completeExclusiveNoExcept(); fetchWorkExclusive(); } catch (Throwable t) @@ -431,24 +422,7 @@ public void run() if (task == null) task = awaitWork(self); - try - { - self.setAccordActiveTask(task); - task.run(); - } - catch (Throwable t) - { - try { task.failExecution(t); } - catch (Throwable t2) - { - try - { - t2.addSuppressed(t); - agent.onException(t2); - } - catch (Throwable t3) { /* empty to ensure we definitely loop so we cleanup the task */ } - } - } + task.runNoExcept(self); } catch (ShutdownException ignore) { @@ -458,10 +432,6 @@ public void run() { agent.onException(t); } - finally - { - self.setAccordActiveTask(null); - } } } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java index 4d2508b87ff6..ee8e79e535c5 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AccordExecutorSimple.java @@ -96,25 +96,7 @@ protected void run() return; } - // TODO (expected): dedup with AbstractLockLoop, and cleanup (executed flag is a bit ugly) - self.setAccordActiveTask(task); - boolean executed = false; - try - { - task.preRunExclusive(); - executed = true; - task.run(); - } - catch (Throwable t) - { - executed = false; - task.failExecution(t); - } - finally - { - cleanupTaskExclusive(task, executed); - self.setAccordActiveTask(null); - } + prepareRunComplete(self, task); } } finally diff --git a/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java b/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java index c171ed08de9b..e93ae0c0840e 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java +++ b/src/java/org/apache/cassandra/service/accord/execution/CancelTask.java @@ -29,9 +29,9 @@ final class CancelTask extends Task } @Override - void submitExclusive() + void submitExclusiveMayThrow() { - cancel.cancelExclusive(); + cancel.tryCancelExclusive(); } @Override @@ -41,13 +41,23 @@ boolean isNewWork() } @Override - String toDescription() + public String description() { - return "Cancel " + cancel.toDescription(); + return "Cancel " + cancel.description(); } - @Override void preRunExclusive() { throw new UnsupportedOperationException(); } - @Override void run() { throw new UnsupportedOperationException(); } - @Override void reportFailure(Throwable fail) { throw new UnsupportedOperationException(); } + @Override + public String briefDescription() + { + return "Cancel " + cancel.briefDescription(); + } + + @Override void unqueueIfQueued() { throw new UnsupportedOperationException(); } + @Override + void maybeCompleteExclusiveMayThrow() { throw new UnsupportedOperationException(); } + @Override void tryCancelExclusive() { throw new UnsupportedOperationException(); } + @Override void reportFailureMayThrow(Throwable fail) { throw new UnsupportedOperationException(); } + @Override boolean runMayThrow() { throw new UnsupportedOperationException(); } @Override public void cancel() { throw new UnsupportedOperationException(); } + @Override AccordExecutor executor() { throw new UnsupportedOperationException(); } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java b/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java index b0bd8d37e6fc..5181f11e1809 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java +++ b/src/java/org/apache/cassandra/service/accord/execution/ExclusiveExecutor.java @@ -33,11 +33,13 @@ import accord.utils.async.Cancellable; import org.apache.cassandra.service.accord.debug.DebugExecution; +import org.apache.cassandra.service.accord.execution.Task.GroupKind; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.EXCLUSIVE_QUEUE_LIMITS; +import static org.apache.cassandra.service.accord.execution.Task.ExecutorQueue.RUNNABLE; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.COMMAND_STORE; import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; -import static org.apache.cassandra.service.accord.execution.TaskQueueRunnable.RUNNABLE; public final class ExclusiveExecutor extends TaskQueueMulti implements ExclusiveAsyncExecutor { @@ -53,40 +55,59 @@ static final class ExclusiveExecutorTask extends Task this.queue = queue; } - @Override void submitExclusive() { throw new UnsupportedOperationException(); } + @Override AccordExecutor executor() { return queue.executor; } + @Override void submitExclusiveMayThrow() { throw new UnsupportedOperationException(); } @Override boolean isNewWork() { throw new UnsupportedOperationException(); } @Override public void cancel() { throw new UnsupportedOperationException(); } + @Override boolean runMayThrow() { throw new UnsupportedOperationException(); } + @Override void unqueueIfQueued() {} + @Override void reportFailureMayThrow(Throwable t) { throw new UnsupportedOperationException(); } + @Override void tryCancelExclusive() { throw new UnsupportedOperationException(); } - @Override - void preRunExclusive() - { - super.preRunExclusive(); - queue.preRunTask(); - } - - @Override - void run() + boolean prepareTask() { - queue.runTask(); + Task task = queue.task; + try + { + task.prepareExclusiveMayThrow(); + task.setStateExclusive(State.PREPARED); + setStateExclusive(State.PREPARED); + return true; + } + catch (Throwable t) + { + task.setStateExclusive(State.FAILED); + task.reportFailureNoExcept(t); + maybeCompleteExclusiveMayThrow(); + return false; + } } @Override - void cleanupExclusive(AccordExecutor executor, boolean executed) + void maybeCompleteExclusiveMayThrow() { - unsafeSetStateExclusive(WAITING_TO_RUN); - queue.cleanupTask(executed); + try + { + unsafeSetStateExclusive(WAITING_TO_RUN); + executor().runnable.cleanup(this); + queue.completeTask(); + } + catch (Throwable t) + { + onException(t); + } } @Override - void reportFailure(Throwable t) + public String description() { - queue.task.reportFailure(t); + return queue.task.description(); } @Override - String toDescription() + public String briefDescription() { - return queue.task.toDescription(); + return queue.task.briefDescription(); } protected boolean isInHeap() @@ -113,48 +134,43 @@ protected boolean isInHeap() ExclusiveExecutor(AccordExecutor executor, int commandStoreId) { - super(RUNNABLE, commandStoreId < 0 ? Task.GroupKind.NONE : Task.GroupKind.EXCLUSIVE, AccordExecutor.EXCLUSIVE_QUEUE_LIMITS); + super(RUNNABLE, commandStoreId < 0 ? GroupKind.NONE : GroupKind.EXCLUSIVE, EXCLUSIVE_QUEUE_LIMITS); this.executor = executor; this.commandStoreId = commandStoreId; this.selfTask = new ExclusiveExecutorTask(this); this.debug = DebugExecution.DebugExclusiveExecutor.maybeDebug(executor.debug, commandStoreId); } - void preRunTask() - { - task.preRunExclusive(); - } - - void runTask() + void runTask(TaskRunner self) { - Thread self = Thread.currentThread(); - if (!ownerUpdater.compareAndSet(this, null, self)) + Thread thread = Thread.currentThread(); + if (!ownerUpdater.compareAndSet(this, null, thread)) { if (DEBUG_EXECUTION) debug.onWaiting(); Invariants.require(waiting == null); - waiting = self; + waiting = thread; outer: do { while (true) { Thread owner = this.owner; - if (owner == self) break outer; + if (owner == thread) break outer; if (owner == null) continue outer; LockSupport.park(); } } - while (!ownerUpdater.compareAndSet(this, null, self)); - Invariants.require(waiting == self); + while (!ownerUpdater.compareAndSet(this, null, thread)); + Invariants.require(waiting == thread); waiting = null; } try { if (stopped && reject(task)) - task.reportFailure(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((SafeTask) task).executionContext())); + task.rejectAtRuntime(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + task.description())); else - task.run(); + task.runNoExcept(self); } finally { @@ -173,12 +189,12 @@ private boolean reject(Task task) return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable)); } - void cleanupTask(boolean executed) + void completeTask() { try { - task.unsetQueue(this); - task.cleanupExclusive(executor, executed); + task.unsetQueue(kind); + task.completeExclusiveNoExcept(); } finally { @@ -212,7 +228,7 @@ void enqueue(Task newTask, boolean incrementArrivals) incrementArrivals(newTask); incrementDispatches(newTask); task = newTask; - task.setQueue(this); + task.setQueue(kind); selfTask.position = newTask.position; selfTask.unsafeSetStateExclusive(WAITING_TO_RUN); executor.runnable.enqueue(selfTask, incrementArrivals); @@ -249,7 +265,7 @@ private void removeCurrentTask(IntrusiveHeapNode remove) // but can for other tasks that don't track their own state decrementDispatches(task); - task.unsetQueue(this); + task.unsetQueue(kind); task = pollMulti(); if (DEBUG_EXECUTION) debug.onSetTask(task); if (executor.runnable.isWaiting(selfTask)) @@ -315,31 +331,23 @@ public AsyncChain flatChain(Callable> call) return AsyncChains.flatChain(this, call); } - Task inherit() - { - Thread thread = Thread.currentThread(); - if (thread == owner) - return Task.unwrap(task); - - return executor.inherit(thread); - } - @Override public void execute(Runnable run) { - Task inherit = inherit(); - PlainRunnable submit = inherit == null ? new PlainRunnable(executor, null, run, this, Task.ExclusiveGroup.OTHER) - : new PlainRunnable(executor, null, run, this, Task.ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); - executor.submit(submit); + Task inherit = executor.inherit(); + PlainRunnable task = new PlainRunnable(executor, null, run, this, Task.ExclusiveGroup.OTHER); + if (inherit != null) inherit.addConsequence(task); + else executor.submitTask(task); } @Override public Cancellable execute(AsyncCallbacks.RunOrFail runOrFail) { - Task inherit = inherit(); - PlainChain submit = inherit == null ? new PlainChain(executor, runOrFail, ExclusiveExecutor.this, Task.ExclusiveGroup.OTHER) - : new PlainChain(executor, runOrFail, ExclusiveExecutor.this, Task.ExclusiveGroup.OTHER, inherit.position, inherit.tranche()); - return executor.submit(submit); + Task inherit = executor.inherit(); + PlainChain task = new PlainChain(executor, runOrFail, ExclusiveExecutor.this, Task.ExclusiveGroup.OTHER); + if (inherit != null) inherit.addConsequence(task); + else executor.submitTask(task); + return task; } @Override @@ -367,7 +375,7 @@ public boolean tryExecuteImmediately(Runnable run) } catch (Throwable t) { - executor.agent.onException(t); + selfTask.onException(t); } finally { diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTask.java b/src/java/org/apache/cassandra/service/accord/execution/IOTask.java index 68c976eb7bac..b3607151fe72 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/IOTask.java +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTask.java @@ -25,40 +25,14 @@ // a task that may be submitted to this executor or another public abstract class IOTask extends Plain implements Cancellable, DebuggableTask { - IOTask(AccordExecutor executor, GlobalGroup group, long position, int tranche) - { - super(executor, group, position, tranche); - } - IOTask(AccordExecutor executor, GlobalGroup group) { super(executor, group); } - abstract void postRunExclusive(); - - @Override - void cleanupExclusive(AccordExecutor executor, boolean executed) - { - super.cleanupExclusive(executor, executed); - postRunExclusive(); - } - @Override ExclusiveExecutor exclusiveExecutor() { return null; } - - @Override - public long creationTimeNanos() - { - return createdAt; - } - - @Override - public long startTimeNanos() - { - return runningAt; - } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java index aa54cfcf2a2b..759ff32ba737 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskLoad.java @@ -20,8 +20,6 @@ import accord.utils.Invariants; -import org.apache.cassandra.utils.Closeable; - import static org.apache.cassandra.service.accord.execution.IOTaskLoad.FailureHolder.NOT_STARTED; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; @@ -50,40 +48,38 @@ static class FailureHolder this.entry = entry; } - void postRunExclusive() + @Override + void maybeCompleteExclusiveMayThrow() { if (!(result instanceof FailureHolder)) executor.onLoadedExclusive(entry, (V) result, null); else executor.onLoadedExclusive(entry, null, ((FailureHolder) result).fail); + super.maybeCompleteExclusiveMayThrow(); } @Override - String toDescription() + public boolean runMayThrow() { - return "Load " + entry.key(); + result = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); + return true; } @Override - public void run() + void reportFailureMayThrow(Throwable t) { - onRunning(); - try (Closeable close = resources.get()) - { - result = entry.owner.parent().adapter().load(entry.owner.commandStore, entry.key()); - } - onRunComplete(); + result = new FailureHolder(t); } @Override - void reportFailure(Throwable t) + public String description() { - result = new FailureHolder(t); + return "Load " + entry.key(); } @Override - public String description() + public String briefDescription() { - return "Loading " + entry; + return description(); } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java index d010b6e1f102..cfe80ee81747 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskSave.java @@ -18,8 +18,6 @@ package org.apache.cassandra.service.accord.execution; -import org.apache.cassandra.utils.Closeable; - import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.SAVE; class IOTaskSave extends IOTask @@ -40,31 +38,22 @@ class IOTaskSave extends IOTask } @Override - void postRunExclusive() + void maybeCompleteExclusiveMayThrow() { executor.onSavedExclusive(entry, identity, failure); + super.maybeCompleteExclusiveMayThrow(); } @Override - String toDescription() - { - return "Save " + entry.key(); - } - - @Override - public void run() + public boolean runMayThrow() { - onRunning(); - try (Closeable close = resources.get()) - { - run.run(); - } - onRunComplete(); + run.run(); failure = null; + return true; } @Override - protected void reportFailure(Throwable t) + void reportFailureMayThrow(Throwable t) { failure = t; } @@ -74,4 +63,10 @@ public String description() { return "Save " + entry; } + + @Override + String briefDescription() + { + return description(); + } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java b/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java index ce6d60d5308d..3476dbde2d9a 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java +++ b/src/java/org/apache/cassandra/service/accord/execution/IOTaskWrapper.java @@ -18,8 +18,6 @@ package org.apache.cassandra.service.accord.execution; -import org.apache.cassandra.utils.Closeable; - class IOTaskWrapper extends IOTask { static abstract class WrappableIOTask @@ -32,43 +30,40 @@ static abstract class WrappableIOTask final WrappableIOTask wrapped; - IOTaskWrapper(AccordExecutor executor, WrappableIOTask wrap, GlobalGroup group, long position, int tranche) + IOTaskWrapper(AccordExecutor executor, WrappableIOTask wrap, GlobalGroup group) { - super(executor, group, position, tranche); + super(executor, group); this.wrapped = wrap; } @Override - String toDescription() + protected boolean runMayThrow() { - return wrapped.description(); + wrapped.runInternal(); + return true; } @Override - protected void run() + void maybeCompleteExclusiveMayThrow() { - onRunning(); - try (Closeable close = resources.get()) - { - wrapped.runInternal(); - } - onRunComplete(); + wrapped.postRunExclusive(); + super.maybeCompleteExclusiveMayThrow(); } @Override - void postRunExclusive() + public String description() { - wrapped.postRunExclusive(); + return wrapped.description(); } @Override - public String description() + public String briefDescription() { - return wrapped.description(); + return description(); } @Override - protected void reportFailure(Throwable fail) + void reportFailureMayThrow(Throwable fail) { wrapped.fail(fail); } diff --git a/src/java/org/apache/cassandra/service/accord/execution/Plain.java b/src/java/org/apache/cassandra/service/accord/execution/Plain.java index aff55550e28a..cc5a3c0ffc60 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/Plain.java +++ b/src/java/org/apache/cassandra/service/accord/execution/Plain.java @@ -20,28 +20,16 @@ import java.util.concurrent.CancellationException; -import accord.utils.Invariants; import accord.utils.async.Cancellable; import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; abstract class Plain extends Task implements Cancellable { final AccordExecutor executor; - Plain(AccordExecutor executor, GlobalGroup group, long position, int tranche) - { - super(group, position, tranche); - this.executor = executor; - } - - Plain(AccordExecutor executor, ExclusiveGroup group, long position, int tranche) - { - super(group, position, tranche); - this.executor = executor; - } - Plain(AccordExecutor executor, GlobalGroup group) { super(group); @@ -57,49 +45,48 @@ abstract class Plain extends Task implements Cancellable abstract ExclusiveExecutor exclusiveExecutor(); @Override - public void cancel() + public final void cancel() { - executor.submit(Task::cancelExclusive, CancelTask::new, this); + executor.submit(Task::tryCancelExclusive, CancelTask::new, this); } - void cancelExclusive() + @Override + final void tryCancelExclusive() { - ExclusiveExecutor exclusiveExecutor = exclusiveExecutor(); - if ((exclusiveExecutor == null ? executor.runnable : exclusiveExecutor).tryUnqueueWaiting(this)) - { - try - { - failExclusive(new CancellationException(), CANCELLED); - } - catch (Throwable t) - { - executor.agent.onException(t); - } - finally - { - executor.cleanupTaskExclusive(this, false); - } - } + tryFailAndCompleteExclusive(new CancellationException(), CANCELLED); } @Override - final void submitExclusive() + final void submitExclusiveMayThrow() { - submitExclusive(exclusiveExecutor(), null); + executor.registerExclusive(this); + setStateExclusive(WAITING_TO_RUN); + ExclusiveExecutor exclusiveExecutor = exclusiveExecutor(); + if (exclusiveExecutor == null) executor.runnable.enqueue(this, true); + else exclusiveExecutor.enqueue(this, true); } - final Cancellable submitExclusive(ExclusiveExecutor exclusiveExecutor, Task parent) + @Override + void maybeCompleteExclusiveMayThrow() { - Invariants.require(executor.isOwningThread()); - setStateExclusive(WAITING_TO_RUN); - - if (parent == null) executor.registerExclusive(this); - else executor.registerConsequenceExclusive(parent, this); - onLoaded(); + if (completeState()) + { + long completeAt = nanoTime(); + executor.elapsedWaitingToRun.increment(runningAt - createdAt, runningAt); + executor.elapsedRunning.increment(completeAt - runningAt, completeAt); + executor.elapsed.increment(completeAt - createdAt, completeAt); + } + } - if (exclusiveExecutor == null) executor.runnable.enqueue(this, true); - else exclusiveExecutor.enqueue(this, true); - return this; + @Override + void unqueueIfQueued() + { + if (isQueued()) + { + ExclusiveExecutor exclusiveExecutor = exclusiveExecutor(); + if (exclusiveExecutor != null) exclusiveExecutor.unqueue(this); + else executor.runnable.unqueue(this); + } } @Override @@ -107,4 +94,10 @@ protected boolean isNewWork() { return true; } + + @Override + AccordExecutor executor() + { + return executor; + } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java b/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java index c729f58e841f..dda6a54e2cef 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainChain.java @@ -22,8 +22,6 @@ import accord.utils.async.AsyncCallbacks; -import org.apache.cassandra.utils.Closeable; - class PlainChain extends Plain { final AsyncCallbacks.RunOrFail runOrFail; @@ -36,52 +34,34 @@ class PlainChain extends Plain this.exclusiveExecutor = exclusiveExecutor; } - PlainChain(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group, long position, int tranche) + @Override + public String description() { - super(executor, group, position, tranche); - this.runOrFail = runOrFail; - this.exclusiveExecutor = exclusiveExecutor; + return runOrFail.toString(); } @Override - ExclusiveExecutor exclusiveExecutor() + public String briefDescription() { - return exclusiveExecutor; + return description(); } @Override - String toDescription() + boolean runMayThrow() { - return runOrFail.toString(); + runOrFail.run(); + return true; } @Override - protected void run() + void reportFailureMayThrow(Throwable fail) { - onRunning(); - try (Closeable close = resources.get()) - { - runOrFail.run(); - } - catch (Throwable t) - { - // shouldn't throw exceptions - executor.agent.onException(t); - } - onRunComplete(); + runOrFail.fail(fail); } @Override - protected void reportFailure(Throwable fail) + ExclusiveExecutor exclusiveExecutor() { - try - { - runOrFail.fail(fail); - } - catch (Throwable t) - { - fail.addSuppressed(t); - executor.agent.onException(fail); - } + return exclusiveExecutor; } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java b/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java index 6a2ab7950b46..efcef616fd0f 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainChainDebuggable.java @@ -35,33 +35,9 @@ class PlainChainDebuggable extends PlainChain implements DebuggableTask this.describe = Invariants.nonNull(describe); } - PlainChainDebuggable(AccordExecutor executor, AsyncCallbacks.RunOrFail runOrFail, @Nullable ExclusiveExecutor exclusiveExecutor, long position, int tranche, Object describe) - { - super(executor, runOrFail, exclusiveExecutor, ExclusiveGroup.OTHER, position, tranche); - this.describe = Invariants.nonNull(describe); - } - - @Override - public long creationTimeNanos() - { - return createdAt; - } - - @Override - public long startTimeNanos() - { - return runningAt; - } - @Override public String description() { return describe.toString(); } - - @Override - public DebuggableTask debuggable() - { - return this; - } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java b/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java index 594ff107bf64..e51c83c964af 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java +++ b/src/java/org/apache/cassandra/service/accord/execution/PlainRunnable.java @@ -22,7 +22,6 @@ import accord.utils.async.Cancellable; -import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.concurrent.AsyncPromise; class PlainRunnable extends Plain implements Cancellable @@ -31,14 +30,6 @@ class PlainRunnable extends Plain implements Cancellable final Runnable run; final @Nullable ExclusiveExecutor exclusiveExecutor; - PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, GlobalGroup group, long position, int tranche) - { - super(executor, group, position, tranche); - this.result = result; - this.run = run; - this.exclusiveExecutor = null; - } - PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, GlobalGroup group) { super(executor, group); @@ -47,48 +38,44 @@ class PlainRunnable extends Plain implements Cancellable this.exclusiveExecutor = null; } - PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group, long position, int tranche) + PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group) { - super(executor, group, position, tranche); + super(executor, group); this.result = result; this.run = run; this.exclusiveExecutor = exclusiveExecutor; } - PlainRunnable(AccordExecutor executor, AsyncPromise result, Runnable run, ExclusiveExecutor exclusiveExecutor, ExclusiveGroup group) + @Override + protected boolean runMayThrow() { - super(executor, group); - this.result = result; - this.run = run; - this.exclusiveExecutor = exclusiveExecutor; + run.run(); + if (result != null) + { + try { result.trySuccess(null); } + catch (Throwable t) { onException(t); } + } + return true; } @Override - String toDescription() + public String description() { // TODO (expected): ensure this is usefully descriptive, or accept a separate description return run.toString(); } @Override - protected void run() + public String briefDescription() { - onRunning(); - try (Closeable close = resources.get()) - { - run.run(); - } - if (result != null) - result.trySuccess(null); - onRunComplete(); + return description(); } @Override - protected void reportFailure(Throwable t) + void reportFailureMayThrow(Throwable t) { if (result != null) result.tryFailure(t); - executor.agent.onException(t); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java index 77c61ecf4047..7027adb4780b 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java @@ -56,6 +56,7 @@ import accord.primitives.AbstractUnseekableKeys; import accord.primitives.Range; import accord.primitives.Ranges; +import accord.primitives.Routable; import accord.primitives.RoutingKeys; import accord.primitives.Timestamp; import accord.primitives.TxnId; @@ -81,7 +82,6 @@ import org.apache.cassandra.service.accord.execution.AccordCacheEntry.Status; import org.apache.cassandra.service.accord.serializers.CommandSerializers; import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.NoSpamLogger; import org.apache.cassandra.utils.concurrent.Condition; @@ -90,6 +90,7 @@ import static accord.local.LoadKeys.SYNC; import static accord.local.LoadKeysFor.RECOVERY; import static accord.local.LoadKeysFor.WRITE; +import static accord.primitives.Routable.Domain.Key; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; import static org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask.SANITY_CHECK; @@ -106,18 +107,26 @@ import static org.apache.cassandra.service.accord.execution.SaferState.preExecute; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.LOAD; import static org.apache.cassandra.service.accord.execution.Task.GlobalGroup.RANGE_LOAD; -import static org.apache.cassandra.service.accord.execution.Task.RunState.PERSISTING; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_FAILED; +import static org.apache.cassandra.service.accord.execution.Task.RunState.NOT_YET_RUN; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_PERSISTING; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_INCOMPLETE; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUNNING; import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED_UNREGISTERED; +import static org.apache.cassandra.service.accord.execution.Task.State.FAILED; import static org.apache.cassandra.service.accord.execution.Task.State.INCOMPLETE; import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_OPTIONAL; import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_REQUIRED; -import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.PREPARED; +import static org.apache.cassandra.service.accord.execution.Task.State.REGISTERED; import static org.apache.cassandra.service.accord.execution.Task.State.SCANNING_RANGES; import static org.apache.cassandra.service.accord.execution.Task.State.WAITING; import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_OPTIONAL; import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_REQUIRED; -import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_OR_RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_OR_PREPARED; import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; public final class SafeTask extends Task implements Cancellable, DebuggableTask { @@ -137,47 +146,46 @@ public static SafeTask create(CommandStore commandStore, ExecutionContext return new SafeTask<>((AccordCommandStore) commandStore, context, function); } - final class NonSyncState extends ExecutionContext.Wrapped implements ExecutionContext + static class NonSyncState extends ExecutionContext.Wrapped implements ExecutionContext { RoutingKeys active; - ObjectHashSet notBlocking; - ObjectHashSet blocking; // cache entries on which we're blocking work + ObjectHashSet blocking, notBlocking; int loaded, processed; - int ready; + boolean alwaysReady; - public NonSyncState() + public NonSyncState(ExecutionContext context) { - super(executionContext); + super(context); } @Override - public Unseekables keys() + public final Unseekables keys() { return active; } - void addLoaded() + final void addLoaded() { ++loaded; } - void onNotHead(AccordCacheEntry entry) + final void onNotHead(AccordCacheEntry entry) { if ((notBlocking == null || !notBlocking.remove((RoutingKey) entry.key())) && blocking != null) blocking.remove((RoutingKey) entry.key()); } - void onNewHead(AccordCacheEntry entry) + final void onNewHead(AccordCacheEntry entry) { ensureNotBlocking().add((RoutingKey) entry.key()); } - void onNewBlockingHead(AccordCacheEntry entry) + final void onNewBlockingHead(AccordCacheEntry entry) { ensureBlocking().add((RoutingKey) entry.key()); } - void onStillHeadNewBlocking(AccordCacheEntry entry) + final void onStillHeadNewBlocking(AccordCacheEntry entry) { notBlocking.remove((RoutingKey) entry.key()); ensureBlocking().add((RoutingKey) entry.key()); @@ -202,33 +210,33 @@ private int readyCount() return (blocking == null ? 0 : blocking.size()) + (notBlocking == null ? 0 : notBlocking.size()); } - boolean isLoaded() + final boolean isLoaded(SafeTask owner) { - return loaded >= Math.min(keys, NONSYNC_MIN_BATCH_SIZE); + return loaded >= Math.min(owner.keys, alwaysReady ? 1 : NONSYNC_MIN_BATCH_SIZE); } - boolean isWaitReady() + final boolean isWaitReady(SafeTask owner) { - if (readyCount() >= Math.min(keys - processed, NONSYNC_MIN_BATCH_SIZE)) + if (readyCount() >= Math.min(owner.keys - processed, alwaysReady ? 1 : NONSYNC_MIN_BATCH_SIZE)) return true; return blocking != null && blocking.size() >= NONSYNC_BLOCKED_LIMIT; } - void preRunExclusive() + void prepareExclusive(SafeTask owner) { try (BufferList keys = new BufferList<>()) { if ((blocking == null || !populate(keys, blocking)) && notBlocking != null) populate(keys, notBlocking); - keys.forEach(key -> preExecute(refs.get(key), SafeTask.this, RELEASE_QUEUE)); + keys.forEach(key -> preExecute(owner.refs.get(key), owner, RELEASE_QUEUE)); keys.sort(RoutingKey::compareTo); active = RoutingKeys.of(keys); } processed += active.size(); - if (processed == keys && isIncremental()) - setIncrementalFinishingExclusive(); + if (processed == owner.keys && owner.isIncremental()) + owner.setIncrementalFinishingExclusive(); } private boolean populate(List keys, ObjectHashSet from) @@ -254,20 +262,35 @@ private boolean populate(List keys, ObjectHashSet from) return false; } - void postRunExclusive() + void postRunExclusive(SafeTask owner) { if (active != null) { for (RoutingKey key : active) - postExecute(refs.remove(key), SafeTask.this); + postExecute(owner.refs.remove(key), owner); active = null; } - ready = readyCount(); + } + } + + static class IncrementalState extends NonSyncState + { + long waiting, running; + + public IncrementalState(ExecutionContext context) + { + super(context); + } + + void updateMetrics(long waiting, long running) + { + this.running += running; + this.waiting += waiting; } } final AccordCommandStore commandStore; - private final ExecutionContext executionContext; + private final ExecutionContext context; private final Function function; // TODO (expected): simple custom map that allows (at least): @@ -296,17 +319,17 @@ void postRunExclusive() int keys; // TODO (expected): not counting keys we add during execution LogLinearDecayingHistograms.Buffer histogramBuffer; - @Nullable RangeTxnScanner rangeScanner; - @Nullable CommandSummaries commandsForRanges; - byte runState; + + @Nullable Object ranges; + long loadedAt; private BiConsumer callback; - public SafeTask(@Nonnull AccordCommandStore commandStore, ExecutionContext executionContext, Function function) + public SafeTask(@Nonnull AccordCommandStore commandStore, ExecutionContext context, Function function) { - super(executionContext, commandStore.executor().uniqueCreatedAt); + super(context, commandStore.executor().uniqueCreatedAt); this.commandStore = commandStore; - this.executionContext = executionContext; + this.context = context; this.function = function; if (logger.isTraceEnabled()) logger.trace("Created {} on {}", this, commandStore); @@ -320,21 +343,19 @@ String loggingId() @Override public String toString() { - return "@[" + commandStore.id() + ',' + commandStore.node().id() + "] " + executionContext.describe() + ' ' + toBriefString(); + return "@[" + commandStore.id() + ',' + commandStore.node().id() + "] " + context.describe() + ' ' + toBriefString(); } public String toBriefString() { - return '{' + loggingId() + ',' + describeState() + '}'; + return '{' + loggingId() + ',' + currentState() + '}'; } - public String toDescription() + public String summarise() { - return toBriefString() + ": " - + (queued() == null ? "unqueued" : describeState()) - + ", primaryTxnId: " + executionContext.primaryTxnId() + return loggingId() + ' ' + context.executionKind() + + ": primaryTxnId: " + context.primaryTxnId() + ", state: " + summarise(refs, SaferState::global); - } private static String summarise(Map map, Function transform) @@ -383,66 +404,79 @@ public AsyncChain chain() @Override protected Cancellable start(BiConsumer callback) { - preSetup(callback); - executor().submit(SafeTask.this); + if (!preSetup(callback)) + executor().submitTask(SafeTask.this); return SafeTask.this; } }; } - private void preSetup(BiConsumer callback) + private boolean preSetup(BiConsumer callback) { Invariants.require(this.callback == null); this.callback = callback; - TaskRunner self = TaskRunner.get(); - Task task = self.accordActiveSelfTask(); - if (task instanceof SafeTask) + Task inherit = executor().inherit(); + if (inherit == null) + return false; + + Invariants.require(inherit.is(PREPARED)); + if (!inherit.is(RUNNING)) + return false; + + if (inherit instanceof SafeTask) { - SafeTask parent = (SafeTask) task; - if (parent.commandStore == commandStore) - preSetup(parent); + SafeTask inheritRefs = (SafeTask) inherit; + if (inheritRefs.commandStore == commandStore) + preSetup(inheritRefs); } + inherit.addConsequence(this); + return true; } // to be invoked only by the CommandStore owning thread, to take references to objects already in use by the current execution - public void preSetup(SafeTask parent) + private void preSetup(SafeTask parent) { - this.position = parent.position; - setInheritedWithTranche(parent.tranche()); - // note we use the caches "unsafely" here deliberately, as we only reference commands we already have references to // so we do not mutate anything, except the atomic counter of references - - LoadKeys loadKeys = loadKeys(executionContext); + LoadKeys loadKeys = loadKeys(context); if (loadKeys != NONE) { - Unseekables parentKeysOrRanges = parent.executionContext.keys(); - Unseekables keysOrRanges = executionContext.keys(); + Unseekables parentKeysOrRanges = parent.context.keys(); + Unseekables keysOrRanges = context.keys(); boolean isKeySubset = parent.isIncremental() ? parent.nonSync.active.containsAll(keysOrRanges) : parentKeysOrRanges.containsAll(keysOrRanges); if (isKeySubset) setInheritedRangeScan(); - setSequencedExclusive(executionContext.executionSequence()); - if (isSequencedByPriorityAtomic()) - { - boolean isTxnIdSubset = executionContext.isTxnIdSubsetOf(parent.executionContext); - Invariants.require(isKeySubset, "Must start ATOMIC tasks from a task declaring a superset of the required keys (for ASYNC/INCR tasks this means the keys active for the batch in question)"); - Invariants.require(isTxnIdSubset, "Must start ATOMIC tasks from a task declaring a superset of the required TxnIds"); - // TODO (required): we're appending to the fifo queue - does this maintain correct order? - setCacheQueuedFifoExclusive(); - } - + setSequencedExclusive(context.executionSequence()); if (loadKeys != SYNC) { setNonSyncExclusive(); - nonSync = new NonSyncState(); if (loadKeys == INCR) { + nonSync = new IncrementalState(context); setIncrementalExclusive(); // forbid BY_PRIORITY sequencing to avoid priority inversion deadlocks on INCR tasks that lock a TxnId but await some key that has a higher priority task (that is waiting on our locked TxnId) - solvable in future if necessary Invariants.require(isSequencedByPriorityAtomic() || isUnsequenced(), "INCR tasks may currently only be ATOMIC or UNSEQUENCED"); } + else + { + nonSync = new NonSyncState(context); + } + } + + if (isSequencedByPriorityAtomic()) + { + boolean isTxnIdSubset = context.isTxnIdSubsetOf(parent.context); + if (!isKeySubset) + { + Invariants.require(keysOrRanges.domain() == Key, "ATOMIC tasks over ranges must declare a subset of their parent's task keys() to avoid a range scan across which we would not impose sequencing"); + // to avoid priority inversion deadlocks, if we are not a strict subset of the parent task we permit running with a single key ready + nonSync.alwaysReady = true; + } + Invariants.require(isTxnIdSubset, "ATOMIC tasks must declare a subset of their parent's task txnIds() to avoid a priority inversion deadlock"); + // TODO (required): we're appending to the fifo queue - does this maintain correct order? + setCacheQueuedFifoExclusive(); } if (keysOrRanges.equals(parentKeysOrRanges)) @@ -473,28 +507,32 @@ public void preSetup(SafeTask parent) } } - for (TxnId txnId : executionContext.txnIds()) + for (TxnId txnId : context.txnIds()) preSetup(txnId, parent.refs, commandStore.cachesUnsafe().commands()); } @Override - void submitExclusive() + void submitExclusiveMayThrow() { Caches caches = commandStore.cachesExclusive(); executor().registerExclusive(this); + setStateExclusive(REGISTERED); boolean hasPreSetup = hasInherited(); - LoadKeys loadKeys = loadKeys(executionContext); + LoadKeys loadKeys = loadKeys(context); if (loadKeys != NONE) { if (loadKeys != SYNC && !hasPreSetup) { setNonSyncExclusive(); - nonSync = new NonSyncState(); - if (loadKeys == INCR) + if (loadKeys != INCR) nonSync = new NonSyncState(context); + else + { + nonSync = new IncrementalState(context); setIncrementalExclusive(); + } } - Unseekables keysOrRanges = executionContext.keys(); + Unseekables keysOrRanges = context.keys(); switch (keysOrRanges.domain()) { case Range: @@ -510,7 +548,7 @@ void submitExclusive() } } - for (TxnId txnId : executionContext.txnIds()) + for (TxnId txnId : context.txnIds()) { if (hasPreSetup && completePresetupExclusive(txnId)) continue; @@ -525,14 +563,15 @@ void submitExclusive() private void setupKeyLoadsExclusive(boolean hasPreSetup, Caches caches, Iterable setupKeys, boolean doNotScanRanges) { - if (executionContext.loadKeys() == NONE) + if (context.loadKeys() == NONE) return; - if (!doNotScanRanges && executionContext.loadKeysFor() == RECOVERY) + if (!doNotScanRanges && context.loadKeysFor() == RECOVERY) { - Invariants.require(rangeScanner == null); - rangeScanner = new RangeTxnScanner(); - rangeScanner.start(); + Invariants.require(ranges == null); + RangeTxnScanner scanner = new RangeTxnScanner(); + ranges = scanner; + scanner.start(); } int waitsForIncrement = isSync() ? 1 : 0; @@ -547,11 +586,12 @@ private void setupKeyLoadsExclusive(boolean hasPreSetup, Caches caches, Iterable private void setupRangeLoadsExclusive(Caches caches) { - if (executionContext.loadKeysFor() == WRITE) + if (context.loadKeysFor() == WRITE) return; - rangeScanner = new RangeTxnAndKeyScanner(caches.commandsForKeys()); - rangeScanner.start(); + RangeTxnAndKeyScanner scanner = new RangeTxnAndKeyScanner(caches.commandsForKeys()); + ranges = scanner; + scanner.start(); } // expects mutual exclusivity only on the command store @@ -679,7 +719,7 @@ private void onSetupOrScannedExclusive() private void onLoadedRequiredExclusive() { - if (isSync() || nonSync.isLoaded()) + if (isSync() || nonSync.isLoaded(this) || isCacheQueuedFifo()) { waitOnCacheQueuesExclusive(); } @@ -702,13 +742,13 @@ boolean isOptional(AccordCacheEntry entry) boolean holdsLocksBetweenRuns() { // TODO (desired): encode as a state bit - return isIncremental() && executionContext.primaryTxnId() != null; + return isIncremental() && context.primaryTxnId() != null; } private void waitOnCacheQueuesExclusive() { Invariants.require(waitingForState == 0); - onLoaded(); + loadedAt = nanoTime(); executor().runnable.incrementArrivals(this); commandStore.exclusiveExecutor().incrementArrivals(this); @@ -731,17 +771,17 @@ else if (status == NOT_RUNNABLE) else { setStateExclusive(WAITING_ON_REQUIRED); - executor().waitingOnCacheQueues.enqueue(this); + executor().waiting.enqueue(this); } } private void waitOnOptionalCacheQueuesExclusive() { - if (isSync() || nonSync.isWaitReady()) waitToRunExclusive(); + if (isSync() || nonSync.isWaitReady(this)) waitToRunExclusive(); else { setStateExclusive(WAITING_ON_OPTIONAL); - executor().waitingOnCacheQueues.enqueue(this); + executor().waiting.enqueue(this); } } @@ -765,7 +805,7 @@ void onLoadOneExclusive(AccordCacheEntry loaded) case WAITING_ON_REQUIRED: case WAITING_ON_OPTIONAL: case WAITING_TO_RUN: - case RUNNING: + case PREPARED: RunnableStatus status = addToCacheQueue(loaded, false); if (status != NOT_RUNNABLE) addQueuedOptionalKey(loaded, status); @@ -796,7 +836,7 @@ void onLoadOneExclusive(AccordCacheEntry loaded) void addLoadedOptionalKey() { nonSync.addLoaded(); - if (is(LOADING_OPTIONAL) && nonSync.isLoaded()) + if (is(LOADING_OPTIONAL) && nonSync.isLoaded(this)) { unqueue(executor().loading); waitOnCacheQueuesExclusive(); @@ -820,16 +860,16 @@ void addQueuedOptionalKey(AccordCacheEntry loaded, RunnableStatus statu break; } - if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady()) + if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady(this)) { - unqueue(executor().waitingOnCacheQueues); + unqueue(executor().waiting); waitToRunExclusive(); } } void onChangeHeadStatus(AccordCacheEntry entry, RunnableStatus status) { - if (isSync() || !entry.isCommandsForKey()) onChangeRequiredHeadStatus(entry, status); + if (isSync() || !entry.isCommandsForKey()) onChangeRequiredHeadStatus(status); if (isNonSync() && entry.isCommandsForKey()) onChangeOptionalHeadStatus(entry, status); } @@ -844,14 +884,14 @@ private void incrementWaitingWhileAlreadyWaiting() // TODO (expected): this is potentially costly; maybe we don't want to swap these in and out (but harder to maintain invariants) unqueue(commandStore.exclusiveExecutor()); setStateExclusive(WAITING_ON_REQUIRED); - executor().waitingOnCacheQueues.enqueue(this); + executor().waiting.enqueue(this); } } Invariants.require(waitingForState < refs.size()); ++waitingForState; } - void onChangeRequiredHeadStatus(AccordCacheEntry entry, RunnableStatus newStatus) + void onChangeRequiredHeadStatus(RunnableStatus newStatus) { if (newStatus == NOT_RUNNABLE) { @@ -862,7 +902,7 @@ else if (newStatus != STILL_RUNNABLE_NEWLY_BLOCKING) Invariants.require(is(WAITING_ON_REQUIRED)); if (--waitingForState == 0) { - unqueue(executor().waitingOnCacheQueues); + unqueue(executor().waiting); waitOnOptionalCacheQueuesExclusive(); } } @@ -870,18 +910,18 @@ else if (newStatus != STILL_RUNNABLE_NEWLY_BLOCKING) void onChangeOptionalHeadStatus(AccordCacheEntry entry, RunnableStatus status) { - Invariants.require(isState(WAITING_OR_RUNNING)); + Invariants.require(isState(WAITING_OR_PREPARED)); switch (status) { default: throw UnhandledEnum.unknown(status); case STILL_RUNNABLE: throw UnhandledEnum.invalid(STILL_RUNNABLE); // onChange -> changed (but this means no change) case NOT_RUNNABLE: nonSync.onNotHead(entry); - if (is(WAITING_TO_RUN) && !nonSync.isWaitReady()) + if (is(WAITING_TO_RUN) && !nonSync.isWaitReady(this)) { unqueue(commandStore.exclusiveExecutor()); setStateExclusive(WAITING_ON_OPTIONAL); - executor().waitingOnCacheQueues.enqueue(this); + executor().waiting.enqueue(this); } return; @@ -898,9 +938,9 @@ void onChangeOptionalHeadStatus(AccordCacheEntry entry, RunnableStatus break; } - if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady()) + if (is(WAITING_ON_OPTIONAL) && nonSync.isWaitReady(this)) { - unqueue(executor().waitingOnCacheQueues); + unqueue(executor().waiting); waitToRunExclusive(); } } @@ -913,136 +953,135 @@ void waitToRunExclusive() public ExecutionContext executionContext() { - return executionContext; + return context; } @Override - protected void preRunExclusive() + protected void prepareExclusiveMayThrow() { - super.preRunExclusive(); - if (rangeScanner != null) + try { - commandsForRanges = rangeScanner.finish(commandStore.cachesExclusive()); - rangeScanner = null; - } + if (ranges instanceof SafeTask.RangeTxnScanner) + ranges = ((SafeTask.RangeTxnScanner)ranges).finish(commandStore.cachesExclusive()); - if (isSync()) - { - refs.forEach((k, v) -> { - preExecute(v, this, RELEASE_QUEUE); - }); - } - else - { - if (!hasIncrementalStarted()) + if (isSync()) + { + refs.forEach((k, v) -> { + preExecute(v, this, RELEASE_QUEUE); + }); + } + else { - TxnId primaryTxnId = executionContext.primaryTxnId(); - if (primaryTxnId != null) + if (!hasIncrementalStarted()) { - LockMode lockMode = holdsLocksBetweenRuns() ? HOLD_QUEUE : RELEASE_QUEUE; - preExecute(refs.get(primaryTxnId), this, lockMode); - TxnId additionalTxnId = executionContext.additionalTxnId(); - if (additionalTxnId != null) - preExecute(refs.get(additionalTxnId), this, lockMode); - } + TxnId primaryTxnId = context.primaryTxnId(); + if (primaryTxnId != null) + { + LockMode lockMode = holdsLocksBetweenRuns() ? HOLD_QUEUE : RELEASE_QUEUE; + preExecute(refs.get(primaryTxnId), this, lockMode); + TxnId additionalTxnId = context.additionalTxnId(); + if (additionalTxnId != null) + preExecute(refs.get(additionalTxnId), this, lockMode); + } - if (isIncremental() && isSequencedByPriorityAtomic() && !isCacheQueuedFifo()) - { - setCacheQueuedFifoExclusive(); - refs.forEach((key, safeState) -> { - AccordCacheEntry entry = global(safeState); - RunnableStatus status = entry.moveToFifo(this); - if (entry.isLoaded()) - { - switch (status) + if (isIncremental() && isSequencedByPriorityAtomic() && !isCacheQueuedFifo()) + { + setCacheQueuedFifoExclusive(); + refs.forEach((key, safeState) -> { + AccordCacheEntry entry = global(safeState); + RunnableStatus status = entry.moveToFifo(this); + if (entry.isLoaded()) { - default: throw UnhandledEnum.unknown(status); - case NOT_RUNNABLE: - case STILL_RUNNABLE: - case STILL_RUNNABLE_NEWLY_BLOCKING: - break; - case NEWLY_RUNNABLE: - nonSync.onNewHead(entry); - break; - case NEWLY_BLOCKING_RUNNABLE: - nonSync.onNewBlockingHead(entry); - break; + switch (status) + { + default: throw UnhandledEnum.unknown(status); + case NOT_RUNNABLE: + case STILL_RUNNABLE: + case STILL_RUNNABLE_NEWLY_BLOCKING: + break; + case NEWLY_RUNNABLE: + nonSync.onNewHead(entry); + break; + case NEWLY_BLOCKING_RUNNABLE: + nonSync.onNewBlockingHead(entry); + break; + } } - } - }); - } + }); + } - if (isIncremental()) - setIncrementalStartedExclusive(); + if (isIncremental()) + setIncrementalStartedExclusive(); + } + nonSync.prepareExclusive(this); } - nonSync.preRunExclusive(); + } + catch (Throwable t) + { + refs.forEach((k, v) -> { v.setAbandoned(); }); + throw t; } } @Override - public void run() + public boolean runMayThrow() { - onRunning(); - SaferCommandStore safeStore = null; - try (Closeable close = resources.get()) + try { - if (Tracing.isTracing()) - Tracing.trace(executionContext.describe()); - - commandStore.begin(safeStore = new SaferCommandStore(this, isSync() ? executionContext : nonSync)); - - R result = function.apply(safeStore); - - boolean finished = !isIncremental() || isIncrementalFinishing(); - if (finished) + SaferCommandStore safeStore = new SaferCommandStore(this, isSync() ? context : nonSync); + commandStore.begin(safeStore); + try { - List changes = new ArrayList<>(); - // TODO (expected): save any TxnId we add so that we don't need to iterate all of refs - refs.forEach((key, value) -> { - if (value instanceof SaferCommand) - { - SaferCommand safeCommand = (SaferCommand) value; - Journal.CommandUpdate diff = safeCommand.update(); - if (diff != null) + if (Tracing.isTracing()) + Tracing.trace(context.describe()); + + R result = function.apply(safeStore); + boolean finished = !isIncremental() || isIncrementalFinishing(); + if (!finished) setRunState(RUN_INCOMPLETE); + else + { + List changes = new ArrayList<>(); + // TODO (expected): save any TxnId we add so that we don't need to iterate all of refs + refs.forEach((key, value) -> { + if (value instanceof SaferCommand) { - changes.add(diff); - maybeSanityCheck(safeCommand); + SaferCommand safeCommand = (SaferCommand) value; + Journal.CommandUpdate diff = safeCommand.update(); + if (diff != null) + { + changes.add(diff); + maybeSanityCheck(safeCommand); + } } - } - }); + }); - boolean flush = !changes.isEmpty() || safeStore.fieldUpdates() != null; - if (flush) - { - setRunState(PERSISTING); - Runnable onFlush = () -> finish(result, null); - safeStore.persistFieldUpdatesInternal(changes.isEmpty() ? onFlush : null); - if (!changes.isEmpty()) - save(changes, onFlush); - finished = false; + boolean flush = !changes.isEmpty() || safeStore.fieldUpdates() != null; + if (flush) + { + setRunState(RUN_PERSISTING); + Runnable onFlush = () -> finish(result); + safeStore.persistFieldUpdatesInternal(changes.isEmpty() ? onFlush : null); + if (!changes.isEmpty()) + save(changes, onFlush); + finished = false; + } } - } - - // TODO (required): exception handling here needs improving - safeStore.postExecute(); - commandStore.complete(safeStore); - safeStore = null; - if (finished) - finish(result, null); + safeStore.postExecute(); + if (finished) + finish(result); + return finished; + } + finally + { + commandStore.complete(safeStore); + } } catch (Throwable t) { - if (safeStore != null) - refs.forEach((k, v) -> v.setAbandoned()); + refs.forEach((k, v) -> v.setAbandoned()); throw t; } - finally - { - if (safeStore != null) - commandStore.complete(safeStore); - onRunComplete(); - } } private void save(List diffs, Runnable onFlush) @@ -1075,40 +1114,77 @@ private void maybeSanityCheck(SaferCommand safeCommand) } } - public void reportFailure(Throwable throwable) + void reportFailureMayThrow(Throwable failure) { - try - { - if (callback != null) - callback.accept(null, throwable); - } - finally + if (callback == null) executor().agent.onException(failure); + else { - commandStore.agent().onException(throwable); + BiConsumer reportTo = callback; + callback = null; + + if (executor().isInLoop()) reportTo.accept(null, failure); + else executor().submit(() -> reportTo.accept(null, failure)); } } @Override - protected void cleanupExclusive(AccordExecutor executor, boolean executed) + void maybeCompleteExclusiveMayThrow() { - if (is(RUNNING) && isNonSync()) + if (isNonSync() && !isEither(NOT_YET_RUN, RUN_FAILED)) { - nonSync.postRunExclusive(); - if (isIncremental() && !isIncrementalFinishing()) + if (is(PREPARED)) { - setStateExclusive(INCOMPLETE); - waitOnOptionalCacheQueuesExclusive(); - return; + nonSync.postRunExclusive(this); + if (isIncremental()) + { + long now = nanoTime(); + ((IncrementalState)nonSync).updateMetrics(now - runningAt, runningAt - loadedAt); + if (!isIncrementalFinishing()) + { + setStateExclusive(INCOMPLETE); + waitOnOptionalCacheQueuesExclusive(); + return; + } + } + } + else + { + Invariants.expect(is(FAILED)); + Invariants.expect(isIncremental()); + setRunState(RUN_FAILED); + refs.forEach((k, v) -> v.setAbandoned()); } } - executor.keys.increment(keys, runningAt); releaseResourcesExclusive(commandStore.cachesExclusive()); - super.cleanupExclusive(executor, executed); - if (histogramBuffer != null) + + AccordExecutor executor = executor(); + if (completeState()) + { + long completeAt = nanoTime(); + executor.elapsedPreparingToRun.increment(loadedAt - createdAt, runningAt); + if (isIncremental()) + { + IncrementalState incrementalState = (IncrementalState) nonSync; + executor.elapsedWaitingToRun.increment(incrementalState.waiting, runningAt); + executor.elapsedRunning.increment(incrementalState.running, completeAt); + } + else + { + executor.elapsedWaitingToRun.increment(runningAt - loadedAt, runningAt); + executor.elapsedRunning.increment(completeAt - runningAt, completeAt); + } + executor.elapsed.increment(completeAt - createdAt, completeAt); + executor.keys.increment(keys, runningAt); + if (histogramBuffer != null) + { + histogramBuffer.flush(runningAt); + histogramBuffer = null; + } + } + else if (histogramBuffer != null) { - histogramBuffer.flush(completeAt); - histogramBuffer = null; + histogramBuffer.clear(); } } @@ -1116,15 +1192,20 @@ protected void cleanupExclusive(AccordExecutor executor, boolean executed) public void cancel() { if (!state().hasStarted()) - executor().submit(Task::cancelExclusive, CancelTask::new, this); + executor().submit(Task::tryCancelExclusive, CancelTask::new, this); } - void cancelExclusive() + void tryCancelExclusive() { State state = state(); switch (state) { default: throw new UnhandledEnum(state); + case UNREGISTERED: + failExclusive(new CancellationException(), CANCELLED_UNREGISTERED); + break; + + case REGISTERED: case SCANNING_RANGES: case LOADING_REQUIRED: case LOADING_OPTIONAL: @@ -1132,41 +1213,53 @@ void cancelExclusive() case WAITING_ON_OPTIONAL: case WAITING_TO_RUN: if (!hasIncrementalStarted()) - { - unqueueIfQueued(); - setStateExclusive(CANCELLED); - if (rangeScanner != null) - rangeScanner.cancelled = true; - try - { - logger.info("Cancelling {}", executionContext); - if (callback != null) - { - if (executor().isInLoop()) callback.accept(null, new CancellationException()); - else executor().submit(() -> callback.accept(null, new CancellationException())); - } - } - finally - { - executor().cleanupTaskExclusive(this, false); - } - break; - } - case UNINITIALIZED: // TODO (expected): preferable to be able to cancel at this stage, even if unlikely to trigger at this phase - case RUNNING: + cancelExclusive(); + break; + + case CANCELLED_UNREGISTERED: case INCOMPLETE: + case PREPARED: case EXECUTED: case CANCELLED: - case FAILED_TO_LOAD: + case FAILED: // cannot safely cancel } } - private void finish(R result, Throwable failure) + void cancelExclusive() + { + if (ranges instanceof SafeTask.RangeTxnScanner) + ((SafeTask.RangeTxnScanner)ranges).cancelled = true; // TODO (expected): should we try to cancel this directly? + failAndCompleteExclusive(new CancellationException(), CANCELLED); + } + + void unqueueIfQueued() + { + TaskQueue expected; + switch (queued()) + { + default: throw UnhandledEnum.unknown(queued()); + case NONE: return; + case LOADING: + expected = executor().loading; + break; + case WAITING: + expected = executor().waiting; + break; + case RUNNABLE: + expected = commandStore.exclusiveExecutor(); + break; + } + expected.unqueue(this); + } + + private void finish(R result) { - setRunState(failure == null ? RunState.SUCCESS : RunState.FAILED); if (callback != null) - callback.accept(result, failure); + { + try { callback.accept(result, null); } + catch (Throwable t) { commandStore.agent().onException(t); } + } } void releaseResourcesExclusive(Caches caches) @@ -1176,12 +1269,12 @@ void releaseResourcesExclusive(Caches caches) try { - if (rangeScanner != null) + if (ranges instanceof SafeTask.RangeTxnScanner) { - rangeScanner.cleanup(caches); - rangeScanner = null; + ((SafeTask.RangeTxnScanner)ranges).cleanup(caches); if (DEBUG_EXECUTION) DebugTask.get(this).onReleasedRangeScanner(); } + ranges = null; refs.forEach((key, safeState) -> { SaferState.postExecute(safeState, this); @@ -1227,7 +1320,7 @@ public void onUpdate(AccordCacheEntry state) } final Set intersectingKeys = new ObjectHashSet<>(); - final Ranges ranges = ((AbstractRanges) executionContext.keys()).toRanges(); + final Ranges ranges = ((AbstractRanges) context.keys()).toRanges(); final AccordCache.Type.Instance commandsForKeyCache; KeyWatcher keyWatcher = new KeyWatcher(); @@ -1353,7 +1446,7 @@ protected void runInternal() ExecutionContext preLoadContext() { - return executionContext; + return context; } @Override @@ -1371,24 +1464,24 @@ protected void fail(Throwable t) public void start() { Caches caches = commandStore.cachesExclusive(); - setStateExclusive(SCANNING_RANGES); - executor().scanningRanges.enqueue(SafeTask.this); + SafeTask.this.setStateExclusive(SCANNING_RANGES); + executor().loading.enqueue(SafeTask.this); startInternal(caches); executor().submitExclusive(SafeTask.this, GlobalGroup.RANGE_SCAN, this); } void startInternal(Caches caches) { - loader = commandStore.rangeIndex().loader(executionContext.primaryTxnId(), executionContext.executeAt(), executionContext.loadKeysFor(), executionContext.keys()); + loader = commandStore.rangeIndex().loader(context.primaryTxnId(), context.executeAt(), context.loadKeysFor(), context.keys()); loader.loadExclusive(guardedSummaries, caches); } public void scannedExclusive() { - Invariants.require(is(SCANNING_RANGES), "Expected SCANNING_RANGES; found %s", SafeTask.this, SafeTask::toDescription); + Invariants.require(is(SCANNING_RANGES), "Expected SCANNING_RANGES; found %s", SafeTask.this, SafeTask::description); scanned = true; scannedInternal(); - unqueue(executor().scanningRanges); + unqueue(executor().loading); // likely to be requeued to same queue, but simpler invariants if we remove here onSetupOrScannedExclusive(); } @@ -1414,7 +1507,7 @@ CommandSummaries finish(Caches caches) @Override public String description() { - return "Scanning range intersections for " + executionContext.reason() + ' ' + toBriefString(); + return "Scanning range intersections for " + context.reason() + ' ' + toBriefString(); } @Override @@ -1425,30 +1518,18 @@ public String toString() } @Override - public DebuggableTask debuggable() - { - return this; - } - - @Override - public long creationTimeNanos() - { - return createdAt; - } - - @Override - public long startTimeNanos() + public String description() { - return runningAt; + return context.describe(); } @Override - public String description() + public String briefDescription() { - return executionContext.describe(); + return context.reason(); } - private AccordExecutor executor() + final AccordExecutor executor() { return commandStore.executor(); } @@ -1459,10 +1540,18 @@ protected boolean isNewWork() return true; } - @Nullable - public RangeTxnScanner rangeScanner() + public @Nullable SafeTask.RangeTxnScanner rangeScanner() + { + if (ranges instanceof SafeTask.RangeTxnScanner) + return ((SafeTask.RangeTxnScanner) ranges); + return null; + } + + public @Nullable CommandSummaries commandsForRanges() { - return rangeScanner; + if (ranges instanceof CommandSummaries) + return (CommandSummaries) ranges; + return null; } private static LoadKeys loadKeys(ExecutionContext context) diff --git a/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java index e1b73cf15e89..89c3f4b46bb2 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SaferCommandStore.java @@ -30,6 +30,7 @@ import accord.impl.AbstractSafeCommandStore; import accord.local.Command; import accord.local.CommandStores; +import accord.local.CommandSummaries; import accord.local.ExecutionContext; import accord.local.NodeCommandStoreService; import accord.local.RedundantBefore; @@ -214,7 +215,8 @@ public LogLinearDecayingHistograms.Buffer histogramBuffer() task.histogramBuffer = commandStore().metricsBuffer; if (task.histogramBuffer == null) task.histogramBuffer = commandStore().metricsBuffer = new LogLinearDecayingHistograms.Buffer(commandStore().executor().histograms); - Invariants.require(task.histogramBuffer.isEmpty()); + if (!Invariants.expect(task.histogramBuffer.isEmpty())) + task.histogramBuffer.clear(); } return task.histogramBuffer; } @@ -254,15 +256,19 @@ private boolean visitForKey(Unseekables keysOrRanges, Predicate void visit(Unseekables keysOrRanges, Timestamp startedBefore, Kinds testKind, ActiveCommandVisitor visitor, P1 p1, P2 p2) { visitForKey(keysOrRanges, cfk -> { cfk.visit(startedBefore, testKind, visitor, p1, p2); return true; }); - if (task.commandsForRanges != null) - task.commandsForRanges.visit(keysOrRanges, startedBefore, testKind, visitor, p1, p2); + CommandSummaries commandsForRanges = task.commandsForRanges(); + if (commandsForRanges != null) + commandsForRanges.visit(keysOrRanges, startedBefore, testKind, visitor, p1, p2); } @Override public boolean visit(Unseekables keysOrRanges, TxnId testTxnId, Kinds testKind, SupersedingCommandVisitor visit) { - return visitForKey(keysOrRanges, cfk -> cfk.visit(testTxnId, testKind, visit)) - && (task.commandsForRanges == null || task.commandsForRanges.visit(keysOrRanges, testTxnId, testKind, visit)); + if (!visitForKey(keysOrRanges, cfk -> cfk.visit(testTxnId, testKind, visit))) + return false; + + CommandSummaries commandsForRanges = task.commandsForRanges(); + return commandsForRanges == null || commandsForRanges.visit(keysOrRanges, testTxnId, testKind, visit); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/execution/Task.java b/src/java/org/apache/cassandra/service/accord/execution/Task.java index 0d2395f8deca..bd0af0d472ab 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/Task.java +++ b/src/java/org/apache/cassandra/service/accord/execution/Task.java @@ -18,10 +18,10 @@ package org.apache.cassandra.service.accord.execution; +import java.util.concurrent.CancellationException; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLong; -import javax.annotation.Nullable; - import accord.local.ExecutionContext; import accord.messages.Accept; import accord.messages.Commit; @@ -33,11 +33,14 @@ import accord.utils.IntrusiveHeapNode; import accord.utils.Invariants; import accord.utils.TinyEnumSet; +import accord.utils.UnhandledEnum; import accord.utils.async.Cancellable; import org.apache.cassandra.concurrent.DebuggableTask; import org.apache.cassandra.concurrent.ExecutorLocals; -import org.apache.cassandra.service.accord.debug.DebugExecution; +import org.apache.cassandra.service.accord.debug.DebugExecution.DebugTask; +import org.apache.cassandra.service.accord.execution.ExclusiveExecutor.ExclusiveExecutorTask; +import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.WithResources; import static accord.local.ExecutionContext.ExecutionSequence.BY_PRIORITY; @@ -48,44 +51,61 @@ import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.APPLY; import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.COMMIT; import static org.apache.cassandra.service.accord.execution.Task.ExclusiveGroup.STABLE; +import static org.apache.cassandra.service.accord.execution.Task.RunState.NOT_YET_RUN; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_FAILED; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUN_INCOMPLETE; +import static org.apache.cassandra.service.accord.execution.Task.RunState.RUNNING; +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED; +import static org.apache.cassandra.service.accord.execution.Task.State.CANCELLED_UNREGISTERED; import static org.apache.cassandra.service.accord.execution.Task.State.EXECUTED; -import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING; -import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING_OR_EXECUTED; +import static org.apache.cassandra.service.accord.execution.Task.State.FAILED; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.LOADING_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.PREPARED; +import static org.apache.cassandra.service.accord.execution.Task.State.PREPARED_OR_EXECUTED; +import static org.apache.cassandra.service.accord.execution.Task.State.SCANNING_RANGES; +import static org.apache.cassandra.service.accord.execution.Task.State.UNREGISTERED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_OPTIONAL; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_ON_REQUIRED; +import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; import static org.apache.cassandra.utils.Clock.Global.nanoTime; -public abstract class Task extends IntrusiveHeapNode implements Cancellable +public abstract class Task extends IntrusiveHeapNode implements Cancellable, DebuggableTask { - private static final int WAITING_ON_OPTIONAL_BIT = 1 << 5; - private static final int WAITING_TO_RUN_BIT = 1 << 6; - private static final int INCOMPLETE_BIT = 1 << 9; + private static final int WAITING_ON_OPTIONAL_BIT = 1 << 7; + private static final int WAITING_TO_RUN_BIT = 1 << 8; + private static final int INCOMPLETE_BIT = 1 << 10; enum State { - UNINITIALIZED(), - SCANNING_RANGES(UNINITIALIZED), - LOADING_REQUIRED(UNINITIALIZED, SCANNING_RANGES), - LOADING_OPTIONAL(UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED), - WAITING_ON_REQUIRED(WAITING_ON_OPTIONAL_BIT | WAITING_TO_RUN_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), - WAITING_ON_OPTIONAL(WAITING_TO_RUN_BIT | INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED), - WAITING_TO_RUN(INCOMPLETE_BIT, UNINITIALIZED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL), - RUNNING(WAITING_TO_RUN), - EXECUTED(RUNNING), - INCOMPLETE(RUNNING), - FAILED_TO_LOAD(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), - FAILED_OTHER(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN), - CANCELLED(SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN, RUNNING), + UNREGISTERED(), + CANCELLED_UNREGISTERED(UNREGISTERED), + REGISTERED(UNREGISTERED), + SCANNING_RANGES(REGISTERED), + LOADING_REQUIRED(REGISTERED, SCANNING_RANGES), + LOADING_OPTIONAL(REGISTERED, SCANNING_RANGES, LOADING_REQUIRED), + WAITING_ON_REQUIRED(WAITING_ON_OPTIONAL_BIT | WAITING_TO_RUN_BIT, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL), + WAITING_ON_OPTIONAL(WAITING_TO_RUN_BIT | INCOMPLETE_BIT, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED), + WAITING_TO_RUN(INCOMPLETE_BIT, UNREGISTERED, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL), + PREPARED(WAITING_TO_RUN), + INCOMPLETE(PREPARED), + EXECUTED(PREPARED), + FAILED(UNREGISTERED, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN, PREPARED, INCOMPLETE), + CANCELLED(UNREGISTERED, REGISTERED, SCANNING_RANGES, LOADING_REQUIRED, LOADING_OPTIONAL, WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN), ; private final int permittedFrom; public static final int WAITING = TinyEnumSet.encode(WAITING_ON_REQUIRED, WAITING_ON_OPTIONAL, WAITING_TO_RUN); - public static final int WAITING_OR_RUNNING = WAITING | TinyEnumSet.encode(RUNNING); - public static final int RUNNING_OR_EXECUTED = WAITING | TinyEnumSet.encode(RUNNING, EXECUTED); + public static final int WAITING_OR_PREPARED = WAITING | TinyEnumSet.encode(PREPARED); + public static final int PREPARED_OR_EXECUTED = TinyEnumSet.encode(PREPARED, EXECUTED); static final State[] VALUES = values(); static { // hack to allow us to create loops in our enum transition declarations Invariants.require(INCOMPLETE_BIT == 1 << INCOMPLETE.ordinal()); + Invariants.require(WAITING_TO_RUN_BIT == 1 << WAITING_TO_RUN.ordinal()); + Invariants.require(WAITING_ON_OPTIONAL_BIT == 1 << WAITING_ON_OPTIONAL.ordinal()); } State() @@ -111,14 +131,14 @@ boolean isPermittedFrom(int prevOrdinal) return (permittedFrom & (1 << prevOrdinal)) != 0; } - boolean isExecuted() + boolean isDone() { return this.compareTo(EXECUTED) >= 0; } boolean hasStarted() { - return this.compareTo(RUNNING) >= 0; + return this.compareTo(PREPARED) >= 0; } static State forOrdinal(int ordinal) @@ -129,7 +149,7 @@ static State forOrdinal(int ordinal) enum RunState { - NONE, PERSISTING, SUCCESS, FAILED; + NOT_YET_RUN, RUNNING, RUN_INCOMPLETE, RUN_PERSISTING, RUN_SUCCESS, RUN_FAILED; private static final RunState[] VALUES = values(); @@ -177,6 +197,28 @@ public enum GroupKind } } + enum ExecutorQueue + { + NONE(), + LOADING(SCANNING_RANGES, LOADING_OPTIONAL, LOADING_REQUIRED), + WAITING(WAITING_ON_OPTIONAL, WAITING_ON_REQUIRED), + RUNNABLE(WAITING_TO_RUN); + + private static final ExecutorQueue[] VALUES = values(); + + final int permittedStates; + + ExecutorQueue(State ... states) + { + this.permittedStates = TinyEnumSet.encode(states); + } + + public static ExecutorQueue forOrdinal(int ordinal) + { + return VALUES[ordinal]; + } + } + private static final int STATE_MASK = 0xf; static final int GROUP_MASK = 0x7; private static final int EXCLUSIVE_GROUP_SHIFT = 4; @@ -185,23 +227,26 @@ public enum GroupKind private static final int NONSYNC_BIT = 1 << 10; private static final int CACHE_QUEUED_BIT = 1 << 11; - private static final int INCREMENTAL_MASK = 0x3 << 12; - private static final int INCREMENTAL = 0x1 << 12; - private static final int INCREMENTAL_STARTED = 0x2 << 12; - private static final int INCREMENTAL_FINISHING = 0x3 << 12; + private static final int HAS_TRANCHE_BIT = 1 << 12; + private static final int HAS_INHERITED_BIT = 1 << 13; + private static final int HAS_INHERITED_RANGE_SCAN_BIT = 1 << 14; + + private static final int EXECUTOR_QUEUE_SHIFT = 16; + private static final int EXECUTOR_QUEUE_UNSHIFTED_MASK = 0x3; + private static final int EXECUTOR_QUEUE_SHIFTED_MASK = EXECUTOR_QUEUE_UNSHIFTED_MASK << EXECUTOR_QUEUE_SHIFT; - private static final int SEQUENCED_SHIFT = 14; + private static final int INCREMENTAL_SHIFT = 18; + private static final int INCREMENTAL_MASK = 0x3 << INCREMENTAL_SHIFT; + private static final int INCREMENTAL = 0x1 << INCREMENTAL_SHIFT; + private static final int INCREMENTAL_STARTED = 0x2 << INCREMENTAL_SHIFT; + private static final int INCREMENTAL_FINISHING = 0x3 << INCREMENTAL_SHIFT; + + private static final int SEQUENCED_SHIFT = 20; private static final int SEQUENCED_MASK = 0x3 << SEQUENCED_SHIFT; private static final int SEQUENCED_PRIORITY = 0x1 << SEQUENCED_SHIFT; private static final int SEQUENCED_ATOMIC = 0x2 << SEQUENCED_SHIFT; private static final int SEQUENCED_ATOMIC_AND_QUEUED = 0x3 << SEQUENCED_SHIFT; - // spare two bits - - private static final int HAS_TRANCHE_BIT = 1 << 18; - private static final int HAS_INHERITED_BIT = 1 << 19; - private static final int HAS_INHERITED_RANGE_SCAN_BIT = 1 << 20; - private static final int TRANCHE_SHIFT = 22; static final int MAX_TRANCHE = 0x3ff; @@ -212,53 +257,40 @@ public enum GroupKind Invariants.require(ExecutionContext.ExecutionSequence.values().length <= 3); } + // TODO (desired): quite heavy to pass-through tracing session state we mostly don't use public final WithResources resources; Task next; + Task consequences; + // the queue position until run is invoked, at which point it is assigned a nanoTime() value long position; int info; - // TODO (expected): do we need this? we should be able to determine the queue from state() if needed for e.g. cancellation - private TaskQueue queued; - public final long createdAt; - // TODO (expected): expose via executors vtable - // TODO (expected): use just one long and some flag bits to indicate which point it represents, and report incrementally - public long loadedAt, runningAt, completeAt; - private byte runState; + long runningAt; + // TODO (desired): expose via executors vtable + private volatile int runState; + + private static final AtomicIntegerFieldUpdater runStateUpdater = AtomicIntegerFieldUpdater.newUpdater(Task.class, "runState"); Task(GlobalGroup group) { - resources = DebugExecution.DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); info = init(group, ExclusiveGroup.OTHER); createdAt = nanoTime(); } Task(ExclusiveGroup group) { - resources = DebugExecution.DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); info = init(GlobalGroup.OTHER, group); createdAt = nanoTime(); } - Task(GlobalGroup group, long position, int tranche) - { - this(group); - this.position = position; - setInheritedWithTranche(tranche); - } - - Task(ExclusiveGroup group, long position, int tranche) - { - this(group); - this.position = position; - setInheritedWithTranche(tranche); - } - protected Task(ExecutionContext context, AtomicLong lastCreatedAt) { - resources = DebugExecution.DebugTask.maybeDebug(ExecutorLocals.propagate(), this); - createdAt = lastCreatedAt.accumulateAndGet(nanoTime(), (prev, next) -> next < prev ? prev + 1 : next); + resources = DebugTask.maybeDebug(ExecutorLocals.propagate(), this); + createdAt = lastCreatedAt.accumulateAndGet(nanoTime(), (prev, next) -> next <= prev ? prev + 1 : next); ExclusiveGroup group = ExclusiveGroup.OTHER; TxnId txnId = context.primaryTxnId(); if (txnId != null) @@ -343,144 +375,241 @@ protected Task(ExecutionContext context, AtomicLong lastCreatedAt) public final Task unwrap() { - if (this instanceof ExclusiveExecutor.ExclusiveExecutorTask) - return ((ExclusiveExecutor.ExclusiveExecutorTask) this).queue.task; + if (this instanceof ExclusiveExecutorTask) + return ((ExclusiveExecutorTask) this).queue.task; + return this; + } + + public DebuggableTask debuggable() + { return this; } - static Task unwrap(Task task) + public final long creationTimeNanos() { - return task == null ? null : task.unwrap(); + return createdAt; } - public DebuggableTask debuggable() + public final long startTimeNanos() { - return null; + if (runState() == NOT_YET_RUN) + return 0; + return runningAt; } - abstract String toDescription(); + abstract void submitExclusiveMayThrow(); + /** Return true if COMPLETED successfully. false indicates more work is being done. Should not throw any exceptions related to reporting success. */ + abstract boolean runMayThrow(); + abstract void maybeCompleteExclusiveMayThrow(); + abstract void tryCancelExclusive(); + abstract void reportFailureMayThrow(Throwable fail); - abstract void submitExclusive(); + abstract AccordExecutor executor(); + abstract void unqueueIfQueued(); + abstract boolean isNewWork(); + abstract String briefDescription(); - /** - * Prepare to run while holding the state cache lock - */ - void preRunExclusive() + final void submitExclusiveNoExcept() { - setStateExclusive(RUNNING); + if (is(UNREGISTERED)) + { + try { submitExclusiveMayThrow(); } + catch (Throwable t) + { + tryFailAndCompleteExclusive(t, State.FAILED); + onException(t); + } + } } /** - * Run the command; the state cache lock may or may not be held depending on the executor implementation + * Prepare to run while holding the state cache lock. + * If returns false, prepare failed and the task should be discarded with no further action. */ - abstract void run(); + final boolean prepareExclusiveNoExcept() + { + if (getClass() == ExclusiveExecutorTask.class) + { + return ((ExclusiveExecutorTask)this).prepareTask(); + } + else + { + try + { + prepareExclusiveMayThrow(); + setStateExclusive(State.PREPARED); + return true; + } + catch (Throwable t) + { + failAndCompleteExclusive(t, State.FAILED); + return false; + } + } + } + + void prepareExclusiveMayThrow() + { + } /** - * Fail the command; the state cache lock may or may not be held depending on the executor implementation + * Run the command; the state cache lock may or may not be held depending on the executor implementation */ - abstract void reportFailure(Throwable fail); - - final void failExclusive(Throwable fail, State newState) + final void runNoExcept(TaskRunner self) { - try + if (getClass() == ExclusiveExecutorTask.class) { - setStateExclusive(newState); + ((ExclusiveExecutorTask)this).queue.runTask(self); } - finally + else { - reportFailure(fail); + onRunning(); + self.setAccordActiveTask(this); + try (Closeable close = resources.get()) + { + if (runMayThrow()) + onSuccess(); + else + Invariants.require(compareTo(RUNNING) > 0); + } + catch (Throwable t) + { + setRunState(RunState.RUN_FAILED); + reportFailureNoExcept(t); + } + finally + { + self.setAccordActiveTask(null); + } } } - final void failExecution(Throwable fail) + final void rejectAtRuntime(Throwable reject) + { + setRunState(RunState.RUN_FAILED); + reportFailureNoExcept(reject); + } + + final void completeExclusiveNoExcept() { - Invariants.require(is(RUNNING)); try { - setRunState(RunState.FAILED); + if (DEBUG_EXECUTION) DebugTask.get(this).onComplete(); + maybeCompleteExclusiveMayThrow(); + } + catch (Throwable t) + { + onException(t); + if (compareTo(EXECUTED) < 0) + failExclusive(t, State.FAILED); } finally { - reportFailure(fail); + if (compareTo(EXECUTED) >= 0) + { + try { submitConsequencesExclusive(is(EXECUTED)); } + catch (Throwable t) { onException(t); } + executor().completedTaskExclusive(this); + } } } - abstract boolean isNewWork(); + final void onException(Throwable t) + { + try { executor().agent.onException(t); } + catch (Throwable t2) { } + } - /** - * Cleanup the command while holding the state cache lock - */ - void cleanupExclusive(AccordExecutor executor, boolean executed) + final void reportFailureNoExcept(Throwable fail) { - if (executed) setStateExclusive(EXECUTED); - else Invariants.require(state().isExecuted()); - executor.unregisterExclusive(this); - completeAt = nanoTime(); - if (runningAt != 0) + try { reportFailureMayThrow(fail); } + catch (Throwable t) { - if (loadedAt == 0) - loadedAt = runningAt; - executor.elapsedWaitingToRun.increment(runningAt - loadedAt, runningAt); - executor.elapsedPreparingToRun.increment(loadedAt - createdAt, runningAt); - executor.elapsedRunning.increment(completeAt - runningAt, completeAt); - executor.elapsed.increment(completeAt - createdAt, completeAt); + try { fail.addSuppressed(t); } + catch (Throwable t2) { } + onException(fail); } - if (DEBUG_EXECUTION) DebugExecution.DebugTask.get(this).onCompleted(executor.debug); } - void cancelExclusive() + // propagate RunState to State + // true if task was executed successfully + final boolean completeState() { + RunState runState = runState(); + boolean success; + switch (runState) + { + default: throw UnhandledEnum.unknown(runState); + case RUN_INCOMPLETE: throw UnhandledEnum.invalid(runState); + case NOT_YET_RUN: + Invariants.expect(state().isDone()); + Invariants.expect(consequences == null); + success = false; + break; + + case RUN_FAILED: + if (compareTo(EXECUTED) < 0) + setStateExclusive(State.FAILED); + success = false; + break; + + case RUNNING: + case RUN_PERSISTING: + case RUN_SUCCESS: + setStateExclusive(EXECUTED); + success = true; + break; + } + return success; } - @Nullable - final TaskQueue queued() + final void tryFailAndCompleteExclusive(Throwable fail, State newState) { - return queued; + if (is(UNREGISTERED)) + failExclusive(fail, CANCELLED_UNREGISTERED); + else if (compareTo(WAITING_TO_RUN) <= 0) + failAndCompleteExclusive(fail, newState); } - final void unqueueIfQueued() + final void failExclusive(Throwable fail, State newState) { - if (queued != null) - { - queued.unqueue(this); - queued = null; - } + unqueueIfQueued(); + setStateExclusive(newState); + reportFailureNoExcept(fail); } - final void unqueue(TaskQueue expected) + final void failAndCompleteExclusive(Throwable fail, State newState) { - Invariants.require(queued == expected, "%s != %s", queued, expected); - queued.unqueue(this); - queued = null; + failExclusive(fail, newState); + completeExclusiveNoExcept(); } - final void unsetQueue(TaskQueue expected) + final void unqueue(TaskQueue expected) { - Invariants.require(queued == expected, "%s != %s", queued, expected); - queued = null; + Invariants.require(queuedOrdinal() == expected.kind.ordinal()); + expected.unqueue(this); + info &= ~EXECUTOR_QUEUE_SHIFTED_MASK; } - final void setQueue(TaskQueue queue) + final void unsetQueue(ExecutorQueue expected) { - Invariants.require(queued == null); - Invariants.require(isCompatible(queue)); - queued = queue; + Invariants.require(expected.ordinal() == queuedOrdinal()); + info &= ~EXECUTOR_QUEUE_SHIFTED_MASK; } final void onRunning() { + Invariants.require(is(PREPARED)); + Invariants.require(is(NOT_YET_RUN) || is(RUN_INCOMPLETE)); runningAt = nanoTime(); - if (DEBUG_EXECUTION) ((DebugExecution.DebugTask) resources).onRunning(); - } - - final void onRunComplete() - { - if (DEBUG_EXECUTION) ((DebugExecution.DebugTask) resources).onRunComplete(); + setRunState(RunState.RUNNING); + if (DEBUG_EXECUTION) ((DebugTask) resources).onRunning(); } - final void onLoaded() + final void onSuccess() { - loadedAt = nanoTime(); + setRunState(RunState.RUN_SUCCESS); + if (DEBUG_EXECUTION) ((DebugTask) resources).onRunComplete(); } final State state() @@ -493,20 +622,20 @@ final RunState runState() return RunState.forOrdinal(runState); } - final Enum describeState() + final Enum currentState() { State state = state(); - if (state == RUNNING || state == EXECUTED) + if (state == State.PREPARED || state == EXECUTED) { RunState runState = runState(); - if (runState == RunState.NONE) + if (runState == NOT_YET_RUN) return state; return runState; } return State.forOrdinal(stateOrdinal()); } - private int stateOrdinal() + final int stateOrdinal() { return info & STATE_MASK; } @@ -516,6 +645,27 @@ final boolean is(State state) return stateOrdinal() == state.ordinal(); } + final int compareTo(State state) + { + return stateOrdinal() - state.ordinal(); + } + + final int compareTo(RunState state) + { + return runState - state.ordinal(); + } + + final boolean is(RunState state) + { + return runState == state.ordinal(); + } + + final boolean isEither(RunState state1, RunState state2) + { + int runState = this.runState; + return runState == state1.ordinal() || runState == state2.ordinal(); + } + final boolean isState(int stateBitSet) { return TinyEnumSet.contains(stateBitSet, stateOrdinal()); @@ -536,11 +686,6 @@ final void override(GlobalGroup group) info = (info & ~(GROUP_MASK << GLOBAL_GROUP_SHIFT)) | (group.ordinal() << GLOBAL_GROUP_SHIFT); } - final int compareTo(State state) - { - return stateOrdinal() - state.ordinal(); - } - final void setStateExclusive(State state) { Invariants.require(state.isPermittedFrom(stateOrdinal()), "%s forbidden from %s", state, this, Task::reportBadStateTransition); @@ -549,13 +694,18 @@ final void setStateExclusive(State state) final void setRunState(RunState state) { - Invariants.require(isState(RUNNING_OR_EXECUTED)); - runState = (byte) state.ordinal(); + Invariants.require(isState(PREPARED_OR_EXECUTED) || (state == RUN_FAILED && is(FAILED))); + setRunState(state.ordinal()); + } + + final void setRunState(int newRunState) + { + runStateUpdater.lazySet(this, newRunState); } private static String reportBadStateTransition(Task task) { - return task.state() + " for " + task.toDescription(); + return task.state() + " for " + task.description(); } final void unsafeSetStateExclusive(State state) @@ -573,10 +723,10 @@ final int exclusiveGroupOrdinal() return (info >>> EXCLUSIVE_GROUP_SHIFT) & GROUP_MASK; } - private boolean isCompatible(TaskQueue queue) + private boolean isCompatible(ExecutorQueue queue) { int self = stateOrdinal(); - return TinyEnumSet.contains(queue.states, self); + return TinyEnumSet.contains(queue.permittedStates, self); } final boolean isSync() @@ -658,6 +808,28 @@ final boolean isCacheQueued() return 0 != (info & CACHE_QUEUED_BIT); } + final boolean isQueued() + { + return 0 != (info & EXECUTOR_QUEUE_SHIFTED_MASK); + } + + final int queuedOrdinal() + { + return (info >>> EXECUTOR_QUEUE_SHIFT) & EXECUTOR_QUEUE_UNSHIFTED_MASK; + } + + final ExecutorQueue queued() + { + return ExecutorQueue.forOrdinal(queuedOrdinal()); + } + + final void setQueue(ExecutorQueue queue) + { + Invariants.require(isCompatible(queue)); + Invariants.require(!isQueued()); + info |= queue.ordinal() << EXECUTOR_QUEUE_SHIFT; + } + // supersedes priority, in whichever order they're called final void setCacheQueuedFifoExclusive() { @@ -682,6 +854,14 @@ final void setTranche(int tranche) info = info | (tranche << TRANCHE_SHIFT) | HAS_TRANCHE_BIT; } + final Task inherit(Task parent) + { + Invariants.require(!hasInherited()); + position = parent.position; + setInheritedWithTranche(parent.tranche()); + return this; + } + final void setInheritedWithTranche(int tranche) { Invariants.require(tranche <= MAX_TRANCHE); @@ -703,6 +883,42 @@ final boolean hasInheritedRangeScan() return (info & HAS_INHERITED_RANGE_SCAN_BIT) != 0; } + void addConsequence(Task task) + { + Task prev = consequences; + Invariants.require(prev == null || prev.is(UNREGISTERED)); + task.next = prev; + consequences = task; + } + + final void submitConsequencesExclusive(boolean success) + { + if (consequences == null) + return; + + Task cur = Task.reverse(consequences); + consequences = null; + + while (cur != null) + { + Task next = cur.next; + cur.next = null; + if (cur.is(UNREGISTERED)) + { + if (success || !(cur instanceof SafeTask || this instanceof SafeTask)) + { + cur.inherit(this); + cur.submitExclusiveNoExcept(); + } + else + { + cur.failExclusive(new CancellationException("Parent task failed"), CANCELLED); + } + } + cur = next; + } + } + static int init(GlobalGroup global, ExclusiveGroup exclusive) { return (global.ordinal() << GLOBAL_GROUP_SHIFT) | (exclusive.ordinal() << EXCLUSIVE_GROUP_SHIFT); diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java b/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java index d9163d5a587c..7b9f2c352cb0 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskInfo.java @@ -22,14 +22,12 @@ import accord.local.ExecutionContext; -import org.apache.cassandra.concurrent.DebuggableTask; - public class TaskInfo implements Comparable { // sorted in name order for reporting to virtual tables public enum Status { - LOADING, RUNNING, SCANNING_RANGES, WAITING_TO_LOAD, WAITING_TO_RUN + RUNNING, LOADING, WAITING_TO_RUN } final Status status; @@ -64,8 +62,8 @@ public long position() if (task instanceof SafeTask) return ((SafeTask) task).executionContext().reason(); - if (task instanceof DebuggableTask) - return ((DebuggableTask) task).description(); + if (task != null) + return task.description(); return null; } diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java index ec483dd6409c..137474286026 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueue.java @@ -19,16 +19,17 @@ package org.apache.cassandra.service.accord.execution; import accord.utils.IntrusivePriorityHeap; -import accord.utils.TinyEnumSet; + +import org.apache.cassandra.service.accord.execution.Task.ExecutorQueue; // has xSingle methods to distinguish from within MultiTaskQueue whether we're invoking the multi or single variation class TaskQueue extends IntrusivePriorityHeap { - final int states; + final ExecutorQueue kind; - public TaskQueue(int states) + public TaskQueue(ExecutorQueue kind) { - this.states = states; + this.kind = kind; } void unqueue(T task) @@ -105,6 +106,6 @@ final boolean isQueuedSingle(T test) @Override public String toString() { - return TinyEnumSet.toString(states, Task.State::forOrdinal); + return kind.toString(); } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java index 9784e51ad905..f3ec1f5e8596 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueMulti.java @@ -21,6 +21,11 @@ import accord.utils.Invariants; import accord.utils.UnhandledEnum; +import org.apache.cassandra.service.accord.execution.Task.ExecutorQueue; +import org.apache.cassandra.service.accord.execution.Task.GroupKind; + +import static org.apache.cassandra.service.accord.execution.Task.ExecutorQueue.RUNNABLE; + /** * A {@link TaskQueue} sub-divided into up to eight per-group sub-queues (groups are phases for the exclusive * executor, or work classes globally) plus a policy for choosing which sub-queue to serve next. The policy balances @@ -130,9 +135,9 @@ abstract class TaskQueueMulti extends TaskQueue int waitingCount; - TaskQueueMulti(int waitingStates, Task.GroupKind groups, long limits) + TaskQueueMulti(ExecutorQueue kind, GroupKind groups, long limits) { - super(waitingStates); + super(kind); this.limits = limits; int queueCount = groups.count; Invariants.require(queueCount <= 8); @@ -149,14 +154,14 @@ final int group(Task task) return (task.info >>> groupShift) & Task.GROUP_MASK; } - void stop(int group) + void stop(long groupOverflowBits) { - stopped |= overflowBit(group); + stopped |= groupOverflowBits; } - void restart(int group) + void restart(long groupOverflowBits) { - stopped &= ~overflowBit(group); + stopped &= ~groupOverflowBits; } final TaskQueue queue(Task task) @@ -172,7 +177,7 @@ final TaskQueue queue(int group) { TaskQueue queue = queues[group]; if (queue == null) - queues[group] = queue = new TaskQueue<>(0); + queues[group] = queue = new TaskQueue<>(RUNNABLE); return queue; } @@ -439,7 +444,7 @@ final T pollMulti() final void enqueueMulti(T task, boolean incrementArrivals) { - task.setQueue(this); + task.setQueue(kind); int group = group(task); if (group < 0) { @@ -492,7 +497,7 @@ boolean tryUnqueueWaiting(T task) private void unqueue(T task, int group, TaskQueue queue) { - task.unsetQueue(this); + task.unsetQueue(kind); boolean dirty = queue.unqueueSingle(task); --waitingCount; if (group >= 0) @@ -565,6 +570,11 @@ final void incrementArrivals(int group) arrivals |= overflow - (overflow >>> 7); } + final boolean hasWaitingToRunExcluding(long groupOverflowBits) + { + return (unsaturatedWithWork() & ~groupOverflowBits) != 0; + } + final void setHasWork(int group) { hasWork |= overflowBit(group); @@ -600,13 +610,18 @@ final int waitingCount() return waitingCount; } - final long lowBit(int group) + static long lowBit(int group) { return 1L << (group * 8); } - final long overflowBit(int group) + static long overflowBit(int group) { return 0x80L << (group * 8); } + + static long overflowBit(Enum group) + { + return overflowBit(group.ordinal()); + } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java index 8aad433fb62a..ddd801b8e449 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueRunnable.java @@ -18,21 +18,16 @@ package org.apache.cassandra.service.accord.execution; -import accord.utils.TinyEnumSet; - -import static org.apache.cassandra.service.accord.execution.Task.State.RUNNING; -import static org.apache.cassandra.service.accord.execution.Task.State.WAITING_TO_RUN; +import static org.apache.cassandra.service.accord.execution.Task.ExecutorQueue.RUNNABLE; final class TaskQueueRunnable extends TaskQueueMulti { - static final int RUNNABLE = TinyEnumSet.encode(WAITING_TO_RUN, RUNNING); - final TaskQueue assigned; TaskQueueRunnable() { super(RUNNABLE, Task.GroupKind.GLOBAL, AccordExecutor.GLOBAL_QUEUE_LIMITS); - this.assigned = new TaskQueue<>(0); + this.assigned = new TaskQueue<>(RUNNABLE); } T poll() @@ -58,7 +53,7 @@ void unqueue(T unqueue) if (group >= 0) decrementActive(group); - unqueue.unsetQueue(this); + unqueue.unsetQueue(kind); assigned.unqueueSingle(unqueue); } else @@ -94,7 +89,7 @@ void cleanup(T task) int group = group(task); if (group >= 0) decrementActive(group); - task.unsetQueue(this); + task.unsetQueue(kind); } } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java index 7dd8b1def0be..4f7757dc25bd 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskQueueStandalone.java @@ -18,29 +18,33 @@ package org.apache.cassandra.service.accord.execution; -import accord.utils.TinyEnumSet; +import org.apache.cassandra.service.accord.execution.Task.ExecutorQueue; final class TaskQueueStandalone extends TaskQueue { - TaskQueueStandalone(int states) + TaskQueueStandalone(ExecutorQueue kind) { - super(states); + super(kind); } - TaskQueueStandalone(Task.State state) + void enqueue(T enqueue) { - super(TinyEnumSet.encode(state)); + enqueue.setQueue(kind); + enqueueSingle(enqueue); } - void enqueue(T enqueue) + boolean tryUnqueue(T unqueue) { - enqueue.setQueue(this); - enqueueSingle(enqueue); + if (!containsNode(unqueue)) + return false; + + unqueue(unqueue); + return true; } void unqueue(T unqueue) { - unqueue.unsetQueue(this); + unqueue.unsetQueue(kind); removeNode(unqueue); } diff --git a/src/java/org/apache/cassandra/utils/NoSpamLogger.java b/src/java/org/apache/cassandra/utils/NoSpamLogger.java index a2bf815b30f5..b14088913cd3 100644 --- a/src/java/org/apache/cassandra/utils/NoSpamLogger.java +++ b/src/java/org/apache/cassandra/utils/NoSpamLogger.java @@ -17,6 +17,11 @@ */ package org.apache.cassandra.utils; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; @@ -64,16 +69,18 @@ public static void unsafeSetClock(Clock clock) CLOCK = clock; } - public class NoSpamLogStatement extends AtomicLong + public static class NoSpamLogStatement extends AtomicLong { private static final long serialVersionUID = 1L; + private final Logger wrapped; private final String statement; private final long minIntervalNanos; - public NoSpamLogStatement(String statement, long minIntervalNanos) + public NoSpamLogStatement(Logger wrapped, String statement, long minIntervalNanos) { super(Long.MIN_VALUE); + this.wrapped = wrapped; this.statement = statement; this.minIntervalNanos = minIntervalNanos; } @@ -159,6 +166,154 @@ public boolean error(Object... objects) } } + public static class NoDuplicateSpamLogStatement + { + private static final long serialVersionUID = 1L; + private static final int PRUNE_SIZE = 32; + + private final Logger wrapped; + private final String statement; + private final long minIntervalNanos; + private final ConcurrentHashMap lastLogged = new ConcurrentHashMap<>(); + private final AtomicLong nextPruneAt = new AtomicLong(); + + public NoDuplicateSpamLogStatement(Logger wrapped, String statement, long minInterval, TimeUnit units) + { + this(wrapped, statement, units.toNanos(minInterval)); + } + + public NoDuplicateSpamLogStatement(Logger wrapped, String statement, long minIntervalNanos) + { + this.wrapped = wrapped; + this.statement = statement; + this.minIntervalNanos = minIntervalNanos; + } + + private boolean shouldLog(long id, long nowNanos) + { + Long expected = lastLogged.getOrDefault(id, Long.MIN_VALUE); + if (nowNanos < expected || !lastLogged.replace(id, expected, nowNanos + minIntervalNanos)) + return false; + + if (lastLogged.size() >= PRUNE_SIZE) + { + long pruneAt = nextPruneAt.get(); + if (nowNanos >= pruneAt && nextPruneAt.compareAndSet(pruneAt, nowNanos + minIntervalNanos)) + { + for (Map.Entry e : lastLogged.entrySet()) + { + if (nowNanos < e.getValue()) + lastLogged.remove(e.getKey(), e.getValue()); + } + } + } + return true; + } + + public boolean log(Level l, long id, long nowNanos, Object... objects) + { + if (!shouldLog(id, nowNanos)) return false; + return logNoCheck(l, objects); + } + + private boolean logNoCheck(Level l, Object... objects) + { + switch (l) + { + case DEBUG: + wrapped.debug(statement, objects); + break; + case INFO: + wrapped.info(statement, objects); + break; + case WARN: + wrapped.warn(statement, objects); + break; + case ERROR: + wrapped.error(statement, objects); + break; + default: + throw new AssertionError(); + } + return true; + } + + public boolean debug(long id, long nowNanos, Object... objects) + { + return log(Level.DEBUG, id, nowNanos, objects); + } + + public boolean debug(long id, Object... objects) + { + return debug(id, CLOCK.nanoTime(), objects); + } + + public boolean info(long id, long nowNanos, Object... objects) + { + return log(Level.INFO, id, nowNanos, objects); + } + + public boolean info(long id, Object... objects) + { + return info(id, CLOCK.nanoTime(), objects); + } + + public boolean warn(long id, long nowNanos, Object... objects) + { + return log(Level.WARN, id, nowNanos, objects); + } + + public boolean warn(long id, Object... objects) + { + return warn(id, CLOCK.nanoTime(), objects); + } + + public boolean error(long id, long nowNanos, Object... objects) + { + return log(Level.ERROR, id, nowNanos, objects); + } + + public boolean error(long id, Object... objects) + { + return error(id, CLOCK.nanoTime(), objects); + } + + public static long exceptionId(Throwable throwable) + { + return exceptionId(throwable, Collections.newSetFromMap(new IdentityHashMap<>())); + } + + private static long exceptionId(Throwable throwable, Set visited) + { + long id = throwable.getClass().hashCode(); + for (StackTraceElement ste : throwable.getStackTrace()) + { + id *= 31; + id += ste.getClassName().hashCode(); + id *= 31; + id += ste.getLineNumber(); + } + + for (Throwable suppressed : throwable.getSuppressed()) + { + if (!visited.add(suppressed)) + continue; + + id *= 31; + id += exceptionId(suppressed, visited); + } + for (Throwable cause = throwable.getCause() ; cause != null ; cause = cause.getCause()) + { + if (!visited.add(cause)) + break; + + id *= 31; + id += exceptionId(cause, visited); + } + return id; + } + } + private static final NonBlockingHashMap wrappedLoggers = new NonBlockingHashMap<>(); @VisibleForTesting @@ -305,7 +460,7 @@ public NoSpamLogStatement getStatement(String key, String s, long minIntervalNan NoSpamLogStatement statement = lastMessage.get(key); if (statement == null) { - statement = new NoSpamLogStatement(s, minIntervalNanos); + statement = new NoSpamLogStatement(wrapped, s, minIntervalNanos); NoSpamLogStatement temp = lastMessage.putIfAbsent(key, statement); if (temp != null) statement = temp; diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java index 97a9fb22fffa..f0c32490b938 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java @@ -285,11 +285,11 @@ private static void flush(AccordCommandStore commandStore) try (AccordExecutor.ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { cacheSize = cache.global.capacity(); - cache.global.setCapacity(0); + commandStore.executor().setCapacity(0); } try (AccordExecutor.ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { - cache.global.setCapacity(cacheSize); + commandStore.executor().setCapacity(cacheSize); } commandsForKey.forceBlockingFlush(FlushReason.UNIT_TESTS); while (commandStore.executor().hasTasks()) diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java index 189b08ed0dea..539d58b2683f 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java @@ -98,7 +98,7 @@ private static PartitionKey key(int k) public void basicCycleTest() throws Throwable { AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); - getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); + getBlocking(commandStore.execute((ExecutionContext.Empty)() -> "Test", unused -> commandStore.executor().setCapacity(0))); TxnId txnId = txnId(1, clock.incrementAndGet(), 1); Txn txn = createWriteTxn(1); @@ -191,7 +191,7 @@ public void basicCycleTest() throws Throwable public void computeDeps() throws Throwable { AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); - getBlocking(commandStore.execute((ExecutionContext.Empty)()->"Test", unused -> commandStore.executor().cacheUnsafe().setCapacity(0))); + getBlocking(commandStore.execute((ExecutionContext.Empty)()->"Test", unused -> commandStore.executor().setCapacity(0))); TxnId txnId1 = txnId(1, clock.incrementAndGet(), 1); Txn txn = createWriteTxn(2); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index c8f9d7cb7b42..1f8f963ccc17 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -379,7 +379,7 @@ public static AccordCommandStore createAccordCommandStore( Node.Id node = new Id(1); Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[] { node }), Sets.newHashSet(node))); AccordCommandStore store = createAccordCommandStore(node, now, topology); - store.execute((ExecutionContext.Empty)()->"Test", safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().cacheUnsafe().setCapacity(1 << 20)); + store.execute((ExecutionContext.Empty)()->"Test", safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().setCapacity(1 << 20)); return store; } diff --git a/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java b/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java index 2da44ed37fee..9f88b7835319 100644 --- a/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java +++ b/test/unit/org/apache/cassandra/service/accord/SafeTaskTest.java @@ -84,7 +84,6 @@ import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.api.TokenKey; -import org.apache.cassandra.service.accord.execution.SaferCommand; import org.apache.cassandra.service.accord.execution.SafeTask; import org.apache.cassandra.utils.AssertionUtils; import org.apache.cassandra.utils.FBUtilities; @@ -101,7 +100,6 @@ import static org.apache.cassandra.service.accord.AccordTestUtils.createPartialTxn; import static org.apache.cassandra.service.accord.AccordTestUtils.keys; import static org.apache.cassandra.service.accord.AccordTestUtils.txnId; -import static org.apache.cassandra.service.accord.execution.AccordExecutionTestUtils.loaded; public class SafeTaskTest { @@ -178,9 +176,6 @@ public void optionalCommandsForKeyTest() throws Throwable private static Command createStableAndPersist(AccordCommandStore commandStore, TxnId txnId, Timestamp executeAt) { Command command = AccordTestUtils.Commands.stable(txnId, createPartialTxn(0), executeAt); - SaferCommand safeCommand = new SaferCommand(loaded(txnId, null)); - safeCommand.set(command); - appendDiffToLog(commandStore).accept(null, command); return command; } @@ -215,12 +210,12 @@ private static Command createStableUsingFastLifeCycle(AccordCommandStore command try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches()) { cacheSize = cache.global.capacity(); - cache.global.setCapacity(0); + commandStore.executor().setCapacity(0); } try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches()) { - cache.global.setCapacity(cacheSize); + commandStore.executor().setCapacity(cacheSize); } while (commandStore.executor().hasTasks()) @@ -265,11 +260,11 @@ private static Command createStableUsingSlowLifeCycle(AccordCommandStore command try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { cacheSize = cache.global.capacity(); - cache.global.setCapacity(0); + commandStore.executor().setCapacity(0); } try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { - cache.global.setCapacity(cacheSize); + commandStore.executor().setCapacity(cacheSize); } while (commandStore.executor().hasTasks()) @@ -287,7 +282,7 @@ public void loadFail() AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl"); try (AccordExecutor.ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();) { - cache.global.setCapacity(0); + commandStore.executor().setCapacity(0); } Gen txnIdGen = rs -> txnId(1, clock.incrementAndGet(), 1); diff --git a/test/unit/org/apache/cassandra/service/accord/execution/TaskLifecycleTest.java b/test/unit/org/apache/cassandra/service/accord/execution/TaskLifecycleTest.java new file mode 100644 index 000000000000..a1a737317d7a --- /dev/null +++ b/test/unit/org/apache/cassandra/service/accord/execution/TaskLifecycleTest.java @@ -0,0 +1,529 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import com.google.common.collect.Sets; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; + +import accord.api.RoutingKey; +import accord.local.ExecutionContext; +import accord.local.Node; +import accord.local.Node.Id; +import accord.local.SafeCommandStore; +import accord.primitives.RoutingKeys; +import accord.primitives.TxnId; +import accord.topology.Shard; +import accord.topology.Topology; +import accord.utils.SortedArrays.SortedArrayList; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.AccordTestUtils; +import org.apache.cassandra.service.accord.TokenRange; +import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.service.accord.execution.AccordExecutor.Mode; +import org.apache.cassandra.utils.concurrent.CountDownLatch; + +import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITH_LOCK; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Lifecycle tests for {@link Task} and its interaction with {@link AccordExecutor}, covering the invariants the + * executor's callers rely on: + *

    + *
  • every task is notified exactly once, and every {@code chain()} eventually completes (success, failure or + * cancellation) - a task that is dropped silently hangs its caller forever;
    • + *
  • a task is completed exactly once, so {@code tasks}/{@link Tranches} accounting returns to zero and + * {@link AccordExecutor#hasTasks()} becomes false (otherwise {@code waitForQuiescence} and + * {@code afterSubmittedAndConsequences} never fire again);
    • + *
  • a task always releases its cache references, even when it fails;
    • + *
  • no failure is reported to the {@link accord.api.Agent} on any of these paths - an agent exception here means + * an internal invariant was broken, not that the user's operation failed.
    • + *
+ * + * Each test creates a private executor + command store so that failures cannot leak between tests. + */ +public class TaskLifecycleTest +{ + private static final long TIMEOUT_SECONDS = 30; + private static final AtomicLong clock = new AtomicLong(0); + + private final List executors = new CopyOnWriteArrayList<>(); + private final List stores = new CopyOnWriteArrayList<>(); + + @BeforeClass + public static void beforeClass() throws Throwable + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), + parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks")); + } + + @After + public void after() + { + stores.forEach(AccordCommandStore::shutdown); + stores.clear(); + executors.forEach(AccordExecutor::shutdown); + executors.clear(); + } + + /** + * An executor plus one command store, and a record of everything reported to the agent. + */ + private class Env + { + final List agentExceptions = new CopyOnWriteArrayList<>(); + final AccordExecutor executor; + final AccordCommandStore store; + + Env(Mode mode) + { + AccordAgent agent = new AccordAgent() + { + @Override + public void onException(Throwable t) + { + agentExceptions.add(t); + } + + @Override + public void onException(Throwable t, String context) + { + agentExceptions.add(t); + } + }; + agent.setup(Id.NONE); + this.executor = new AccordExecutorSyncSubmit(0, mode, "TaskLifecycleTest", agent); + executors.add(executor); + this.store = newStore(executor); + stores.add(store); + } + + void assertQuiescent() + { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + while (executor.hasTasks() && System.nanoTime() < deadline) + Thread.yield(); + assertThat(executor.hasTasks()).describedAs("executor still has registered tasks").isFalse(); + assertThat(executor.unsafeRunningCount()).describedAs("tasks still assigned to a runner").isZero(); + } + + void assertNoAgentExceptions() + { + assertThat(agentExceptions).describedAs("internal failures reported to the agent").isEmpty(); + } + } + + private static AccordCommandStore newStore(AccordExecutor executor) + { + TableMetadata metadata = Schema.instance.getTableMetadata("ks", "tbl"); + TokenRange range = TokenRange.fullRange(metadata.id, Murmur3Partitioner.instance); + Node.Id node = new Id(1); + Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[]{ node }), Sets.newHashSet(node))); + AccordCommandStore store = AccordTestUtils.createAccordCommandStore(node, clock::incrementAndGet, topology, executor); + // NOTE: capacity must be set via the executor, so that its derived maxWorkingCapacityInBytes is refreshed + executor.executeDirectlyWithLock(() -> { + executor.setCapacity(1 << 20); + executor.setWorkingSetSize(1 << 20); + }); + return store; + } + + private TxnId nextTxnId() + { + return AccordTestUtils.txnId(1, clock.incrementAndGet(), 1); + } + + private static Consumer noop() + { + return ignore -> {}; + } + + private static void await(CountDownLatch latch, String what) throws InterruptedException + { + assertThat(latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)).describedAs(what).isTrue(); + } + + /** + * A task that fails must cancel *every* consequence it accumulated, and must still be unregistered. + */ + @Test + public void failedTaskCancelsAllConsequences() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + int count = 4; + List> children = new ArrayList<>(); + CountDownLatch childrenDone = CountDownLatch.newCountDownLatch(count); + for (int i = 0 ; i < count ; ++i) + children.add(new AtomicReference<>()); + + AtomicReference parentFailure = new AtomicReference<>(); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + RuntimeException failure = new RuntimeException("deliberate failure"); + + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parent"), (Consumer) safe -> { + for (int i = 0 ; i < count ; ++i) + { + AtomicReference child = children.get(i); + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "child" + i), noop(), + (r, f) -> { child.set(f == null ? new AssertionError("child should not have run") : f); childrenDone.decrement(); }); + } + throw failure; + }, (r, f) -> { parentFailure.set(f); parentDone.decrement(); }); + + await(parentDone, "parent was notified"); + assertThat(parentFailure.get()).isSameAs(failure); + await(childrenDone, "every consequence was notified"); + for (int i = 0 ; i < count ; ++i) + assertThat(children.get(i).get()).describedAs("child" + i).isInstanceOf(CancellationException.class); + + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * The happy path counterpart: consequences of a successful task all run. + */ + @Test + public void successfulTaskSubmitsAllConsequences() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + int count = 4; + AtomicInteger ran = new AtomicInteger(); + CountDownLatch childrenDone = CountDownLatch.newCountDownLatch(count); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parent"), (Consumer) safe -> { + for (int i = 0 ; i < count ; ++i) + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "child" + i), + (Consumer) s -> ran.incrementAndGet(), + (r, f) -> { assertThat(f).isNull(); childrenDone.decrement(); }); + }, (r, f) -> { assertThat(f).isNull(); parentDone.decrement(); }); + + await(parentDone, "parent was notified"); + await(childrenDone, "every consequence was notified"); + assertThat(ran.get()).isEqualTo(count); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * Cancelling one consequence must not disturb its siblings. Here the cancellation is submitted from within the + * parent, so it is applied after the parent has completed (i.e. once the consequence has been submitted). + */ + @Test + public void cancellingOneConsequenceDoesNotAffectSiblings() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + AtomicReference cancelled = new AtomicReference<>(); + CountDownLatch cancelledDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch siblingDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parent"), (Consumer) safe -> { + Cancellable cancel = env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "cancelled"), noop(), + (r, f) -> { cancelled.set(f); cancelledDone.decrement(); }); + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "sibling"), noop(), + (r, f) -> { assertThat(f).isNull(); siblingDone.decrement(); }); + cancel.cancel(); + }, (r, f) -> { assertThat(f).isNull(); parentDone.decrement(); }); + + await(parentDone, "parent was notified"); + await(siblingDone, "sibling of the cancelled consequence ran"); + await(cancelledDone, "cancelled consequence was notified"); + assertThat(cancelled.get()).isInstanceOf(CancellationException.class); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * As above, but the cancellation arrives from a foreign thread while the parent is still running, so the + * consequence is terminated before it has ever been submitted. + */ + @Test + public void externallyCancellingOneConsequenceDoesNotAffectSiblings() throws Throwable + { + Env env = new Env(RUN_WITHOUT_LOCK); // the parent blocks, so it must not hold the executor lock + AtomicReference toCancel = new AtomicReference<>(); + AtomicReference cancelled = new AtomicReference<>(); + CountDownLatch cancelledDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch siblingDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch consequencesAdded = CountDownLatch.newCountDownLatch(1); + CountDownLatch releaseParent = CountDownLatch.newCountDownLatch(1); + + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parent"), (Consumer) safe -> { + toCancel.set(env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "cancelled"), noop(), + (r, f) -> { cancelled.set(f); cancelledDone.decrement(); })); + env.store.execute(ExecutionContext.unsequenced(nextTxnId(), "sibling"), noop(), + (r, f) -> { assertThat(f).isNull(); siblingDone.decrement(); }); + consequencesAdded.decrement(); + releaseParent.awaitUninterruptibly(); + }, (r, f) -> { assertThat(f).isNull(); parentDone.decrement(); }); + + await(consequencesAdded, "consequences were registered"); + toCancel.get().cancel(); + releaseParent.decrement(); + + await(parentDone, "parent was notified"); + await(siblingDone, "sibling of the cancelled consequence ran"); + await(cancelledDone, "cancelled consequence was notified"); + assertThat(cancelled.get()).isInstanceOf(CancellationException.class); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * A task whose prepare fails must be failed and completed, and the {@link ExclusiveExecutor} it was dispatched + * from must go on to dispatch the next task. + */ + @Test + public void prepareFailureCompletesTaskAndDispatchesNext() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + ExclusiveExecutor exclusive = env.executor.newExclusiveExecutor(0); + + CountDownLatch failed = CountDownLatch.newCountDownLatch(1); + CountDownLatch ran = CountDownLatch.newCountDownLatch(1); + RuntimeException failure = new RuntimeException("deliberate prepare failure"); + TestTask first = new TestTask(env.executor, exclusive, failure, failed); + TestTask second = new TestTask(env.executor, exclusive, null, ran); + + env.executor.executeDirectlyWithLock(() -> { + first.submitExclusiveNoExcept(); + second.submitExclusiveNoExcept(); + }); + + await(failed, "the task that failed to prepare was notified"); + assertThat(first.failure).isSameAs(failure); + await(ran, "the next queued task was dispatched"); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * A chain submitted to a command store hosted by a *different* executor must be submitted to that executor + * independently, not attached as a consequence of the running task (whose executor's lock we hold). + */ + @Test + public void consequenceOnAnotherExecutorIsSubmittedIndependently() throws Throwable + { + Env a = new Env(RUN_WITHOUT_LOCK); + Env b = new Env(RUN_WITHOUT_LOCK); + + CountDownLatch childDone = CountDownLatch.newCountDownLatch(1); + CountDownLatch parentDone = CountDownLatch.newCountDownLatch(1); + + a.store.execute(ExecutionContext.unsequenced(nextTxnId(), "parentOnA"), (Consumer) safe -> { + b.store.execute(ExecutionContext.unsequenced(nextTxnId(), "childOnB"), noop(), + (r, f) -> { assertThat(f).isNull(); childDone.decrement(); }); + }, (r, f) -> { assertThat(f).isNull(); parentDone.decrement(); }); + + await(parentDone, "parent was notified"); + await(childDone, "child on the other executor ran"); + a.assertNoAgentExceptions(); + b.assertNoAgentExceptions(); + a.assertQuiescent(); + b.assertQuiescent(); + } + + /** + * An incremental task with more keys than a single batch can hold must run several batches and then complete once. + */ + @Test + public void incrementalTaskRunsMultipleBatches() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + AtomicInteger batches = new AtomicInteger(); + AtomicReference failure = new AtomicReference<>(); + CountDownLatch done = CountDownLatch.newCountDownLatch(1); + + env.store.execute(ExecutionContext.unsequencedIncrementalWrite(keys(0, 200), "incremental"), + (Consumer) safe -> batches.incrementAndGet(), + (r, f) -> { failure.set(f); done.decrement(); }); + + await(done, "incremental task completed"); + assertThat(failure.get()).isNull(); + assertThat(batches.get()).describedAs("expected several batches").isGreaterThan(1); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * An incremental task can be failed while it is parked between batches - this is what a late failure of one of + * its (optional) key loads does, see {@link AccordExecutor#onLoadedExclusive}. It must release its cache + * references, exactly as it would if it were failed before it first ran. + */ + @Test + public void incrementalTaskFailedBetweenBatchesReleasesResources() throws Throwable + { + Env env = new Env(RUN_WITHOUT_LOCK); + AtomicInteger batches = new AtomicInteger(); + AtomicReference failure = new AtomicReference<>(); + CountDownLatch done = CountDownLatch.newCountDownLatch(1); + RuntimeException loadFailure = new RuntimeException("simulated load failure"); + + Cancellable submitted = + env.store.execute(ExecutionContext.unsequencedIncrementalWrite(keys(1000, 1200), "incremental"), + (Consumer) safe -> { + batches.incrementAndGet(); + try { Thread.sleep(5); } catch (InterruptedException e) { throw new RuntimeException(e); } + }, + (r, f) -> { failure.set(f); done.decrement(); }); + + SafeTask task = (SafeTask) submitted; + AtomicReference failedIn = new AtomicReference<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + while (failedIn.get() == null && failure.get() == null && System.nanoTime() < deadline) + { + env.executor.executeDirectlyWithLock(() -> { + Task.State state = task.state(); + if (batches.get() > 0 && task.isState(Task.State.WAITING)) + { + failedIn.set(state); + task.tryFailAndCompleteExclusive(loadFailure, Task.State.FAILED); + } + }); + } + + assertThat(failedIn.get()).describedAs("did not observe the task parked between batches").isNotNull(); + await(done, "task was notified of the failure"); + assertThat(failure.get()).isSameAs(loadFailure); + int stillHeld = task.refs == null ? 0 : task.refs.size(); + assertThat(stillHeld).describedAs("cache references were not released").isZero(); + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + /** + * The executor throttles loading while the cache is over its working-set budget, but must always be able to make + * *some* progress: if the only work that can run is loading, it must run regardless of the budget. + */ + @Test + public void progressesWithZeroCacheCapacity() throws Throwable + { + Env env = new Env(RUN_WITH_LOCK); + env.executor.executeDirectlyWithLock(() -> { + env.executor.setCapacity(0); + env.executor.setWorkingSetSize(0); + }); + + for (int i = 0 ; i < 2 ; ++i) + { + AtomicReference failure = new AtomicReference<>(); + CountDownLatch done = CountDownLatch.newCountDownLatch(1); + env.store.execute(ExecutionContext.unsequencedIncrementalWrite(keys(2000 + i * 200, 2200 + i * 200), "zeroCapacity" + i), + noop(), (r, f) -> { failure.set(f); done.decrement(); }); + await(done, "task completed with a zero capacity cache"); + assertThat(failure.get()).isNull(); + } + + env.assertNoAgentExceptions(); + env.assertQuiescent(); + } + + private static RoutingKeys keys(int from, int to) + { + TableMetadata metadata = Schema.instance.getTableMetadata("ks", "tbl"); + List keys = new ArrayList<>(to - from); + for (int i = from ; i < to ; ++i) + keys.add(AccordTestUtils.key(metadata, i).toUnseekable()); + return RoutingKeys.of(keys); + } + + /** + * A minimal {@link Plain} task: optionally fails during prepare, otherwise records that it ran. + */ + private static class TestTask extends Plain + { + final ExclusiveExecutor exclusiveExecutor; + final RuntimeException failPrepareWith; + final CountDownLatch notified; + volatile Throwable failure; + + TestTask(AccordExecutor executor, ExclusiveExecutor exclusiveExecutor, RuntimeException failPrepareWith, CountDownLatch notified) + { + super(executor, ExclusiveGroup.OTHER); + this.exclusiveExecutor = exclusiveExecutor; + this.failPrepareWith = failPrepareWith; + this.notified = notified; + } + + @Override + ExclusiveExecutor exclusiveExecutor() + { + return exclusiveExecutor; + } + + @Override + void prepareExclusiveMayThrow() + { + if (failPrepareWith != null) + throw failPrepareWith; + } + + @Override + boolean runMayThrow() + { + if (failPrepareWith != null) + throw new AssertionError("should not have run"); + notified.decrement(); + return true; + } + + @Override + void reportFailureMayThrow(Throwable fail) + { + failure = fail; + notified.decrement(); + } + + @Override + public String description() + { + return "TestTask[failPrepare=" + (failPrepareWith != null) + ']'; + } + + @Override + String briefDescription() + { + return description(); + } + } +} From 988235c73607cda1b82aa9f7a62e2e12f1b42f21 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Thu, 30 Jul 2026 12:55:36 +0100 Subject: [PATCH 07/10] fix ThreadLocalTaskRunner reentrancy claude tests fix negative histogram values under simulation, and catch and refuse bad values in other cases --- modules/accord | 2 +- .../metrics/LogLinearDecayingHistograms.java | 7 + .../cassandra/metrics/LogLinearHistogram.java | 9 + .../service/accord/api/AccordAgent.java | 11 +- .../accord/execution/AbstractLockLoop.java | 4 +- .../service/accord/execution/SafeTask.java | 2 +- .../service/accord/execution/Task.java | 2 +- .../service/accord/execution/TaskRunner.java | 10 +- .../AccordCommandStoreExecutorTest.java | 441 ++++++++++++++++++ .../simulator/test/AccordExecutorTest.java | 77 ++- 10 files changed, 542 insertions(+), 23 deletions(-) create mode 100644 test/simulator/test/org/apache/cassandra/service/accord/execution/AccordCommandStoreExecutorTest.java diff --git a/modules/accord b/modules/accord index 4d091037c7b7..138e96dd0f0b 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 4d091037c7b751631d6ce204e0c192e2301162da +Subproject commit 138e96dd0f0b9e315be21a8659f1aa81ec48637a diff --git a/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java b/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java index 4a1779482785..46f568145014 100644 --- a/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java +++ b/src/java/org/apache/cassandra/metrics/LogLinearDecayingHistograms.java @@ -59,6 +59,9 @@ public Buffer(LogLinearDecayingHistograms histograms) private void add(long histogramIndex, long value) { + if (!Invariants.expect(value >= 0, "Negative value reported: %s", value)) + return; + Invariants.require(histogramIndex <= HISTOGRAM_INDEX_MASK); Invariants.require(value >= 0); if (value <= LARGE_VALUE) value <<= HISTOGRAM_INDEX_BITS; @@ -117,6 +120,9 @@ public void increment(long value) public void increment(long value, long at) { + if (!Invariants.expect(value >= 0, "Negative value reported: %s", value)) + return; + int index = index(value); if (bufferCount >= buffer.length) { @@ -127,6 +133,7 @@ public void increment(long value, long at) updateDecay(at); long v = Double.doubleToRawLongBits(increment) & VALUE_MASK; v |= histogramIndex | ((long)index << HISTOGRAM_INDEX_BITS); + Invariants.require(Double.longBitsToDouble(v & VALUE_MASK) > 0); buffer[bufferCount++] = v; } diff --git a/src/java/org/apache/cassandra/metrics/LogLinearHistogram.java b/src/java/org/apache/cassandra/metrics/LogLinearHistogram.java index 10c727c7c438..bae5779e6617 100644 --- a/src/java/org/apache/cassandra/metrics/LogLinearHistogram.java +++ b/src/java/org/apache/cassandra/metrics/LogLinearHistogram.java @@ -54,6 +54,9 @@ public LogLinearHistogram(long expectedMaxValue) public void increment(long value) { + if (!Invariants.expect(value >= 0, "Negative value reported: %s", value)) + return; + int index = index(value); buckets(index)[index]++; ++totalCount; @@ -61,6 +64,9 @@ public void increment(long value) public void decrement(long value) { + if (!Invariants.expect(value >= 0, "Negative value reported: %s", value)) + return; + int index = index(value); buckets(index)[index]--; --totalCount; @@ -68,6 +74,9 @@ public void decrement(long value) public void replace(long decrement, long increment) { + if (!Invariants.expect(decrement >= 0 && increment >= 0, "Negative value(s?) reported: %s and %s", decrement, increment)) + return; + int decrementIndex = index(decrement); int incrementIndex = index(increment); long[] buckets = buckets(Math.max(decrementIndex, incrementIndex)); diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java index 3188fe00294e..fc3ffa975b48 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java @@ -39,6 +39,7 @@ import accord.api.RoutingKey; import accord.api.Tracing; import accord.coordinate.Coordination; +import accord.coordinate.CoordinationFailed; import accord.coordinate.Exhausted; import accord.coordinate.Preempted; import accord.coordinate.Timeout; @@ -222,13 +223,19 @@ public static void handleException(Throwable t) return; AccordSystemMetrics.metrics.errors.inc(); - if (t instanceof CancellationException || t instanceof TimeoutException || t instanceof Timeout || t instanceof Preempted || t instanceof Exhausted || t instanceof LogUnavailableException) - // TODO (required): leaky logger, permitting multiple messages per time period and reporting how many were dropped + if (expectedException(t)) // TODO (required): leaky logger, permitting multiple messages per time period and reporting how many were dropped noSpamException.warn(exceptionId(t), t); else JVMStabilityInspector.uncaughtException(Thread.currentThread(), t); } + public static boolean expectedException(Throwable t) + { + if (t instanceof CancellationException) + return t.getCause() == null; + return t instanceof TimeoutException || t instanceof LogUnavailableException || t instanceof CoordinationFailed; + } + @Override public void onException(Throwable t) { diff --git a/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java index 41738b8dd3f4..7018ae63185b 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java +++ b/src/java/org/apache/cassandra/service/accord/execution/AbstractLockLoop.java @@ -24,7 +24,6 @@ import accord.api.Agent; -import org.apache.cassandra.concurrent.CassandraThread; import org.apache.cassandra.service.accord.execution.Loops.LoopTask; import static org.apache.cassandra.service.accord.debug.DebugExecution.DEBUG_EXECUTION; @@ -248,7 +247,8 @@ protected LoopTask runWithoutLock(String name) @Override public void run() { - CassandraThread self = (CassandraThread) Thread.currentThread(); + Thread thread = Thread.currentThread(); + TaskRunner self = TaskRunner.get(thread); self.setAccordActiveExecutor(AbstractLockLoop.this); setWrapped(self); diff --git a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java index 7027adb4780b..5eee015214b5 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java @@ -748,7 +748,7 @@ boolean holdsLocksBetweenRuns() private void waitOnCacheQueuesExclusive() { Invariants.require(waitingForState == 0); - loadedAt = nanoTime(); + loadedAt = Math.max(createdAt, nanoTime()); executor().runnable.incrementArrivals(this); commandStore.exclusiveExecutor().incrementArrivals(this); diff --git a/src/java/org/apache/cassandra/service/accord/execution/Task.java b/src/java/org/apache/cassandra/service/accord/execution/Task.java index bd0af0d472ab..6a3e50309f9f 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/Task.java +++ b/src/java/org/apache/cassandra/service/accord/execution/Task.java @@ -601,7 +601,7 @@ final void onRunning() { Invariants.require(is(PREPARED)); Invariants.require(is(NOT_YET_RUN) || is(RUN_INCOMPLETE)); - runningAt = nanoTime(); + runningAt = Math.max(createdAt, nanoTime()); setRunState(RunState.RUNNING); if (DEBUG_EXECUTION) ((DebugTask) resources).onRunning(); } diff --git a/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java b/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java index 2dffb11e1850..0d9705a1864b 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java +++ b/src/java/org/apache/cassandra/service/accord/execution/TaskRunner.java @@ -48,6 +48,7 @@ static TaskRunner get(Thread thread) final class ThreadLocalTaskRunner implements TaskRunner { private AccordExecutor lockedExecutor; + private int lockedExecutorDepth; private AccordExecutor activeExecutor; volatile Task activeTask; @@ -74,16 +75,17 @@ public AccordExecutor accordLockedExecutor() @Override public boolean tryEnterAccordLockedExecutor(AccordExecutor newLockedExecutor) { - if (lockedExecutor != null) - return false; - lockedExecutor = newLockedExecutor; + if (lockedExecutor == null) lockedExecutor = newLockedExecutor; + else if (lockedExecutor != newLockedExecutor) return false; + ++lockedExecutorDepth; return true; } @Override public void exitAccordLockedExecutor() { - lockedExecutor = null; + if (--lockedExecutorDepth == 0) + lockedExecutor = null; } @Override diff --git a/test/simulator/test/org/apache/cassandra/service/accord/execution/AccordCommandStoreExecutorTest.java b/test/simulator/test/org/apache/cassandra/service/accord/execution/AccordCommandStoreExecutorTest.java new file mode 100644 index 000000000000..2d1a68a0d231 --- /dev/null +++ b/test/simulator/test/org/apache/cassandra/service/accord/execution/AccordCommandStoreExecutorTest.java @@ -0,0 +1,441 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.LockSupport; +import java.util.function.LongSupplier; +import java.util.function.ToLongFunction; + +import org.junit.Test; + +import accord.api.AsyncExecutor; +import accord.api.ExclusiveAsyncExecutor; +import accord.api.ProgressLog; +import accord.api.Result; +import accord.api.RoutingKey; +import accord.api.Scheduler; +import accord.coordinate.Coordinations; +import accord.impl.DefaultLocalListeners; +import accord.impl.TestAgent; +import accord.impl.DefaultLocalListeners.NotifySink; +import accord.impl.DefaultRemoteListeners; +import accord.impl.basic.InMemoryJournal; +import accord.local.CommandStores.RangesForEpoch; +import accord.local.DurableBefore; +import accord.local.TimeService; +import accord.local.durability.DurabilityService; +import accord.local.ExecutionContext; +import accord.local.LoadKeys; +import accord.local.LoadKeysFor; +import accord.local.Node.Id; +import accord.local.NodeCommandStoreService; +import accord.local.SafeCommandStore; +import accord.local.SafeState; +import accord.primitives.Ballot; +import accord.primitives.Range; +import accord.primitives.Ranges; +import accord.primitives.Route; +import accord.primitives.RoutingKeys; +import accord.primitives.Timestamp; +import accord.primitives.TxnId; +import accord.primitives.Writes; +import accord.topology.TopologyManager; +import accord.utils.DefaultRandom; +import accord.utils.Invariants; +import accord.utils.async.Cancellable; + +import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.TokenRange; +import org.apache.cassandra.service.accord.api.TokenKey; +import org.apache.cassandra.simulator.test.SimulationTestBase; +import org.apache.cassandra.utils.concurrent.CountDownLatch; +import org.apache.cassandra.utils.concurrent.Future; + +import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.service.accord.execution.AccordExecutor.Mode.RUN_WITHOUT_LOCK; + +/** + * Simulator driven test of {@link SafeTask} execution on a real {@link AccordCommandStore}, i.e. of the parts of the + * executor {@link AccordExecutorTest} cannot reach: cache references, the per-key/per-txnId queues, and the ordering + * guarantees they provide. + * + *

The command store is real, but everything below it is synthetic: an in-memory journal, and cache load functions + * that return {@code null} (an uninitialised value) rather than reading from {@code system_accord}. So no schema, no + * cluster metadata, no commit log - only the executor and task machinery is under test. + * + *

Scope (phase 1)

+ * A single executor and command store; {@code SYNC} key-domain contexts; no nesting, no cancellation, no failures. + * Verifies: + *
    + *
  • liveness: every submission is notified exactly once, and successfully;
    • + *
  • declared access: while running, a task holds a reference for every key and txnId it declared;
    • + *
  • mutual exclusion: two tasks that declare the same key or txnId never execute concurrently - the + * central guarantee of the cache-entry queues;
    • + *
  • no leaks: once everything has been notified, no cache entry is still referenced and no task retains + * its references.
    • + *
+ * + *

Notes

+ *
    + *
  • only the {@code SIGNAL} and {@code ASYNC} submission models can be simulated; {@code SYNC}/{@code SEMI_SYNC} + * (the production default) need the simulator to intercept {@link java.util.concurrent.locks.ReentrantLock}, + * see the TODO in {@link AccordExecutorTest};
    • + *
  • only one test method may be run per JVM (pre-existing limitation of {@link AccordExecutorTest}); ant forks + * per test method.
    • + *
+ */ +public class AccordCommandStoreExecutorTest extends SimulationTestBase +{ + private static final int KEYS = 8; + private static final int TXN_IDS = 8; + private static final int SUBMIT_THREADS = 8; + private static final int OUTER_LOOP = 5; + private static final int INNER_LOOP = 10; + private static final int MAX_KEYS_PER_TASK = 3; + private static final int MAX_TXN_IDS_PER_TASK = 2; + private static final int MAX_TASKS = SUBMIT_THREADS * OUTER_LOOP * INNER_LOOP; + + @Test + public void signalLoopTest() + { + test(() -> new AccordExecutorSignalLoop(1, RUN_WITHOUT_LOCK, 4, -1, -1, TimeUnit.MICROSECONDS, i -> "Loop" + i, new RecordingAgent())); + } + + @Test + public void asyncSubmitTest() + { + test(() -> new AccordExecutorAsyncSubmit(1, RUN_WITHOUT_LOCK, 4, i -> "Loop" + i, new RecordingAgent())); + } + + private void test(SerializableSupplier executorSupplier) + { + simulate(arr(() -> { + try + { + DatabaseDescriptor.daemonInitialization(); + AccordExecutor executor = executorSupplier.get(); + Env env = new Env(executor); + + for (float loadDelayChance : new float[]{ 0f, 0.1f }) + { + for (float sleepChance : new float[]{ 0f, 0.1f }) + { + System.out.printf("loadDelayChance %.2f, sleepChance %.2f%n", loadDelayChance, sleepChance); + env.loadDelayChance = loadDelayChance; + env.round(sleepChance); + } + } + } + catch (Throwable t) + { + throw new RuntimeException(t); + } + }), + () -> {}, 1L); + } + + /** + * Records everything reported to the agent: on these paths any report indicates a broken internal invariant, + * not a failed operation. We do not use {@link AccordAgent}, as reporting an exception there touches + * {@code AccordSystemMetrics}, which requires a started {@code AccordService}. + */ + public static class RecordingAgent extends TestAgent + { + static final List exceptions = new CopyOnWriteArrayList<>(); + + @Override + public void onException(Throwable t) + { + exceptions.add(t); + System.out.println("### agent.onException: " + t); + t.printStackTrace(System.out); + } + + @Override + public void onException(Throwable t, String context) + { + onException(t); + } + } + + static class Env + { + final AccordExecutor executor; + final AccordCommandStore store; + final RoutingKey[] keys = new RoutingKey[KEYS]; + final TxnId[] txnIds = new TxnId[TXN_IDS]; + + /** key/txnId ordinal -> id of the task currently executing with it, or -1 */ + final AtomicIntegerArray keyOwner = new AtomicIntegerArray(KEYS); + final AtomicIntegerArray txnIdOwner = new AtomicIntegerArray(TXN_IDS); + + /** reset for every round: task ids are round-local */ + AtomicInteger nextTaskId = new AtomicInteger(); + /** task id -> number of times its callback has been invoked; must be exactly one for every submission */ + AtomicIntegerArray notifications = new AtomicIntegerArray(MAX_TASKS); + final List failures = new CopyOnWriteArrayList<>(); + SafeTask[] tasks = new SafeTask[MAX_TASKS]; + + volatile float loadDelayChance; + + Env(AccordExecutor executor) + { + this.executor = executor; + TableId tableId = TableId.fromUUID(new java.util.UUID(0, 1)); + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + this.store = newCommandStore(tableId, partitioner, executor); + for (int i = 0 ; i < KEYS ; ++i) + keys[i] = new TokenKey(tableId, partitioner.getToken(Int32Type.instance.decompose(i))); + for (int i = 0 ; i < TXN_IDS ; ++i) + txnIds[i] = TxnId.fromValues(1, 1 + i, 0, new Id(1)); + for (int i = 0 ; i < KEYS ; ++i) + keyOwner.set(i, -1); + for (int i = 0 ; i < TXN_IDS ; ++i) + txnIdOwner.set(i, -1); + + // synthetic loads: an uninitialised value, optionally after some simulated latency + executor.cacheUnsafe().types().forEach(type -> type.unsafeSetLoadFunction((ignoreStore, ignoreKey) -> { + maybePark(loadDelayChance); + return null; + })); + } + + void round(float sleepChance) throws ExecutionException, InterruptedException + { + nextTaskId = new AtomicInteger(); + notifications = new AtomicIntegerArray(MAX_TASKS); + tasks = new SafeTask[MAX_TASKS]; + ExecutorPlus submit = executorFactory().pooled("submit", SUBMIT_THREADS); + try + { + List> submitting = new ArrayList<>(); + for (int i = 0 ; i < SUBMIT_THREADS ; ++i) + { + int id = i; + submitting.add(submit.submit(() -> { + for (int outer = 0 ; outer < OUTER_LOOP ; ++outer) + { + CountDownLatch inner = CountDownLatch.newCountDownLatch(INNER_LOOP); + for (int j = 0 ; j < INNER_LOOP ; ++j) + submitOne(sleepChance, inner); + inner.awaitUninterruptibly(); + System.out.println("Loop " + id + '.' + outer); + } + })); + } + for (Future f : submitting) + f.get(); + } + finally + { + submit.shutdown(); + } + + verifyRoundComplete(); + } + + private void submitOne(float sleepChance, CountDownLatch done) + { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + int taskId = nextTaskId.getAndIncrement(); + int[] keyOrdinals = distinct(rnd, KEYS, 1 + rnd.nextInt(MAX_KEYS_PER_TASK)); + int[] txnIdOrdinals = distinct(rnd, TXN_IDS, rnd.nextInt(1 + MAX_TXN_IDS_PER_TASK)); + + RoutingKeys declaredKeys = keys(keyOrdinals); + TxnId primary = txnIdOrdinals.length > 0 ? txnIds[txnIdOrdinals[0]] : null; + TxnId additional = txnIdOrdinals.length > 1 ? txnIds[txnIdOrdinals[1]] : null; + ExecutionContext context = ExecutionContext.contextFor(primary, additional, declaredKeys, LoadKeys.SYNC, LoadKeysFor.READ_WRITE, "task" + taskId); + + Cancellable submitted = + store.execute(context, (java.util.function.Consumer) safeStore -> body(taskId, keyOrdinals, txnIdOrdinals, safeStore, sleepChance), + (success, fail) -> { + notifications.incrementAndGet(taskId); + if (fail != null) + failures.add(fail); + done.decrement(); + }); + tasks[taskId] = (SafeTask) submitted; + } + + private void body(int taskId, int[] keyOrdinals, int[] txnIdOrdinals, SafeCommandStore safeStore, float sleepChance) + { + SafeTask task = ((SaferCommandStore) safeStore).task; + + // declared access: we must hold a reference for everything we declared, and be able to look it up + for (int k : keyOrdinals) + { + SafeState ref = task.refs.get(keys[k]); + Invariants.require(ref != null, "task %d declared key %d but holds no reference for it", taskId, k); + Invariants.require(SaferState.global(ref).references() > 0, "task %d holds an unreferenced entry for key %d", taskId, k); + safeStore.ifLoadedAndInitialised(keys[k]); + } + for (int t : txnIdOrdinals) + { + SafeState ref = task.refs.get(txnIds[t]); + Invariants.require(ref != null, "task %d declared txnId %d but holds no reference for it", taskId, t); + Invariants.require(SaferState.global(ref).references() > 0, "task %d holds an unreferenced entry for txnId %d", taskId, t); + safeStore.ifInitialised(txnIds[t]); + } + + // mutual exclusion: nobody else may be executing with any key or txnId we declared + int keysTaken = 0, txnIdsTaken = 0; + try + { + while (keysTaken < keyOrdinals.length) + { + int k = keyOrdinals[keysTaken]; + Invariants.require(keyOwner.compareAndSet(k, -1, taskId), + "task %d ran concurrently with task %d, which also declared key %d", taskId, keyOwner.get(k), k); + ++keysTaken; + } + while (txnIdsTaken < txnIdOrdinals.length) + { + int t = txnIdOrdinals[txnIdsTaken]; + Invariants.require(txnIdOwner.compareAndSet(t, -1, taskId), + "task %d ran concurrently with task %d, which also declared txnId %d", taskId, txnIdOwner.get(t), t); + ++txnIdsTaken; + } + + maybePark(sleepChance); + } + finally + { + while (keysTaken > 0) + keyOwner.set(keyOrdinals[--keysTaken], -1); + while (txnIdsTaken > 0) + txnIdOwner.set(txnIdOrdinals[--txnIdsTaken], -1); + } + } + + private void verifyRoundComplete() + { + Invariants.require(failures.isEmpty(), "%s", failures); + Invariants.require(RecordingAgent.exceptions.isEmpty(), "%s", RecordingAgent.exceptions); + + int lastTaskId = nextTaskId.get(); + Invariants.require(lastTaskId == MAX_TASKS, "submitted %d tasks, expected %d", lastTaskId, MAX_TASKS); + for (int taskId = 0 ; taskId < lastTaskId ; ++taskId) + { + Invariants.require(notifications.get(taskId) == 1, "task %d was notified %d times", taskId, notifications.get(taskId)); + Invariants.require(tasks[taskId].refs == null, "task %d did not release its references", taskId); + } + for (int i = 0 ; i < KEYS ; ++i) + Invariants.require(keyOwner.get(i) == -1, "key %d is still owned by task %d", i, keyOwner.get(i)); + for (int i = 0 ; i < TXN_IDS ; ++i) + Invariants.require(txnIdOwner.get(i) == -1, "txnId %d is still owned by task %d", i, txnIdOwner.get(i)); + + try (AccordCommandStore.ExclusiveCaches caches = store.lockCaches()) + { + for (AccordCacheEntry entry : caches.commands()) + Invariants.require(entry.references() == 0, "%s is still referenced", entry); + for (AccordCacheEntry entry : caches.commandsForKeys()) + Invariants.require(entry.references() == 0, "%s is still referenced", entry); + } + } + + private RoutingKeys keys(int[] ordinals) + { + RoutingKey[] result = new RoutingKey[ordinals.length]; + for (int i = 0 ; i < ordinals.length ; ++i) + result[i] = keys[ordinals[i]]; + return RoutingKeys.of(result); + } + } + + private static void maybePark(float chance) + { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + if (chance > 0 && rnd.nextFloat() < chance) + LockSupport.parkNanos(rnd.nextInt(10000, 100000)); + } + + /** {@code count} distinct ordinals in [0..limit), in ascending order (so a task never declares a key twice) */ + private static int[] distinct(ThreadLocalRandom rnd, int limit, int count) + { + int mask = 0; + for (int i = 0 ; i < count ; ++i) + mask |= 1 << rnd.nextInt(limit); + int[] result = new int[Integer.bitCount(mask)]; + for (int i = 0, ordinal = 0 ; mask != 0 ; ++ordinal, mask >>>= 1) + { + if ((mask & 1) != 0) + result[i++] = ordinal; + } + return result; + } + + private static AccordCommandStore newCommandStore(TableId tableId, IPartitioner partitioner, AccordExecutor executor) + { + AtomicLong clock = new AtomicLong(); + LongSupplier now = clock::incrementAndGet; + Id nodeId = new Id(1); + NodeCommandStoreService node = new NodeCommandStoreService() + { + private final ToLongFunction elapsed = TimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.MICROSECONDS, this::now); + private long stamp = 0; + + @Override public AsyncExecutor someExecutor() { return null; } + @Override public ExclusiveAsyncExecutor someExclusiveExecutor() { return null; } + @Override public accord.api.Timeouts timeouts() { return null; } + @Override public DurableBefore durableBefore() { return DurableBefore.EMPTY; } + @Override public DurabilityService durability() { return null; } + @Override public Id id() { return nodeId; } + @Override public long epoch() { return 1; } + @Override public long now() { return now.getAsLong(); } + @Override public long uniqueNow(long atLeast) { return now.getAsLong(); } + @Override public long elapsed(TimeUnit units) { return elapsed.applyAsLong(units); } + @Override public TopologyManager topology() { throw new UnsupportedOperationException(); } + @Override public Coordinations coordinations() { return new Coordinations(); } + @Override public Scheduler scheduler() { return null; } + @Override public long currentStamp() { return stamp; } + @Override public void updateStamp() { ++stamp; } + @Override public boolean isReplaying() { return false; } + @Override public void reportLocalExecution(TxnId txnId, Route route, Ballot ballot, Timestamp applyAt, Writes writes, Result result) {} + }; + + Range range = TokenRange.fullRange(tableId, partitioner); + RangesForEpoch rangesForEpoch = new RangesForEpoch(1, Ranges.of(range)); + AccordCommandStore store = new AccordCommandStore(0, node, new RecordingAgent(), null, + cs -> new ProgressLog.NoOpProgressLog(), + cs -> new DefaultLocalListeners(null, new DefaultRemoteListeners.NoOpRemoteListeners(), new NotifySink.NoOpNotifySink()), + rangesForEpoch, + new InMemoryJournal(nodeId, new DefaultRandom(1)), + executor); + executor.executeDirectlyWithLock(() -> { + executor.setCapacity(8 << 20); + executor.setWorkingSetSize(4 << 20); + }); + return store; + } +} diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java index 3d441ae04e15..e075a0da7c78 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordExecutorTest.java @@ -87,7 +87,7 @@ static class Submitted { final AtomicInteger nextId = new AtomicInteger(); final AtomicInteger doneBefore = new AtomicInteger(); - final ConcurrentHashMap>> consequences = new ConcurrentHashMap<>(); + final ConcurrentHashMap consequences = new ConcurrentHashMap<>(); boolean isDone() { @@ -106,11 +106,8 @@ boolean isDoneBetween(int from, int before) { for (int id = from ; id < before ; ++id) { - for (Future future : consequences.get(id)) - { - if (!future.isDone()) - return false; - } + if (!consequences.get(id).isDone()) + return false; } return true; } @@ -119,6 +116,34 @@ class Consequences extends ConcurrentLinkedQueue> { volatile boolean started; + /** + * The number of submissions in this group that have not yet invoked their callback. + *

+ * We cannot decide this by iterating the queue: each level of the recursion appends the next level's + * future while it runs, i.e. strictly before its own future completes, so the queue is still growing + * while it is being consumed - and {@link ConcurrentLinkedQueue}'s iterator prefetches, so an iterator + * that has reached the tail terminates and never sees the later additions. A counter is exact, because + * for the same reason it can only reach zero once the whole group has finished: a consequence is + * registered before its parent's future completes. + */ + final AtomicInteger outstanding = new AtomicInteger(); + + void submitted(Future future) + { + outstanding.incrementAndGet(); + add(future); + } + + void completed() + { + outstanding.decrementAndGet(); + } + + boolean isDone() + { + return outstanding.get() == 0; + } + void ensureStarted() { if (started) @@ -171,14 +196,29 @@ static class Control extends ConcurrentLinkedQueue void submit(AsyncExecutor executor, Consequences consequences, Consumer run) { AsyncPromise future = new AsyncPromise<>(); - consequences.add(future); + consequences.submitted(future); Cancellable cancel = executor.chain(() -> run.accept(consequences)).begin((success, fail) -> { - if (fail == null) future.trySuccess(null); - else + try + { + if (fail == null) future.trySuccess(null); + else + { + future.tryFailure(fail); + // replace the cancelled work with an equivalent submission, so that cancelling does not + // simply reduce the amount of work we perform. + // + // NOTE: this callback may be invoked from inside the executor (e.g. while cancelling, which + // happens with the executor's lock held), so we must not run the body here: it re-enters the + // executor (afterSubmittedAndConsequences, and the lock itself), which is forbidden for a + // thread that already holds it. Submit it as ordinary work instead, so it runs in a legal + // context - this also widens the set of interleavings we explore. + if (fail instanceof CancellationException) + submit(executor, submitted.allocate(), run); + } + } + finally { - future.tryFailure(fail); - if (fail instanceof CancellationException) - run.accept(submitted.allocate()); + consequences.completed(); } }); consequences.ensureStarted(); @@ -248,8 +288,16 @@ public void executorTest(SerializableSupplier supplier, int subm for (Future f : done) f.get(); + // the awaits in submitLoop only wait for the submissions they can see, and the + // deeper levels of each group are only registered as they run, so work may still + // be in flight here; drain the executor before verifying that everything we + // recorded has in fact run + executor.waitForQuiescence(); if (!submitted.isDone()) throw new AssertionError(); + // nothing is running, so we can now safely inspect every future we recorded + for (int id = 0 ; id < submitted.nextId.get() ; ++id) + await(submitted.consequences.get(id), CancellationException.class); } } } @@ -350,6 +398,11 @@ private static void submitUntil(Lock lock, Executor executor, float sleepChance, }); } + /** + * Waits for those submissions that are visible in {@code await}; note that this deliberately does not wait for + * the whole tree of consequences (which is still being appended to as each level runs), so that submission + * threads continue to overlap their outer loops - {@link Submitted#isDone} is verified once the executor drains. + */ private static void await(Collection> await, @Nullable Class ignore) throws InterruptedException, ExecutionException { for (Future future : await) From ba683a02fde1834e7f8c7ce5528827703bd1aaca Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Tue, 19 May 2026 15:07:59 +0100 Subject: [PATCH 08/10] Improve Rebootstrap: - Distinguish corrupt/incomplete cases; latter case only refuses uncertain data - Ensures quorum durability before marking own log faulty - Refuse unsafe operations more selectively once log marked faulty - Simplify and merge logic with standard bootstrap --- .gitmodules | 2 +- .../managing/configuration/configuration.adoc | 2 +- modules/accord | 2 +- .../apache/cassandra/concurrent/Stage.java | 2 +- .../apache/cassandra/config/AccordConfig.java | 42 +- .../cassandra/config/DatabaseDescriptor.java | 14 + .../db/virtual/AccordDebugKeyspace.java | 27 +- .../exceptions/ExceptionSerializer.java | 11 +- .../apache/cassandra/journal/Compactor.java | 7 +- .../cassandra/journal/SegmentCompactor.java | 6 +- .../service/accord/AccordCommandStore.java | 8 +- .../service/accord/AccordDataStore.java | 81 +- .../accord/AccordFetchCoordinator.java | 29 +- .../service/accord/AccordKeyspace.java | 3 +- .../service/accord/AccordMessageSink.java | 8 + .../service/accord/AccordResult.java | 4 +- .../service/accord/AccordService.java | 131 ++- .../service/accord/IAccordService.java | 8 +- .../service/accord/api/AccordAgent.java | 62 +- .../accord/api/AccordWaitStrategies.java | 8 +- .../accord/interop/AccordInteropAdapter.java | 20 +- .../interop/AccordInteropExecution.java | 16 +- .../journal/AbstractSegmentCompactor.java | 10 + .../serializers/CommandStoreSerializers.java | 5 +- .../accord/serializers/FetchSerializers.java | 20 +- .../tcm/sequences/BootstrapAndJoin.java | 3 +- test/conf/logback-info.xml | 56 + .../test/accord/AccordCQLTestBase.java | 27 +- .../accord/AccordIncrementalRepairTest.java | 4 +- .../test/accord/AccordTestBase.java | 7 +- .../test/accord/load/AccordLoadTestBase.java | 976 ++++++++++++++++++ .../load/AccordRebootstrapLoadTest.java | 66 ++ .../test/accord/load/AccordYcsbLoadTest.java | 137 +++ .../test/accord/load/LoadSettings.java | 338 ++++++ .../fuzz/topology/AccordRebootstrapTest.java | 2 +- .../service/accord/AccordMessageSinkTest.java | 2 +- .../service/accord/AccordTestUtils.java | 16 +- .../accord/SimulatedAccordCommandStore.java | 7 + .../CommandsForKeySerializerTest.java | 2 + 39 files changed, 1945 insertions(+), 226 deletions(-) create mode 100644 test/conf/logback-info.xml create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordLoadTestBase.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordRebootstrapLoadTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordYcsbLoadTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/accord/load/LoadSettings.java diff --git a/.gitmodules b/.gitmodules index 4b1eea6020b2..c00814bf0741 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "modules/accord"] path = modules/accord url = https://github.com/belliottsmith/cassandra-accord.git - branch = executor-fair + branch = rerebootstrap diff --git a/doc/modules/cassandra/pages/managing/configuration/configuration.adoc b/doc/modules/cassandra/pages/managing/configuration/configuration.adoc index 9b4df248b26f..0933da73fd6c 100644 --- a/doc/modules/cassandra/pages/managing/configuration/configuration.adoc +++ b/doc/modules/cassandra/pages/managing/configuration/configuration.adoc @@ -153,7 +153,7 @@ configuration changes in the near future. `@Replaces` is the annotation to be used when you make changes to any configuration parameters in `Config` class and `cassandra.yaml`, and you want to add backward compatibility with previous Cassandra versions. `Converters` class enumerates the different methods used for backward compatibility. `IDENTITY` is the one used for name change only. For more information about the other Converters, please, check the JavaDoc in the class. -For backward compatibility virtual table `Settings` contains both the old and the new +For backward compatibility virtual table `LoadSettings` contains both the old and the new parameters with the old and the new value format. Only exception at the moment are the following three parameters: `key_cache_save_period`, `row_cache_save_period` and `counter_cache_save_period` which appear only once with the new value format. The old names and value format still can be used at least until the next major release. Deprecation warning is emitted on startup. diff --git a/modules/accord b/modules/accord index 138e96dd0f0b..673cf2ff93a4 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 138e96dd0f0b9e315be21a8659f1aa81ec48637a +Subproject commit 673cf2ff93a4b32a392c91feacda09bb48ac16e7 diff --git a/src/java/org/apache/cassandra/concurrent/Stage.java b/src/java/org/apache/cassandra/concurrent/Stage.java index 557c1ca0a48b..4e99729e0f73 100644 --- a/src/java/org/apache/cassandra/concurrent/Stage.java +++ b/src/java/org/apache/cassandra/concurrent/Stage.java @@ -47,7 +47,7 @@ public enum Stage MUTATION (true, "MutationStage", "request", DatabaseDescriptor::getConcurrentWriters, DatabaseDescriptor::setConcurrentWriters, Stage::multiThreadedLowSignalStage), COUNTER_MUTATION (true, "CounterMutationStage", "request", DatabaseDescriptor::getConcurrentCounterWriters, DatabaseDescriptor::setConcurrentCounterWriters, Stage::multiThreadedLowSignalStage), VIEW_MUTATION (true, "ViewMutationStage", "request", DatabaseDescriptor::getConcurrentViewWriters, DatabaseDescriptor::setConcurrentViewWriters, Stage::multiThreadedLowSignalStage), - ACCORD_MIGRATION (false, "AccordMigrationStage", "request", DatabaseDescriptor::getAccordConcurrentOps, DatabaseDescriptor::setConcurrentAccordOps, Stage::multiThreadedLowSignalStage), + ACCORD_MIGRATION (false, "AccordMigrationStage", "request", DatabaseDescriptor::getAccordConcurrentMigrationOps, DatabaseDescriptor::setConcurrentAccordMigrationOps, Stage::multiThreadedLowSignalStage), GOSSIP (true, "GossipStage", "internal", () -> 1, null, Stage::singleThreadedStage), REQUEST_RESPONSE (false, "RequestResponseStage", "request", FBUtilities::getAvailableProcessors, null, Stage::multiThreadedLowSignalStage), ANTI_ENTROPY (false, "AntiEntropyStage", "internal", () -> 1, null, Stage::singleThreadedStage), diff --git a/src/java/org/apache/cassandra/config/AccordConfig.java b/src/java/org/apache/cassandra/config/AccordConfig.java index 9813f3ffa016..50d00d2b6f37 100644 --- a/src/java/org/apache/cassandra/config/AccordConfig.java +++ b/src/java/org/apache/cassandra/config/AccordConfig.java @@ -34,7 +34,7 @@ import org.apache.cassandra.service.accord.serializers.Version; import org.apache.cassandra.service.consensus.TransactionalMode; -import static org.apache.cassandra.config.AccordConfig.CatchupMode.NORMAL; +import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.HLC_FIFO; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_POOL_PER_SHARD; import static org.apache.cassandra.config.AccordConfig.QueueSubmissionModel.SYNC; import static org.apache.cassandra.config.AccordConfig.RangeIndexMode.in_memory; @@ -214,6 +214,10 @@ public enum QueueBalancingModel * once this number of tasks are blocked behind it, regardless of batch_size. */ public Integer queue_nonsync_blocked_limit; + /** + * The number of threads that may be used to execute distributed requests for migration tasks + */ + public volatile OptionaldPositiveInt migration_concurrency = OptionaldPositiveInt.UNDEFINED; /** * If set, the signal loop does not match park/unpark pairs, but instead consumers perform timed-park spin waits @@ -258,6 +262,7 @@ public enum QueueBalancingModel public String expire_epoch_wait = "10s"; // we don't want to wait ages for durability as it blocks other durability progress; even this might be too long, as we can always retry public String expire_durability = "10s*attempts <= 30s"; + public String slow_durability = "10s"; public String slow_syncpoint_preaccept = "10s"; public String slow_txn_preaccept = "30ms <= p50*2 <= 1000ms"; public String slow_read = "30ms <= p50*2 <= 1000ms"; @@ -294,12 +299,12 @@ public enum TransactionalRangeMigration */ public volatile TransactionalRangeMigration range_migration = TransactionalRangeMigration.auto; - public enum CatchupMode + public enum CatchupFallbackMode { - DISABLED, - NORMAL, - FALLBACK_TO_HARD, - HARD + IGNORE, + EXIT, + REBOOTSTRAP, + REBOOTSTRAP_AND_CATCHUP } /** @@ -344,16 +349,19 @@ public enum CatchupMode public int commands_for_key_prune_interval = 64; public DurationSpec.IntSecondsBound max_conflicts_prune_delta = new DurationSpec.IntSecondsBound(1); + public int catchup_on_start_max_attempts = 5; + public boolean catchup_on_start = true; public DurationSpec.IntSecondsBound catchup_on_start_success_latency = new DurationSpec.IntSecondsBound(60); public DurationSpec.IntSecondsBound catchup_on_start_fail_latency = new DurationSpec.IntSecondsBound(900); - public int catchup_on_start_max_attempts = 5; - // TODO (required): roll this back to catchup_on_start_exit_on_failure: true - public boolean catchup_on_start_exit_on_failure = false; - public CatchupMode catchup_on_start = NORMAL; + // TODO (required): default this to EXIT or REBOOTSTRAP + public CatchupFallbackMode catchup_on_start_on_timeout = CatchupFallbackMode.IGNORE; + public CatchupFallbackMode catchup_on_start_on_error = CatchupFallbackMode.IGNORE; + public CatchupFallbackMode catchup_on_start_on_rebootstrap_fallback = CatchupFallbackMode.IGNORE; + public DurationSpec.IntSecondsBound shutdown_grace_period = new DurationSpec.IntSecondsBound(15 * 60); + public boolean execute_waiting_on_start = true; public DurationSpec.IntSecondsBound execute_waiting_on_start_timeout = new DurationSpec.IntSecondsBound(0); public boolean execute_waiting_on_start_fail_on_timeout = false; - public DurationSpec.IntSecondsBound shutdown_grace_period = new DurationSpec.IntSecondsBound(15 * 60); public enum RangeIndexMode { in_memory, journal_sai } public RangeIndexMode range_index_mode = in_memory; @@ -391,7 +399,17 @@ public enum ReplayMode * Replay journal entries for commands that are not durable to the data or command stores. * THIS MODE IS NOT YET SAFE TO RUN */ - NON_DURABLE + NON_DURABLE, + + /** + * Don't replay, simply rebootstrap, marking our log as incomplete. + */ + REBOOTSTRAP_INCOMPLETE, + + /** + * Don't replay, simply rebootstrap, marking our log as corrupted/unavailable. + */ + REBOOTSTRAP_RESET } public enum ReplaySavePoint diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 5bc6de7ac1e4..765ee10d4548 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -2967,6 +2967,20 @@ public static void setConcurrentAccordOps(int concurrent_operations) conf.accord.queue_thread_count = new OptionaldPositiveInt(concurrent_operations); } + public static int getAccordConcurrentMigrationOps() + { + return conf.accord.migration_concurrency.or(2 * FBUtilities.getAvailableProcessors()); + } + + public static void setConcurrentAccordMigrationOps(int concurrent_operations) + { + if (concurrent_operations < 0) + { + throw new IllegalArgumentException("Concurrent accord operations must be non-negative"); + } + conf.accord.migration_concurrency = new OptionaldPositiveInt(concurrent_operations); + } + public static int getFlushWriters() { return conf.memtable_flush_writers; diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index 5b3e9f79e5de..9b149de9f2d7 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -62,7 +62,8 @@ import accord.impl.progresslog.DefaultProgressLog; import accord.impl.progresslog.DefaultProgressLog.ModeFlag; import accord.impl.progresslog.TxnStateKind; -import accord.local.CatchupHard; +import accord.local.BootstrapReason; +import accord.local.Catchup; import accord.local.Cleanup; import accord.local.Command; import accord.local.CommandStore; @@ -166,6 +167,8 @@ import static accord.coordinate.Infer.InvalidIf.NotKnownToBeInvalid; import static accord.impl.CommandChange.Field.ACCEPTED; import static accord.impl.CommandChange.Field.PROMISED; +import static accord.local.BootstrapReason.LOG_CORRUPTED; +import static accord.local.BootstrapReason.LOG_INCOMPLETE; import static accord.local.RedundantStatus.Property.GC_BEFORE; import static accord.local.RedundantStatus.Property.LOCALLY_APPLIED; import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_COMMAND_STORE; @@ -183,6 +186,7 @@ import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest; import static org.apache.cassandra.db.virtual.AbstractLazyVirtualTable.OnTimeout.BEST_EFFORT; import static org.apache.cassandra.db.virtual.AbstractLazyVirtualTable.OnTimeout.FAIL; +import static org.apache.cassandra.db.virtual.AccordDebugKeyspace.CommandStoreOpsTable.CommandStoreOp.REBOOTSTRAP_CORRUPTED; import static org.apache.cassandra.db.virtual.VirtualTable.Sorted.ASC; import static org.apache.cassandra.db.virtual.VirtualTable.Sorted.SORTED; import static org.apache.cassandra.db.virtual.VirtualTable.Sorted.UNSORTED; @@ -1988,8 +1992,10 @@ enum CommandStoreOp UNSET_PROGRESS_LOG_MODE("Unset the specified progress log mode."), TRY_EXECUTE_LISTENING("Try to execute all of the transactions (and their dependencies) that have registered listeners on other transactions."), REPLAY("Run journal replay for all transactions"), - REBOOTSTRAP("Rebootstrap the command store. This invalidates the local journal, synchronises its data via data repair and rejoins the distributed state machine."), - HARD_CATCHUP("Hard catchup the command store. This invalidates the local journal for any ranges not up to date with some quorum, synchronises its data via data repair and rejoins the distributed state machine."), + // TODO (expected): specify ranges + REBOOTSTRAP_CORRUPTED("Rebootstrap the command store. This invalidates the local journal, synchronises its data via data repair and rejoins the distributed state machine."), + REBOOTSTRAP_INCOMPLETE("Rebootstrap the command store. This invalidates the local journal, synchronises its data via data repair and rejoins the distributed state machine."), + REBOOTSTRAP_IF_BEHIND("Hard catchup the command store. This invalidates the local journal for any ranges not up to date with some quorum, synchronises its data via data repair and rejoins the distributed state machine."), ; final String description; @@ -2082,13 +2088,15 @@ protected void applyRowUpdate(Object[] partitionKeys, Object[] clusteringKeys, C }; break; } - case REBOOTSTRAP: - allFunction = () -> node.commandStores().rebootstrap(node); - function = commandStore -> commandStore.rebootstrap(node); + case REBOOTSTRAP_CORRUPTED: + case REBOOTSTRAP_INCOMPLETE: + BootstrapReason reason = op == REBOOTSTRAP_CORRUPTED ? LOG_CORRUPTED : LOG_INCOMPLETE; + allFunction = () -> node.commandStores().rebootstrap(node, reason); + function = commandStore -> commandStore.rebootstrap(node, reason); break; - case HARD_CATCHUP: - allFunction = () -> CatchupHard.catchup(node, Arrays.asList(node.commandStores().all())).beginAsResult(); - function = commandStore -> CatchupHard.catchup(node, Collections.singletonList(commandStore)).beginAsResult(); + case REBOOTSTRAP_IF_BEHIND: + allFunction = () -> Catchup.rebootstrapIfBehind(node, Arrays.asList(node.commandStores().all())).beginAsResult(); + function = commandStore -> Catchup.rebootstrapIfBehind(node, Collections.singletonList(commandStore)).beginAsResult(); break; } @@ -2484,7 +2492,6 @@ private static BucketMode checkBucketMode(Object value) { throw new InvalidRequestException("Unknown bucket_mode '" + value + '\''); } - } private static int checkNonNegative(Object value, String field, int ifNull) diff --git a/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java b/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java index 834abca49dd0..50a5393d1084 100644 --- a/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java +++ b/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java @@ -72,10 +72,13 @@ public String toString() static String getMessageWithOriginatingHost(Throwable t, boolean isFirstException) { - if (isFirstException) - return "Remote exception from host " + FBUtilities.getBroadcastAddressAndPort().toString() + (t.getLocalizedMessage() != null ? " - " + t.getLocalizedMessage() : ""); - else - return t.getLocalizedMessage(); + String message; + if (isFirstException) message = "Remote exception from host " + FBUtilities.getBroadcastAddressAndPort().toString() + (t.getLocalizedMessage() != null ? " - " + t.getLocalizedMessage() : ""); + else message = t.getLocalizedMessage(); + + if (message.length() > Short.MAX_VALUE) + message = message.substring(0, Short.MAX_VALUE); + return message; } private static final IVersionedSerializer stackTraceElementSerializer = new IVersionedSerializer() diff --git a/src/java/org/apache/cassandra/journal/Compactor.java b/src/java/org/apache/cassandra/journal/Compactor.java index ffc48ff5f2e1..428866098c34 100644 --- a/src/java/org/apache/cassandra/journal/Compactor.java +++ b/src/java/org/apache/cassandra/journal/Compactor.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.journal; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -29,6 +28,7 @@ import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.concurrent.Shutdownable; +import org.apache.cassandra.journal.SegmentCompactor.Stopped; import org.apache.cassandra.utils.concurrent.WaitQueue; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; @@ -100,9 +100,9 @@ public void run() compacted.signalAll(); } - catch (IOException e) + catch (Stopped e) { - throw new RuntimeException("Could not compact segments: " + toCompact); + logger.info("Cancelling compaction of {}", toCompact); } } @@ -118,6 +118,7 @@ public void shutdown() logger.debug("Shutting down " + executor); if (scheduled != null) scheduled.cancel(false); + segmentCompactor.stop(); executor.shutdown(); } diff --git a/src/java/org/apache/cassandra/journal/SegmentCompactor.java b/src/java/org/apache/cassandra/journal/SegmentCompactor.java index 7b84ea82e12d..01897a52c19f 100644 --- a/src/java/org/apache/cassandra/journal/SegmentCompactor.java +++ b/src/java/org/apache/cassandra/journal/SegmentCompactor.java @@ -17,18 +17,20 @@ */ package org.apache.cassandra.journal; -import java.io.IOException; import java.util.Collection; public interface SegmentCompactor { SegmentCompactor NOOP = (SegmentCompactor) (segments) -> segments; + class Stopped extends RuntimeException {} + static SegmentCompactor noop() { //noinspection unchecked return (SegmentCompactor) NOOP; } - Collection> compact(Collection> segments) throws IOException; + Collection> compact(Collection> segments) throws Stopped; + default void stop() {} } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index 2dbbc07e0061..3fe7550439ea 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -254,8 +254,10 @@ public AccordCommandStore(int id, maybeLoadRedundantBefore(journal.loadRedundantBefore(id())); maybeLoadBootstrapBeganAt(journal.loadBootstrapBeganAt(id())); maybeLoadSafeToRead(journal.loadSafeToRead(id())); - - tableId = (TableId)rangesForEpoch.all().stream().map(r -> r.start().prefix()).reduce((a, b) -> { + maybeLoadRangesForEpoch(journal.loadRangesForEpoch(id())); + RangesForEpoch ranges = this.rangesForEpoch; + Invariants.require(ranges != null && !ranges.all().isEmpty(), "CommandStore %d created with no ranges", id); + tableId = (TableId)ranges.all().stream().map(r -> r.start().prefix()).reduce((a, b) -> { Invariants.require(a.equals(b), "CommandStore created with multiple distinct TableId (%s and %s)", a, b); return a; }).orElseThrow(() -> Invariants.illegalState("CommandStore %d created with no ranges", id)); @@ -824,7 +826,7 @@ AsyncChain> restoreState() { File rjbf = new File(savePoint, "reject_before"); if (rjbf.exists()) - mxc = mxc.with(readOne(rjbf, rejectBefore)); + mxc = mxc.update(readOne(rjbf, rejectBefore)); } dll = readList(new File(savePoint, "listeners"), txnListener); dpl = readList(new File(savePoint, "progress_log"), progressLogState); diff --git a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java index 0e757ba418d5..9e227e5f5ab3 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java @@ -20,24 +20,23 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.api.DataStore; import accord.local.CommandStore; +import accord.local.CommandStores.RestrictedStoreSelector; +import accord.local.MapReduceConsumeCommandStores; import accord.local.Node; import accord.local.RedundantBefore; import accord.local.SafeCommandStore; import accord.primitives.Ranges; -import accord.primitives.SyncPoint; +import accord.primitives.Timestamp; import accord.primitives.TxnId; import accord.utils.Invariants; -import accord.utils.Reduce; +import accord.utils.SortedArrays.SortedArrayList; import accord.utils.UnhandledEnum; -import accord.utils.async.AsyncResult; import accord.utils.async.AsyncResults; import org.apache.cassandra.concurrent.ScheduledExecutors; @@ -56,10 +55,9 @@ import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.utils.progress.ProgressEvent; +import org.apache.cassandra.utils.progress.ProgressListener; -import static accord.local.durability.DurabilityService.SyncLocal.NoLocal; -import static accord.local.durability.DurabilityService.SyncRemote.Quorum; -import static accord.primitives.Txn.Kind.ExclusiveSyncPoint; import static accord.utils.Invariants.illegalArgument; public class AccordDataStore implements DataStore @@ -94,12 +92,12 @@ private void ensureDurableInternal(CommandStore commandStore, RedundantBefore re AccordDurableOnFlush.notifyOnDurable(cfs, commandStore, ReportDurable.of(redundantBefore, flags)); } - public FetchResult image(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback) + public FetchResult image(Node node, SafeCommandStore safeStore, Ranges ranges, TxnId atLeast, SortedArrayList readable, FetchRanges callback) { AccordFetchCoordinator coordinator; try { - coordinator = new AccordFetchCoordinator(node, ranges, syncPoint, callback, safeStore.commandStore()); + coordinator = new AccordFetchCoordinator(node, ranges, atLeast, readable, callback, safeStore.commandStore()); } catch (Throwable t) { @@ -110,11 +108,10 @@ public FetchResult image(Node node, SafeCommandStore safeStore, Ranges ranges, S return coordinator.result(); } - public FetchResult sync(Node node, SafeCommandStore safeStore, Map rangesById, FetchRanges callback) + @Override + public FetchResult sync(Node node, SafeCommandStore safeStore, Ranges ranges, TxnId atLeast, SortedArrayList readable, FetchRanges callback) { TableId tableId = ((AccordCommandStore)safeStore.commandStore()).tableId(); - Invariants.require(rangesById.values().stream().flatMap(Ranges::stream).allMatch(r -> tableId.equals(r.prefix()))); - Ranges ranges = rangesById.values().stream().reduce(Ranges::with).get(); ClusterMetadata cm = ClusterMetadata.current(); TableMetadata tableMetadata = cm.schema.getTableMetadata(tableId); @@ -137,27 +134,22 @@ public void abort(Ranges ranges) // TODO (expected): add some automatic slicing of ranges and retry/back-off logic; but for now, // since this is done at the command store level, and this is already a slice of a node, this should be fine SyncResult syncResult = new SyncResult(); + ProgressListener listener = new ProgressListener() + { + StartingRangeFetch starting = callback.starting(ranges); + { Invariants.require(starting != null); } - logger.info("Requesting quorum durability before initiating repair"); - List> syncs = new ArrayList<>(); - for (Map.Entry e : rangesById.entrySet()) - syncs.add(node.durability().sync("Sync Data Store", ExclusiveSyncPoint, e.getKey(), e.getValue(), NoLocal, Quorum, 1L, TimeUnit.HOURS)); - - AsyncResults.reduce(syncs, Reduce.toNull()).invoke((success, fail) -> { - if (fail != null) + @Override + public void progress(String tag, ProgressEvent event) { - logger.error("{} failed to achieve quorum durability before repair for rebootstrap of {}", safeStore.commandStore(), ranges); - syncResult.tryFailure(fail); - return; - } - - RepairCoordinator coord = StorageService.instance.newRepairCoordinator(tableMetadata.keyspace, options(tableMetadata, ranges)); - coord.addProgressListener((tag, event) -> { switch (event.getType()) { default: throw new UnhandledEnum(event.getType()); + case SUCCESS: + callback.fetched(ranges); + syncResult.trySuccess(null); case START: - callback.starting(ranges); + reportStarted(); break; case PROGRESS: case COMPLETE: @@ -169,17 +161,32 @@ public void abort(Ranges ranges) callback.fail(ranges, ex); syncResult.tryFailure(ex); break; - case SUCCESS: - callback.fetched(ranges); - syncResult.trySuccess(null); - break; } - }); + } - ScheduledExecutors.optionalTasks.submit(coord).addCallback((s, f) -> { - if (f != null) - syncResult.tryFailure(f); - }); + private void reportStarted() + { + StartingRangeFetch start = this.starting; + if (start == null) + return; + this.starting = null; + node.commandStores().mapReduceConsume(new RestrictedStoreSelector(ranges, 0, Long.MAX_VALUE), new MapReduceConsumeCommandStores(ranges) + { + @Override public Timestamp reduce(Timestamp o1, Timestamp o2) { return Timestamp.max(o1, o2); } + @Override public void accept(Timestamp result, Throwable failure) { start.started(result); } + @Override public TxnId primaryTxnId() { return null; } + @Override public String reason() { return "Compute MaxConflict to report for fetch"; } + @Override protected Timestamp applyInternal(SafeCommandStore safeStore) { return safeStore.commandStore().maxConflict(TxnId.NONE, ranges); } + }); + } + }; + + RepairCoordinator coord = StorageService.instance.newRepairCoordinator(tableMetadata.keyspace, options(tableMetadata, ranges)); + coord.addProgressListener(listener); + + ScheduledExecutors.optionalTasks.submit(coord).addCallback((s, f) -> { + if (f != null) + syncResult.tryFailure(f); }); return syncResult; diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java index a14d9b3cd468..630f81ccbf54 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java @@ -38,7 +38,6 @@ import accord.local.CommandStore; import accord.local.Node; import accord.local.SafeCommandStore; -import accord.primitives.PartialDeps; import accord.primitives.PartialTxn; import accord.primitives.Participants; import accord.primitives.Range; @@ -46,12 +45,12 @@ import accord.primitives.Routable; import accord.primitives.Seekable; import accord.primitives.Seekables; -import accord.primitives.SyncPoint; import accord.primitives.Timestamp; import accord.primitives.Txn; import accord.primitives.TxnId; import accord.topology.TopologyException; import accord.utils.Invariants; +import accord.utils.SortedArrays.SortedArrayList; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; @@ -395,9 +394,9 @@ public long serializedSize(Update t, TableMetadatasAndKeys seekables, Version ve private final Map streams = new HashMap<>(); - public AccordFetchCoordinator(Node node, Ranges ranges, SyncPoint syncPoint, DataStore.FetchRanges fetchRanges, CommandStore commandStore) throws TopologyException + public AccordFetchCoordinator(Node node, Ranges ranges, TxnId atLeast, SortedArrayList readable, DataStore.FetchRanges fetchRanges, CommandStore commandStore) throws TopologyException { - super(node, node.someExclusiveExecutor(), ranges, syncPoint, fetchRanges, commandStore); + super(node, node.someExclusiveExecutor(), ranges, atLeast, readable, fetchRanges, commandStore); } @Override @@ -431,13 +430,6 @@ protected void onDone(Ranges success, Throwable failure) super.onDone(success, failure); } - @Override - protected PartialTxn rangeReadTxn(Ranges ranges) - { - StreamingRead read = new StreamingRead(FBUtilities.getBroadcastAddressAndPort(), ranges); - return new PartialTxn.InMemory(Txn.Kind.Read, ranges, read, noopQuery, null, TableMetadatasAndKeys.none(Routable.Domain.Range)); - } - @Override protected synchronized void onReadOk(Node.Id from, CommandStore commandStore, Data data, Ranges received) { @@ -463,9 +455,9 @@ protected synchronized void onReadOk(Node.Id from, CommandStore commandStore, Da public static class AccordFetchRequest extends FetchRequest { - public AccordFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges, PartialDeps partialDeps, PartialTxn partialTxn) + public AccordFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges, PartialTxn partialTxn) { - super(sourceEpoch, syncId, ranges, partialDeps, partialTxn); + super(sourceEpoch, syncId, ranges, partialTxn); } @Override @@ -479,8 +471,15 @@ protected AsyncChain beginRead(SafeCommandStore safeStore, Timestamp execu } @Override - protected FetchRequest newFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges, PartialDeps partialDeps, PartialTxn partialTxn) + protected FetchRequest newFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges) { - return new AccordFetchRequest(sourceEpoch, syncId, ranges, partialDeps, partialTxn); + return new AccordFetchRequest(sourceEpoch, syncId, ranges, rangeReadTxn(ranges)); } + + private PartialTxn rangeReadTxn(Ranges ranges) + { + StreamingRead read = new StreamingRead(FBUtilities.getBroadcastAddressAndPort(), ranges); + return new PartialTxn.InMemory(Txn.Kind.Read, ranges, read, noopQuery, null, TableMetadatasAndKeys.none(Routable.Domain.Range)); + } + } diff --git a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java index b0efd33b1bc0..a7bdedefdbc5 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java +++ b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java @@ -106,6 +106,7 @@ import org.apache.cassandra.schema.Types; import org.apache.cassandra.schema.UserFunctions; import org.apache.cassandra.schema.Views; +import org.apache.cassandra.service.accord.api.AccordAgent; import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.accord.serializers.CommandSerializers; import org.apache.cassandra.utils.AbstractIterator; @@ -341,7 +342,7 @@ private static PartitionUpdate makeUpdate(int storeId, TokenKey key, CommandsFor { ByteBuffer bytes; if (serialized instanceof ByteBuffer) bytes = (ByteBuffer) serialized; - else bytes = Serialize.toBytesWithoutKey(commandsForKey.maximalPrune()); // TODO (expected): we only need to strip pruned, not prune additional txns + else bytes = Serialize.toBytesWithoutKey(commandsForKey.maybePrune(0, AccordAgent.cfkHlcPruneDelta(DatabaseDescriptor.getAccord()))); // TODO (expected): we only need to strip pruned, not prune additional txns return makeUpdate(storeId, key, timestampMicros, bytes); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java index 18dafb1908ee..2ddac1b52c3a 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java +++ b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java @@ -254,6 +254,14 @@ public Cancellable send(Node.Id to, Request request, int attempt, AsyncExecutor slowAtNanos = nowNanos + slow.computeWait(attempt, NANOSECONDS); break; } + + case ACCORD_WAIT_UNTIL_APPLIED_REQ: + { + TimeoutStrategy slow = slowPreaccept(txnId); + if (slow != null) + slowAtNanos = nowNanos + slow.computeWait(attempt, NANOSECONDS); + break; + } } Message message = Message.out(verb, request, expiresAtNanos); diff --git a/src/java/org/apache/cassandra/service/accord/AccordResult.java b/src/java/org/apache/cassandra/service/accord/AccordResult.java index 033ccdd88af9..59440ee1748b 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordResult.java +++ b/src/java/org/apache/cassandra/service/accord/AccordResult.java @@ -201,7 +201,9 @@ else if (fail instanceof RequestTimeoutException || fail instanceof TimeoutExcep } else { - logger.error("Unexpected exception", fail); + AccordAgent.handleException(fail); + if (!AccordAgent.expectedException(fail)) + logger.error("Unexpected exception", fail); JVMStabilityInspector.inspectThrowable(fail); report = bookkeeping.newFailed(txnId, keysOrRanges); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index 19a0f9da098f..e198670b5f55 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -56,8 +56,9 @@ import accord.impl.SizeOfIntersectionSorter; import accord.impl.progresslog.DefaultProgressLog; import accord.impl.progresslog.DefaultProgressLogs; +import accord.local.BootstrapReason; import accord.local.Catchup; -import accord.local.CatchupHard; +import accord.local.Catchup.Unsuccessful; import accord.local.CommandStores; import accord.local.Node; import accord.local.Node.Id; @@ -92,7 +93,6 @@ import org.apache.cassandra.concurrent.Shutdownable; import org.apache.cassandra.config.AccordConfig; -import org.apache.cassandra.config.AccordConfig.CatchupMode; import org.apache.cassandra.config.AccordConfig.JournalConfig.ReplaySavePoint; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; @@ -166,6 +166,8 @@ import static accord.api.ProtocolModifiers.FastExecution.MAY_BYPASS_SAFESTORE; import static accord.coordinate.Coordination.CoordinationKind.Client; import static accord.impl.progresslog.DefaultProgressLog.ModeFlag.CATCH_UP; +import static accord.local.BootstrapReason.LOG_CORRUPTED; +import static accord.local.BootstrapReason.LOG_INCOMPLETE; import static accord.local.durability.DurabilityService.SyncLocal.Self; import static accord.local.durability.DurabilityService.SyncRemote.All; import static accord.messages.SimpleReply.Ok; @@ -179,8 +181,9 @@ import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorThreadTag.JOB; import static org.apache.cassandra.concurrent.ExecutorFactory.SystemThreadTag.DAEMON; -import static org.apache.cassandra.config.AccordConfig.CatchupMode.FALLBACK_TO_HARD; -import static org.apache.cassandra.config.AccordConfig.CatchupMode.HARD; +import static org.apache.cassandra.config.AccordConfig.CatchupFallbackMode.REBOOTSTRAP; +import static org.apache.cassandra.config.AccordConfig.JournalConfig.ReplayMode.REBOOTSTRAP_INCOMPLETE; +import static org.apache.cassandra.config.AccordConfig.JournalConfig.ReplayMode.REBOOTSTRAP_RESET; import static org.apache.cassandra.config.AccordConfig.JournalConfig.ReplayMode.RESET; import static org.apache.cassandra.config.DatabaseDescriptor.getAccord; import static org.apache.cassandra.config.DatabaseDescriptor.getAccordGlobalDurabilityCycle; @@ -534,7 +537,16 @@ public synchronized void localStartup() if (state != State.INIT) return; - boolean rebootstrap = false; + BootstrapReason rebootstrap = null; + if (getAccord().journal.replay == REBOOTSTRAP_RESET) + { + rebootstrap = LOG_CORRUPTED; + } + else if (getAccord().journal.replay == REBOOTSTRAP_INCOMPLETE) + { + rebootstrap = LOG_INCOMPLETE; + } + else { long startMarker = ReplayMarkers.readStartMarker(); long stopMarker = ReplayMarkers.readStopMarker(); @@ -553,7 +565,7 @@ public synchronized void localStartup() case REBOOTSTRAP: logger.info("Stop marker is older than start marker ({}<{}). Rebootstrapping.", stopMarker, startMarker); - rebootstrap = true; + rebootstrap = LOG_INCOMPLETE; } } } @@ -594,7 +606,7 @@ public synchronized void localStartup() // restore save points before starting the journal so we can validate consistency between journal and save point state (where possible) Long2LongHashMap minSegments; - if (rebootstrap) minSegments = null; + if (rebootstrap != null) minSegments = null; else { switch (getAccord().journal.replaySavePoint) @@ -616,11 +628,11 @@ public synchronized void localStartup() } node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start()); - if (rebootstrap) + if (rebootstrap != null) { // rebootstrap expects the durability service to process visibility sync points for command stores node.durability().start(); - getBlocking(node.commandStores().rebootstrap(node)); + getBlocking(node.commandStores().rebootstrap(node, rebootstrap)); } else { @@ -707,81 +719,100 @@ private void distributedStartupInternal() void catchup() { AccordConfig spec = DatabaseDescriptor.getAccord(); - if (spec.catchup_on_start == CatchupMode.DISABLED) + if (!spec.catchup_on_start) { logger.info("Catchup disabled; continuing to startup"); return; } - CatchupMode mode = spec.catchup_on_start; BootstrapState bootstrapState = SystemKeyspace.getBootstrapState(); if (bootstrapState == COMPLETED) { node.commandStores().forAllUnsafe(commandStore -> ((DefaultProgressLog)commandStore.unsafeProgressLog()).setMode(CATCH_UP)); try { + AccordConfig.CatchupFallbackMode onError = spec.catchup_on_start_on_error; + AccordConfig.CatchupFallbackMode onTimeout = spec.catchup_on_start_on_timeout; + long maxLatencyNanos = spec.catchup_on_start_fail_latency.toNanoseconds(); int attempts = 1; while (true) { - logger.info("Catchup ({}) with quorum...", mode); + logger.info("Catchup with quorum..."); long start = nanoTime(); long failAt = start + maxLatencyNanos; - Future f; + Future f; { - AsyncChain submit; - if (mode == HARD) - { - if (!node.durability().isStarted()) - node.durability().start(); - submit = CatchupHard.catchup(node); - } - else submit = Catchup.catchup(node); + AsyncChain submit = Catchup.catchup(node, failAt, NANOSECONDS); f = toFuture(submit); } - if (!f.awaitUntilThrowUncheckedOnInterrupt(failAt)) - { - if (spec.catchup_on_start_exit_on_failure) - { - logger.error("Catchup exceeded maximum latency of {}ns; shutting down", maxLatencyNanos); - throw new RuntimeException("Could not catchup with peers"); - } - logger.error("Catchup exceeded maximum latency of {}ns; continuing to startup", maxLatencyNanos); - break; - } + f.awaitThrowUncheckedOnInterrupt(); Throwable failed = f.cause(); + Unsuccessful result = f.getNow(); + if (failed != null) { - if (spec.catchup_on_start_exit_on_failure) - throw new RuntimeException("Could not catchup with peers", failed); - - logger.error("Could not catchup with peers; continuing to startup"); - break; + switch (onError) + { + default: throw new UnhandledEnum(onError); + case EXIT: + logger.error("Could not catchup with peers; exiting", failed); + throw new RuntimeException("Could not catchup with peers", failed); + case IGNORE: + logger.error("Could not catchup with peers; continuing to startup", failed); + return; + case REBOOTSTRAP: + case REBOOTSTRAP_AND_CATCHUP: + logger.error("Could not catchup with peers; rebootstrapping", failed); + getBlocking(node.commandStores().rebootstrap(node, LOG_INCOMPLETE)); + if (onError == REBOOTSTRAP) + return; + ++attempts; + onError = onTimeout = spec.catchup_on_start_on_rebootstrap_fallback; + continue; + } } long end = nanoTime(); double seconds = NANOSECONDS.toMillis(end - start)/1000.0; logger.info("Finished catchup with all quorums. {}s elapsed.", String.format("%.2f", seconds)); - if (seconds <= spec.catchup_on_start_success_latency.toSeconds()) - break; - - if (++attempts > spec.catchup_on_start_max_attempts) + if (result == null) { - if (spec.catchup_on_start_exit_on_failure) + if (seconds <= spec.catchup_on_start_success_latency.toSeconds()) + return; + + if (attempts < spec.catchup_on_start_max_attempts) { - logger.error("Catchup was slow, aborting after {} attempts and shutting down", attempts); - throw new RuntimeException("Could not catchup with peers"); + logger.info("Catchup was slow, so we may behind again; retrying"); + ++attempts; + continue; } - - logger.info("Catchup was slow; continuing to startup after {} attempts.", attempts - 1); - break; } - logger.info("Catchup was slow, so we may behind again; retrying"); - if (mode == FALLBACK_TO_HARD) - mode = HARD; + switch (onTimeout) + { + default: throw new UnhandledEnum(onTimeout); + case EXIT: + if (result == null) logger.error("Catchup was slow; aborting after {} attempts and shutting down", attempts); + else logger.error("Catchup was incomplete for {}; aborting after {} attempts and shutting down", result.ranges, attempts); + throw new RuntimeException("Could not catchup with peers"); + case IGNORE: + if (result == null) logger.info("Catchup was slow, continuing to startup after {} attempts", attempts); + else logger.info("Catchup was incomplete for {}, continuing to startup after {} attempts", result.ranges, attempts); + return; + case REBOOTSTRAP: + case REBOOTSTRAP_AND_CATCHUP: + if (result == null) logger.info("Catchup was slow, rebootstrapping after {} attempts", attempts); + else logger.info("Catchup was incomplete for {}, rebootstrapping after {} attempts", result.ranges, attempts); + getBlocking(node.commandStores().rebootstrap(node, result == null ? null : result.ranges, LOG_INCOMPLETE)); + if (onTimeout == REBOOTSTRAP) + return; + ++attempts; + onError = onTimeout = spec.catchup_on_start_on_rebootstrap_fallback; + continue; + } } } finally @@ -954,7 +985,7 @@ public ShardDurability.ImmutableView shardDurability() } @Override - public AsyncResult sync(Object requestedBy, TxnId minBound, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) + public AsyncResult sync(Object requestedBy, TxnId minBound, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) { return node.durability().sync(requestedBy, ExclusiveSyncPoint, minBound, ranges, include, syncLocal, syncRemote, timeout, timeoutUnits); } diff --git a/src/java/org/apache/cassandra/service/accord/IAccordService.java b/src/java/org/apache/cassandra/service/accord/IAccordService.java index abf7336b2241..79bde24b575f 100644 --- a/src/java/org/apache/cassandra/service/accord/IAccordService.java +++ b/src/java/org/apache/cassandra/service/accord/IAccordService.java @@ -88,8 +88,8 @@ public interface IAccordService IVerbHandler requestHandler(); IVerbHandler responseHandler(); - AsyncResult sync(Object requestedBy, @Nullable TxnId minBound, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits); - AsyncChain sync(@Nullable TxnId minBound, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote); + AsyncResult sync(Object requestedBy, @Nullable TxnId minBound, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits); + AsyncChain sync(@Nullable TxnId minBound, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote); AsyncChain maxConflict(Ranges ranges); @Nonnull @@ -402,13 +402,13 @@ public IVerbHandler responseHandler() } @Override - public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) + public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ranges ranges, @Nullable Collection include, SyncLocal syncLocal, SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) { return delegate.sync(requestedBy, onOrAfter, ranges, include, syncLocal, syncRemote, timeout, timeoutUnits); } @Override - public AsyncChain sync(@Nullable TxnId onOrAfter, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote) + public AsyncChain sync(@Nullable TxnId onOrAfter, Keys keys, SyncLocal syncLocal, SyncRemote syncRemote) { return delegate.sync(onOrAfter, keys, syncLocal, syncRemote); } diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java index fc3ffa975b48..cabb3de079af 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java @@ -40,9 +40,6 @@ import accord.api.Tracing; import accord.coordinate.Coordination; import accord.coordinate.CoordinationFailed; -import accord.coordinate.Exhausted; -import accord.coordinate.Preempted; -import accord.coordinate.Timeout; import accord.local.Command; import accord.local.CommandStore; import accord.local.LogUnavailableException; @@ -51,6 +48,7 @@ import accord.local.SafeCommandStore; import accord.local.TimeService; import accord.messages.MessageType; +import accord.primitives.Ballot; import accord.primitives.Keys; import accord.primitives.Participants; import accord.primitives.Ranges; @@ -266,6 +264,11 @@ public boolean rejectPreAccept(TimeService time, TxnId txnId) // TODO (expected): we probably want additional configuration here so we can prune on shorter time horizons when we have a lot of transactions on a single key @Override public long cfkHlcPruneDelta() + { + return cfkHlcPruneDelta(config); + } + + public static long cfkHlcPruneDelta(AccordConfig config) { return config.commands_for_key_prune_delta.to(MICROSECONDS); } @@ -325,24 +328,12 @@ public ReplicaEventListener replicaEvents() @Override public long slowCoordinatorDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int attempt) { - SafeCommand safeCommand = safeStore.unsafeGetNoCleanup(txnId); - if (safeCommand == null) - { - noSpamLogger.warn("{} invoked slowCoordinatorDelay for {} without having it in cache", safeStore.commandStore(), txnId, new RuntimeException()); - return recover(txnId).computeWait(attempt, units); - } - - Command command = safeCommand.current(); - if (command == null) - { - noSpamLogger.warn("{} invoked slowCoordinatorDelay for {} without knowing the command", safeStore.commandStore(), txnId, new RuntimeException()); - return recover(txnId).computeWait(attempt, units); - } - + Command command = command(safeStore, txnId, "slowCoordinatorDelay"); + Ballot promised = promised(command); // TODO (expected): make this a configurable calculation on normal request latencies (like ContentionStrategy) long nowMicros = MILLISECONDS.toMicros(Clock.Global.currentTimeMillis()); - long mostRecentStart = mostRecentStart(command, nowMicros); + long mostRecentStart = mostRecentStart(txnId, promised, nowMicros); long waitMicros = recover(txnId).computeWait(attempt, MICROSECONDS); long startTime = mostRecentStart + waitMicros; if (startTime < nowMicros) @@ -355,7 +346,7 @@ public long slowCoordinatorDelay(Node node, SafeCommandStore safeStore, TxnId tx } RoutingKey homeKey = command.route().homeKey(); - Shard shard = node.topology().active().forEpochIfKnown(homeKey, command.txnId().epoch()); + Shard shard = node.topology().active().forEpochIfKnown(homeKey, txnId.epoch()); startTime = nonClashingStartTime(startTime, shard == null ? null : shard.nodes, node.id(), ONE_SECOND, random); long delayMicros = Math.max(1, startTime - nowMicros); @@ -363,15 +354,15 @@ public long slowCoordinatorDelay(Node node, SafeCommandStore safeStore, TxnId tx return units.convert(delayMicros, MICROSECONDS); } - private static long mostRecentStart(Command command, long nowMicros) + private static long mostRecentStart(TxnId txnId, Ballot promised, long nowMicros) { // TODO (expected): make this a configurable calculation on normal request latencies (like ContentionStrategy) - long promisedHlc = command.promised().hlc(); + long promisedHlc = promised.hlc(); if (promisedHlc > nowMicros + ONE_MINUTE) promisedHlc = 0; - long result = Math.max(command.txnId().hlc(), promisedHlc); + long result = Math.max(txnId.hlc(), promisedHlc); if (result > nowMicros + ONE_SECOND) - noSpamLogger.warn("max({},{})>{}", command.txnId(), command.promised(), nowMicros); + noSpamLogger.warn("max({},{})>{}", txnId, promised, nowMicros); return result; } @@ -405,25 +396,36 @@ public static long nonClashingStartTime(long startTime, SortedList node return newStartTime; } - @Override - public long slowReplicaDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int attempt, BlockedUntil blockedUntil, TimeUnit units) + private static Ballot promised(@Nullable Command command) + { + return command == null ? Ballot.ZERO : command.promised(); + } + + private static Command command(SafeCommandStore safeStore, TxnId txnId, String source) { SafeCommand safeCommand = safeStore.unsafeGetNoCleanup(txnId); if (safeCommand == null) { - noSpamLogger.warn("{} invoked slowReplicaDelay for {} without having it in cache", safeStore.commandStore(), txnId, new RuntimeException()); - return fetch(txnId).computeWait(attempt, units); + noSpamLogger.warn("{} invoked {} for {} without having it in cache", safeStore.commandStore(), source, txnId, new RuntimeException()); + return null; } Command command = safeCommand.current(); if (command == null) { - noSpamLogger.warn("{} invoked slowReplicaDelay for {} without knowing the command", safeStore.commandStore(), txnId, new RuntimeException()); - return fetch(txnId).computeWait(attempt, units); + noSpamLogger.warn("{} invoked {} for {} without knowing the command", safeStore.commandStore(), source, txnId, new RuntimeException()); + return null; } + return command; + } + + @Override + public long slowReplicaDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int attempt, BlockedUntil blockedUntil, TimeUnit units) + { + Ballot promised = promised(command(safeStore, txnId, "slowReplicaDelay")); long nowMicros = MILLISECONDS.toMicros(Clock.Global.currentTimeMillis()); - long mostRecentStart = mostRecentStart(command, nowMicros); + long mostRecentStart = mostRecentStart(txnId, promised, nowMicros); long waitMicros = fetch(txnId).computeWait(attempt, units); long startTime = mostRecentStart + waitMicros; if (startTime < nowMicros) diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java b/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java index cf7d8ff5ae2a..eb1fcbfbfa79 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java @@ -38,7 +38,7 @@ public class AccordWaitStrategies { - static TimeoutStrategy slowTxnPreaccept, slowSyncPointPreaccept, slowRead; + static TimeoutStrategy slowTxnPreaccept, slowSyncPointPreaccept, slowRead, slowDurability; static TimeoutStrategy expireTxn, expireSyncPoint, expireDurability, expireEpochWait; static TimeoutStrategy fetchTxn, fetchSyncPoint; static RetryStrategy recoverTxn, recoverSyncPoint, retrySyncPoint, retryDurability, retryBootstrap, retryJoinBootstrap; @@ -88,6 +88,7 @@ public static RetryStrategy retryFetchTopology() setSlowRead(config.slow_read); setSlowTxnPreaccept(config.slow_txn_preaccept); setSlowSyncPointPreaccept(config.slow_syncpoint_preaccept); + setSlowDurability(config.slow_durability); setExpireTxn(config.expire_txn); setExpireSyncPoint(config.expire_syncpoint); setExpireDurability(config.expire_durability); @@ -119,6 +120,11 @@ public static void setSlowSyncPointPreaccept(String spec) slowSyncPointPreaccept = TimeoutStrategy.parse(spec, none()); } + public static void setSlowDurability(String spec) + { + slowDurability = TimeoutStrategy.parse(spec, none()); + } + public static void setExpireTxn(String spec) { expireTxn = TimeoutStrategy.parse(spec, rw(accordReadMetrics, accordWriteMetrics)); diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java index 697a094e608f..adaf531a5b1d 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java @@ -47,6 +47,7 @@ import org.apache.cassandra.service.accord.txn.AccordUpdate; import org.apache.cassandra.service.accord.txn.TxnRead; +import static accord.api.ProtocolModifiers.sendMinimal; import static accord.messages.Apply.Kind.Maximal; import static accord.messages.Apply.Kind.Minimal; @@ -55,12 +56,13 @@ public class AccordInteropAdapter extends TxnAdapter private static final Logger logger = LoggerFactory.getLogger(AccordInteropAdapter.class); public static final class AccordInteropFactory extends DefaultFactory { - final AccordInteropAdapter standard, recovery; + final AccordInteropAdapter minimal, maximal, recovery; public AccordInteropFactory(AccordEndpointMapper endpointMapper) { - standard = new AccordInteropAdapter(endpointMapper, Minimal); - recovery = new AccordInteropAdapter(endpointMapper, Maximal); + minimal = new AccordInteropAdapter(endpointMapper, Minimal, true); + maximal = new AccordInteropAdapter(endpointMapper, Maximal, true); + recovery = new AccordInteropAdapter(endpointMapper, Maximal, false); } @Override @@ -68,18 +70,18 @@ public CoordinationAdapter get(TxnId txnId, Kind step) { if (txnId.isSyncPoint()) return super.get(txnId, step); - return (CoordinationAdapter) (step == Kind.Recovery ? recovery : standard); + return (CoordinationAdapter) (step == Kind.Recovery ? recovery : sendMinimal() ? minimal : maximal); } }; private final AccordEndpointMapper endpointMapper; - private final Apply.Kind applyKind; + private final boolean isUserFacing; - private AccordInteropAdapter(AccordEndpointMapper endpointMapper, Apply.Kind applyKind) + private AccordInteropAdapter(AccordEndpointMapper endpointMapper, Apply.Kind applyKind, boolean isUserFacing) { - super(Minimal); + super(applyKind); this.endpointMapper = endpointMapper; - this.applyKind = applyKind; + this.isUserFacing = isUserFacing; } @Override @@ -92,7 +94,7 @@ public void execute(Node node, ExclusiveAsyncExecutor executor, Topologies any, @Override public void persist(Node node, ExclusiveAsyncExecutor executor, Topologies any, Route require, Route sendTo, FullRoute route, Ballot ballot, CoordinationFlags flags, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, boolean informDurableOnDone, BiConsumer callback) { - if (applyKind == Minimal && doInteropPersist(node, executor, any, require, sendTo, ballot, txnId, txn, executeAt, deps, writes, result, route, informDurableOnDone, callback)) + if (isUserFacing && doInteropPersist(node, executor, any, require, sendTo, ballot, txnId, txn, executeAt, deps, writes, result, route, informDurableOnDone, callback)) return; super.persist(node, executor, any, require, sendTo, route, ballot, flags, txnId, txn, executeAt, deps, writes, result, informDurableOnDone, callback); diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java index 262e98af4e62..874c2e305af3 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java @@ -94,6 +94,7 @@ import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.transport.Dispatcher; +import static accord.api.ProtocolModifiers.sendMinimal; import static accord.coordinate.CoordinationAdapter.Factory.Kind.Standard; import static accord.primitives.Txn.Kind.Write; import static accord.topology.SelectShards.LIVE; @@ -144,6 +145,7 @@ public AccordInteropExecution(Node node, TxnId txnId, Txn txn, AccordUpdate.Kind ExclusiveAsyncExecutor executor, ConsistencyLevel consistencyLevel, AccordEndpointMapper endpointMapper) throws TopologyException { requireArgument(!txn.read().keys().isEmpty() || updateKind == AccordUpdate.Kind.UNRECOVERABLE_REPAIR); + // TODO (required): this does not support privileged coordinator optimisation, as must apply local stable before any other message is sent this.node = node; this.txnId = txnId; this.txn = txn; @@ -217,8 +219,7 @@ public void sendReadCommand(Message message, InetAddressAndPort to, // TODO (desired): It would be better to use the re-use the command from the transaction but it's fragile // to try and figure out exactly what changed for things like read repair and short read protection // Also this read scope doesn't reflect the contents of this particular read and is larger than it needs to be - // TODO (required): understand interop and whether StableFastPath is appropriate - AccordInteropStableThenRead commit = new AccordInteropStableThenRead(id, allTopologies, txnId, Kind.StableFastPath, executeAt, txn, deps, route, message.payload); + AccordInteropStableThenRead commit = new AccordInteropStableThenRead(id, allTopologies, txnId, commitKind(), executeAt, txn, deps, route, message.payload); node.send(id, commit, executor, new AccordInteropRead.ReadCallback(id, to, message, callback, this), null); } @@ -376,7 +377,7 @@ private void sendStableToUncontacted() { InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(to); if (endpoint != null && !contacted.contains(endpoint)) - node.send(to, new Commit(Kind.StableFastPath, to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); + node.send(to, new Commit(commitKind(), to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); } } @@ -387,7 +388,7 @@ public void start() for (Node.Id to : allTopologies.nodes()) { if (!executeTopology.contains(to)) - node.send(to, new Commit(Kind.StableFastPath, to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); + node.send(to, new Commit(commitKind(), to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); } } AsyncChain result; @@ -415,6 +416,11 @@ public void start() }); } + private static Commit.Kind commitKind() + { + return sendMinimal() ? Kind.StableFastPath : Kind.StableWithTxnAndDeps; + } + private AsyncChain executeUnrecoverableRepairUpdate() { return AsyncChains.chain(Stage.ACCORD_MIGRATION.executor(), () -> { @@ -423,7 +429,7 @@ private AsyncChain executeUnrecoverableRepairUpdate() // and can be extended similar to MessageType which allows additional types not from Accord to be added // This commit won't necessarily execute before the interop read repair message so there could be an insufficient which is fine for (Node.Id to : executeTopology.nodes()) - node.send(to, new Commit(Kind.StableFastPath, to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); + node.send(to, new Commit(commitKind(), to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps), null); repairUpdate.runBRR(AccordInteropExecution.this); return new TxnData(); }); diff --git a/src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java b/src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java index f2caf355371a..32f17625fd62 100644 --- a/src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java +++ b/src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java @@ -93,6 +93,7 @@ boolean considerWritingKey() // Only valid in the scope of a single `compact` call private JournalKey prevKey; private DecoratedKey prevDecoratedKey; + private volatile boolean stopped; @Override public Collection> compact(Collection> segments) @@ -129,6 +130,9 @@ public Collection> compact(Collection reader; while ((reader = readers.poll()) != null) { + if (stopped) + throw new Stopped(); + if (key == null || !reader.key().equals(key)) { maybeWritePartition(key, merger, serializer, firstDescriptor, firstOffset); @@ -232,6 +236,12 @@ private void maybeWritePartition(JournalKey key, Merger merger, MergeSerializer< } } + @Override + public void stop() + { + stopped = true; + } + private static int normalize(int cmp) { if (cmp == 0) diff --git a/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java index 0b3a75f90cb6..7c19d05d0799 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java @@ -39,6 +39,7 @@ import accord.local.MaxConflicts; import accord.local.MaxDecidedRX; import accord.local.RedundantBefore; +import accord.local.RedundantStatus; import accord.primitives.Range; import accord.primitives.Ranges; import accord.primitives.SaveStatus; @@ -274,7 +275,9 @@ public void serialize(RedundantBefore.Bounds b, DataOutputPlus out) throws IOExc private short cast(long v) { - if ((v & ~0xFFFF) != 0) + if (v < 0 || v >= (1 << RedundantStatus.Property.WAS_OWNED.ordinal())) + throw new IllegalStateException("Should not be serializing WAS_OWNED or NOT_OWNED statuses"); + if ((v & ~0xFFFF) != 0) // retain this throw new IllegalStateException("Cannot serialize RedundantStatus larger than 0xFFFF. Requires serialization changes."); return (short)v; } diff --git a/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java index 2acd8e2464e3..c4137805460d 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java @@ -25,7 +25,11 @@ import accord.impl.AbstractFetchCoordinator.FetchResponse; import accord.messages.ReadData.CommitOrReadNack; import accord.messages.ReadData.ReadReply; +import accord.primitives.PartialDeps; +import accord.primitives.PartialTxn; import accord.primitives.Ranges; +import accord.primitives.TxnId; +import accord.utils.Invariants; import accord.utils.VIntCoding; import org.apache.cassandra.db.TypeSizes; @@ -51,18 +55,20 @@ public void serialize(FetchRequest request, DataOutputPlus out, Version version) out.writeUnsignedVInt(request.executeAtEpoch); CommandSerializers.txnId.serialize(request.txnId, out); KeySerializers.ranges.serialize((Ranges) request.scope, out); - DepsSerializers.partialDeps.serialize(request.partialDeps, out); + DepsSerializers.partialDeps.serialize(PartialDeps.NONE, out); StreamingTxn.serializer.serialize(request.read, out, version); } @Override public FetchRequest deserialize(DataInputPlus in, Version version) throws IOException { - return new AccordFetchRequest(in.readUnsignedVInt(), - CommandSerializers.txnId.deserialize(in), - KeySerializers.ranges.deserialize(in), - DepsSerializers.partialDeps.deserialize(in), - StreamingTxn.serializer.deserialize(in, version)); + long epoch = in.readUnsignedVInt(); + TxnId txnId = CommandSerializers.txnId.deserialize(in); + Ranges ranges = KeySerializers.ranges.deserialize(in); + PartialDeps deps = DepsSerializers.partialDeps.deserialize(in); + Invariants.require(deps.isEmpty()); + PartialTxn txn = StreamingTxn.serializer.deserialize(in, version); + return new AccordFetchRequest(epoch, txnId, ranges, txn); } @Override @@ -71,7 +77,7 @@ public long serializedSize(FetchRequest request, Version version) return TypeSizes.sizeofUnsignedVInt(request.executeAtEpoch) + CommandSerializers.txnId.serializedSize(request.txnId) + KeySerializers.ranges.serializedSize((Ranges) request.scope) - + DepsSerializers.partialDeps.serializedSize(request.partialDeps) + + DepsSerializers.partialDeps.serializedSize(PartialDeps.NONE) + StreamingTxn.serializer.serializedSize(request.read, version); } }; diff --git a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java index 07147fac1783..1629b94fa3f7 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java +++ b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java @@ -74,6 +74,7 @@ import org.apache.cassandra.utils.concurrent.FutureCombiner; import org.apache.cassandra.utils.vint.VIntCoding; +import static accord.local.BootstrapReason.GAIN_OWNERSHIP; import static com.google.common.collect.ImmutableList.of; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.cassandra.tcm.MultiStepOperation.Kind.JOIN; @@ -396,7 +397,7 @@ public static boolean bootstrap(final Collection tokens, Node node = service.node(); node.commandStores().forAllUnsafe(commandStore -> { - AsyncResult ready = commandStore.resumeBootstrap(node); + AsyncResult ready = commandStore.resumeBootstrap(node, GAIN_OWNERSHIP); bootstraps.add(AccordService.toFuture(ready.flatMap(e -> e.reads))); }); } diff --git a/test/conf/logback-info.xml b/test/conf/logback-info.xml new file mode 100644 index 000000000000..bc121ca6490f --- /dev/null +++ b/test/conf/logback-info.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + ./build/test/logs/${cassandra.testtag}/${suitename}/${cluster_id}/${instance_id}/system.log + + %-5level [%thread] ${instance_id} %date{"yyyy-MM-dd'T'HH:mm:ss,SSS", UTC} %F:%L - %msg%n + + + INFO + + true + + + + + %-5level %date{"HH:mm:ss,SSS"} %msg%n + + + INFO + + + + + + + + + + diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java index f38b84ede287..03c16edd2a56 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java @@ -81,6 +81,7 @@ import org.apache.cassandra.distributed.api.QueryResults; import org.apache.cassandra.distributed.api.SimpleQueryResult; import org.apache.cassandra.distributed.shared.AssertUtils; +import org.apache.cassandra.distributed.shared.FutureUtils; import org.apache.cassandra.distributed.test.sai.SAIUtil; import org.apache.cassandra.distributed.util.QueryResultUtil; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -93,6 +94,7 @@ import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode; import org.apache.cassandra.utils.AssertionUtils; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.FailingConsumer; import org.apache.cassandra.utils.Pair; @@ -116,8 +118,10 @@ public abstract class AccordCQLTestBase extends AccordTestBase { private static final Logger logger = LoggerFactory.getLogger(AccordCQLTestBase.class); + private static final int CAS_SIMULATOR_LITE_CONCURRENCY = 20; - protected AccordCQLTestBase(TransactionalMode transactionalMode) { + protected AccordCQLTestBase(TransactionalMode transactionalMode) + { super(transactionalMode); } @@ -131,7 +135,9 @@ protected Logger logger() public static void setupClass() throws IOException { AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL) - .set("paxos_variant", PaxosVariant.v2.name())), 2); + .set("paxos_variant", PaxosVariant.v2.name()) + .set("accord.migration_concurrency", "" + (1 + CAS_SIMULATOR_LITE_CONCURRENCY)) + ), 2); SHARED_CLUSTER.schemaChange("CREATE TYPE " + KEYSPACE + ".person (height int, age int)"); } @@ -3332,15 +3338,18 @@ public void testCASSimulatorLite() throws Exception coordinator.execute("INSERT INTO " + qualifiedAccordTableName + " (pk, count, seq1, seq2) VALUES (1, 0, '', []) USING TIMESTAMP 0", ConsistencyLevel.ALL); ListType LIST_TYPE = ListType.getInstance(Int32Type.instance, true); - ExecutorService es = Executors.newCachedThreadPool(); - List> futures = new ArrayList<>(); - for (int ii = 0; ii < 10; ii++) + Future[] futures = new Future[CAS_SIMULATOR_LITE_CONCURRENCY]; + long[] starts = new long[CAS_SIMULATOR_LITE_CONCURRENCY]; + for (int id = 0; id < CAS_SIMULATOR_LITE_CONCURRENCY; id++) { - int id = ii; - futures.add(es.submit(() -> coordinator.execute("UPDATE " + qualifiedAccordTableName + " SET count = count + 1, seq1 = seq1 + ?, seq2 = seq2 + ? WHERE pk = ? IF EXISTS", ConsistencyLevel.ALL, id + ",", ByteBufferUtil.getArray(LIST_TYPE.decompose(singletonList(id))), 1))); + starts[id] = Clock.Global.nanoTime(); + futures[id] = FutureUtils.map(coordinator.asyncExecuteWithResult("UPDATE " + qualifiedAccordTableName + " SET count = count + 1, seq1 = seq1 + ?, seq2 = seq2 + ? WHERE pk = ? IF EXISTS", ConsistencyLevel.ALL, id + ",", ByteBufferUtil.getArray(LIST_TYPE.decompose(singletonList(id))), 1), SimpleQueryResult::toObjectArrays); + } + for (int id = 0; id < CAS_SIMULATOR_LITE_CONCURRENCY; id++) + { + futures[id].get(); + System.out.println(String.format("Completed in %.3fs", (Clock.Global.nanoTime() - starts[id]) * 0.000000001)); } - for (Future f : futures) - f.get(); Object[][] result = coordinator.execute("SELECT pk, count, seq1, seq2 FROM " + qualifiedAccordTableName + " WHERE pk = 1", ConsistencyLevel.SERIAL); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java index d3c1fadfc4e4..1802efa82a74 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java @@ -89,7 +89,7 @@ public BarrierRecordingService(IAccordService delegate) } @Override - public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) + public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ranges ranges, @Nullable Collection include, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote, long timeout, TimeUnit timeoutUnits) { return delegate.sync(requestedBy, onOrAfter, ranges, include, syncLocal, syncRemote, 10L, TimeUnit.MINUTES).map(v -> { executedBarriers = true; @@ -98,7 +98,7 @@ public AsyncResult sync(Object requestedBy, @Nullable TxnId onOrAfter, Ran } @Override - public AsyncChain sync(@Nullable TxnId onOrAfter, Keys keys, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote) + public AsyncChain sync(@Nullable TxnId onOrAfter, Keys keys, DurabilityService.SyncLocal syncLocal, DurabilityService.SyncRemote syncRemote) { return delegate.sync(onOrAfter, keys, syncLocal, syncRemote).map(v -> { executedBarriers = true; diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java index 763cbd4d5a97..c3ec06b6a74f 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java @@ -170,7 +170,12 @@ public void tearDown() throws Exception { SHARED_CLUSTER.filters().reset(); for (IInvokableInstance instance : SHARED_CLUSTER) - instance.runOnInstance(() -> AccordService.instance().node().commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start())); + { + instance.runOnInstance(() -> { + if (AccordService.isStarted()) + AccordService.instance().node().commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start()); + }); + } truncateSystemTables(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordLoadTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordLoadTestBase.java new file mode 100644 index 000000000000..fc1763a27323 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordLoadTestBase.java @@ -0,0 +1,976 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.load; + +import java.io.IOException; +import java.security.SecureRandom; +import java.time.Duration; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceArray; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Snapshot; +import com.codahale.metrics.Timer; +import com.google.common.util.concurrent.RateLimiter; + +import org.agrona.collections.IntArrayList; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import accord.local.Catchup; +import accord.local.CommandStore; +import accord.local.Node; +import accord.local.ExecutionContext; +import accord.local.SafeCommand; +import accord.primitives.PartialDeps; +import accord.primitives.TxnId; +import accord.utils.Functions; +import accord.utils.Invariants; +import accord.utils.UnhandledEnum; + +import org.apache.cassandra.concurrent.NamedThreadFactory; +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.ICoordinator; +import org.apache.cassandra.distributed.api.IInstance; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IIsolatedExecutor; +import org.apache.cassandra.distributed.api.IMessage; +import org.apache.cassandra.distributed.api.IMessageFilters; +import org.apache.cassandra.distributed.test.accord.AccordTestBase; +import org.apache.cassandra.distributed.test.accord.load.LoadSettings.ClusterChaos; +import org.apache.cassandra.metrics.AccordCoordinatorMetrics; +import org.apache.cassandra.metrics.AccordExecutorMetrics; +import org.apache.cassandra.metrics.ShardedDecayingHistograms.ShardedDecayingHistogram; +import org.apache.cassandra.metrics.ShardedHistogram; +import org.apache.cassandra.metrics.SnapshottingTimer; +import org.apache.cassandra.net.ArtificialLatency; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.service.accord.AccordKeyspace; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.api.AccordAgent; +import org.apache.cassandra.service.accord.debug.AccordTracing; +import org.apache.cassandra.service.accord.debug.AccordTracing.Message; +import org.apache.cassandra.service.accord.debug.CoordinationKinds; +import org.apache.cassandra.service.accord.debug.TxnKindsAndDomains; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.EstimatedHistogram; +import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; +import org.apache.cassandra.utils.concurrent.WaitQueue; + +import static accord.coordinate.Coordination.CoordinationKind.Client; +import static accord.coordinate.Coordination.CoordinationKind.Execute; +import static accord.local.BootstrapReason.LOG_CORRUPTED; +import static accord.local.BootstrapReason.LOG_INCOMPLETE; +import static java.lang.System.currentTimeMillis; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS; +import static org.apache.cassandra.distributed.test.accord.load.LoadSettings.ClusterChaos.REBOOTSTRAP_RESET; +import static org.apache.cassandra.service.accord.debug.AccordTracing.BucketMode.LEAKY; +import static org.apache.cassandra.service.accord.debug.AccordTracing.BucketMode.RING; +import static org.apache.cassandra.service.accord.debug.AccordTracing.BucketMode.SLOWEST; + +public class AccordLoadTestBase extends AccordTestBase +{ + private static long CHAOS_TIMEOUT_NANOS = TimeUnit.MINUTES.toNanos(10L); + private static final Logger logger = LoggerFactory.getLogger(AccordLoadTestBase.class); + + @Before + public void setup() + { + setupCluster(); + super.setup(); + } + + public void setupCluster() + { + setupCluster(5); + } + + public void setupCluster(int nodeCount) + { + setupCluster(nodeCount, config -> { + config.with(Feature.NETWORK, Feature.GOSSIP) + .set("accord.shard_durability_target_splits", "8") + .set("accord.shard_durability_max_splits", "16") + .set("accord.shard_durability_cycle", "1m") + .set("accord.queue_submission_model", "SIGNAL") + .set("accord.command_store_shard_count", "8") + .set("accord.queue_thread_count", "4") + .set("accord.queue_shard_count", "1") + .set("accord.replica_execution", "ALL") + .set("accord.send_stable", "TO_ALL_REPLICA_EXECUTABLE_ELSE_FOR_READS") + .set("accord.send_minimal", "false") + .set("accord.catchup_on_start_fail_latency", "2m"); + }); + } + + public void setupCluster(int nodeCount, Consumer configure) + { + Invariants.require(SHARED_CLUSTER == null); + try { SHARED_CLUSTER = createCluster(nodeCount, builder -> builder.withDCs(nodeCount).withConfig(configure)); } + catch (IOException e) { throw new RuntimeException(e); } + CassandraRelevantProperties.SIMULATOR_STARTED.setString(Long.toString(MILLISECONDS.toSeconds(currentTimeMillis()))); + } + + public void testLoad(final LoadSettings settings) throws Exception + { + Cluster cluster = SHARED_CLUSTER; + cluster.schemaChange("CREATE TABLE " + qualifiedAccordTableName + " (k int, v int, PRIMARY KEY(k)) WITH transactional_mode = 'full'"); +// long seed = new SecureRandom().nextLong(); + long seed = 3705626102508196273L; + try + { + final ConcurrentHashMap verbs = new ConcurrentHashMap<>(); + cluster.filters().outbound().messagesMatching(new IMessageFilters.Matcher() + { + @Override + public boolean matches(int i, int i1, IMessage iMessage) + { + verbs.computeIfAbsent(Verb.fromId(iMessage.verb()), ignore -> new AtomicInteger()).incrementAndGet(); + return false; + } + }).drop(); + + boolean waitForTransactions = settings.totalTransactions < Long.MAX_VALUE; + boolean waitForClusterChaos = settings.totalClusterChaos < Long.MAX_VALUE; + + int clientCount = settings.clients < 0 ? cluster.size() : settings.clients; + long nextRepairAt = settings.repairInterval; + long nextCompactionAt = settings.compactionInterval; + long nextJournalFlushAt = settings.journalFlushInterval; + long nextDataFlushAt = settings.dataFlushInterval; + long nextCfkFlushAt = settings.cfkFlushInterval; + long nextChaosAt = settings.clusterChaosInterval; + final ExecutorService chaosExecutor = Executors.newFixedThreadPool(settings.clusterChaosConcurrency, new NamedThreadFactory("ClusterChaos")); + final ExecutorService clientExecutor = Executors.newFixedThreadPool(clientCount, new NamedThreadFactory("Client")); + final BitSet initialised = new BitSet(); + final Supplier clusterChaos = settings.clusterChaos.apply(seed); + + List chaosActive = new ArrayList<>(); + cluster.get(1).nodetoolResult("cms", "reconfigure", "datacenter1:1", "datacenter2:1", "datacenter3:1").asserts().success(); + if (settings.cfkCompactionPeriodSeconds < Integer.MAX_VALUE && settings.cfkCompactionPeriodSeconds > 0) + { + cluster.forEach(i -> i.acceptOnInstance(period -> { + ((AccordService) AccordService.instance()).journal().compactor().updateCompactionPeriod(period, SECONDS); + }, settings.cfkCompactionPeriodSeconds)); + } + + if (settings.artificialLatencies != null) + { + for (int i = 0 ; i < cluster.size() ; ++i) + { + StringBuilder str = new StringBuilder(); + for (int j = 0 ; j < settings.artificialLatencies[i].length ; ++j) + { + if (j > 0) + str.append(","); + str.append("datacenter") + .append(j + 1) + .append(':') + .append(settings.artificialLatencies[i][j]) + .append("ms"); + } + cluster.get(i + 1).acceptOnInstance(latencies -> { + ArtificialLatency.setArtificialLatencies(latencies); + ArtificialLatency.setArtificialLatencyOnlyPermittedConsistencyLevels(false); + ArtificialLatency.setArtificialLatencyVerbs(ArtificialLatency.recommendedVerbs()); + ArtificialLatency.setEnabled(true); + }, str.toString()); + } + } + + if (settings.traceSlowest > 0f) + { + float traceSlowest = settings.traceSlowest; + for (int i = 0 ; i < cluster.size() ; ++i) + { + cluster.get(i + 1).runOnInstance(() -> { + AccordTracing tracing = ((AccordAgent) AccordService.unsafeInstance().agent()).tracing(); + tracing.setPattern(1, pattern -> pattern.withChance(traceSlowest) + .withKinds(TxnKindsAndDomains.parse("{K*}")) + .withTraceNew(CoordinationKinds.ALL), + SLOWEST, -1, 2, LEAKY, 10, 1, CoordinationKinds.ALL); + }); + } + } + + if (settings.traceLast > 0) + { + int traceLast = settings.traceLast; + for (int i = 0 ; i < cluster.size() ; ++i) + { + cluster.get(i + 1).runOnInstance(() -> { + AccordTracing tracing = ((AccordAgent) AccordService.unsafeInstance().agent()).tracing(); + tracing.setPattern(2, pattern -> pattern.withKinds(TxnKindsAndDomains.parse("{KW}")) + .withTraceNew(CoordinationKinds.ALL), + RING, -1, traceLast, LEAKY, 10, 1, CoordinationKinds.ALL); + }); + } + } + + final AtomicBoolean stop = new AtomicBoolean(); + final AtomicBoolean pauseOrStop = new AtomicBoolean(); + final WaitQueue waitQueue = WaitQueue.newWaitQueue(); + final Random random = new Random(); + final Semaphore completed = new Semaphore(0); + final AtomicIntegerArray coordinatorIndexes = new AtomicIntegerArray(clientCount); + final Set chaosCandidates = new ConcurrentSkipListSet<>(); + for (int i = 1; i <= cluster.size() ; ++i) + chaosCandidates.add(i); + final List> clients = new ArrayList<>(); + final AtomicReferenceArray rateLimiters = new AtomicReferenceArray<>(clientCount); + final AtomicReference readHistogram = new AtomicReference<>(new EstimatedHistogram(200)); + final AtomicReference writeHistogram = new AtomicReference<>(new EstimatedHistogram(200)); + final List chaosHistory = new ArrayList<>(); + if (settings.clients >= cluster.size()) + throw new IllegalArgumentException("Cannot have more clients than nodes"); + if (settings.clusterChaosInterval < Integer.MAX_VALUE && settings.clients + 1 >= cluster.size()) + throw new IllegalArgumentException("If restarting, cannot have as many clients as nodes, as must reroute client requests during restart"); + + int clientRatePerSecond = Math.min(settings.ratePerSecond, settings.minRatePerSecond) / clientCount; + for (int client = 0 ; client < clientCount ; ++client) + { + rateLimiters.set(client, RateLimiter.create(clientRatePerSecond)); + final int clientIndex = client; + coordinatorIndexes.set(client, client + 1); + clients.add(clientExecutor.submit(() -> { + final Semaphore inFlight = new Semaphore(settings.clientConcurrency); + while (true) + { + while (pauseOrStop.get()) + { + if (stop.get()) + break; + + WaitQueue.Signal signal = waitQueue.register(); + if (pauseOrStop.get()) signal.awaitThrowUncheckedOnInterrupt(); + else signal.cancel(); + } + + int coordinatorIdx = coordinatorIndexes.get(clientIndex); + ICoordinator coordinator = cluster.coordinator(coordinatorIdx); + try + { + rateLimiters.get(clientIndex).acquire(); + inFlight.acquire(); + long commandStart = System.nanoTime(); + IntArrayList keys = new IntArrayList(settings.keysPerOperation, -1); + for (int i = 0 ; i < settings.keysPerOperation ; ++i) + { + int k = settings.keySelector.getAsInt(); + if (!keys.containsInt(k)) + keys.add(k); + } + if (!keys.intStream().allMatch(initialised::get)) + { + coordinator.executeWithResult((success, fail) -> { + inFlight.release(); + completed.release(); + if (fail == null) + { + long elapsed = System.nanoTime() - commandStart; + writeHistogram.get().add(NANOSECONDS.toMicros(elapsed)); + synchronized (initialised) + { + keys.forEachInt(initialised::set); + } + } + else + { + logger.error("{}", fail.toString()); + } + }, "UPDATE " + qualifiedAccordTableName + " SET v = 0 WHERE k IN ?", ConsistencyLevel.SERIAL, ConsistencyLevel.QUORUM, keys); + } + else if (random.nextFloat() < settings.readRatio) + { + coordinator.executeWithResult((success, fail) -> { + inFlight.release(); + completed.release(); + if (fail == null) + readHistogram.get().add(NANOSECONDS.toMicros(System.nanoTime() - commandStart)); + }, "BEGIN TRANSACTION\n" + + "SELECT * FROM " + qualifiedAccordTableName + " WHERE k IN ?;\n" + + "COMMIT TRANSACTION;", ConsistencyLevel.SERIAL, keys + ); + } + else + { + coordinator.executeWithResult((success, fail) -> { + inFlight.release(); + completed.release(); + if (fail == null) + { + long elapsed = System.nanoTime() - commandStart; + writeHistogram.get().add(NANOSECONDS.toMicros(elapsed)); + } + else + logger.error("{}", fail.toString()); + }, "BEGIN TRANSACTION\n" + + // "UPDATE " + qualifiedAccordTableName + " SET v = ? WHERE k = ?;\n" + + "UPDATE " + qualifiedAccordTableName + " SET v += ? WHERE k IN ?;\n" + + "COMMIT TRANSACTION;", ConsistencyLevel.SERIAL, ConsistencyLevel.QUORUM, random.nextInt(100), keys); + } + } + catch (RejectedExecutionException e) + { + inFlight.release(); + } + catch (InterruptedException e) + { + throw new UncheckedInterruptedException(e); + } + } + })); + } + + int targetClientRatePerSecond = settings.ratePerSecond / clientCount; + int nextRateLimitIncrease = settings.increaseRatePerSecondInterval; + long remainingTransactions = settings.totalTransactions; + int remainingClusterChaos = settings.totalClusterChaos; + while (true) + { + long batchStart = System.nanoTime(); + int batchSize = 0; + + if (completed.tryAcquire(settings.batchSize, settings.batchPeriodNanos, NANOSECONDS)) + batchSize = settings.batchSize; + batchSize += completed.drainPermits(); + + if (clientRatePerSecond < targetClientRatePerSecond) + { + if ((nextRateLimitIncrease -= batchSize) <= 0) + { + clientRatePerSecond = Math.min(clientRatePerSecond * 2, targetClientRatePerSecond); + for (int i = 0 ; i < clientCount ; ++i) + rateLimiters.set(i, RateLimiter.create(clientRatePerSecond)); + nextRateLimitIncrease = settings.increaseRatePerSecondInterval; + } + } + + if ((nextRepairAt -= batchSize) <= 0) + { + nextRepairAt += settings.repairInterval; + System.out.println("repairing..."); + cluster.coordinator(1).instance().nodetool("repair", qualifiedAccordTableName); + } + + if ((nextCompactionAt -= batchSize) <= 0) + { + nextCompactionAt += settings.compactionInterval; + compactJournalCfs(cluster); + } + + if ((nextJournalFlushAt -= batchSize) <= 0) + { + nextJournalFlushAt += settings.journalFlushInterval; + flushJournal(cluster); + } + + if ((nextDataFlushAt -= batchSize) <= 0) + { + nextDataFlushAt += settings.dataFlushInterval; + flushData(cluster); + } + + if ((nextCfkFlushAt -= batchSize) <= 0) + { + nextCfkFlushAt += settings.cfkFlushInterval; + flushCfk(cluster); + } + + if ((nextChaosAt -= batchSize) <= 0) + { + Iterator iter = chaosActive.iterator(); + while (iter.hasNext()) + { + ChaosActive chaos = iter.next(); + if (chaos.future.isDone()) + { + chaos.future.get(); + iter.remove(); + Invariants.require(cluster.size() <= chaosActive.size() + chaosCandidates.size()); + } + else + { + long elapsedNanos = Clock.Global.nanoTime() - chaos.startedAt; + if (elapsedNanos >= CHAOS_TIMEOUT_NANOS) + throw new AssertionError("Chaos " + chaos.kind + " has been running for " + NANOSECONDS.toSeconds(elapsedNanos) + "s with seed " + seed); + } + } + + if (chaosActive.size() < settings.clusterChaosConcurrency) + { + if (remainingClusterChaos > 0) + { + --remainingClusterChaos; + nextChaosAt += settings.clusterChaosInterval; + ClusterChaos chaos = clusterChaos.get(); + chaosActive.add(new ChaosActive(chaos, chaos(cluster, coordinatorIndexes, chaosCandidates, chaosExecutor, random, chaos, chaosHistory))); + } + else if (chaosActive.isEmpty() && (remainingTransactions <= 0 || !waitForTransactions)) + { + break; + } + } + } + + if ((remainingTransactions -= batchSize) <= 0 && (remainingClusterChaos <= 0 || !waitForClusterChaos)) + break; + + long nowMillis = System.currentTimeMillis(); + EstimatedHistogram reads = readHistogram.getAndSet(new EstimatedHistogram(200)); + EstimatedHistogram writes = writeHistogram.getAndSet(new EstimatedHistogram(200)); + + maybePrintSlowestTraces(cluster, settings); + maybePrintLastTraces(cluster, pauseOrStop, settings, waitQueue); + printInstanceMetrics(nowMillis, cluster); + printRates(nowMillis, batchStart, batchSize, reads, writes); + printVerbs(nowMillis, verbs); + } + } + catch (Throwable t) + { + t.printStackTrace(); + System.exit(1); + } + + logger.info("Workload completed successfully"); + } + + static void safeForEach(Cluster cluster, IIsolatedExecutor.SerializableRunnable run) + { + safeForEach(cluster, ignore -> run.run(), null); + } + + static

void safeForEach(Cluster cluster, IIsolatedExecutor.SerializableConsumer

consumer, P param) + { + for (IInvokableInstance i : cluster) + { + try + { + if (!i.isShutdown()) + { + i.acceptOnInstance(consumer, param); + } + } + catch (Throwable t) + { + logger.error("", t); + } + } + } + + private void compactJournalCfs(Cluster cluster) + { + System.out.println("compacting journal cfs..."); + for (IInvokableInstance i : cluster) + { + try { i.nodetool("compact", "system_accord.journal"); } + catch (Throwable t) { logger.error("", t); } + } + } + + private void flushJournal(Cluster cluster) + { + System.out.println("flushing journal..."); + safeForEach(cluster, () -> { + if (AccordService.started()) + ((AccordService) AccordService.instance()).journal().closeCurrentSegmentForTestingIfNonEmpty(); + }); + } + + private void flushData(Cluster cluster) + { + System.out.println("flushing data..."); + safeForEach(cluster, name -> { + Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata(KEYSPACE, name).id).forceFlush(UNIT_TESTS); + }, accordTableName); + } + + private void flushCfk(Cluster cluster) + { + System.out.println("flushing cfk..."); + safeForEach(cluster, () -> { + if (CommitLog.instance.isStarted()) + AccordKeyspace.AccordColumnFamilyStores.commandsForKey.forceFlush(UNIT_TESTS); + }); + } + + private static class ChaosActive + { + final ClusterChaos kind; + final Future future; + final long startedAt; + + private ChaosActive(ClusterChaos kind, Future future) + { + this.kind = kind; + this.future = future; + this.startedAt = Clock.Global.nanoTime(); + } + } + + private static Future chaos(Cluster cluster, AtomicIntegerArray coordinatorIndexes, Set candidates, ExecutorService chaosExecutor, Random random, ClusterChaos chaos, List history) + { + List snapshot = new ArrayList<>(candidates); + int nodeIdx; + { + int i = random.nextInt(snapshot.size()); + Integer remove = snapshot.get(i); + candidates.remove(remove); + snapshot = new ArrayList<>(candidates); + Invariants.require(snapshot.size() > 0); + nodeIdx = remove; + } + + for (int i = 0; i < coordinatorIndexes.length(); ++i) + { + if (nodeIdx == coordinatorIndexes.get(i)) + { + int j = random.nextInt(snapshot.size()); + int replaceIdx = snapshot.get(j); + coordinatorIndexes.set(j, replaceIdx); + } + } + + String describe = String.format("%s node %d...", chaos, nodeIdx); + history.add(describe); + System.out.println("========= BEGIN CHAOS =========="); + System.out.println(describe); + System.out.println(candidates); + System.out.println("========= BEGIN CHAOS =========="); + switch (chaos) + { + default: throw UnhandledEnum.unknown(chaos); + case REBOOTSTRAP_INCOMPLETE: + case REBOOTSTRAP_RESET: + { + IInvokableInstance node = cluster.get(nodeIdx); + return node.asyncAcceptsOnInstance((Set cnds) -> { + try + { + Node accordNode = AccordService.instance().node(); + AccordService.getBlocking(accordNode.commandStores().rebootstrap(accordNode, chaos == REBOOTSTRAP_RESET ? LOG_CORRUPTED : LOG_INCOMPLETE)); + } + finally + { + Invariants.require(cnds.add(nodeIdx)); + System.out.println("========== END CHAOS ==========="); + System.out.println(describe); + System.out.println("========== END CHAOS ==========="); + } + }).apply(candidates); + } + case REBOOTSTRAP_IF_BEHIND: + { + IInvokableInstance node = cluster.get(nodeIdx); + return node.asyncAcceptsOnInstance((Set cmds) -> { + try + { + AccordService.getBlocking(Catchup.rebootstrapIfBehind(AccordService.instance().node())); + } + finally + { + Invariants.require(cmds.add(nodeIdx)); + System.out.println("========== END CHAOS ==========="); + System.out.println(String.format("%s node %d...", chaos, nodeIdx)); + System.out.println("========== END CHAOS ==========="); + } + }).apply(candidates); + } + case RESTART: + case RESTART_AND_REBOOTSTRAP_INCOMPLETE: + case RESTART_AND_REBOOTSTRAP_RESET: + case RESTART_AND_REBOOTSTRAP_AFTER_TIMEOUT: + { + return chaosExecutor.submit(() -> { + IInvokableInstance node = cluster.get(nodeIdx); + try + { + node.shutdown().get(); + switch (chaos) + { + case RESTART_AND_REBOOTSTRAP_AFTER_TIMEOUT: + node.config().set("accord.catchup_on_start_on_timeout", "REBOOTSTRAP"); + node.config().set("accord.catchup_on_start_success_latency", "0s"); + node.config().set("accord.catchup_on_start_fail_latency", "0s"); + break; + case RESTART_AND_REBOOTSTRAP_INCOMPLETE: + node.config().set("accord.journal.replay", "REBOOTSTRAP_INCOMPLETE"); + break; + case RESTART_AND_REBOOTSTRAP_RESET: + node.config().set("accord.journal.replay", "REBOOTSTRAP_RESET"); + break; + } + node.startup(); + return null; + } + catch (InterruptedException | ExecutionException e) + { + throw new RuntimeException(e); + } + finally + { + Invariants.require(candidates.add(nodeIdx)); + node.config().set("accord.catchup_on_start_success_latency", "60s"); + node.config().set("accord.catchup_on_start_fail_latency", "120s"); + node.config().set("accord.catchup_on_start_on_timeout", "IGNORE"); + node.config().set("accord.journal.replay", "PART_NON_DURABLE"); + System.out.println("========== END CHAOS ==========="); + System.out.println(String.format("%s node %d...", chaos, nodeIdx)); + System.out.println("========== END CHAOS ==========="); + } + }); + } + } + } + + + private static void printRates(long nowMillis, long batchStartNanos, long batchSize, EstimatedHistogram reads, EstimatedHistogram writes) + { + System.out.println(String.format("%tT.%tL rate: %.2f/s (%d total)", nowMillis, nowMillis, (((float)batchSize * 1000) / NANOSECONDS.toMillis(System.nanoTime() - batchStartNanos)), batchSize)); + System.out.println(String.format("%tT.%tL reads : %d %d %d %d %d %d", nowMillis, nowMillis, reads.percentile(.25)/1000, reads.percentile(.5)/1000, reads.percentile(.95)/1000, reads.percentile(.99)/1000, reads.percentile(.999)/1000, reads.percentile(1)/1000)); + System.out.println(String.format("%tT.%tL writes: %d %d %d %d %d %d", nowMillis, nowMillis, writes.percentile(.25)/1000, writes.percentile(.5)/1000, writes.percentile(.95)/1000, writes.percentile(.99)/1000, writes.percentile(.999)/1000, writes.percentile(1)/1000)); + } + + private static void printInstanceMetrics(long nowMillis, Cluster cluster) + { + safeForEach(cluster, () -> { + refresh(AccordExecutorMetrics.INSTANCE.elapsedRunning); + refresh(AccordExecutorMetrics.INSTANCE.elapsed); + System.out.println(String.format("%tT.%tL (%d %d %d %d %d %d)ms (%d %d %d %d %d %d)ms (%d %d %d %d %.0f, %d %d %d)us %d %d %d", nowMillis, nowMillis, + getLatency(AccordCoordinatorMetrics.readMetrics.preacceptLatency, 0.5), + getLatency(AccordCoordinatorMetrics.readMetrics.executeLatency, 0.5), + getLatency(AccordCoordinatorMetrics.readMetrics.applyLatency, 0.5), + getLatency(AccordCoordinatorMetrics.readMetrics.preacceptLatency, 0.999), + getLatency(AccordCoordinatorMetrics.readMetrics.executeLatency, 0.999), + getLatency(AccordCoordinatorMetrics.readMetrics.applyLatency, 0.999), + getLatency(AccordCoordinatorMetrics.writeMetrics.preacceptLatency, 0.95), + getLatency(AccordCoordinatorMetrics.writeMetrics.executeLatency, 0.5), + getLatency(AccordCoordinatorMetrics.writeMetrics.applyLatency, 0.5), + getLatency(AccordCoordinatorMetrics.writeMetrics.preacceptLatency, 0.999), + getLatency(AccordCoordinatorMetrics.writeMetrics.executeLatency, 0.999), + getLatency(AccordCoordinatorMetrics.writeMetrics.applyLatency, 0.999), + getLatency(AccordExecutorMetrics.INSTANCE.elapsedRunning, 0.5), + getLatency(AccordExecutorMetrics.INSTANCE.elapsedRunning, 0.9), + getLatency(AccordExecutorMetrics.INSTANCE.elapsedRunning, 1.0), + getCount(AccordExecutorMetrics.INSTANCE.elapsedRunning), + getTotal(AccordExecutorMetrics.INSTANCE.elapsedRunning), + getLatency(AccordExecutorMetrics.INSTANCE.elapsed, 0.5), + getLatency(AccordExecutorMetrics.INSTANCE.elapsed, 0.9), + getLatency(AccordExecutorMetrics.INSTANCE.elapsed, 0.999), + AccordExecutorMetrics.INSTANCE.running.getValue(), + AccordExecutorMetrics.INSTANCE.waitingToRun.getValue(), + AccordExecutorMetrics.INSTANCE.preparingToRun.getValue() + )); + clear(AccordExecutorMetrics.INSTANCE.elapsedRunning); + clear(AccordExecutorMetrics.INSTANCE.elapsed); + }); + } + + private static void printVerbs(long nowMillis, Map verbs) + { + class VerbCount + { + final Verb verb; + final int count; + + VerbCount(Verb verb, int count) + { + this.verb = verb; + this.count = count; + } + } + List verbCounts = new ArrayList<>(); + for (Map.Entry e : verbs.entrySet()) + { + int count = e.getValue().getAndSet(0); + if (count != 0) verbCounts.add(new VerbCount(e.getKey(), count)); + } + verbCounts.sort(Comparator.comparing(v -> -v.count)); + + StringBuilder verbSummary = new StringBuilder(); + for (VerbCount vs : verbCounts) + { + { + if (verbSummary.length() > 0) + verbSummary.append(", "); + verbSummary.append(vs.verb); + verbSummary.append(": "); + verbSummary.append(vs.count); + } + } + System.out.println(String.format("%tT.%tL verbs: %s", nowMillis, nowMillis, verbSummary)); + } + + private static void maybePrintSlowestTraces(Cluster cluster, LoadSettings settings) + { + if (settings.traceSlowest > 0f) + { + safeForEach(cluster, () -> { + AccordTracing tracing = ((AccordAgent)AccordService.instance().agent()).tracing(); + + tracing.forEach(Functions.alwaysTrue(), (txnId, state) -> { + state.forEach(event -> { + if (event.elapsedNanos() < MILLISECONDS.toNanos(100)) + return; + + for (Message message : event.messages()) + { + long multiplier = message.atNanos < event.doneAtNanos() ? 1 : -1; + System.out.println(String.format("%s %s %s %s %s %s", txnId, event.kind, multiplier * (message.atNanos - event.atNanos)/1000000, message.nodeId, message.commandStoreId, message.message)); + } + }); + }); + tracing.eraseAll(); + }); + } + } + + private static void maybePrintLastTraces(Cluster cluster, AtomicBoolean pauseOrStop, LoadSettings settings, WaitQueue waitQueue) + { + if (settings.traceLast > 0) + { + pauseOrStop.set(true); + Map>> print = new HashMap<>(); + for (int i = 1 ; i <= cluster.size() ; ++i) + { + cluster.get(i).acceptOnInstance(out -> { + AccordService service = (AccordService)AccordService.instance(); + AccordTracing tracing = ((AccordAgent)AccordService.instance().agent()).tracing(); + PriorityQueue candidates = new PriorityQueue<>(Comparator.comparingLong(c -> -c.elapsedMicros)); + tracing.forEach(Functions.alwaysTrue(), (txnId, events) -> { + events.forEach(event -> { + if (event.kind == Client) + { + long doneAtMicros = event.doneAtMicros(); + long elapsedMicros = doneAtMicros - event.txnId().hlc(); + if (elapsedMicros > 350000 && elapsedMicros < 390000) + candidates.add(new SortedByElapsed(txnId, elapsedMicros)); + } + }); + }); + + AtomicInteger storeId = new AtomicInteger(); + while (!candidates.isEmpty()) + { + SortedByElapsed sortedCandidate = candidates.poll(); + if (sortedCandidate.elapsedMicros < 300000) + return; + + TxnId candidate = sortedCandidate.txnId; + storeId.lazySet(-1); + tracing.forEach(candidate, events -> { + events.forEach(event -> { + if (storeId.get() >= 0) + return; + for (Message message : event.messages()) + { + if (message.nodeId < 0 && message.commandStoreId >= 0) + { + storeId.set(message.commandStoreId); + break; + } + } + }); + }); + + if (storeId.get() >= 0) + { + CommandStore commandStore = service.node().commandStores().forId(storeId.get()); + List> result = AccordService.getBlocking(commandStore.submit(ExecutionContext.unsequenced(candidate, "LoadTest"), safeStore -> { + SafeCommand safeCommand = safeStore.unsafeGet(candidate); + PartialDeps deps = safeCommand.current().partialDeps(); + if (deps == null) + return null; + List> infos = new ArrayList<>(); + for (TxnId txnId : deps.txnIds()) + { + List info = new ArrayList<>(); + info.add(txnId.toString()); + infos.add(info); + } + List info = new ArrayList<>(); + info.add(candidate.toString()); + infos.add(info); + return infos; + })); + + if (result != null) + { + for (List info : result) + { + TxnId txnId = TxnId.parse(info.get(0)); + AccordService.getBlocking(commandStore.execute(ExecutionContext.unsequenced(txnId, "LoadTest"), safeStore -> { + SafeCommand safeCommand = safeStore.unsafeGet(txnId); + if (safeCommand.current().executeAt != null) + info.add(safeCommand.current().executeAt.toString()); + })); + } + + out.put(candidate.toString(), result); + return; + } + } + } + }, print); + } + + for (int i = 1 ; i <= cluster.size() ; ++i) + { + cluster.get(i).acceptOnInstance(out -> { + AccordTracing tracing = ((AccordAgent)AccordService.instance().agent()).tracing(); + for (Map.Entry>> e : out.entrySet()) + { + TxnId parentId = TxnId.parse(e.getKey()); + for (List infos : e.getValue()) + { + TxnId depId = TxnId.parse(infos.get(0)); + tracing.forEach(depId, events -> { + events.forEach(event -> { + infos.add(event.kind + ": [" + (event.idMicros - parentId.hlc()) + "..." + (event.doneAtMicros() - parentId.hlc()) + "][" + (event.idMicros - depId.hlc()) + "..." + (event.doneAtMicros() - depId.hlc()) + "]"); + if (event.kind == Execute) + { + for (Message message : event.messages()) + { + if (message.nodeId == parentId.node.id) + { + long atMicros = (event.idMicros + (message.atNanos - event.atNanos)/1000) - parentId.hlc(); + infos.add(atMicros + ": " + message.message); + } + } + } + }); + }); + } + } + }, print); + } + + for (Map.Entry>> e : print.entrySet()) + { + System.out.println("======" + e.getKey() + "======"); + for (List infos : e.getValue()) + System.out.println(infos); + } + if (!print.isEmpty()) + System.out.println(); + pauseOrStop.set(false); + waitQueue.signalAll(); + } + } + + private static void refresh(Histogram histogram) + { + if (histogram instanceof ShardedHistogram) + ((ShardedHistogram) histogram).refresh(); + if (histogram instanceof ShardedDecayingHistogram) + ((ShardedDecayingHistogram) histogram).refresh(); + } + + private static long getLatency(Histogram histogram, double percentile) + { + return (long)(histogram.getSnapshot().getValue(percentile) / 1000); + } + + private static long getCount(Histogram histogram) + { + return histogram.getSnapshot().size(); + } + + private static double getTotal(Histogram histogram) + { + Snapshot snapshot = histogram.getSnapshot(); + return (snapshot.getMean() * 0.0001d * snapshot.size()); + } + + private static void clear(Histogram histogram) + { + if (histogram instanceof ShardedHistogram) + ((ShardedHistogram) histogram).clear(); + if (histogram instanceof ShardedDecayingHistogram) + ((ShardedDecayingHistogram) histogram).clear(); + } + + private static long getLatency(Timer timer, double percentile) + { + if (timer instanceof SnapshottingTimer) + return (long) (((SnapshottingTimer) timer).getPercentileSnapshot().getValue(percentile) / 1000); + return (long)(timer.getSnapshot().getValue(0.999) / 1000); + } + + private static long getSize(Timer timer) + { + if (timer instanceof SnapshottingTimer) + return ((SnapshottingTimer) timer).getPercentileSnapshot().size(); + return timer.getSnapshot().size(); + } + + @Override + protected Logger logger() + { + return logger; + } + + static class SortedByElapsed + { + final TxnId txnId; + final long elapsedMicros; + + SortedByElapsed(TxnId txnId, long elapsedMicros) + { + this.txnId = txnId; + this.elapsedMicros = elapsedMicros; + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordRebootstrapLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordRebootstrapLoadTest.java new file mode 100644 index 000000000000..4ddbf72fe05b --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordRebootstrapLoadTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.load; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.distributed.api.Feature; + +import static org.apache.cassandra.distributed.test.accord.load.LoadSettings.ycsbZipfian; + +public class AccordRebootstrapLoadTest extends AccordLoadTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(AccordRebootstrapLoadTest.class); + + @Override + protected Logger logger() + { + return logger; + } + + public void setupCluster(int nodeCount) + { + setupCluster(nodeCount, config -> { + config.with(Feature.NETWORK, Feature.GOSSIP) + .set("accord.shard_durability_target_splits", "8") + .set("accord.shard_durability_max_splits", "16") + .set("accord.shard_durability_cycle", "1m") + .set("accord.queue_submission_model", "SIGNAL") + .set("accord.command_store_shard_count", "8") + .set("accord.queue_thread_count", "4") + .set("accord.queue_shard_count", "1") + .set("accord.send_minimal", "false") // TODO (expected): only required because of misordering of preaccept, accept, commit if queued together + .set("accord.catchup_on_start_fail_latency", "2m"); + }); + } + + @Test + public void testLoad() throws Exception + { + testLoad(new LoadSettings.Builder() + .setKeySelector(ycsbZipfian(100_000)) + .setRatePerSecond(200) + .setClusterChaosInterval(10000) + .setClusterChaosConcurrency(2) + .setTotalClusterChaos(10) + .build()); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordYcsbLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordYcsbLoadTest.java new file mode 100644 index 000000000000..e72cd30aad48 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/load/AccordYcsbLoadTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.load; + +import java.util.Arrays; + +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.distributed.shared.DistributedTestBase; + +import static org.apache.cassandra.distributed.test.accord.load.LoadSettings.ycsbZipfian; + +public class AccordYcsbLoadTest extends AccordLoadTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(AccordYcsbLoadTest.class); + + private static final int[][] LATENCIES = new int[][] { + new int[] { 0, 44, 64, 43, 84 }, + new int[] { 44, 0, 30, 3, 45 }, + new int[] { 64, 30, 0, 28, 37 }, + new int[] { 43, 3, 28, 0, 49 }, + new int[] { 84, 45, 37, 49, 0 } + }; + + private static LoadSettings.Builder withArtificialLatencies(LoadSettings.Builder builder) + { + return builder.setArtificialLatencies(LATENCIES); + } + + private static LoadSettings.Builder ycsbA(LoadSettings.Builder builder, int keyCount) + { + return builder.setKeySelector(ycsbZipfian(keyCount)) + .setReadRatio(0.5f); + } + + private static LoadSettings.Builder ycsbB(LoadSettings.Builder builder, int keyCount) + { + return builder.setKeySelector(ycsbZipfian(keyCount)) + .setReadRatio(0.95f); + } + + private static LoadSettings.Builder ycsbC(LoadSettings.Builder builder, int keyCount) + { + return builder.setKeySelector(ycsbZipfian(keyCount)) + .setReadRatio(1.0f); + + } + + @Override + protected Logger logger() + { + return logger; + } + + private static void computeWorstLatencies() + { + int[] qs = new int[LATENCIES.length]; + for (int i = 0 ; i < qs.length ; ++i) + { + int[] copy = LATENCIES[i].clone(); + Arrays.sort(copy); + qs[i] = copy[copy.length/2]; + } + int[] ws = new int[qs.length]; + for (int i = 0 ; i < qs.length ; ++i) + { + int iw = Integer.MIN_VALUE; + for (int j = 0; j < qs.length ; ++j) + iw = Math.max(iw, qs[i] + 3*qs[j] + LATENCIES[i][j]); + ws[i] = iw; + } + System.out.println(Arrays.toString(ws)); + Arrays.fill(ws, 0); + for (int i = 0 ; i < qs.length ; ++i) + { + for (int j = 0 ; j < qs.length ; ++j) + { + if (j == i) continue; + if (qs[j] > 2*qs[i]) continue; + int w = qs[i] + 4*qs[j] + LATENCIES[i][j]; + if (w > ws[i]) + ws[i] = w; + } + } + System.out.println(Arrays.toString(ws)); + } + + @Ignore + @Test + public void testLoad() throws Exception + { + testLoad(ycsbA(new LoadSettings.Builder(), 100_000) + .setRatePerSecond(1600).setMinRatePerSecond(200) + .setIncreaseRatePerSecondInterval(5000) + .build()); + } + + public static void main(String[] args) throws Throwable + { + computeWorstLatencies(); + + DistributedTestBase.beforeClass(); + AccordYcsbLoadTest test = new AccordYcsbLoadTest(); + try + { + test.setupCluster(); + test.setup(); + test.testLoad(withArtificialLatencies(ycsbA(new LoadSettings.Builder(), 100_000) + .setRatePerSecond(1600).setMinRatePerSecond(200) + .setIncreaseRatePerSecondInterval(5000) + ).build()); + } + finally + { + test.tearDown(); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/load/LoadSettings.java b/test/distributed/org/apache/cassandra/distributed/test/accord/load/LoadSettings.java new file mode 100644 index 000000000000..004df8294158 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/load/LoadSettings.java @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.load; + +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.IntSupplier; +import java.util.function.LongFunction; +import java.util.function.Supplier; + +import org.apache.commons.math3.distribution.ZipfDistribution; +import org.apache.commons.math3.random.JDKRandomGenerator; + +import accord.utils.DefaultRandom; + +import static java.util.concurrent.TimeUnit.SECONDS; + +class LoadSettings +{ + enum ClusterChaos { + RESTART, + RESTART_AND_REBOOTSTRAP_INCOMPLETE, RESTART_AND_REBOOTSTRAP_RESET, + RESTART_AND_REBOOTSTRAP_AFTER_TIMEOUT, + + REBOOTSTRAP_INCOMPLETE, REBOOTSTRAP_RESET, + REBOOTSTRAP_IF_BEHIND + } + + final int repairInterval; + final int compactionInterval; + final int journalFlushInterval; + final int cfkFlushInterval; + final int cfkCompactionPeriodSeconds; + final int dataFlushInterval; + final int clusterChaosInterval; + final int clusterChaosDecay; + final int clusterChaosConcurrency; + final LongFunction> clusterChaos; + final int batchSize; + final long batchPeriodNanos; + final int clientConcurrency; + final int clients; + final int ratePerSecond; + final int minRatePerSecond; + final int increaseRatePerSecondInterval; + final int keysPerOperation; + final float readRatio; + final IntSupplier keySelector; + final boolean readBeforeWrite; + final float traceSlowest; + final int traceLast; + final long totalTransactions; + final int totalClusterChaos; + final int[][] artificialLatencies; + + LoadSettings(Builder builder) + { + this.repairInterval = builder.repairInterval; + this.compactionInterval = builder.compactionInterval; + this.journalFlushInterval = builder.journalFlushInterval; + this.cfkFlushInterval = builder.cfkFlushInterval; + this.cfkCompactionPeriodSeconds = builder.cfkCompactionPeriodSeconds; + this.dataFlushInterval = builder.dataFlushInterval; + this.clusterChaosInterval = builder.clusterChaosInterval; + this.clusterChaosDecay = builder.clusterChaosDecay; + this.clusterChaosConcurrency = builder.clusterChaosConcurrency; + this.clusterChaos = builder.clusterChaos; + this.batchSize = builder.batchSize; + this.batchPeriodNanos = builder.batchPeriodNanos; + this.clientConcurrency = builder.clientConcurrency; + this.clients = builder.clients; + this.ratePerSecond = builder.ratePerSecond; + this.minRatePerSecond = builder.minRatePerSecond; + this.increaseRatePerSecondInterval = builder.increaseRatePerSecondInterval; + this.keysPerOperation = builder.keysPerOperation; + this.readRatio = builder.readRatio; + this.keySelector = builder.keySelector; + this.readBeforeWrite = builder.readBeforeWrite; + this.artificialLatencies = builder.artificialLatencies; + this.traceSlowest = builder.traceSlowest; + this.traceLast = builder.traceLast; + this.totalTransactions = builder.totalTransactions; + this.totalClusterChaos = builder.totalClusterChaos; + } + + // interval is measured in terms of *operations* unless otherwise specified + public static class Builder + { + int repairInterval = Integer.MAX_VALUE; + int compactionInterval = Integer.MAX_VALUE; + int journalFlushInterval = Integer.MAX_VALUE; + int cfkFlushInterval = Integer.MAX_VALUE; + int cfkCompactionPeriodSeconds = 0; + int dataFlushInterval = Integer.MAX_VALUE; + int clusterChaosInterval = Integer.MAX_VALUE; + int clusterChaosDecay = 1; + int clusterChaosConcurrency = 1; + LongFunction> clusterChaos = seed -> new DefaultRandom(seed).randomWeightedPicker(LoadSettings.ClusterChaos.values()); + int batchSize = 1000; + long batchPeriodNanos = SECONDS.toNanos(10); + int clientConcurrency = 50; + int clients = -1; + int ratePerSecond = 1000; + int minRatePerSecond = 50; + int increaseRatePerSecondInterval = 1000; + int keysPerOperation = 1; + float readRatio = 0.5f; + IntSupplier keySelector; + boolean readBeforeWrite; + float traceSlowest; + int traceLast; + int[][] artificialLatencies; + long totalTransactions = Long.MAX_VALUE; + int totalClusterChaos = Integer.MAX_VALUE; + + public Builder setRepairInterval(int repairInterval) + { + this.repairInterval = repairInterval; + return this; + } + + public Builder setCompactionInterval(int compactionInterval) + { + this.compactionInterval = compactionInterval; + return this; + } + + public Builder setJournalFlushInterval(int journalFlushInterval) + { + this.journalFlushInterval = journalFlushInterval; + return this; + } + + public Builder setCfkFlushInterval(int cfkFlushInterval) + { + this.cfkFlushInterval = cfkFlushInterval; + return this; + } + + public Builder setCfkCompactionPeriodSeconds(int cfkCompactionPeriodSeconds) + { + this.cfkCompactionPeriodSeconds = cfkCompactionPeriodSeconds; + return this; + } + + public Builder setDataFlushInterval(int dataFlushInterval) + { + this.dataFlushInterval = dataFlushInterval; + return this; + } + + public Builder setClusterChaosInterval(int clusterChaosInterval) + { + this.clusterChaosInterval = clusterChaosInterval; + return this; + } + + public Builder setClusterChaosDecay(int clusterChaosDecay) + { + this.clusterChaosDecay = clusterChaosDecay; + return this; + } + + public Builder setClusterChaosConcurrency(int clusterChaosConcurrency) + { + this.clusterChaosConcurrency = clusterChaosConcurrency; + return this; + } + + public Builder setClusterChaos(LongFunction> clusterChaos) + { + this.clusterChaos = clusterChaos; + return this; + } + + public Builder setTotalTransactions(long totalTransactions) + { + this.totalTransactions = totalTransactions; + return this; + } + + public Builder setTotalClusterChaos(int totalClusterChaos) + { + this.totalClusterChaos = totalClusterChaos; + return this; + } + + public Builder setBatchSize(int batchSize) + { + this.batchSize = batchSize; + return this; + } + + public Builder setBatchPeriodNanos(long batchPeriodNanos) + { + this.batchPeriodNanos = batchPeriodNanos; + return this; + } + + public Builder setClientConcurrency(int clientConcurrency) + { + this.clientConcurrency = clientConcurrency; + return this; + } + + public Builder setClients(int clients) + { + this.clients = clients; + return this; + } + + public Builder setRatePerSecond(int ratePerSecond) + { + this.ratePerSecond = ratePerSecond; + return this; + } + + public Builder setMinRatePerSecond(int minRatePerSecond) + { + this.minRatePerSecond = minRatePerSecond; + return this; + } + + public Builder setIncreaseRatePerSecondInterval(int increaseRatePerSecondInterval) + { + this.increaseRatePerSecondInterval = increaseRatePerSecondInterval; + return this; + } + + public Builder setKeysPerOperation(int keysPerOperation) + { + this.keysPerOperation = keysPerOperation; + return this; + } + + public Builder setReadRatio(float readRatio) + { + this.readRatio = readRatio; + return this; + } + + public Builder setReadBeforeWrite(boolean readBeforeWrite) + { + this.readBeforeWrite = readBeforeWrite; + return this; + } + + public Builder setTraceSlowest(float traceSlowest) + { + this.traceSlowest = traceSlowest; + return this; + } + + public Builder setTraceLast(int traceLast) + { + this.traceLast = traceLast; + return this; + } + + public Builder setKeySelector(IntSupplier keySelector) + { + this.keySelector = keySelector; + return this; + } + + public Builder setArtificialLatencies(int[][] artificialLatencies) + { + this.artificialLatencies = artificialLatencies; + return this; + } + + public LoadSettings build() + { + return new LoadSettings(this); + } + } + + static IntSupplier ycsbZipfian(int keyCount) + { + ZipfDistribution distribution = new ZipfDistribution(new JDKRandomGenerator(), keyCount, 0.99); + int count = distribution.inverseCumulativeProbability(0.65f); + float[] probs = new float[count]; + for (int i = 0 ; i < probs.length ; ++i) + probs[i] = (float) distribution.cumulativeProbability(i); + // zipf is slow to compute, so we cache the first 65% of the distribution then use uniform probability; this is good enough for our purposes + float max = probs[probs.length - 1]; + float inv_incr = probs.length >= keyCount ? 0f : 1f / ((1f-max)/(keyCount - probs.length)); + Random random = new Random(); + return () -> { + float v = random.nextFloat(); + if (v < max) + { + int i = Arrays.binarySearch(probs, v); + if (i < 0) i = -1 - i; + return i; + } + else + { + return (int)((v - max)*inv_incr); + } + }; + } + + static IntSupplier roundrobin(int keyCount) + { + AtomicInteger next = new AtomicInteger(); + return () -> { + int v = next.incrementAndGet(); + if (v < keyCount) + return v; + return next.updateAndGet(i -> i > keyCount ? 0 : i + 1); + }; + } + + static IntSupplier uniform(int keyCount) + { + Random random = new Random(); + return () -> random.nextInt(keyCount); + } +} diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java index 640ce44740a1..04121c55d41b 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/AccordRebootstrapTest.java @@ -65,7 +65,7 @@ public void rebootstrapFuzzTest() throws Throwable try (Cluster cluster = builder.withNodes(3) .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(100)) .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(100, "dc0", "rack0")) - .withConfig((config) -> config.with(Feature.NETWORK, Feature.GOSSIP)) + .withConfig((config) -> config.with(Feature.NETWORK, Feature.GOSSIP).set("accord.catchup_on_start_fail_latency", "60s")) .start()) { IInvokableInstance cmsInstance = cluster.get(1); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java b/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java index b4af888878e0..8ab7e56bffab 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java @@ -84,7 +84,7 @@ public void bootstrapRead() TxnId id = nextTxnId(epoch, txn); Ranges ranges = Ranges.of(IntKey.range(40, 50)); PartialTxn partialTxn = txn.slice(ranges, true); - Request request = new AccordFetchRequest(epoch, id, ranges, PartialDeps.NONE, partialTxn); + Request request = new AccordFetchRequest(epoch, id, ranges, partialTxn); checkRequestReplies(request, new AbstractFetchCoordinator.FetchResponse(null, null, id), diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index 1f8f963ccc17..c5689e813ac0 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -44,6 +44,7 @@ import accord.api.Result; import accord.api.Result.PersistableResult; import accord.api.RoutingKey; +import accord.api.Scheduler; import accord.api.Timeouts; import accord.coordinate.Coordinations; import accord.impl.DefaultLocalListeners; @@ -324,21 +325,11 @@ public static AccordCommandStore createAccordCommandStore( { NodeCommandStoreService time = new NodeCommandStoreService() { - @Override - public AsyncExecutor someExecutor() - { - return null; - } - - @Override - public ExclusiveAsyncExecutor someExclusiveExecutor() - { - return null; - } - private ToLongFunction elapsed = TimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.MICROSECONDS, this::now); private long stamp = 0; + @Override public AsyncExecutor someExecutor() { return null; } + @Override public ExclusiveAsyncExecutor someExclusiveExecutor() { return null; } @Override public Timeouts timeouts() { return null; } @Override public DurableBefore durableBefore() { return DurableBefore.EMPTY; } @Override public DurabilityService durability() { return null; } @@ -349,6 +340,7 @@ public ExclusiveAsyncExecutor someExclusiveExecutor() @Override public long elapsed(TimeUnit timeUnit) { return elapsed.applyAsLong(timeUnit); } @Override public TopologyManager topology() { throw new UnsupportedOperationException(); } @Override public Coordinations coordinations() { return new Coordinations(); } + @Override public Scheduler scheduler() { return null; } @Override public long currentStamp() { return stamp; } @Override public void updateStamp() {++stamp;} @Override public boolean isReplaying() { return false; } diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java index 165115d8fdc0..ae5b6468e3b8 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java @@ -42,6 +42,7 @@ import accord.api.RemoteListeners; import accord.api.Result; import accord.api.RoutingKey; +import accord.api.Scheduler; import accord.api.Timeouts; import accord.coordinate.Coordinations; import accord.impl.DefaultLocalListeners; @@ -268,6 +269,12 @@ public void reportLocalExecution(TxnId txnId, Route route, Ballot ballot, Tim } @Override public Coordinations coordinations() { return new Coordinations(); } + + @Override + public Scheduler scheduler() + { + return null; + } }; TestAgent.RethrowAgent agent = new TestAgent.RethrowAgent() diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java index 448583ac00c7..3b975207b7fa 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java @@ -56,6 +56,7 @@ import accord.api.ProgressLog; import accord.api.Result; import accord.api.RoutingKey; +import accord.api.Scheduler; import accord.api.Timeouts; import accord.coordinate.Coordinations; import accord.impl.AbstractReplayer; @@ -711,6 +712,7 @@ public TestSafeCommandStore(ExecutionContext context) @Override public long now() { return 0; } @Override public long elapsed(TimeUnit unit) { return 0; } @Override public Coordinations coordinations() { return new Coordinations(); } + @Override public Scheduler scheduler() { return null; } }; } @Override public boolean visit(Unseekables keysOrRanges, TxnId testTxnId, Kind.Kinds testKind, SupersedingCommandVisitor visit) { return false; } @Override public void visit(Unseekables keysOrRanges, Timestamp startedBefore, Kind.Kinds testKind, ActiveCommandVisitor visit, P1 p1, P2 p2) { } From 25f6d75be81dbb276f9beed1c1904fd82aa1cc69 Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Thu, 30 Jul 2026 16:31:20 +0100 Subject: [PATCH 09/10] fixup --- .../managing/configuration/configuration.adoc | 2 +- modules/accord | 2 +- .../apache/cassandra/config/AccordConfig.java | 12 +++- .../exceptions/ExceptionSerializer.java | 10 ++- .../service/accord/AccordDataStore.java | 9 ++- .../accord/AccordFetchCoordinator.java | 9 --- .../service/accord/AccordService.java | 68 ++++++++++++++----- .../service/accord/execution/SafeTask.java | 5 +- .../service/accord/execution/Task.java | 1 - .../serializers/CommandStoreSerializers.java | 6 +- .../accord/AccordJournalIntegrationTest.java | 6 +- .../journal/AccordJournalReplayTest.java | 2 +- ...rnalAccessRouteIndexOnStartupRaceTest.java | 2 +- ...atchupTest.java => AccordCatchupTest.java} | 4 +- 14 files changed, 94 insertions(+), 44 deletions(-) rename test/distributed/org/apache/cassandra/fuzz/topology/{AccordHardCatchupTest.java => AccordCatchupTest.java} (99%) diff --git a/doc/modules/cassandra/pages/managing/configuration/configuration.adoc b/doc/modules/cassandra/pages/managing/configuration/configuration.adoc index 0933da73fd6c..9b4df248b26f 100644 --- a/doc/modules/cassandra/pages/managing/configuration/configuration.adoc +++ b/doc/modules/cassandra/pages/managing/configuration/configuration.adoc @@ -153,7 +153,7 @@ configuration changes in the near future. `@Replaces` is the annotation to be used when you make changes to any configuration parameters in `Config` class and `cassandra.yaml`, and you want to add backward compatibility with previous Cassandra versions. `Converters` class enumerates the different methods used for backward compatibility. `IDENTITY` is the one used for name change only. For more information about the other Converters, please, check the JavaDoc in the class. -For backward compatibility virtual table `LoadSettings` contains both the old and the new +For backward compatibility virtual table `Settings` contains both the old and the new parameters with the old and the new value format. Only exception at the moment are the following three parameters: `key_cache_save_period`, `row_cache_save_period` and `counter_cache_save_period` which appear only once with the new value format. The old names and value format still can be used at least until the next major release. Deprecation warning is emitted on startup. diff --git a/modules/accord b/modules/accord index 673cf2ff93a4..f4c8ae02f939 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 673cf2ff93a4b32a392c91feacda09bb48ac16e7 +Subproject commit f4c8ae02f9393c5eb14720b5074bd45135ee5b00 diff --git a/src/java/org/apache/cassandra/config/AccordConfig.java b/src/java/org/apache/cassandra/config/AccordConfig.java index 50d00d2b6f37..944b8c9d9226 100644 --- a/src/java/org/apache/cassandra/config/AccordConfig.java +++ b/src/java/org/apache/cassandra/config/AccordConfig.java @@ -27,6 +27,7 @@ import accord.api.ProtocolModifiers.FastExecution; import accord.api.ProtocolModifiers.ReplicaExecution; import accord.api.ProtocolModifiers.SendStableMessages; +import accord.api.ProtocolModifiers.UniqueTimestampOnConflict; import accord.primitives.TxnId; import accord.utils.Invariants; @@ -34,7 +35,6 @@ import org.apache.cassandra.service.accord.serializers.Version; import org.apache.cassandra.service.consensus.TransactionalMode; -import static org.apache.cassandra.config.AccordConfig.QueuePriorityModel.HLC_FIFO; import static org.apache.cassandra.config.AccordConfig.QueueShardModel.THREAD_POOL_PER_SHARD; import static org.apache.cassandra.config.AccordConfig.QueueSubmissionModel.SYNC; import static org.apache.cassandra.config.AccordConfig.RangeIndexMode.in_memory; @@ -173,6 +173,13 @@ public enum QueueBalancingModel BLENDED_PRIORITY_PHASE_FAIR, } + public enum UniqueTimestampReservations + { + NONE, + SMALL_SHARED, + HISTOGRAM + } + public QueueShardModel queue_shard_model = THREAD_POOL_PER_SHARD; public QueueSubmissionModel queue_submission_model = SYNC; @@ -334,6 +341,9 @@ public enum CatchupFallbackMode public Boolean send_minimal; // note: simulator incompatible (for now) public Boolean precise_micros; + public UniqueTimestampReservations unique_timestamp_reservations; + public UniqueTimestampOnConflict unique_timestamp_on_conflict; + public DurationSpec.IntMillisecondsBound unique_timestamp_reservation_range; public boolean ephemeral_reads = true; public boolean state_cache_listener_jfr_enabled = false; diff --git a/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java b/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java index 50a5393d1084..63c70a544795 100644 --- a/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java +++ b/src/java/org/apache/cassandra/exceptions/ExceptionSerializer.java @@ -44,6 +44,9 @@ */ public class ExceptionSerializer { + // allow plenty of room for UTF8 encoding + private static final int MAX_MESSAGE_CHAR_LENGTH = Short.MAX_VALUE / 4; + public static class RemoteException extends RuntimeException { public final String originalClass; @@ -76,8 +79,11 @@ static String getMessageWithOriginatingHost(Throwable t, boolean isFirstExceptio if (isFirstException) message = "Remote exception from host " + FBUtilities.getBroadcastAddressAndPort().toString() + (t.getLocalizedMessage() != null ? " - " + t.getLocalizedMessage() : ""); else message = t.getLocalizedMessage(); - if (message.length() > Short.MAX_VALUE) - message = message.substring(0, Short.MAX_VALUE); + if (message == null) + return message; + + if (message.length() > MAX_MESSAGE_CHAR_LENGTH) + message = message.substring(0, MAX_MESSAGE_CHAR_LENGTH); return message; } diff --git a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java index 9e227e5f5ab3..ebd2911673d3 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java @@ -148,6 +148,7 @@ public void progress(String tag, ProgressEvent event) case SUCCESS: callback.fetched(ranges); syncResult.trySuccess(null); + // fall-through to ensure started case START: reportStarted(); break; @@ -157,7 +158,7 @@ public void progress(String tag, ProgressEvent event) break; case ABORT: case ERROR: - RuntimeException ex = new RuntimeException(String.format("Repair failed (%s):", event.getType(), event.getMessage())); + RuntimeException ex = new RuntimeException(String.format("Repair failed (%s): %s", event.getType(), event.getMessage())); callback.fail(ranges, ex); syncResult.tryFailure(ex); break; @@ -173,7 +174,11 @@ private void reportStarted() node.commandStores().mapReduceConsume(new RestrictedStoreSelector(ranges, 0, Long.MAX_VALUE), new MapReduceConsumeCommandStores(ranges) { @Override public Timestamp reduce(Timestamp o1, Timestamp o2) { return Timestamp.max(o1, o2); } - @Override public void accept(Timestamp result, Throwable failure) { start.started(result); } + @Override public void accept(Timestamp result, Throwable failure) + { + if (failure != null) syncResult.tryFailure(failure); + else start.started(result); + } @Override public TxnId primaryTxnId() { return null; } @Override public String reason() { return "Compute MaxConflict to report for fetch"; } @Override protected Timestamp applyInternal(SafeCommandStore safeStore) { return safeStore.commandStore().maxConflict(TxnId.NONE, ranges); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java index 630f81ccbf54..74266b0c15aa 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java @@ -459,15 +459,6 @@ public AccordFetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges, Partial { super(sourceEpoch, syncId, ranges, partialTxn); } - - @Override - protected AsyncChain beginRead(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn, Participants execute) - { - AsyncChain result = super.beginRead(safeStore, executeAt, txn, execute); - // TODO (required): verify that streaming snapshots have all been created by now, so we won't stream any data that arrives after this - readStarted(safeStore); - return result; - } } @Override diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index e198670b5f55..ff5752dca000 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -63,6 +63,9 @@ import accord.local.Node; import accord.local.Node.Id; import accord.local.ShardDistributor.EvenSplit; +import accord.local.TimeService; +import accord.local.UniqueTimeService; +import accord.local.UniqueTimeService.AtomicUniqueTime; import accord.local.UniqueTimeService.AtomicUniqueTimeWithStaleReservation; import accord.local.durability.DurabilityService; import accord.local.durability.ShardDurability; @@ -93,7 +96,9 @@ import org.apache.cassandra.concurrent.Shutdownable; import org.apache.cassandra.config.AccordConfig; +import org.apache.cassandra.config.AccordConfig.CatchupFallbackMode; import org.apache.cassandra.config.AccordConfig.JournalConfig.ReplaySavePoint; +import org.apache.cassandra.config.AccordConfig.UniqueTimestampReservations; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; @@ -175,16 +180,20 @@ import static accord.primitives.Txn.Kind.Write; import static accord.primitives.TxnId.MediumPath.NoMediumPath; import static accord.primitives.TxnId.MediumPath.TrackStable; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorThreadTag.JOB; import static org.apache.cassandra.concurrent.ExecutorFactory.SystemThreadTag.DAEMON; +import static org.apache.cassandra.config.AccordConfig.CatchupFallbackMode.EXIT; import static org.apache.cassandra.config.AccordConfig.CatchupFallbackMode.REBOOTSTRAP; +import static org.apache.cassandra.config.AccordConfig.CatchupFallbackMode.REBOOTSTRAP_AND_CATCHUP; import static org.apache.cassandra.config.AccordConfig.JournalConfig.ReplayMode.REBOOTSTRAP_INCOMPLETE; import static org.apache.cassandra.config.AccordConfig.JournalConfig.ReplayMode.REBOOTSTRAP_RESET; import static org.apache.cassandra.config.AccordConfig.JournalConfig.ReplayMode.RESET; +import static org.apache.cassandra.config.AccordConfig.UniqueTimestampReservations.HISTOGRAM; import static org.apache.cassandra.config.DatabaseDescriptor.getAccord; import static org.apache.cassandra.config.DatabaseDescriptor.getAccordGlobalDurabilityCycle; import static org.apache.cassandra.config.DatabaseDescriptor.getAccordShardDurabilityCycle; @@ -479,7 +488,7 @@ public AccordService(Id localId) this.node = new Node(localId, messageSink, topologyService, - time, new AtomicUniqueTimeWithStaleReservation(time), + time, uniqueTimeService(time), () -> dataStore, new KeyspaceSplitter(new EvenSplit<>(getAccord().commandStoreShardCount(), getPartitioner().accordSplitter())), agent, @@ -529,6 +538,8 @@ public static void applyProtocolModifiers(AccordConfig config) ProtocolModifiers.Configure.setSendStableMessages(config.send_stable); if (config.send_minimal != null) ProtocolModifiers.Configure.setSendMinimal(config.send_minimal); + if (config.unique_timestamp_on_conflict != null) + ProtocolModifiers.Configure.setUniqueTimestampOnConflict(config.unique_timestamp_on_conflict); } @Override @@ -718,8 +729,8 @@ private void distributedStartupInternal() void catchup() { - AccordConfig spec = DatabaseDescriptor.getAccord(); - if (!spec.catchup_on_start) + AccordConfig config = DatabaseDescriptor.getAccord(); + if (!config.catchup_on_start) { logger.info("Catchup disabled; continuing to startup"); return; @@ -731,22 +742,17 @@ void catchup() node.commandStores().forAllUnsafe(commandStore -> ((DefaultProgressLog)commandStore.unsafeProgressLog()).setMode(CATCH_UP)); try { - AccordConfig.CatchupFallbackMode onError = spec.catchup_on_start_on_error; - AccordConfig.CatchupFallbackMode onTimeout = spec.catchup_on_start_on_timeout; + CatchupFallbackMode onError = config.catchup_on_start_on_error; + CatchupFallbackMode onTimeout = config.catchup_on_start_on_timeout; - long maxLatencyNanos = spec.catchup_on_start_fail_latency.toNanoseconds(); + long maxLatencyNanos = config.catchup_on_start_fail_latency.toNanoseconds(); int attempts = 1; while (true) { logger.info("Catchup with quorum..."); long start = nanoTime(); long failAt = start + maxLatencyNanos; - Future f; - { - AsyncChain submit = Catchup.catchup(node, failAt, NANOSECONDS); - f = toFuture(submit); - } - + Future f = toFuture(Catchup.catchup(node, failAt, NANOSECONDS)); f.awaitThrowUncheckedOnInterrupt(); Throwable failed = f.cause(); Unsuccessful result = f.getNow(); @@ -769,7 +775,7 @@ void catchup() if (onError == REBOOTSTRAP) return; ++attempts; - onError = onTimeout = spec.catchup_on_start_on_rebootstrap_fallback; + onError = onTimeout = rebootstrapFallbackMode(config); continue; } } @@ -780,10 +786,10 @@ void catchup() if (result == null) { - if (seconds <= spec.catchup_on_start_success_latency.toSeconds()) + if (seconds <= config.catchup_on_start_success_latency.toSeconds()) return; - if (attempts < spec.catchup_on_start_max_attempts) + if (attempts < config.catchup_on_start_max_attempts) { logger.info("Catchup was slow, so we may behind again; retrying"); ++attempts; @@ -810,7 +816,7 @@ void catchup() if (onTimeout == REBOOTSTRAP) return; ++attempts; - onError = onTimeout = spec.catchup_on_start_on_rebootstrap_fallback; + onError = onTimeout = rebootstrapFallbackMode(config); continue; } } @@ -826,6 +832,18 @@ void catchup() } } + private static CatchupFallbackMode rebootstrapFallbackMode(AccordConfig config) + { + CatchupFallbackMode mode = config.catchup_on_start_on_rebootstrap_fallback; + if (mode == REBOOTSTRAP_AND_CATCHUP) + { + CatchupFallbackMode newMode = EXIT; + logger.warn("Converting {} to {} after first failed rebootstrap, to avoid infinite loop", mode, newMode); + mode = newMode; + } + return mode; + } + /** * Startup is broken up in two phases: local and distributed startup. During local startup, we replay up to * the latest epoch known to the node prior to restart. After that, we replay journal itself, and only after @@ -1469,4 +1487,22 @@ public Params journalConfiguration() { return journal.configuration(); } + + private static UniqueTimeService uniqueTimeService(TimeService time) + { + AccordConfig config = getAccord(); + UniqueTimestampReservations reservations = config.unique_timestamp_reservations; + if (reservations == null) + reservations = HISTOGRAM; + + switch (reservations) + { + default: throw UnhandledEnum.unknown(reservations); + case NONE: return new AtomicUniqueTime(time); + case SMALL_SHARED: return new AtomicUniqueTimeWithStaleReservation(time); + case HISTOGRAM: + long millis = config.unique_timestamp_reservation_range == null ? 50 : config.unique_timestamp_reservation_range.toMilliseconds(); + return new UniqueTimeService.AtomicUniqueAutoStaleTimes(time, millis, MILLISECONDS); + } + } } diff --git a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java index 5eee015214b5..3b4274a022eb 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java +++ b/src/java/org/apache/cassandra/service/accord/execution/SafeTask.java @@ -467,13 +467,16 @@ private void preSetup(SafeTask parent) if (isSequencedByPriorityAtomic()) { - boolean isTxnIdSubset = context.isTxnIdSubsetOf(parent.context); if (!isKeySubset) { Invariants.require(keysOrRanges.domain() == Key, "ATOMIC tasks over ranges must declare a subset of their parent's task keys() to avoid a range scan across which we would not impose sequencing"); // to avoid priority inversion deadlocks, if we are not a strict subset of the parent task we permit running with a single key ready nonSync.alwaysReady = true; } + + boolean isTxnIdSubset = true; + for (TxnId txnId : context.txnIds()) + isTxnIdSubset &= parent.refs.containsKey(txnId); Invariants.require(isTxnIdSubset, "ATOMIC tasks must declare a subset of their parent's task txnIds() to avoid a priority inversion deadlock"); // TODO (required): we're appending to the fifo queue - does this maintain correct order? setCacheQueuedFifoExclusive(); diff --git a/src/java/org/apache/cassandra/service/accord/execution/Task.java b/src/java/org/apache/cassandra/service/accord/execution/Task.java index 6a3e50309f9f..c377ea334659 100644 --- a/src/java/org/apache/cassandra/service/accord/execution/Task.java +++ b/src/java/org/apache/cassandra/service/accord/execution/Task.java @@ -262,7 +262,6 @@ public static ExecutorQueue forOrdinal(int ordinal) Task next; Task consequences; - // the queue position until run is invoked, at which point it is assigned a nanoTime() value long position; int info; diff --git a/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java index 7c19d05d0799..f394b68df393 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java @@ -273,13 +273,13 @@ public void serialize(RedundantBefore.Bounds b, DataOutputPlus out) throws IOExc } } - private short cast(long v) + private int cast(long v) { if (v < 0 || v >= (1 << RedundantStatus.Property.WAS_OWNED.ordinal())) throw new IllegalStateException("Should not be serializing WAS_OWNED or NOT_OWNED statuses"); if ((v & ~0xFFFF) != 0) // retain this throw new IllegalStateException("Cannot serialize RedundantStatus larger than 0xFFFF. Requires serialization changes."); - return (short)v; + return (int) v; } @Override @@ -301,7 +301,7 @@ public RedundantBefore.Bounds deserialize(DataInputPlus in) throws IOException bounds[i] = CommandSerializers.txnId.deserialize(in); int[] statuses = new int[count * 2]; for (int i = 0 ; i < statuses.length ; ++i) - statuses[i] = in.readShort(); + statuses[i] = in.readShort() & 0xFFFF; return new RedundantBefore.Bounds(range, startEpoch, endEpoch, bounds, statuses, staleUntilAtLeast); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java index f8e2f82d2d09..c1bd8d6b4805 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java @@ -47,7 +47,7 @@ public void saveLoadSanityCheck() throws Throwable { try (WithProperties wp = new WithProperties().set(CassandraRelevantProperties.DTEST_ACCORD_JOURNAL_SANITY_CHECK_ENABLED, "true"); Cluster cluster = init(Cluster.build(1) - .withConfig(config -> config.set("accord.catchup_on_start", "DISABLED")) + .withConfig(config -> config.set("accord.catchup_on_start", "false")) .withoutVNodes() .start())) { @@ -96,7 +96,7 @@ public void memtableStateReloadingTest() throws Throwable { try (Cluster cluster = Cluster.build(1) .withoutVNodes() - .withConfig(config -> config.set("accord.catchup_on_start", "DISABLED")) + .withConfig(config -> config.set("accord.catchup_on_start", "false")) .start()) { cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};"); @@ -128,7 +128,7 @@ public void restartWithEpochChanges() throws IOException .withoutVNodes() .withConfig(c -> { c.with(GOSSIP).with(NETWORK); - c.set("accord.catchup_on_start", "DISABLED"); + c.set("accord.catchup_on_start", "false"); }) .start()) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java index 44bcc0f30688..08779458fcc2 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java @@ -98,7 +98,7 @@ public void replayCommandWithOnlyDurableSyncPointDependency() throws Throwable .set("accord.retry_syncpoint", "1s*attempts") .set("accord.retry_durability", "1s*attempts") .set("accord.journal.replay_save_point", "NO") - .set("accord.catchup_on_start", "DISABLED") + .set("accord.catchup_on_start", "false") .with(NETWORK, GOSSIP)) .start()) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/JournalAccessRouteIndexOnStartupRaceTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/JournalAccessRouteIndexOnStartupRaceTest.java index 04f53770c555..6ab73888794f 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/JournalAccessRouteIndexOnStartupRaceTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/JournalAccessRouteIndexOnStartupRaceTest.java @@ -66,7 +66,7 @@ public class JournalAccessRouteIndexOnStartupRaceTest extends TestBaseImpl public void test() throws IOException { try (Cluster cluster = Cluster.build(1) - .withConfig(config -> config.set("accord.catchup_on_start", "DISABLED")) + .withConfig(config -> config.set("accord.catchup_on_start", "false")) .withInstanceInitializer(BBHelper::install).start()) { IInvokableInstance node = cluster.get(1); diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/AccordHardCatchupTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/AccordCatchupTest.java similarity index 99% rename from test/distributed/org/apache/cassandra/fuzz/topology/AccordHardCatchupTest.java rename to test/distributed/org/apache/cassandra/fuzz/topology/AccordCatchupTest.java index 746d3b5bb7f8..6f2135b79468 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/AccordHardCatchupTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/AccordCatchupTest.java @@ -46,7 +46,7 @@ import static org.apache.cassandra.distributed.shared.ClusterUtils.waitForCMSToQuiesce; import static org.apache.cassandra.harry.checker.TestHelper.withRandom; -public class AccordHardCatchupTest extends FuzzTestBase +public class AccordCatchupTest extends FuzzTestBase { private static final int WRITES = 10; private static final int POPULATION = 1000; @@ -117,7 +117,7 @@ public void hardCatchupFuzzTest() throws Throwable writeAndValidate.run(); history.customThrowing(() -> { - cluster.get(2).config().set("accord.catchup_on_start", "HARD"); + cluster.get(2).config().set("accord.catchup_on_start", "true"); cluster.get(2).startup(); cluster.get(2).logs().watchFor(".*Catchup.*"); cluster.get(1).logs().watchFor("/127.0.0.2:.* is now UP"); From 9893f3b2fd5c19fa11fa6e30bfa34c16e483746f Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Fri, 31 Jul 2026 15:17:05 +0100 Subject: [PATCH 10/10] UNFINISHED.md - Claude generated summary of remaining work --- UNFINISHED.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 UNFINISHED.md diff --git a/UNFINISHED.md b/UNFINISHED.md new file mode 100644 index 000000000000..34fb6438c94a --- /dev/null +++ b/UNFINISHED.md @@ -0,0 +1,28 @@ +### D1. Partial log unavailability (the piece you're deferring) + + Current state, so it's recoverable later: + - Cleanup.isUnavailable (Cleanup.java:229-233) — any(LOG_UNAVAILABLE) or all(LOG_INCOMPLETE) && saveStatus < Stable, with your TODO (required): how does this interact with lost ownership?. + - SafeCommandStore.registerTransitiveDeps (~:565) — per-range subtraction via redundantBefore.removeLogUnavailableOrIncomplete(txnId, tmp), with TODO (required): if we only part-filter we're still going to have problems, as we won't be able to + update the transaction. + - RedundantBefore.Bounds.depBound — folds LOG_INCOMPLETE/LOG_UNAVAILABLE into the dep bound when UNREADY is ahead of the applied bound; this is what keeps deps sound after a restart, when the in-memory refuses map is gone. + - MapReduceCommandStores.Refuse{NONE,DEPS,ALL} + per-message abort() — the in-memory, coarse (whole-scope) refusal, only alive between unsafeRefuseRequests and unsafeAcceptRequests. + + These four must agree on one predicate. Concretely the questions to settle: + 1. Is any ever right for LOG_UNAVAILABLE? The dangerous arm isn't FULL (throwing is merely unavailability) but non-FULL, where logUnavailable(...) returns ERASE — a compaction decision that discards the record for ranges whose log is intact. + If nothing else changes, splitting just that arm (throw on any, erase only on all) is strictly safer and is a one-line intermediate. + 2. Lost ownership / retired ranges break all. Ranges that are WAS_OWNED or locally retired never carry LOG_INCOMPLETE, so all(...) is false and we don't refuse even though the owned remainder is incomplete. Your "treat pre-log-point ranges as + locally-retired-like" idea is exactly the missing normalisation, and note the patch already does the dual of it in SafeCommandStore.refuses(participants) (removeLocallyRetired(participants) before folding). A shared helper — "the + participants we must have a complete log for" = participants − retired − wasOwned − logIncomplete/Unavailable — would let Cleanup, refuses() and registerTransitive share one definition instead of three. + 3. Recovery cannot answer partially. BeginRecovery refuses on refuses.max != NONE, which only covers the in-memory window, not the post-restart window. There is no wire representation for "valid only for a sub-range" — but ReadOk.unavailable + already is exactly that for reads, so a RecoverNack{unavailable} (and the analogous field on the deps replies) is the smallest precedent-following addition; it would also let Commit/Apply stop depending on the coarse Refuse.MinMax. + 4. Future distinction you mentioned: if you do split further, the axis that matters to consumers looks like "records absent below the bound" (incomplete) vs "records possibly wrong below the bound" (corrupted) vs "records absent but decisions + recoverable from peers". Worth writing that down in BootstrapReason/RedundantStatus.Property javadoc before adding a third status, since the persisted bit space is nearly full (see N9). + + ### D2. Empty / short DurabilityResults + + Now benign-ish but still wrong: DurabilityService.submit's canUseDurableBefore && isAlreadyMet shortcut calls reportSuccess() while success == DurabilityResults.EMPTY, so markBootstrapping marks nothing and the attempt completes with missing + == valid → onFailedBootstrap → retry. Since each retry re-derives a fresh maxConflict and can be satisfied by durableBefore again, an idle range can retry indefinitely, with the unhelpful "Unknown failure" from complete()'s fail runnable. + Options: (a) populate DurabilityResults.of(ranges, request.min, ) on the shortcut path; (b) make the shortcut ineligible when the requester needs per-range bounds/readable (a flag on the request); (c) fail prepareToBootstrap explicitly + if byTxnId() doesn't cover the requested ranges, so the diagnostic names the cause. Also sync(...)/close(...) still return success(null) for empty ranges — an empty DurabilityResults would be safer for every caller that dereferences the value. + Worth adding an Invariants.require(success.byTxnId() covers commitRanges) at the markBootstrapping site so this class of shortfall is loud rather than a retry loop. +