Reorder task structure and add padding

kpamnany · kpamnany · commit 788f46fe9599 · 2024-12-04T14:09:47.000-05:00
To ensure that the task metrics do not cross a cache line boundary.
Also rename `cpu_time_ns` to `running_time_ns`.
diff --git a/base/boot.jl b/base/boot.jl
@@ -175,15 +175,33 @@
 #end
 
 #mutable struct Task
-#    parent::Task
+#    next::Any
+#    queue::Any
 #    storage::Any
-#    state::Symbol
 #    donenotify::Any
 #    result::Any
-#    exception::Any
-#    backtrace::Any
 #    scope::Any
 #    code::Any
+#    @atomic _state::UInt8
+#    sticky::UInt8
+#    priority::UInt16
+#    @atomic _isexception::UInt8
+#    pad00::UInt8
+#    pad01::UInt8
+#    pad02::UInt8
+#    rngState0::UInt64
+#    rngState1::UInt64
+#    rngState2::UInt64
+#    rngState3::UInt64
+#    rngState4::UInt64
+#    const metrics_enabled::Bool
+#    pad10::UInt8
+#    pad11::UInt8
+#    pad12::UInt8
+#    @atomic first_enqueued_at::UInt64
+#    @atomic last_started_running_at::UInt64
+#    @atomic running_time_ns::UInt64
+#    @atomic finished_at::UInt64
 #end
 
 export
diff --git a/base/experimental.jl b/base/experimental.jl
@@ -510,7 +510,7 @@ disable_new_worlds() = ccall(:jl_disable_new_worlds, Cvoid, ())
 
 Enable or disable the collection of per-task metrics.
 A `Task` created when `Base.Experimental.task_metrics(true)` is in effect will have
-[`Base.Experimental.task_cpu_time_ns`](@ref) and [`Base.Experimental.task_wall_time_ns`](@ref)
+[`Base.Experimental.task_running_time_ns`](@ref) and [`Base.Experimental.task_wall_time_ns`](@ref)
 timing information available.
 
 !!! note
@@ -526,7 +526,7 @@ function task_metrics(b::Bool)
 end
 
 """
-    Base.Experimental.task_cpu_time_ns(t::Task) -> Union{UInt64, Nothing}
+    Base.Experimental.task_running_time_ns(t::Task) -> Union{UInt64, Nothing}
 
 Return the total nanoseconds that the task `t` has spent running.
 This metric is only updated when `t` yields or completes unless `t` is the current task, in
@@ -543,14 +543,14 @@ See [`Base.Experimental.task_metrics`](@ref).
 !!! compat "Julia 1.12"
     This method was added in Julia 1.12.
 """
-function task_cpu_time_ns(t::Task=current_task())
+function task_running_time_ns(t::Task=current_task())
     t.metrics_enabled || return nothing
     if t == current_task()
         # These metrics fields can't update while we're running.
         # But since we're running we need to include the time since we last started running!
-        return t.cpu_time_ns + (time_ns() - t.last_started_running_at)
+        return t.running_time_ns + (time_ns() - t.last_started_running_at)
     else
-        return t.cpu_time_ns
+        return t.running_time_ns
     end
 end
 
@@ -560,7 +560,7 @@ end
 Return the total nanoseconds that the task `t` was runnable.
 This is the time since the task first entered the run queue until the time at which it
 completed, or until the current time if the task has not yet completed.
-See also [`Base.Experimental.task_cpu_time_ns`](@ref).
+See also [`Base.Experimental.task_running_time_ns`](@ref).
 
 Returns `nothing` if task timings are not enabled.
 See [`Base.Experimental.task_metrics`](@ref).
diff --git a/base/task.jl b/base/task.jl
@@ -1070,15 +1070,15 @@ immediately yields to `t` before calling the scheduler.
 Throws a `ConcurrencyViolationError` if `t` is the currently running task.
 """
 function yield(t::Task, @nospecialize(x=nothing))
-    current = current_task()
+    ct = current_task()
     # [task] user_time -yield-> wait_time
-    record_cpu_time!(current)
-    t === current && throw(ConcurrencyViolationError("Cannot yield to currently running task!"))
+    record_running_time!(ct)
+    t === ct && throw(ConcurrencyViolationError("Cannot yield to currently running task!"))
     (t._state === task_state_runnable && t.queue === nothing) || throw(ConcurrencyViolationError("yield: Task not runnable"))
     # [task] created -scheduled-> wait_time
     maybe_record_enqueued!(t)
     t.result = x
-    enq_work(current)
+    enq_work(ct)
     set_next_task(t)
     return try_yieldto(ensure_rescheduled)
 end
@@ -1092,8 +1092,9 @@ call to `yieldto`. This is a low-level call that only switches tasks, not consid
 or scheduling in any way. Its use is discouraged.
 """
 function yieldto(t::Task, @nospecialize(x=nothing))
+    ct = current_task()
     # [task] user_time -yield-> wait_time
-    record_cpu_time!(current_task())
+    record_running_time!(ct)
     # TODO: these are legacy behaviors; these should perhaps be a scheduler
     # state error instead.
     if t._state === task_state_done
@@ -1133,8 +1134,9 @@ end
 
 # yield to a task, throwing an exception in it
 function throwto(t::Task, @nospecialize exc)
+    ct = current_task()
     # [task] user_time -yield-> wait_time
-    record_cpu_time!(current_task())
+    record_running_time!(ct)
     # [task] created -scheduled-unfairly-> wait_time
     maybe_record_enqueued!(t)
     t.result = exc
@@ -1189,8 +1191,9 @@ checktaskempty = Partr.multiq_check_empty
 end
 
 function wait()
+    ct = current_task()
     # [task] user_time -yield-or-done-> wait_time
-    record_cpu_time!(current_task())
+    record_running_time!(ct)
     GC.safepoint()
     W = workqueue_for(Threads.threadid())
     poptask(W)
@@ -1206,10 +1209,10 @@ else
     pause() = ccall(:pause, Cvoid, ())
 end
 
-# update the `cpu_time_ns` field of `t` to include the time since it last started running.
-function record_cpu_time!(t::Task)
+# update the `running_time_ns` field of `t` to include the time since it last started running.
+function record_running_time!(t::Task)
     if t.metrics_enabled && !istaskdone(t)
-        @atomic :monotonic t.cpu_time_ns += time_ns() - t.last_started_running_at
+        @atomic :monotonic t.running_time_ns += time_ns() - t.last_started_running_at
     end
     return t
 end
diff --git a/src/jltypes.c b/src/jltypes.c
@@ -3746,46 +3746,58 @@ void jl_init_types(void) JL_GC_DISABLED
                         NULL,
                         jl_any_type,
                         jl_emptysvec,
-                        jl_perm_symsvec(21,
+                        jl_perm_symsvec(27,
                                         "next",
                                         "queue",
                                         "storage",
                                         "donenotify",
                                         "result",
                                         "scope",
                                         "code",
+                                        "_state",
+                                        "sticky",
+                                        "priority",
+                                        "_isexception",
+                                        "pad00",
+                                        "pad01",
+                                        "pad02",
                                         "rngState0",
                                         "rngState1",
                                         "rngState2",
                                         "rngState3",
                                         "rngState4",
-                                        "_state",
-                                        "sticky",
-                                        "_isexception",
-                                        "priority",
                                         "metrics_enabled",
+                                        "pad10",
+                                        "pad11",
+                                        "pad12",
                                         "first_enqueued_at",
                                         "last_started_running_at",
-                                        "cpu_time_ns",
+                                        "running_time_ns",
                                         "finished_at"),
-                        jl_svec(21,
+                        jl_svec(27,
                                 jl_any_type,
                                 jl_any_type,
                                 jl_any_type,
                                 jl_any_type,
                                 jl_any_type,
                                 jl_any_type,
                                 jl_any_type,
+                                jl_uint8_type,
+                                jl_bool_type,
+                                jl_uint16_type,
+                                jl_bool_type,
+                                jl_uint8_type,
+                                jl_uint8_type,
+                                jl_uint8_type,
                                 jl_uint64_type,
                                 jl_uint64_type,
                                 jl_uint64_type,
                                 jl_uint64_type,
                                 jl_uint64_type,
-                                jl_uint8_type,
-                                jl_bool_type,
-                                jl_bool_type,
-                                jl_uint16_type,
                                 jl_bool_type,
+                                jl_uint8_type,
+                                jl_uint8_type,
+                                jl_uint8_type,
                                 jl_uint64_type,
                                 jl_uint64_type,
                                 jl_uint64_type,
@@ -3795,10 +3807,10 @@ void jl_init_types(void) JL_GC_DISABLED
     XX(task);
     jl_value_t *listt = jl_new_struct(jl_uniontype_type, jl_task_type, jl_nothing_type);
     jl_svecset(jl_task_type->types, 0, listt);
-    // Set field 17 (metrics_enabled) as const
-    // Set fields 13 (_state) and 18-21 (metric counters) as atomic
-    const static uint32_t task_constfields[1]  = { 0b000010000000000000000 };
-    const static uint32_t task_atomicfields[1] = { 0b111100001000000000000 };
+    // Set field 20 (metrics_enabled) as const
+    // Set fields 8 (_state) and 24-27 (metric counters) as atomic
+    const static uint32_t task_constfields[1]  = { 0b00000000000010000000000000000000 };
+    const static uint32_t task_atomicfields[1] = { 0b00000111100000000000000010000000 };
     jl_task_type->name->constfields = task_constfields;
     jl_task_type->name->atomicfields = task_atomicfields;
 
diff --git a/src/julia.h b/src/julia.h
@@ -2276,24 +2276,23 @@ typedef struct _jl_task_t {
     jl_value_t *result;
     jl_value_t *scope;
     jl_function_t *start;
-    // 4 byte padding on 32-bit systems
-    // uint32_t padding0;
-    uint64_t rngState[JL_RNG_SIZE];
     _Atomic(uint8_t) _state;
     uint8_t sticky; // record whether this Task can be migrated to a new thread
-    _Atomic(uint8_t) _isexception; // set if `result` is an exception to throw or that we exited with
-    // 1 byte padding
-    // uint8_t padding1;
-    // multiqueue priority
     uint16_t priority;
+    _Atomic(uint8_t) _isexception; // set if `result` is an exception to throw or that we exited with
+    uint8_t pad0[3];
+    // === 64 bytes (cache line)
+    uint64_t rngState[JL_RNG_SIZE];
     // flag indicating whether or not to record timing metrics for this task
     uint8_t metrics_enabled;
+    uint8_t pad1[3];
     // timestamp this task first entered the run queue
     _Atomic(uint64_t) first_enqueued_at;
     // timestamp this task was most recently scheduled to run
     _Atomic(uint64_t) last_started_running_at;
     // time this task has spent running; updated when it yields or finishes.
-    _Atomic(uint64_t) cpu_time_ns;
+    _Atomic(uint64_t) running_time_ns;
+    // === 64 bytes (cache line)
     // timestamp this task finished (i.e. entered state DONE or FAILED).
     _Atomic(uint64_t) finished_at;
 
diff --git a/src/task.c b/src/task.c
@@ -318,7 +318,7 @@ void JL_NORETURN jl_finish_task(jl_task_t *ct)
         assert(jl_atomic_load_relaxed(&ct->first_enqueued_at) != 0);
         uint64_t now = jl_hrtime();
         jl_atomic_store_relaxed(&ct->finished_at, now);
-        jl_atomic_fetch_add_relaxed(&ct->cpu_time_ns, now - jl_atomic_load_relaxed(&ct->last_started_running_at));
+        jl_atomic_fetch_add_relaxed(&ct->running_time_ns, now - jl_atomic_load_relaxed(&ct->last_started_running_at));
     }
     if (jl_atomic_load_relaxed(&ct->_isexception))
         jl_atomic_store_release(&ct->_state, JL_TASK_STATE_FAILED);
@@ -1156,7 +1156,7 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, jl_value_t *completion
     t->metrics_enabled = jl_atomic_load_relaxed(&jl_task_metrics_enabled) != 0;
     jl_atomic_store_relaxed(&t->first_enqueued_at, 0);
     jl_atomic_store_relaxed(&t->last_started_running_at, 0);
-    jl_atomic_store_relaxed(&t->cpu_time_ns, 0);
+    jl_atomic_store_relaxed(&t->running_time_ns, 0);
     jl_atomic_store_relaxed(&t->finished_at, 0);
     jl_timing_task_init(t);
 
@@ -1614,7 +1614,7 @@ jl_task_t *jl_init_root_task(jl_ptls_t ptls, void *stack_lo, void *stack_hi)
     ct->ptls = ptls;
     ct->world_age = 1; // OK to run Julia code on this task
     ct->reentrant_timing = 0;
-    jl_atomic_store_relaxed(&ct->cpu_time_ns, 0);
+    jl_atomic_store_relaxed(&ct->running_time_ns, 0);
     jl_atomic_store_relaxed(&ct->finished_at, 0);
     ct->metrics_enabled = jl_atomic_load_relaxed(&jl_task_metrics_enabled) != 0;
     if (ct->metrics_enabled) {
