Make load/store! increment size of the loaded/stored types.

Author: chriselrod

.gitignore

@@ -4,3 +4,4 @@
 Manifest.toml
 /docs/build/
 src/*~
+*#*

Project.toml

@@ -1,7 +1,7 @@
 name = "ThreadingUtilities"
 uuid = "8290d209-cae3-49c0-8002-c8c24d57dab5"
 authors = ["Chris Elrod <[email protected]> and contributors"]
-version = "0.2.5"
+version = "0.3.0"
 
 [deps]
 VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"

src/ThreadingUtilities.jl

@@ -1,7 +1,7 @@
 module ThreadingUtilities
 
 using VectorizationBase:
-    pause, StaticInt, StridedPointer, offsets, cache_linesize, align, vload, vstore!, num_threads, assume
+    pause, StaticInt, StridedPointer, offsets, cache_linesize, align, __vload, __vstore!, num_threads, assume, False, register_size
 
 @enum ThreadState::UInt begin
     SPIN = 0   # 0: spinning
@@ -11,7 +11,7 @@ using VectorizationBase:
     STUP = 4   # 4: problem being setup. Any reason to have two lock flags?
 end
 const TASKS = Task[]
-const THREADBUFFERSIZE = 32
+const THREADBUFFERSIZE = 64
 const THREADPOOL = UInt[]
 const THREADPOOLPTR =  Ref{Ptr{UInt}}(C_NULL);
 

src/threadtasks.jl

@@ -44,8 +44,8 @@ function wake_thread!(tid)
 end
 
 # 1-based tid
-@inline function __wait(tid)
-    p = taskpointer(tid)
+@inline __wait(tid::Integer) = __wait(taskpointer(tid))
+@inline function __wait(p::Ptr{UInt})
     # note: based on relative values (SPIN = 0, WAIT = 1)
     # thus it should spin for as long as the task is doing anything else
     while reinterpret(UInt, _atomic_load(p)) > reinterpret(UInt, WAIT)

src/utils.jl

@@ -1,10 +1,24 @@
 # To add support for loading/storing...
-@inline function load(p::Ptr{UInt}, ::Type{T}) where {T}
-    reinterpret(T, vload(p))
+@inline function load(p::Ptr{UInt}, ::Type{T}) where {T<:Ptr}
+    reinterpret(T, __vload(p, False(), register_size()))
 end
-@inline function store!(p::Ptr{UInt}, x)
-    vstore!(p, reinterpret(UInt, x))
+@inline function load(p::Ptr{UInt32}, ::Type{T}) where {T<:Union{UInt32,Int32,Float32}}
+    reinterpret(T, __vload(p, False(), register_size()))
 end
+@inline function load(p::Ptr{UInt64}, ::Type{T}) where {T<:Union{UInt64,Int64,Float64}}
+    reinterpret(T, __vload(p, False(), register_size()))
+end
+@inline load(p::Ptr{UInt}, ::Type{T}) where {T} = unsafe_load(Base.unsafe_convert(Ptr{T}, p))
+@inline function store!(p::Ptr{UInt}, x::T) where {T <: Ptr}
+    __vstore!(p, reinterpret(UInt, x), False(), False(), False(), register_size())
+end
+@inline function store!(p::Ptr{UInt32}, x::T) where {T <: Union{UInt32,Int32,Float32}}
+    __vstore!(p, reinterpret(UInt, x), False(), False(), False(), register_size())
+end
+@inline function store!(p::Ptr{UInt64}, x::T) where {T <: Union{UInt64,Int64,Float64}}
+    __vstore!(p, reinterpret(UInt, x), False(), False(), False(), register_size())
+end
+@inline store!(p::Ptr{UInt}, x::T) where {T} = (unsafe_store!(Base.unsafe_convert(Ptr{T}, p), x); nothing)
 
 @inline load(p::Ptr{UInt}, ::Type{StaticInt{N}}, i) where {N} = i, StaticInt{N}()
 @inline store!(p::Ptr{UInt}, ::StaticInt, i) = i
@@ -53,12 +67,11 @@ end
 end
 
 @inline function load(p::Ptr{UInt}, ::Type{T}, i) where {T}
-    i += 1
-    i, load(p + i * sizeof(UInt), T)
+    i + sizeof(T), load(p + i, T)
 end
 @inline function store!(p::Ptr{UInt}, x, i)
-    store!(p + sizeof(UInt)*(i += 1), x)
-    i
+    store!(p + i, x)
+    i + sizeof(x)
 end
 
 @generated function load(p::Ptr{UInt}, ::Type{T}, i) where {T<:Tuple}
@@ -81,4 +94,5 @@ end
     store!(p, first(tup), i)
 end
 @inline store!(p::Ptr{UInt}, tup::Tuple{}, i) = i
+@inline store!(p::Ptr{UInt}, tup::Nothing, i) = i
 

test/internals.jl

@@ -1,3 +1,4 @@
 @testset "Internals" begin
     @test ThreadingUtilities.store!(pointer(UInt[]), (), 1) == 1
+    @test ThreadingUtilities.store!(pointer(UInt[]), nothing, 1) == 1
 end

test/threadingutilities.jl

@@ -1,6 +1,6 @@
 struct Copy{P} end
 function (::Copy{P})(p::Ptr{UInt}) where {P}
-    _, (ptry,ptrx,N) = ThreadingUtilities.load(p, P, 1)
+    _, (ptry,ptrx,N) = ThreadingUtilities.load(p, P, 2*sizeof(UInt))
     N > 0 || throw("This function throws if N == 0 for testing purposes.")
     @simd ivdep for n ∈ 1:N
         vstore!(ptry, vload(ptrx, (n,)), (n,))
@@ -18,7 +18,7 @@ function setup_copy!(p, y, x)
     py = stridedpointer(y)
     px = stridedpointer(x)
     fptr = copy_ptr(py, px)
-    offset = ThreadingUtilities.store!(p, fptr, 0)
+    offset = ThreadingUtilities.store!(p, fptr, sizeof(UInt))
     ThreadingUtilities.store!(p, (py,px,N), offset)
 end
 
@@ -46,21 +46,23 @@ function test_copy(tid, N = 100_000)
     x = similar(a) .= NaN;
     y = similar(b) .= NaN;
     z = similar(c) .= NaN;
-    launch_thread_copy!(tid, x, a)
-    yield()
-    # sleep(1e-3)
-    ThreadingUtilities.__wait(tid)
-    launch_thread_copy!(tid, y, b)
-    yield()
-    # sleep(1e-3)
-    ThreadingUtilities.__wait(tid)
-    launch_thread_copy!(tid, z, c)
-    yield()
-    # sleep(1e-3)
-    ThreadingUtilities.__wait(tid)
-    @test a == x
-    @test b == y
-    @test c == z
+    GC.@preserve a b c x y z begin
+        launch_thread_copy!(tid, x, a)
+        yield()
+        # sleep(1e-3)
+        ThreadingUtilities.__wait(tid)
+        launch_thread_copy!(tid, y, b)
+        yield()
+        # sleep(1e-3)
+        ThreadingUtilities.__wait(tid)
+        launch_thread_copy!(tid, z, c)
+        yield()
+        # sleep(1e-3)
+        ThreadingUtilities.__wait(tid)
+        @test a == x
+        @test b == y
+        @test c == z
+    end
 end
 
 @testset "ThreadingUtilities.jl" begin