Add build jobgraph support

tests/apis/rules_inject_deps/xmake.lua

@@ -1,10 +1,6 @@
 rule("cppfront")
     set_extensions(".cpp2")
-    on_load(function (target)
-        local rule = target:rule("c++.build"):clone()
-        rule:add("deps", "cppfront", {order = true})
-        target:rule_add(rule)
-    end)
+    add_orders("cppfront", "c++.build")
     on_build_file(function (target, sourcefile, opt)
         print("build cppfront file")
         local objectfile = target:objectfile(sourcefile:gsub("cpp2", "cpp"))

tests/apis/rules_order/xmake.lua

@@ -1,7 +1,15 @@
 
 rule("markdown")
-    add_deps("man", {order = true})
     set_extensions(".md", ".markdown")
+    add_orders("man", "markdown")
+
+    before_build(function (target)
+        print("before_build: markdown")
+    end)
+    after_build(function (target)
+        print("after_build: markdown")
+    end)
+
     before_build_file(function (target, sourcefile)
         print("before_build_file: %s", sourcefile)
     end)
@@ -14,6 +22,14 @@ rule("markdown")
 
 rule("man")
     set_extensions(".man")
+
+    before_build(function (target)
+        print("before_build: man")
+    end)
+    after_build(function (target)
+        print("after_build: man")
+    end)
+
     before_build_file(function (target, sourcefile)
         print("before_build_file: %s", sourcefile)
     end)
@@ -26,7 +42,7 @@ rule("man")
 
 target("test")
     set_kind("binary")
-    add_rules("markdown")
+    add_rules("markdown", "man")
     add_files("src/*.c")
     add_files("src/*.md")
     add_files("src/*.man")

tests/modules/async/run_callback.lua

@@ -0,0 +1,19 @@
+import("core.base.scheduler")
+import("async.runjobs")
+
+function _jobfunc(index, total, opt)
+    print("%s: run job (%d/%d)", scheduler.co_running(), index, total)
+    local dt = os.mclock()
+    os.sleep(1000)
+    dt = os.mclock() - dt
+    print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt)
+end
+
+function main()
+    print("==================================== test callback ====================================")
+    local t = os.mclock()
+    runjobs("test", _jobfunc, {total = 100, comax = 6, timeout = 1000, timer = function (running_jobs_indices)
+        print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ","))
+    end})
+end
+

tests/modules/async/run_jobgraph.lua

@@ -0,0 +1,53 @@
+import("core.base.scheduler")
+import("async.jobgraph")
+import("async.runjobs")
+
+function _jobfunc(index, total, opt)
+    print("%s: run job (%d/%d)", scheduler.co_running(), index, total)
+    local dt = os.mclock()
+    os.sleep(1000)
+    dt = os.mclock() - dt
+    print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt)
+end
+
+function _test_basic()
+    print("==================================== test basic ====================================")
+    local jobs = jobgraph.new()
+    jobs:add("job/root", _jobfunc)
+    for i = 1, 3 do
+        jobs:add("job/" .. i, _jobfunc)
+        for j = 1, 50 do
+            jobs:add("job/" .. i .. "/" .. j, _jobfunc)
+            jobs:add_orders("job/" .. i .. "/" .. j, "job/" .. i, "job/root")
+        end
+    end
+    t = os.mclock()
+    runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices)
+        print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ","))
+    end})
+end
+
+function _test_group()
+    print("==================================== test group ====================================")
+    local jobs = jobgraph.new()
+    jobs:add("job/root", _jobfunc)
+    for i = 1, 3 do
+        jobs:add("job/" .. i, _jobfunc, {groups = "bar"})
+        jobs:group("foo", function ()
+            for j = 1, 50 do
+                jobs:add("job/" .. i .. "/" .. j, _jobfunc)
+            end
+        end)
+    end
+    jobs:add_orders("foo", "bar", "job/root")
+    t = os.mclock()
+    runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices)
+        print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ","))
+    end})
+end
+
+function main()
+    _test_basic()
+    _test_group()
+end
+

tests/modules/async/run_jobpool.lua

@@ -0,0 +1,28 @@
+import("core.base.scheduler")
+import("private.async.jobpool")
+import("async.runjobs")
+
+function _jobfunc(index, total, opt)
+    print("%s: run job (%d/%d)", scheduler.co_running(), index, total)
+    local dt = os.mclock()
+    os.sleep(1000)
+    dt = os.mclock() - dt
+    print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt)
+end
+
+function main()
+    print("==================================== test jobpool ====================================")
+    local jobs = jobpool.new()
+    local root = jobs:addjob("job/root", _jobfunc)
+    for i = 1, 3 do
+        local job = jobs:addjob("job/" .. i, _jobfunc, {rootjob = root})
+        for j = 1, 50 do
+            jobs:addjob("job/" .. i .. "/" .. j, _jobfunc, {rootjob = job})
+        end
+    end
+    t = os.mclock()
+    runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices)
+        print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ","))
+    end})
+end
+

tests/modules/graph/test.lua

@@ -1,6 +1,6 @@
 import("core.base.graph")
 
-function test_topological_sort(t)
+function test_topo_sort(t)
     local edges = {
         {0, 5},
         {0, 2},
@@ -18,7 +18,7 @@ function test_topological_sort(t)
     for _, e in ipairs(edges) do
         dag:add_edge(e[1], e[2])
     end
-    local order_path = dag:topological_sort()
+    local order_path = dag:topo_sort()
     local orders = {}
     for i, v in ipairs(order_path) do
         orders[v] = i
@@ -28,7 +28,7 @@ function test_topological_sort(t)
     end
 
     dag = dag:reverse()
-    order_path = dag:topological_sort()
+    order_path = dag:topo_sort()
     orders = {}
     for i, v in ipairs(order_path) do
         orders[v] = i
@@ -38,6 +38,138 @@ function test_topological_sort(t)
     end
 end
 
+function test_paritail_topo_sort(t)
+    local function partiail_topo_sort(dag)
+        dag:partial_topo_sort_reset()
+
+        local node, has_cycle
+        local order_vertices = {}
+        while true do
+            node, has_cycle = dag:partial_topo_sort_next()
+            if node then
+                table.insert(order_vertices, node)
+                dag:partial_topo_sort_remove(node)
+            else
+                if has_cycle then
+                    raise("has cycle!")
+                end
+                break
+            end
+        end
+
+        return order_vertices, has_cycle
+    end
+
+    local edges = {
+        {0, 5},
+        {0, 2},
+        {0, 1},
+        {3, 6},
+        {3, 5},
+        {3, 4},
+        {5, 4},
+        {6, 4},
+        {6, 0},
+        {3, 2},
+        {1, 4},
+        {2, 9},
+    }
+    local dag = graph.new(true)
+    for _, e in ipairs(edges) do
+        dag:add_edge(e[1], e[2])
+    end
+    local order_path = partiail_topo_sort(dag)
+    local orders = {}
+    for i, v in ipairs(order_path) do
+        orders[v] = i
+    end
+    for _, e in ipairs(edges) do
+        t:require(orders[e[1]] < orders[e[2]])
+    end
+
+    dag = dag:reverse()
+    order_path = partiail_topo_sort(dag)
+    orders = {}
+    for i, v in ipairs(order_path) do
+        orders[v] = i
+    end
+    for _, e in ipairs(edges) do
+        t:require(orders[e[1]] > orders[e[2]])
+    end
+end
+
+function test_paritail_topo_sort_dynamic(t)
+    local function partiail_topo_sort(dag)
+        dag:partial_topo_sort_reset()
+
+        local node, has_cycle
+        local order_vertices = {}
+        local dynamic_adjust = false
+        while true do
+            node, has_cycle = dag:partial_topo_sort_next()
+            if node then
+                if not dynamic_adjust then
+                    dag:add_edge(1, 4)
+                    dag:remove_vertex(6)
+                end
+                table.insert(order_vertices, node)
+                dag:partial_topo_sort_remove(node)
+                if not dynamic_adjust then
+                    dag:add_edge(2, 9)
+                    dynamic_adjust = true
+                end
+            else
+                if has_cycle then
+                    raise("has cycle!")
+                end
+                break
+            end
+        end
+
+        assert(#order_vertices == #dag:vertices(), "vertices count not matched, %d != %d", #order_vertices, #dag:vertices())
+        return order_vertices, has_cycle
+    end
+
+    local edges = {
+        {0, 5},
+        {0, 2},
+        {0, 1},
+        {3, 6},
+        {3, 5},
+        {3, 4},
+        {5, 4},
+        {6, 4},
+        {6, 0},
+        {3, 2},
+    }
+    local dag = graph.new(true)
+    for _, e in ipairs(edges) do
+        dag:add_edge(e[1], e[2])
+    end
+    local order_path = partiail_topo_sort(dag)
+    local orders = {}
+    for i, v in ipairs(order_path) do
+        orders[v] = i
+    end
+    edges = {
+        {0, 5},
+        {0, 2},
+        {0, 1},
+        -- {3, 6},
+        {3, 5},
+        {3, 4},
+        {5, 4},
+        -- {6, 4},
+        -- {6, 0},
+        {3, 2},
+        {1, 4},
+        {2, 9}
+    }
+    for _, e in ipairs(edges) do
+        t:require(orders[e[1]] < orders[e[2]])
+    end
+end
+
 function test_find_cycle(t)
     local edges = {
         {9, 1},
@@ -52,5 +184,8 @@ function test_find_cycle(t)
     end
     local cycle = dag:find_cycle()
     t:are_equal(cycle, {1, 6, 0})
+
+    local _, has_cycle = dag:topo_sort()
+    t:require(has_cycle)
 end
 

tests/modules/queue/test.lua

@@ -0,0 +1,35 @@
+import("core.base.queue")
+
+function test_push(t)
+    local d = queue.new()
+    d:push(1)
+    d:push(2)
+    d:push(3)
+    d:push(4)
+    d:push(5)
+    t:are_equal(d:first(), 1)
+    t:are_equal(d:last(), 5)
+    local idx = 1
+    for item in d:items() do
+        t:are_equal(item, idx)
+        idx = idx + 1
+    end
+end
+
+function test_pop(t)
+    local d = queue.new()
+    d:push(1)
+    d:push(2)
+    d:push(3)
+    d:push(4)
+    d:push(5)
+    d:pop()
+    t:are_equal(d:first(), 2)
+    t:are_equal(d:last(), 5)
+    local idx = 2
+    for item in d:items() do
+        t:are_equal(item, idx)
+        idx = idx + 1
+    end
+end
+