Add async support in os APIs

core/src/xmake/engine.c

@@ -769,11 +769,25 @@ static tb_bool_t xm_engine_save_arguments(xm_engine_t* engine, tb_int_t argc, tb
     return tb_true;
 }
 
-static tb_size_t xm_engine_get_program_file(xm_engine_t* engine, tb_char_t** argv, tb_char_t* path, tb_size_t maxn)
+static tb_bool_t xm_engine_get_program_file(xm_engine_t* engine, tb_char_t** argv, tb_char_t* path, tb_size_t maxn)
 {
     // check
     tb_assert_and_check_return_val(engine && path && maxn, tb_false);
 
+    /* we cache it, because the current path will be changed in thread.
+     *
+     * The executable file compiled using cosmocc on macOS might retrieve a relative path to the programfile.
+     * If the root directory has been changed, the retrieved programfile will be a non-existent path.
+     */
+    static tb_char_t s_program_filepath[TB_PATH_MAXN] = {0};
+    if (s_program_filepath[0])
+    {
+        tb_strlcpy(path, s_program_filepath, maxn);
+        lua_pushstring(engine->lua, s_program_filepath);
+        lua_setglobal(engine->lua, "_PROGRAM_FILE");
+        return tb_true;
+    }
+
     tb_bool_t ok = tb_false;
     do
     {
@@ -813,7 +827,10 @@ static tb_size_t xm_engine_get_program_file(xm_engine_t* engine, tb_char_t** arg
         if (!_NSGetExecutablePath(path, &bufsize))
             ok = tb_true;
 #elif defined(XM_PROC_SELF_FILE)
-        // get the executale file path as program directory
+        /* get the executale file path as program directory
+         *
+         * @see it may be a relative path
+         */
         ssize_t size = readlink(XM_PROC_SELF_FILE, path, (size_t)maxn);
         if (size > 0 && size < maxn)
         {
@@ -878,6 +895,9 @@ static tb_size_t xm_engine_get_program_file(xm_engine_t* engine, tb_char_t** arg
         // trace
         tb_trace_d("programfile: %s", path);
 
+        // cache it
+        tb_strlcpy(s_program_filepath, path, sizeof(s_program_filepath));
+
         // save the directory to the global variable: _PROGRAM_FILE
         lua_pushstring(engine->lua, path);
         lua_setglobal(engine->lua, "_PROGRAM_FILE");
@@ -909,6 +929,15 @@ static tb_bool_t xm_engine_get_program_directory(xm_engine_t* engine, tb_char_t*
     // check
     tb_assert_and_check_return_val(engine && path && maxn, tb_false);
 
+    static tb_char_t s_program_directory[TB_PATH_MAXN] = {0};
+    if (s_program_directory[0])
+    {
+        tb_strlcpy(path, s_program_directory, maxn);
+        lua_pushstring(engine->lua, s_program_directory);
+        lua_setglobal(engine->lua, "_PROGRAM_DIR");
+        return tb_true;
+    }
+
     tb_bool_t ok = tb_false;
     tb_char_t data[TB_PATH_MAXN] = {0};
     do
@@ -994,6 +1023,9 @@ static tb_bool_t xm_engine_get_program_directory(xm_engine_t* engine, tb_char_t*
         // trace
         tb_trace_d("programdir: %s", path);
 
+        // cache it
+        tb_strlcpy(s_program_directory, path, sizeof(s_program_directory));
+
         // save the directory to the global variable: _PROGRAM_DIR
         lua_pushstring(engine->lua, path);
         lua_setglobal(engine->lua, "_PROGRAM_DIR");

core/src/xmake/thread/event_wait.c

@@ -15,7 +15,7 @@
  * Copyright (C) 2015-present, Xmake Open Source Community.
  *
  * @author      ruki
- * @file        thread_event_unlock.c
+ * @file        thread_event_wait.c
  *
  */
 

tests/modules/os/async_copy.lua

@@ -0,0 +1,34 @@
+local function _prepare_source(dir)
+    os.mkdir(dir)
+    io.writefile(path.join(dir, "foo.txt"), "foo")
+    io.writefile(path.join(dir, "bar.txt"), "bar")
+end
+
+function main()
+    local root = os.tmpfile() .. ".os_async_copy"
+    local srcdir = path.join(root, "src")
+    local dstdir = path.join(root, "dst")
+
+    _prepare_source(srcdir)
+    print("source prepared: %s", srcdir)
+
+    local files = os.files(path.join(srcdir, "*.txt"), {async = true})
+    assert(files and #files == 2)
+    print("async enumerate: %d files", #files)
+
+    os.cp(srcdir, dstdir, {async = true})
+    assert(os.isdir(dstdir))
+    print("async copy done: %s", dstdir)
+
+    files = os.files(path.join(dstdir, "*.txt"), {async = true})
+    assert(files and #files == 2)
+
+    os.rm(dstdir, {async = true})
+    assert(not os.isdir(dstdir))
+    print("dst removed async")
+
+    os.rm(srcdir, {async = true})
+    os.tryrm(root)
+    print("async copy test finished")
+end
+

tests/modules/os/async_scheduler.lua

@@ -0,0 +1,32 @@
+import("core.base.scheduler")
+
+local function _prepare_workspace(root)
+    os.mkdir(root)
+    for idx = 1, 4 do
+        io.writefile(path.join(root, string.format("file%d.txt", idx)), "xmake")
+    end
+end
+
+function main()
+    local root = os.tmpfile() .. ".os_async_sched"
+    _prepare_workspace(root)
+    print("workspace prepared: %s", root)
+
+    local group = "os_async_scheduler"
+    scheduler.co_group_begin(group, function ()
+        for idx = 1, 4 do
+            scheduler.co_start(function ()
+                local matches = os.files(path.join(root, string.format("file%d.txt", idx)), {async = true})
+                assert(matches and #matches == 1)
+                print("async match %d finished", idx)
+            end)
+        end
+
+        scheduler.co_start(function ()
+            print("async find all files in programdir...")
+            local files = os.files(path.join(os.programdir(), "**"), {async = true})
+            print("files: %d", #files)
+            print("async find all finished")
+        end)
+    end)
+end

tests/modules/os/test.lua

@@ -149,3 +149,25 @@ function test_args(t)
     t:are_equal(os.args({'-DTEST="hello world"', '-DTEST2="hello world2"'}), '"-DTEST=\\\"hello world\\\"" "-DTEST2=\\\"hello world2\\\""')
 end
 
+function test_async(t)
+    local tmpdir = os.tmpfile() .. ".dir"
+    local tmpdir2 = os.tmpfile() .. ".dir"
+    io.writefile(path.join(tmpdir, "foo.txt"), "foo")
+    io.writefile(path.join(tmpdir, "bar.txt"), "bar")
+    local files = os.files(path.join(tmpdir, "*.txt"), {async = true})
+    t:require(files and #files == 2)
+
+    os.cp(tmpdir, tmpdir2, {async = true, detach = true})
+    t:require(not os.isdir(tmpdir2))
+
+    os.cp(tmpdir, tmpdir2, {async = true})
+    t:require(os.isdir(tmpdir2))
+
+    t:require(os.isdir(tmpdir))
+    os.rm(tmpdir, {async = true})
+    t:require(not os.isdir(tmpdir))
+
+    t:require(os.isdir(tmpdir2))
+    os.rm(tmpdir2, {async = true, detach = true})
+    t:require(os.isdir(tmpdir2))
+end

tests/runner.lua

@@ -18,15 +18,13 @@ function main(script, opt)
     local context = test_context(script)
 
     local root = path.directory(script)
-
     local verbose = option.get("verbose") or option.get("diagnosis")
 
     -- trace
     cprint(">> [%d/%d]: testing %s ...", opt.index, opt.total, path.relative(root))
 
     -- get test functions
     local data = import("test", { rootdir = root, anonymous = true })
-
     if data.main then
         -- ignore everthing when we found a main function
         data = { test_main = data.main }
@@ -37,9 +35,12 @@ function main(script, opt)
 
     -- run test
     local succeed_count = 0
+    local start_time = os.mclock()
     for k, v in pairs(data) do
         if k:startswith("test") and type(v) == "function" then
-            if verbose then print(">>     running %s ...", k) end
+            if verbose then
+                print(">>     running %s ...", k)
+            end
             context.func = v
             context.funcname = k
             local result = try
@@ -69,7 +70,9 @@ function main(script, opt)
             succeed_count = succeed_count + 1
         end
     end
-    if verbose then print(">>   finished %d test method(s) ...", succeed_count) end
+    if verbose then
+        print(">>   finished %d test method(s), spent %0.02fs", succeed_count, (os.mclock() - start_time) / 1000)
+    end
 
     -- leave script directory
     os.cd(old_dir)

xmake/core/_xmake_main.lua

@@ -40,6 +40,12 @@ xmake._EMBED            = _EMBED
 xmake._THREAD_CALLBACK  = _THREAD_CALLBACK
 xmake._THREAD_CALLINFO  = _THREAD_CALLINFO
 
+-- we need not any arguments in sub-thread.
+-- because it always uses CommandLineToArgvW() on windows, so we need to reset it.
+if _THREAD_CALLBACK then
+    xmake._ARGV = {}
+end
+
 -- In order to be compatible with updates from lower versions of engine core
 -- @see https://github.com/xmake-io/xmake/issues/1694#issuecomment-925507210
 if xmake._LUAJIT == nil then

xmake/core/base/option.lua

@@ -314,21 +314,12 @@ end
 function option.taskmenu(task)
     assert(option._MENU)
 
-    -- the current task
     task = task or option.taskname() or "main"
-
-    -- get the task menu
     local taskmenu = option._MENU[task]
     if type(taskmenu) == "function" then
-
-        -- load this task menu
         taskmenu = taskmenu()
-
-        -- save this task menu
         option._MENU[task] = taskmenu
     end
-
-    -- get it
     return taskmenu
 end
 

xmake/core/base/os.lua

@@ -54,6 +54,16 @@ os.SYSERR_NOT_PERM    = 1
 os.SYSERR_NOT_FILEDIR = 2
 os.SYSERR_NOT_ACCESS  = 3
 
+-- get the async task
+function os._async_task()
+    local async_task = os._ASYNC_TASK
+    if async_task == nil then
+        async_task = require("base/private/async_task")
+        os._ASYNC_TASK = async_task
+    end
+    return async_task
+end
+
 -- copy single file or directory
 function os._cp(src, dst, rootdir, opt)
     opt = opt or {}
@@ -390,7 +400,15 @@ end
 --              end)
 -- @endcode
 --
-function os.match(pattern, mode, callback)
+function os.match(pattern, mode, opt)
+
+    -- do it in the asynchronous task
+    if type(opt) == "table" and opt.async and xmake.in_main_thread() then
+        return os._async_task().match(pattern, mode)
+    end
+
+    -- extract callback
+    local callback = type(opt) == "function" and opt or (type(opt) == "table" and opt.callback or nil)
 
     -- support path instance
     pattern = tostring(pattern)
@@ -483,18 +501,18 @@ end
 --
 -- @note only return {} without count to simplify code, e.g. table.unpack(os.dirs(""))
 --
-function os.dirs(pattern, callback)
-    return (os.match(pattern, 'd', callback))
+function os.dirs(pattern, opt)
+    return (os.match(pattern, 'd', opt))
 end
 
 -- match files
-function os.files(pattern, callback)
-    return (os.match(pattern, 'f', callback))
+function os.files(pattern, opt)
+    return (os.match(pattern, 'f', opt))
 end
 
 -- match files and directories
-function os.filedirs(pattern, callback)
-    return (os.match(pattern, 'a', callback))
+function os.filedirs(pattern, opt)
+    return (os.match(pattern, 'a', opt))
 end
 
 -- copy files or directories and we can reserve the source directory structure
@@ -511,6 +529,11 @@ function os.cp(srcpath, dstpath, opt)
         return false, string.format("invalid arguments!")
     end
 
+    -- do it in the asynchronous task
+    if opt and opt.async and xmake.in_main_thread() then
+        return os._async_task().cp(srcpath, dstpath, {detach = opt.detach})
+    end
+
     -- reserve the source directory structure if opt.rootdir is given
     local rootdir = opt and opt.rootdir
     if rootdir then
@@ -564,14 +587,19 @@ end
 
 -- remove files or directories
 function os.rm(filepath, opt)
+    opt = opt or {}
 
     -- check arguments
     if not filepath then
         return false, string.format("invalid arguments!")
     end
 
+    -- do it in the asynchronous task
+    if opt.async and xmake.in_main_thread() then
+       return os._async_task().rm(filepath, {detach = opt.detach})
+    end
+
     -- remove file or directories
-    opt = opt or {}
     filepath = tostring(filepath)
     local filepathes = os._match_wildcard_pathes(filepath)
     if type(filepathes) == "string" then
@@ -617,6 +645,12 @@ end
 function os.cd(dir)
     assert(dir)
 
+    -- we can only change directory in main thread
+    if not xmake.in_main_thread() then
+        local thread = require("base/thread")
+        os.raise("we cannot change directory in non-main thread(%s)", thread.running() or "unknown")
+    end
+
     -- support path instance
     dir = tostring(dir)
 
@@ -682,13 +716,18 @@ function os.mkdir(dir)
 end
 
 -- remove directories
-function os.rmdir(dir)
+function os.rmdir(dir, opt)
 
     -- check arguments
     if not dir then
         return false, string.format("invalid arguments!")
     end
 
+    -- do it in the asynchronous task
+    if opt and opt.async and xmake.in_main_thread() then
+        return os._async_task().rmdir(dir, {detach = opt.detach})
+    end
+
     -- support path instance
     dir = tostring(dir)
 
@@ -835,17 +874,10 @@ function os.runv(program, argv, opt)
             errors = string.format("cannot runv(%s), %s", cmd, errors and errors or "unknown reason")
         end
 
-        -- remove the temporary log file
         os.rm(logfile)
-
-        -- failed
         return false, errors
     end
-
-    -- remove the temporary log file
     os.rm(logfile)
-
-    -- ok
     return true
 end