diff --git a/core/src/tbox/tbox b/core/src/tbox/tbox index 9bb164af891..c3a4be464e2 160000 --- a/core/src/tbox/tbox +++ b/core/src/tbox/tbox @@ -1 +1 @@ -Subproject commit 9bb164af89169f9194be7663795d954b0578831c +Subproject commit c3a4be464e2e42efdff83ca4abf22134facf5bf8 diff --git a/core/src/xmake/engine.c b/core/src/xmake/engine.c index 0e79140e6b8..61ba79af182 100644 --- a/core/src/xmake/engine.c +++ b/core/src/xmake/engine.c @@ -769,11 +769,25 @@ static tb_bool_t xm_engine_save_arguments(xm_engine_t* engine, tb_int_t argc, tb return tb_true; } -static tb_size_t xm_engine_get_program_file(xm_engine_t* engine, tb_char_t** argv, tb_char_t* path, tb_size_t maxn) +static tb_bool_t xm_engine_get_program_file(xm_engine_t* engine, tb_char_t** argv, tb_char_t* path, tb_size_t maxn) { // check tb_assert_and_check_return_val(engine && path && maxn, tb_false); + /* we cache it, because the current path will be changed in thread. + * + * The executable file compiled using cosmocc on macOS might retrieve a relative path to the programfile. + * If the root directory has been changed, the retrieved programfile will be a non-existent path. + */ + static tb_char_t s_program_filepath[TB_PATH_MAXN] = {0}; + if (s_program_filepath[0]) + { + tb_strlcpy(path, s_program_filepath, maxn); + lua_pushstring(engine->lua, s_program_filepath); + lua_setglobal(engine->lua, "_PROGRAM_FILE"); + return tb_true; + } + tb_bool_t ok = tb_false; do { @@ -813,7 +827,10 @@ static tb_size_t xm_engine_get_program_file(xm_engine_t* engine, tb_char_t** arg if (!_NSGetExecutablePath(path, &bufsize)) ok = tb_true; #elif defined(XM_PROC_SELF_FILE) - // get the executale file path as program directory + /* get the executale file path as program directory + * + * @see it may be a relative path + */ ssize_t size = readlink(XM_PROC_SELF_FILE, path, (size_t)maxn); if (size > 0 && size < maxn) { @@ -878,6 +895,9 @@ static tb_size_t xm_engine_get_program_file(xm_engine_t* engine, tb_char_t** arg // trace tb_trace_d("programfile: %s", path); + // cache it + tb_strlcpy(s_program_filepath, path, sizeof(s_program_filepath)); + // save the directory to the global variable: _PROGRAM_FILE lua_pushstring(engine->lua, path); lua_setglobal(engine->lua, "_PROGRAM_FILE"); @@ -909,6 +929,15 @@ static tb_bool_t xm_engine_get_program_directory(xm_engine_t* engine, tb_char_t* // check tb_assert_and_check_return_val(engine && path && maxn, tb_false); + static tb_char_t s_program_directory[TB_PATH_MAXN] = {0}; + if (s_program_directory[0]) + { + tb_strlcpy(path, s_program_directory, maxn); + lua_pushstring(engine->lua, s_program_directory); + lua_setglobal(engine->lua, "_PROGRAM_DIR"); + return tb_true; + } + tb_bool_t ok = tb_false; tb_char_t data[TB_PATH_MAXN] = {0}; do @@ -994,6 +1023,9 @@ static tb_bool_t xm_engine_get_program_directory(xm_engine_t* engine, tb_char_t* // trace tb_trace_d("programdir: %s", path); + // cache it + tb_strlcpy(s_program_directory, path, sizeof(s_program_directory)); + // save the directory to the global variable: _PROGRAM_DIR lua_pushstring(engine->lua, path); lua_setglobal(engine->lua, "_PROGRAM_DIR"); diff --git a/core/src/xmake/thread/event_wait.c b/core/src/xmake/thread/event_wait.c index 0831036c628..d6d628311ad 100644 --- a/core/src/xmake/thread/event_wait.c +++ b/core/src/xmake/thread/event_wait.c @@ -15,7 +15,7 @@ * Copyright (C) 2015-present, Xmake Open Source Community. * * @author ruki - * @file thread_event_unlock.c + * @file thread_event_wait.c * */ diff --git a/tests/modules/os/async_copy.lua b/tests/modules/os/async_copy.lua new file mode 100644 index 00000000000..751a940aed3 --- /dev/null +++ b/tests/modules/os/async_copy.lua @@ -0,0 +1,34 @@ +local function _prepare_source(dir) + os.mkdir(dir) + io.writefile(path.join(dir, "foo.txt"), "foo") + io.writefile(path.join(dir, "bar.txt"), "bar") +end + +function main() + local root = os.tmpfile() .. ".os_async_copy" + local srcdir = path.join(root, "src") + local dstdir = path.join(root, "dst") + + _prepare_source(srcdir) + print("source prepared: %s", srcdir) + + local files = os.files(path.join(srcdir, "*.txt"), {async = true}) + assert(files and #files == 2) + print("async enumerate: %d files", #files) + + os.cp(srcdir, dstdir, {async = true}) + assert(os.isdir(dstdir)) + print("async copy done: %s", dstdir) + + files = os.files(path.join(dstdir, "*.txt"), {async = true}) + assert(files and #files == 2) + + os.rm(dstdir, {async = true}) + assert(not os.isdir(dstdir)) + print("dst removed async") + + os.rm(srcdir, {async = true}) + os.tryrm(root) + print("async copy test finished") +end + diff --git a/tests/modules/os/async_scheduler.lua b/tests/modules/os/async_scheduler.lua new file mode 100644 index 00000000000..0a201f8f71f --- /dev/null +++ b/tests/modules/os/async_scheduler.lua @@ -0,0 +1,32 @@ +import("core.base.scheduler") + +local function _prepare_workspace(root) + os.mkdir(root) + for idx = 1, 4 do + io.writefile(path.join(root, string.format("file%d.txt", idx)), "xmake") + end +end + +function main() + local root = os.tmpfile() .. ".os_async_sched" + _prepare_workspace(root) + print("workspace prepared: %s", root) + + local group = "os_async_scheduler" + scheduler.co_group_begin(group, function () + for idx = 1, 4 do + scheduler.co_start(function () + local matches = os.files(path.join(root, string.format("file%d.txt", idx)), {async = true}) + assert(matches and #matches == 1) + print("async match %d finished", idx) + end) + end + + scheduler.co_start(function () + print("async find all files in programdir...") + local files = os.files(path.join(os.programdir(), "**"), {async = true}) + print("files: %d", #files) + print("async find all finished") + end) + end) +end diff --git a/tests/modules/os/test.lua b/tests/modules/os/test.lua index 92890a234dc..1207cb615e7 100644 --- a/tests/modules/os/test.lua +++ b/tests/modules/os/test.lua @@ -149,3 +149,25 @@ function test_args(t) t:are_equal(os.args({'-DTEST="hello world"', '-DTEST2="hello world2"'}), '"-DTEST=\\\"hello world\\\"" "-DTEST2=\\\"hello world2\\\""') end +function test_async(t) + local tmpdir = os.tmpfile() .. ".dir" + local tmpdir2 = os.tmpfile() .. ".dir" + io.writefile(path.join(tmpdir, "foo.txt"), "foo") + io.writefile(path.join(tmpdir, "bar.txt"), "bar") + local files = os.files(path.join(tmpdir, "*.txt"), {async = true}) + t:require(files and #files == 2) + + os.cp(tmpdir, tmpdir2, {async = true, detach = true}) + t:require(not os.isdir(tmpdir2)) + + os.cp(tmpdir, tmpdir2, {async = true}) + t:require(os.isdir(tmpdir2)) + + t:require(os.isdir(tmpdir)) + os.rm(tmpdir, {async = true}) + t:require(not os.isdir(tmpdir)) + + t:require(os.isdir(tmpdir2)) + os.rm(tmpdir2, {async = true, detach = true}) + t:require(os.isdir(tmpdir2)) +end diff --git a/tests/runner.lua b/tests/runner.lua index 5b8b0f5f59a..5d154674dd2 100644 --- a/tests/runner.lua +++ b/tests/runner.lua @@ -18,7 +18,6 @@ function main(script, opt) local context = test_context(script) local root = path.directory(script) - local verbose = option.get("verbose") or option.get("diagnosis") -- trace @@ -26,7 +25,6 @@ function main(script, opt) -- get test functions local data = import("test", { rootdir = root, anonymous = true }) - if data.main then -- ignore everthing when we found a main function data = { test_main = data.main } @@ -37,9 +35,12 @@ function main(script, opt) -- run test local succeed_count = 0 + local start_time = os.mclock() for k, v in pairs(data) do if k:startswith("test") and type(v) == "function" then - if verbose then print(">> running %s ...", k) end + if verbose then + print(">> running %s ...", k) + end context.func = v context.funcname = k local result = try @@ -69,7 +70,9 @@ function main(script, opt) succeed_count = succeed_count + 1 end end - if verbose then print(">> finished %d test method(s) ...", succeed_count) end + if verbose then + print(">> finished %d test method(s), spent %0.02fs", succeed_count, (os.mclock() - start_time) / 1000) + end -- leave script directory os.cd(old_dir) diff --git a/xmake/core/_xmake_main.lua b/xmake/core/_xmake_main.lua index 00538c86456..139e6d20c9c 100644 --- a/xmake/core/_xmake_main.lua +++ b/xmake/core/_xmake_main.lua @@ -40,6 +40,12 @@ xmake._EMBED = _EMBED xmake._THREAD_CALLBACK = _THREAD_CALLBACK xmake._THREAD_CALLINFO = _THREAD_CALLINFO +-- we need not any arguments in sub-thread. +-- because it always uses CommandLineToArgvW() on windows, so we need to reset it. +if _THREAD_CALLBACK then + xmake._ARGV = {} +end + -- In order to be compatible with updates from lower versions of engine core -- @see https://github.com/xmake-io/xmake/issues/1694#issuecomment-925507210 if xmake._LUAJIT == nil then diff --git a/xmake/core/base/option.lua b/xmake/core/base/option.lua index d258a1451da..6d46bddef8f 100644 --- a/xmake/core/base/option.lua +++ b/xmake/core/base/option.lua @@ -314,21 +314,12 @@ end function option.taskmenu(task) assert(option._MENU) - -- the current task task = task or option.taskname() or "main" - - -- get the task menu local taskmenu = option._MENU[task] if type(taskmenu) == "function" then - - -- load this task menu taskmenu = taskmenu() - - -- save this task menu option._MENU[task] = taskmenu end - - -- get it return taskmenu end diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua index 647c76c5069..dcd5ab5d1ab 100644 --- a/xmake/core/base/os.lua +++ b/xmake/core/base/os.lua @@ -54,6 +54,16 @@ os.SYSERR_NOT_PERM = 1 os.SYSERR_NOT_FILEDIR = 2 os.SYSERR_NOT_ACCESS = 3 +-- get the async task +function os._async_task() + local async_task = os._ASYNC_TASK + if async_task == nil then + async_task = require("base/private/async_task") + os._ASYNC_TASK = async_task + end + return async_task +end + -- copy single file or directory function os._cp(src, dst, rootdir, opt) opt = opt or {} @@ -390,7 +400,15 @@ end -- end) -- @endcode -- -function os.match(pattern, mode, callback) +function os.match(pattern, mode, opt) + + -- do it in the asynchronous task + if type(opt) == "table" and opt.async and xmake.in_main_thread() then + return os._async_task().match(pattern, mode) + end + + -- extract callback + local callback = type(opt) == "function" and opt or (type(opt) == "table" and opt.callback or nil) -- support path instance pattern = tostring(pattern) @@ -483,18 +501,18 @@ end -- -- @note only return {} without count to simplify code, e.g. table.unpack(os.dirs("")) -- -function os.dirs(pattern, callback) - return (os.match(pattern, 'd', callback)) +function os.dirs(pattern, opt) + return (os.match(pattern, 'd', opt)) end -- match files -function os.files(pattern, callback) - return (os.match(pattern, 'f', callback)) +function os.files(pattern, opt) + return (os.match(pattern, 'f', opt)) end -- match files and directories -function os.filedirs(pattern, callback) - return (os.match(pattern, 'a', callback)) +function os.filedirs(pattern, opt) + return (os.match(pattern, 'a', opt)) end -- copy files or directories and we can reserve the source directory structure @@ -511,6 +529,11 @@ function os.cp(srcpath, dstpath, opt) return false, string.format("invalid arguments!") end + -- do it in the asynchronous task + if opt and opt.async and xmake.in_main_thread() then + return os._async_task().cp(srcpath, dstpath, {detach = opt.detach}) + end + -- reserve the source directory structure if opt.rootdir is given local rootdir = opt and opt.rootdir if rootdir then @@ -564,14 +587,19 @@ end -- remove files or directories function os.rm(filepath, opt) + opt = opt or {} -- check arguments if not filepath then return false, string.format("invalid arguments!") end + -- do it in the asynchronous task + if opt.async and xmake.in_main_thread() then + return os._async_task().rm(filepath, {detach = opt.detach}) + end + -- remove file or directories - opt = opt or {} filepath = tostring(filepath) local filepathes = os._match_wildcard_pathes(filepath) if type(filepathes) == "string" then @@ -617,6 +645,12 @@ end function os.cd(dir) assert(dir) + -- we can only change directory in main thread + if not xmake.in_main_thread() then + local thread = require("base/thread") + os.raise("we cannot change directory in non-main thread(%s)", thread.running() or "unknown") + end + -- support path instance dir = tostring(dir) @@ -682,13 +716,18 @@ function os.mkdir(dir) end -- remove directories -function os.rmdir(dir) +function os.rmdir(dir, opt) -- check arguments if not dir then return false, string.format("invalid arguments!") end + -- do it in the asynchronous task + if opt and opt.async and xmake.in_main_thread() then + return os._async_task().rmdir(dir, {detach = opt.detach}) + end + -- support path instance dir = tostring(dir) @@ -835,17 +874,10 @@ function os.runv(program, argv, opt) errors = string.format("cannot runv(%s), %s", cmd, errors and errors or "unknown reason") end - -- remove the temporary log file os.rm(logfile) - - -- failed return false, errors end - - -- remove the temporary log file os.rm(logfile) - - -- ok return true end diff --git a/xmake/core/base/private/async_task.lua b/xmake/core/base/private/async_task.lua new file mode 100644 index 00000000000..ca176816e0d --- /dev/null +++ b/xmake/core/base/private/async_task.lua @@ -0,0 +1,466 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file async_task.lua +-- + +-- define module: async_task +local async_task = async_task or {} + +-- load modules +local os = require("base/os") +local utils = require("base/utils") +local thread = require("base/thread") +local option = require("base/option") +local path = require("base/path") +local pipe_event = require("base/private/pipe_event") + +-- the task status +local is_stopped = false +local is_started = false + +-- the task event and queue +local task_event = nil +local task_queue = nil +local task_mutex = nil + +-- object pool for sharedata +local sharedata_pool = {} + +function async_task._absolute_dirs(searchdirs) + local dirs = {} + if searchdirs then + for _, directory in ipairs(searchdirs) do + local dir = tostring(directory) + if #dir > 0 then + table.insert(dirs, path.absolute(dir)) + end + end + end + return dirs +end + +function async_task._get_event() + return pipe_event.new("async_task") +end + +function async_task._put_event(event) + if event then + event:close() + end +end + +-- get sharedata from pool or create new one +function async_task._get_sharedata() + local sharedata = table.remove(sharedata_pool) + if not sharedata then + sharedata = thread.sharedata() + else + sharedata:clear() + end + return sharedata +end + +-- return sharedata to pool +function async_task._put_sharedata(sharedata) + if sharedata then + table.insert(sharedata_pool, sharedata) + end +end + +-- the asynchronous task loop +function async_task._loop(event, queue, mutex, is_stopped, is_diagnosis) + local os = require("base/os") + local try = require("sandbox/modules/try") + local thread = require("base/thread") + + local function dprint(...) + if is_diagnosis then + print(...) + end + end + + local function _restore_thread_objects(cmd) + if cmd.event_data then + local event, errors = thread._deserialize_object(cmd.event_data) + assert(event, errors or "failed to deserialize event") + cmd.event = event + end + if cmd.result_data then + local result, errors = thread._deserialize_object(cmd.result_data) + assert(result, errors or "failed to deserialize result") + cmd.result = result + end + end + + local function _runcmd_cp(cmd) + os.cp(cmd.srcpath, cmd.dstpath) + end + local function _runcmd_rm(cmd) + os.rm(cmd.filepath) + end + local function _runcmd_rmdir(cmd) + os.rmdir(cmd.dir) + end + local function _runcmd_match(cmd) + return os.match(cmd.pattern, cmd.mode) + end + local function _runcmd_find_file(cmd) + local find_file = require("sandbox/modules/import/lib/detect/find_file") + return find_file._find_from_directories(cmd.name, cmd.searchdirs or {}, cmd.suffixes or {}) + end + local function _runcmd_find_path(cmd) + local find_path = require("sandbox/modules/import/lib/detect/find_path") + return find_path._find_from_directories(cmd.name, cmd.searchdirs or {}, cmd.suffixes or {}) + end + local function _runcmd_find_directory(cmd) + local find_directory = require("sandbox/modules/import/lib/detect/find_directory") + return find_directory._find_from_directories(cmd.name, cmd.searchdirs or {}, cmd.suffixes or {}) + end + local function _runcmd_find_library(cmd) + local find_library = require("sandbox/modules/import/lib/detect/find_library") + return find_library._find_from_directories(cmd.names or {}, cmd.searchdirs or {}, cmd.kinds or {}, cmd.suffixes or {}, cmd.opt or {}) + end + local runops = { + cp = _runcmd_cp, + rm = _runcmd_rm, + rmdir = _runcmd_rmdir, + match = _runcmd_match, + find_file = _runcmd_find_file, + find_path = _runcmd_find_path, + find_directory = _runcmd_find_directory, + find_library = _runcmd_find_library + } + local function _runcmd(cmd) + + _restore_thread_objects(cmd) + + local ok = true + local errors + local result_data + local runop = runops[cmd.kind] + if runop then + try + { + function () + local ret = runop(cmd) + if ret then + result_data = ret + end + end, + catch + { + function (errs) + ok = false + errors = tostring(errs) + end + } + } + end + + -- notify completion if event is provided + if cmd.event and cmd.result then + cmd.result:set({ok = ok, errors = errors, data = result_data}) + cmd.event:post() + end + end + + dprint("async_task: started") + while not is_stopped:get() do + if event:wait(-1) > 0 then + + -- fetch all tasks from queue at once + local cmds = {} + mutex:lock() + while not queue:empty() do + local cmd = queue:pop() + if cmd then + table.insert(cmds, cmd) + end + end + mutex:unlock() + + -- execute tasks without holding lock + dprint("async_task: handling tasks(%d) ..", #cmds) + for _, cmd in ipairs(cmds) do + _runcmd(cmd) + end + end + end + dprint("async_task: exited") +end + +-- start the asynchronous task +function async_task._start() + assert(task_queue == nil and task_event == nil and task_mutex == nil) + task_event = thread.event() + task_queue = thread.queue() + task_mutex = thread.mutex() + local task_is_stopped = thread.sharedata() + local task_thread = thread.new(async_task._loop, { + name = "core.base.async_task", internal = true, + argv = {task_event, task_queue, task_mutex, task_is_stopped, option.get("diagnosis")}}) + local ok, errors = task_thread:start() + if not ok then + return false, errors + end + os.atexit(function (errors) + if task_thread then + is_stopped = true + + -- Perhaps the thread hasn't started yet. + -- Let's wait a while and let it finish executing the tasks in the current queue. + utils.dprint("async_task: wait the pending tasks(%d) for exiting ..", task_queue:size()) + task_mutex:lock() + local is_empty = task_queue:empty() + task_mutex:unlock() + if not is_empty then + task_event:post() + os.sleep(300) + end + task_is_stopped:set(true) + task_event:post() + + utils.dprint("async_task: wait thread for exiting ..") + task_thread:wait(-1) + utils.dprint("async_task: wait finished") + end + end) + return true +end + +-- ensure the asynchronous task is started +function async_task._ensure_started() + if is_stopped then + return false, string.format("async_task has been stopped") + end + if not is_started then + local ok, errors = async_task._start() + if ok then + is_started = true + end + return ok, errors + end + assert(task_queue and task_event) + return true +end + +-- post task and wait for result +function async_task._post_task(cmd, is_detach, return_data) + local cmd_event + local cmd_result + + -- create pipe and result for non-detach mode + if not is_detach then + cmd_event = async_task._get_event() + if not cmd_event then + return false, "failed to acquire event" + end + cmd_result = async_task._get_sharedata() + cmd.result_data = thread._serialize_object(cmd_result) + if not cmd.result_data then + async_task._put_sharedata(cmd_result) + async_task._put_event(cmd_event) + return false, "failed to serialize sharedata" + end + cmd.event_data = thread._serialize_object(cmd_event) + if not cmd.event_data then + async_task._put_sharedata(cmd_result) + async_task._put_event(cmd_event) + return false, "failed to serialize event" + end + end + + task_mutex:lock() + task_queue:push(cmd) + task_mutex:unlock() + + task_event:post() + + if is_detach then + return true + end + + local wait_ok, wait_errors = cmd_event:wait(-1) + + local result + if wait_ok then + result = cmd_result:get() + end + async_task._put_sharedata(cmd_result) + async_task._put_event(cmd_event) + + if not wait_ok then + return false, wait_errors or "wait event failed" + end + + if result and result.ok then + if return_data then + local data = result.data + if type(data) == "table" then + return data, #data + elseif data ~= nil then + return data + else + return nil, 0 + end + else + return true + end + else + if return_data then + return nil, 0 + else + return false, result and result.errors or "unknown error" + end + end +end + +-- copy files or directories +function async_task.cp(srcpath, dstpath, opt) + opt = opt or {} + local ok, errors = async_task._ensure_started() + if not ok then + return false, errors + end + local cmd = {kind = "cp", srcpath = path.absolute(tostring(srcpath)), dstpath = path.absolute(tostring(dstpath))} + return async_task._post_task(cmd, opt.detach, false) +end + +-- remove files or directories +function async_task.rm(filepath, opt) + opt = opt or {} + local ok, errors = async_task._ensure_started() + if not ok then + return false, errors + end + local cmd = {kind = "rm", filepath = path.absolute(tostring(filepath))} + return async_task._post_task(cmd, opt.detach, false) +end + +-- remove directories +function async_task.rmdir(dir, opt) + opt = opt or {} + local ok, errors = async_task._ensure_started() + if not ok then + return false, errors + end + local cmd = {kind = "rmdir", dir = path.absolute(tostring(dir))} + return async_task._post_task(cmd, opt.detach, false) +end + +-- match files or directories +function async_task.match(pattern, mode) + local ok, errors = async_task._ensure_started() + if not ok then + return nil, errors + end + local cmd = {kind = "match", pattern = path.absolute(tostring(pattern)), mode = mode} + return async_task._post_task(cmd, false, true) +end + +-- find file +function async_task.find_file(name, searchdirs, opt) + opt = opt or {} + local ok, errors = async_task._ensure_started() + if not ok then + return nil, errors + end + local dirs = async_task._absolute_dirs(searchdirs) + local suffixes = {} + if opt.suffixes then + for _, suffix in ipairs(opt.suffixes) do + table.insert(suffixes, tostring(suffix)) + end + end + local cmd = {kind = "find_file", name = tostring(name), searchdirs = dirs, suffixes = suffixes} + return async_task._post_task(cmd, false, true) +end + +-- find path +function async_task.find_path(name, searchdirs, opt) + opt = opt or {} + local ok, errors = async_task._ensure_started() + if not ok then + return nil, errors + end + local dirs = async_task._absolute_dirs(searchdirs) + local suffixes = {} + if opt.suffixes then + for _, suffix in ipairs(opt.suffixes) do + table.insert(suffixes, tostring(suffix)) + end + end + local cmd = {kind = "find_path", name = tostring(name), searchdirs = dirs, suffixes = suffixes} + return async_task._post_task(cmd, false, true) +end + +-- find directory +function async_task.find_directory(name, searchdirs, opt) + opt = opt or {} + local ok, errors = async_task._ensure_started() + if not ok then + return nil, errors + end + local dirs = async_task._absolute_dirs(searchdirs) + local suffixes = {} + if opt.suffixes then + for _, suffix in ipairs(opt.suffixes) do + table.insert(suffixes, tostring(suffix)) + end + end + local cmd = {kind = "find_directory", name = tostring(name), searchdirs = dirs, suffixes = suffixes} + return async_task._post_task(cmd, false, true) +end + +-- find library +function async_task.find_library(names, searchdirs, kinds, opt) + opt = opt or {} + local ok, errors = async_task._ensure_started() + if not ok then + return nil, errors + end + local dirs = async_task._absolute_dirs(searchdirs) + local suffixes = {} + if opt.suffixes then + for _, suffix in ipairs(opt.suffixes) do + table.insert(suffixes, tostring(suffix)) + end + end + local names_list = {} + if names then + if type(names) == "table" then + for _, name in ipairs(names) do + table.insert(names_list, tostring(name)) + end + else + table.insert(names_list, tostring(names)) + end + end + local kinds_list = {} + if kinds then + for _, kind in ipairs(kinds) do + table.insert(kinds_list, tostring(kind)) + end + end + local cmd = {kind = "find_library", names = names_list, searchdirs = dirs, kinds = kinds_list, suffixes = suffixes, opt = {plat = opt.plat}} + return async_task._post_task(cmd, false, true) +end + +-- return module: async_task +return async_task + diff --git a/xmake/core/base/private/pipe_event.lua b/xmake/core/base/private/pipe_event.lua new file mode 100644 index 00000000000..8d12787666b --- /dev/null +++ b/xmake/core/base/private/pipe_event.lua @@ -0,0 +1,140 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file pipe_event.lua +-- + +-- define module +local pipe_event = pipe_event or {} +local _instance = _instance or {} + +-- load modules +local pipe = require("base/pipe") +local libc = require("base/libc") +local bytes = require("base/bytes") +local table = require("base/table") + +function _instance.new(name) + local event = table.inherit(_instance) + event._PIPE_EVENT = true + event._BUFFER = bytes(2) + event._NAME = name or "pipe_event" + local reader, writer, errors = pipe.openpair("AA") + if not reader or not writer then + if reader then reader:close() end + if writer then writer:close() end + return nil, errors or "failed to open pipe" + end + event._READER = reader + event._WRITER = writer + event._WRITER_PTR = nil + return event +end + +function _instance:name() + return self._NAME +end + +function _instance:post() + local writer = self._WRITER + if not writer then + return false, "pipe event writer closed" + end + local ok, errors = writer:write("1", {block = true}) + if ok < 0 then + return false, errors or "pipe event post failed" + end + writer:close() + self._WRITER = nil + self._WRITER_PTR = nil + return true +end + +function _instance:wait(timeout) + if not self._READER then + return false, "pipe event reader closed" + end + local read, read_errors = self._READER:read(self._BUFFER, 1, {block = true, timeout = timeout}) + if read < 0 then + return false, read_errors + end + return read +end + +function _instance:close() + if self._READER then + self._READER:close() + end + if self._WRITER then + self._WRITER:close() + end + self._READER = nil + self._WRITER = nil + self._WRITER_PTR = nil +end + +-- return pipe cdata for serialization (writer pointer is stable for passing across threads) +function _instance:cdata() + if self._WRITER then + return self._WRITER:cdata() + end + return self._WRITER_PTR +end + +function _instance:__gc() + self:close() +end + +function _instance:_serialize() + if not self._WRITER and self._WRITER_PTR then + return {ptr = self._WRITER_PTR, name = self:name()} + end + if not self._WRITER then + return nil + end + local ptr = libc.dataptr(self._WRITER:cdata(), {ffi = false}) + if not ptr then + return nil + end + self._WRITER._PIPE = nil + self._WRITER = nil + self._WRITER_PTR = ptr + return {ptr = ptr, name = self:name()} +end + +function _instance:_deserialize(data) + if not data or not data.ptr then + return false, "invalid pipe event data" + end + self:close() + local writer = pipe.new(libc.ptraddr(data.ptr, {ffi = false})) + if not writer then + return false, "invalid pipe pointer" + end + self._NAME = data.name or self._NAME or "pipe_event" + self._WRITER = writer + self._WRITER_PTR = data.ptr + return true +end + +function pipe_event.new(name) + return _instance.new(name) +end + +return pipe_event + + diff --git a/xmake/core/base/thread.lua b/xmake/core/base/thread.lua index 3856feb812c..26ac672c37a 100644 --- a/xmake/core/base/thread.lua +++ b/xmake/core/base/thread.lua @@ -28,14 +28,15 @@ local _queue = _queue or {} local _sharedata = _sharedata or {} -- load modules -local io = require("base/io") -local libc = require("base/libc") -local pipe = require("base/pipe") -local bytes = require("base/bytes") -local table = require("base/table") -local string = require("base/string") -local scheduler = require("base/scheduler") -local sandbox = require("sandbox/sandbox") +local io = require("base/io") +local libc = require("base/libc") +local pipe = require("base/pipe") +local pipe_event = require("base/private/pipe_event") +local bytes = require("base/bytes") +local table = require("base/table") +local string = require("base/string") +local scheduler = require("base/scheduler") +local sandbox = require("sandbox/sandbox") -- the thread status thread.STATUS_READY = 1 @@ -52,6 +53,7 @@ function _thread.new(callback, opt) instance._CALLBACK = callback instance._STACKSIZE = opt.stacksize or 0 instance._STATUS = thread.STATUS_READY + instance._INTERNAL = opt.internal setmetatable(instance, _thread) return instance end @@ -102,41 +104,28 @@ function _thread:start() local argv = {} for _, arg in ipairs(self._ARGV) do if type(arg) == "table" then - -- is mutex? we can only pass cdata address - if arg._MUTEX and arg.cdata then - thread.mutex_incref(arg:cdata()) - arg = {mutex = true, name = arg:name(), caddr = libc.dataptr(arg:cdata(), {ffi = false})} - -- is event? we can only pass cdata address - elseif arg._EVENT and arg.cdata then - thread.event_incref(arg:cdata()) - arg = {event = true, name = arg:name(), caddr = libc.dataptr(arg:cdata(), {ffi = false})} - -- is semaphore? we can only pass cdata address - elseif arg._SEMAPHORE and arg.cdata then - thread.semaphore_incref(arg:cdata()) - arg = {semaphore = true, name = arg:name(), caddr = libc.dataptr(arg:cdata(), {ffi = false})} - -- is queue? we can only pass cdata address - elseif arg._QUEUE and arg.cdata then - thread.queue_incref(arg:cdata()) - arg = {queue = true, name = arg:name(), caddr = libc.dataptr(arg:cdata(), {ffi = false})} - -- is sharedata? we can only pass cdata address - elseif arg._SHAREDATA and arg.cdata then - thread.sharedata_incref(arg:cdata()) - arg = {sharedata = true, name = arg:name(), caddr = libc.dataptr(arg:cdata())} + -- try to serialize thread object (mutex, event, semaphore, queue, sharedata) + local serialized = thread._serialize_object(arg) + if serialized then + arg = serialized end end table.insert(argv, arg) end -- init callback info + local is_internal = self._INTERNAL local callback = string._dump(self._CALLBACK) - local callinfo = {name = self:name(), argv = argv} + local callinfo = {name = self:name(), argv = argv, internal = is_internal} -- we need a pipe pair to wait and listen thread exit event - local rpipe, wpipe = pipe.openpair("AA") - self._RPIPE = rpipe - callinfo.wpipe = libc.dataptr(wpipe:cdata(), {ffi = false}) - -- we need to suppress gc to free it, because it has been transfer to thread in another lua state instance - wpipe._PIPE = nil + if not is_internal then + local rpipe, wpipe = pipe.openpair("AA") + self._RPIPE = rpipe + callinfo.wpipe = libc.dataptr(wpipe:cdata(), {ffi = false}) + -- we need to suppress gc to free it, because it has been transfer to thread in another lua state instance + wpipe._PIPE = nil + end -- serialize and pass callback and arguments to this thread -- we do not use string.serialize to serialize callback, because it's slower (deserialize) @@ -206,8 +195,13 @@ function _thread:wait(timeout) ok = read errors = data_or_errors end - else - ok, errors = thread.thread_wait(self:cdata(), timeout) + end + if not rpipe then + local waitok, wait_errors = thread.thread_wait(self:cdata(), timeout) + if ok == nil or ok > 0 then + ok = waitok + errors = wait_errors + end end if ok < 0 then return -1, errors or string.format("%s: failed to resume thread!", self) @@ -781,23 +775,90 @@ function _sharedata:__gc() end end --- new a thread --- --- @param callback the thread callback --- @param opt the thread options, e.g. {name = "", argv = {}, stacksize = 8192} --- --- @return the thread instance --- -function thread.new(callback, opt) - if callback == nil then - return nil, "invalid thread, callback is nil" +-- serialize thread object for passing through queue or table (private helper) +-- this is used when you need to pass thread objects (mutex, event, semaphore, queue, sharedata) +-- through a queue or embed them in a table +-- returns a table with serialized caddr that can be pushed to queue +function thread._serialize_object(obj) + if not obj or type(obj) ~= "table" or not obj.cdata then + return nil + end + + local result = {} + -- detect object type by checking internal marker + if obj._MUTEX then + thread.mutex_incref(obj:cdata()) + result.mutex = true + result.name = obj:name() + result.caddr = libc.dataptr(obj:cdata(), {ffi = false}) + elseif obj._EVENT then + thread.event_incref(obj:cdata()) + result.event = true + result.name = obj:name() + result.caddr = libc.dataptr(obj:cdata(), {ffi = false}) + elseif obj._SEMAPHORE then + thread.semaphore_incref(obj:cdata()) + result.semaphore = true + result.name = obj:name() + result.caddr = libc.dataptr(obj:cdata(), {ffi = false}) + elseif obj._QUEUE then + thread.queue_incref(obj:cdata()) + result.queue = true + result.name = obj:name() + result.caddr = libc.dataptr(obj:cdata(), {ffi = false}) + elseif obj._SHAREDATA then + thread.sharedata_incref(obj:cdata()) + result.sharedata = true + result.name = obj:name() + result.caddr = libc.dataptr(obj:cdata(), {ffi = false}) + elseif obj._PIPE_EVENT then + local data = obj:_serialize() + if not data then + return nil + end + result.pipe_event = true + result.ptr = data.ptr + result.name = data.name + result.caddr = data.ptr + else + return nil + end + + return result +end + +-- deserialize thread object from serialized data (private helper) +-- this is used to restore thread objects (mutex, event, semaphore, queue, sharedata) +-- from serialized caddr received through queue or from table +function thread._deserialize_object(data) + if not data or type(data) ~= "table" or not data.caddr then + return nil + end + + local cdata = libc.ptraddr(data.caddr, {ffi = false}) + if data.mutex then + return _mutex.new(data.name, cdata) + elseif data.event then + return _event.new(data.name, cdata) + elseif data.semaphore then + return _semaphore.new(data.name, cdata) + elseif data.queue then + return _queue.new(data.name, cdata) + elseif data.sharedata then + return _sharedata.new(data.name, cdata) + elseif data.pipe_event then + local event = pipe_event.new(data.name) + if not event then + return nil, "failed to create pipe event" + end + local ok, errors = event:_deserialize({ptr = data.ptr, name = data.name}) + if not ok then + return nil, errors + end + return event end - return _thread.new(callback, opt) -end --- get the running thread name -function thread.running() - return thread._RUNNING + return nil end -- run thread @@ -808,6 +869,7 @@ function thread._run_thread(callback_str, callinfo_str) local argv local threadname local wpipe + local is_internal = false if callinfo_str then local result, errors = string.deserialize(callinfo_str) if not result then @@ -817,7 +879,10 @@ function thread._run_thread(callback_str, callinfo_str) if callinfo then argv = callinfo.argv threadname = callinfo.name - wpipe = pipe.new(libc.ptraddr(callinfo.wpipe, {ffi = false})) + is_internal = callinfo.internal + if callinfo.wpipe then + wpipe = pipe.new(libc.ptraddr(callinfo.wpipe, {ffi = false})) + end end end @@ -850,23 +915,24 @@ function thread._run_thread(callback_str, callinfo_str) return false, errors end + -- if it's an internal thread, we need to bind some additional internal interfaces. + if is_internal then + sandbox_inst:api_register_builtin("require", require) + end + -- save the running thread name thread._RUNNING = threadname - -- translate arguments (mutex, ...) + -- translate arguments (mutex, event, semaphore, queue, sharedata, ...) if argv then local newargv = {} for _, arg in ipairs(argv) do - if type(arg) == "table" and arg.mutex and arg.caddr then - arg = _mutex.new(arg.name, libc.ptraddr(arg.caddr, {ffi = false})) - elseif type(arg) == "table" and arg.event and arg.caddr then - arg = _event.new(arg.name, libc.ptraddr(arg.caddr, {ffi = false})) - elseif type(arg) == "table" and arg.semaphore and arg.caddr then - arg = _semaphore.new(arg.name, libc.ptraddr(arg.caddr, {ffi = false})) - elseif type(arg) == "table" and arg.queue and arg.caddr then - arg = _queue.new(arg.name, libc.ptraddr(arg.caddr, {ffi = false})) - elseif type(arg) == "table" and arg.sharedata and arg.caddr then - arg = _sharedata.new(arg.name, libc.ptraddr(arg.caddr, {ffi = false})) + if type(arg) == "table" and arg.caddr then + -- try to deserialize thread object + local obj = thread._deserialize_object(arg) + if obj then + arg = obj + end end table.insert(newargv, arg) end @@ -887,6 +953,25 @@ function thread._run_thread(callback_str, callinfo_str) return ok, errors end +-- new a thread +-- +-- @param callback the thread callback +-- @param opt the thread options, e.g. {name = "", argv = {}, stacksize = 8192} +-- +-- @return the thread instance +-- +function thread.new(callback, opt) + if callback == nil then + return nil, "invalid thread, callback is nil" + end + return _thread.new(callback, opt) +end + +-- get the running thread name +function thread.running() + return thread._RUNNING +end + -- open a mutex function thread.mutex(name) local mutex = thread.mutex_init() diff --git a/xmake/core/base/utils.lua b/xmake/core/base/utils.lua index d20b3c7746d..ac9884851e0 100644 --- a/xmake/core/base/utils.lua +++ b/xmake/core/base/utils.lua @@ -193,6 +193,20 @@ function utils.vprintf(format, ...) end end +-- print the diagnosis information +function utils.dprint(format, ...) + if option.get("diagnosis") and format ~= nil then + utils.print(format, ...) + end +end + +-- print the diagnosis information without newline +function utils.dprintf(format, ...) + if option.get("diagnosis") and format ~= nil then + utils.printf(format, ...) + end +end + -- print the error information function utils.error(format, ...) if format ~= nil then diff --git a/xmake/core/base/xmake.lua b/xmake/core/base/xmake.lua index 2b41e9a9dde..d25e60147d6 100644 --- a/xmake/core/base/xmake.lua +++ b/xmake/core/base/xmake.lua @@ -67,6 +67,11 @@ function xmake.is_embed() return xmake._EMBED or false end +-- in main thread? +function xmake.in_main_thread() + return xmake._THREAD_CALLBACK == nil +end + -- get command arguments function xmake.argv() return xmake._ARGV diff --git a/xmake/core/main.lua b/xmake/core/main.lua index 3f0a4484f2a..1c51342b94d 100644 --- a/xmake/core/main.lua +++ b/xmake/core/main.lua @@ -52,9 +52,11 @@ local menu = -- the tasks: xmake [task] , function () local tasks = task.tasks() or {} - local ok, project_tasks = pcall(project.tasks) - if ok then - table.join2(tasks, project_tasks) + if xmake.in_main_thread() then + local ok, project_tasks = pcall(project.tasks) + if ok then + table.join2(tasks, project_tasks) + end end return task.menu(tasks) end @@ -218,7 +220,7 @@ function main._init() xmake._PROJECT_FILE = projectfile -- enter the project directory - if os.isdir(os.projectdir()) then + if os.isdir(os.projectdir()) and xmake.in_main_thread() then os.cd(os.projectdir()) end else @@ -302,33 +304,36 @@ function main.entry() return main._exit(ok, errors) end - -- check run command as root - if main._limit_root() then - if os.isroot() then - errors = [[Running xmake as root is extremely dangerous and no longer supported. -As xmake does not drop privileges on installation you would be giving all -build scripts full access to your system. -Or you can add `--root` option or XMAKE_ROOT=y to allow run as root temporarily. - ]] - return main._exit(false, errors) + if xmake.in_main_thread() then + + -- check run command as root + if main._limit_root() then + if os.isroot() then + errors = [[Running xmake as root is extremely dangerous and no longer supported. + As xmake does not drop privileges on installation you would be giving all + build scripts full access to your system. + Or you can add `--root` option or XMAKE_ROOT=y to allow run as root temporarily. + ]] + return main._exit(false, errors) + end end - end - -- show help? - if main._show_help() then - return main._exit(true) - end + -- show help? + if main._show_help() then + return main._exit(true) + end - -- save command lines to history and we need to make sure that the .xmake directory is not generated everywhere - local skip_history = (os.getenv('XMAKE_SKIP_HISTORY') or ''):trim() - if os.projectfile() and os.isfile(os.projectfile()) and os.isdir(config.directory()) and skip_history == '' then - local cmdlines = table.wrap(localcache.get("history", "cmdlines")) - if #cmdlines > 64 then - table.remove(cmdlines, 1) + -- save command lines to history and we need to make sure that the .xmake directory is not generated everywhere + local skip_history = (os.getenv('XMAKE_SKIP_HISTORY') or ''):trim() + if os.projectfile() and os.isfile(os.projectfile()) and os.isdir(config.directory()) and skip_history == '' then + local cmdlines = table.wrap(localcache.get("history", "cmdlines")) + if #cmdlines > 64 then + table.remove(cmdlines, 1) + end + table.insert(cmdlines, option.cmdline()) + localcache.set("history", "cmdlines", cmdlines) + localcache.save("history") end - table.insert(cmdlines, option.cmdline()) - localcache.set("history", "cmdlines", cmdlines) - localcache.save("history") end -- enable scheduler @@ -362,3 +367,4 @@ end -- return module: main return main + diff --git a/xmake/core/project/project.lua b/xmake/core/project/project.lua index 8b94cbff768..dc54507a929 100644 --- a/xmake/core/project/project.lua +++ b/xmake/core/project/project.lua @@ -1276,6 +1276,11 @@ end -- get the project menu function project.menu() + -- we cannot only get it in main thread + if not xmake.in_main_thread() then + return {} + end + -- attempt to load options from the project file local options = nil local errors = nil diff --git a/xmake/core/sandbox/modules/import/lib/detect/find_directory.lua b/xmake/core/sandbox/modules/import/lib/detect/find_directory.lua index 4c6048428a5..ec737785b9c 100644 --- a/xmake/core/sandbox/modules/import/lib/detect/find_directory.lua +++ b/xmake/core/sandbox/modules/import/lib/detect/find_directory.lua @@ -28,6 +28,66 @@ local utils = require("base/utils") local table = require("base/table") local raise = require("sandbox/modules/raise") local vformat = require("sandbox/modules/vformat") +local xmake = require("base/xmake") + +-- expand search paths +function sandbox_lib_detect_find_directory._expand_paths(paths) + local results = {} + for _, _path in ipairs(table.wrap(paths)) do + if type(_path) == "function" then + local ok, result_or_errors = sandbox.load(_path) + if ok then + _path = result_or_errors or "" + else + raise(result_or_errors) + end + else + _path = vformat(_path) + end + for _, _s_path in ipairs(table.wrap(_path)) do + _s_path = tostring(_s_path) + if #_s_path > 0 then + table.insert(results, _s_path) + end + end + end + return results +end + +-- normalize suffixes +function sandbox_lib_detect_find_directory._normalize_suffixes(suffixes) + local results = {} + for _, suffix in ipairs(table.wrap(suffixes)) do + suffix = tostring(suffix) + if #suffix > 0 then + table.insert(results, suffix) + end + end + return results +end + +-- find directory from directories list +function sandbox_lib_detect_find_directory._find_from_directories(name, directories, suffixes) + directories = table.wrap(directories) + suffixes = table.wrap(suffixes) + if #suffixes > 0 then + for _, directory in ipairs(directories) do + for _, suffix in ipairs(suffixes) do + local results = os.dirs(path.join(directory, suffix, name), function (file, isdir) return false end) + if results and #results > 0 then + return results[1] + end + end + end + else + for _, directory in ipairs(directories) do + local results = os.dirs(path.join(directory, name), function (file, isdir) return false end) + if results and #results > 0 then + return results[1] + end + end + end +end -- find directory -- @@ -51,43 +111,17 @@ function sandbox_lib_detect_find_directory.main(name, paths, opt) -- init options opt = opt or {} - -- init paths - paths = table.wrap(paths) + local suffixes = sandbox_lib_detect_find_directory._normalize_suffixes(opt.suffixes) + local directories = sandbox_lib_detect_find_directory._expand_paths(paths) - -- append suffixes to paths - local suffixes = table.wrap(opt.suffixes) - if #suffixes > 0 then - local paths_new = {} - for _, parent in ipairs(paths) do - for _, suffix in ipairs(suffixes) do - table.insert(paths_new, path.join(parent, suffix)) - end - end - paths = paths_new + if opt.async and xmake.in_main_thread() then + local result, _ = os._async_task().find_directory(name, directories, {suffixes = suffixes}) + return result end - -- find file - for _, _path in ipairs(paths) do - - -- format path for builtin variables - if type(_path) == "function" then - local ok, results = sandbox.load(_path) - if ok then - _path = results or "" - else - raise(results) - end - else - _path = vformat(_path) - end - - -- find the first directory - local results = os.dirs(path.join(_path, name), function (file, isdir) return false end) - if results and #results > 0 then - return results[1] - end - end + return sandbox_lib_detect_find_directory._find_from_directories(name, directories, suffixes) end -- return module return sandbox_lib_detect_find_directory + diff --git a/xmake/core/sandbox/modules/import/lib/detect/find_file.lua b/xmake/core/sandbox/modules/import/lib/detect/find_file.lua index 8c82d55466b..71e580ba168 100644 --- a/xmake/core/sandbox/modules/import/lib/detect/find_file.lua +++ b/xmake/core/sandbox/modules/import/lib/detect/find_file.lua @@ -29,6 +29,72 @@ local table = require("base/table") local profiler = require("base/profiler") local raise = require("sandbox/modules/raise") local vformat = require("sandbox/modules/vformat") +local xmake = require("base/xmake") + +-- expand search paths +function sandbox_lib_detect_find_file._expand_paths(paths) + local results = {} + for _, _path in ipairs(table.wrap(paths)) do + if type(_path) == "function" then + local ok, result_or_errors = sandbox.load(_path) + if ok then + _path = result_or_errors or "" + else + raise(result_or_errors) + end + elseif type(_path) == "string" then + if _path:match("^%$%(env .+%)$") then + _path = path.splitenv(vformat(_path)) + else + _path = vformat(_path) + end + end + for _, _s_path in ipairs(table.wrap(_path)) do + _s_path = tostring(_s_path) + if #_s_path > 0 then + table.insert(results, _s_path) + end + end + end + return results +end + +-- normalize suffixes +function sandbox_lib_detect_find_file._normalize_suffixes(suffixes) + local results = {} + for _, suffix in ipairs(table.wrap(suffixes)) do + suffix = tostring(suffix) + if #suffix > 0 then + table.insert(results, suffix) + end + end + return results +end + +-- find from directories list +function sandbox_lib_detect_find_file._find_from_directories(name, directories, suffixes) + local results + suffixes = table.wrap(suffixes) + directories = table.wrap(directories) + if #suffixes > 0 then + for _, directory in ipairs(directories) do + for _, suffix in ipairs(suffixes) do + local filedir = path.join(directory, suffix) + results = sandbox_lib_detect_find_file._find(filedir, name) + if results then + return results + end + end + end + else + for _, directory in ipairs(directories) do + results = sandbox_lib_detect_find_file._find(directory, name) + if results then + return results + end + end + end +end -- find the given file path or directory function sandbox_lib_detect_find_file._find(filedir, name) @@ -72,48 +138,17 @@ function sandbox_lib_detect_find_file.main(name, paths, opt) -- find file profiler:enter("find_file", name) - local results - local suffixes = table.wrap(opt.suffixes) - for _, _path in ipairs(table.wrap(paths)) do - - -- format path for builtin variables - if type(_path) == "function" then - local ok, result_or_errors = sandbox.load(_path) - if ok then - _path = result_or_errors or "" - else - raise(result_or_errors) - end - elseif type(_path) == "string" then - if _path:match("^%$%(env .+%)$") then - _path = path.splitenv(vformat(_path)) - else - _path = vformat(_path) - end - end + local suffixes = sandbox_lib_detect_find_file._normalize_suffixes(opt.suffixes) + local directories = sandbox_lib_detect_find_file._expand_paths(paths) - for _, _s_path in ipairs(table.wrap(_path)) do - if #_s_path > 0 then - -- find file with suffixes - if #suffixes > 0 then - for _, suffix in ipairs(suffixes) do - local filedir = path.join(_s_path, suffix) - results = sandbox_lib_detect_find_file._find(filedir, name) - if results then - goto found - end - end - else - -- find file in the given path - results = sandbox_lib_detect_find_file._find(_s_path, name) - if results then - goto found - end - end - end - end + -- run in async task? + if opt.async and xmake.in_main_thread() then + local result, _ = os._async_task().find_file(name, directories, {suffixes = suffixes}) + profiler:leave("find_file", name) + return result end -::found:: + + local results = sandbox_lib_detect_find_file._find_from_directories(name, directories, suffixes) profiler:leave("find_file", name) return results end diff --git a/xmake/core/sandbox/modules/import/lib/detect/find_library.lua b/xmake/core/sandbox/modules/import/lib/detect/find_library.lua index e072dded1c6..ab81e88b816 100644 --- a/xmake/core/sandbox/modules/import/lib/detect/find_library.lua +++ b/xmake/core/sandbox/modules/import/lib/detect/find_library.lua @@ -30,8 +30,40 @@ local config = require("project/config") local target = require("project/target") local raise = require("sandbox/modules/raise") local import = require("sandbox/modules/import") +local xmake = require("base/xmake") local find_file = import("lib.detect.find_file") +-- find library from directories list +function sandbox_lib_detect_find_library._find_from_directories(names, directories, kinds, suffixes, opt) + names = table.wrap(names) + directories = table.wrap(directories) + kinds = table.wrap(kinds) + suffixes = table.wrap(suffixes) + if #kinds == 0 then + kinds = {"static", "shared"} + end + opt = opt or {} + local plat = opt.plat + for _, name in ipairs(names) do + for _, kind in ipairs(kinds) do + local filename = target.filename(name, kind, {plat = plat}) + local filepath = find_file._find_from_directories(filename, directories, suffixes) + if plat == "mingw" then + if not filepath and kind == "shared" then + filepath = find_file._find_from_directories(filename .. ".a", directories, suffixes) + end + if not filepath then + filepath = find_file._find_from_directories(target.filename(name, kind, {plat = "windows"}), directories, suffixes) + end + end + if filepath then + local linkname = target.linkname(path.filename(filepath), {plat = plat}) + return {kind = kind, filename = path.filename(filepath), linkdir = path.directory(filepath), link = linkname} + end + end + end +end + -- find library -- -- @param names the library names @@ -56,27 +88,16 @@ function sandbox_lib_detect_find_library.main(names, paths, opt) -- find library file from the given paths opt = opt or {} - local kinds = opt.kind or {"static", "shared"} - for _, name in ipairs(table.wrap(names)) do - for _, kind in ipairs(table.wrap(kinds)) do - local filepath = find_file(target.filename(name, kind, {plat = opt.plat}), paths, opt) - if opt.plat == "mingw" then - if not filepath and kind == "shared" then - -- for implib/mingw, e.g. libxxx.dll.a - filepath = find_file(target.filename(name, kind, {plat = opt.plat}) .. ".a", paths, opt) - end - if not filepath then - -- in order to be compatible with mingw/windows library with .lib - filepath = find_file(target.filename(name, kind, {plat = "windows"}), paths, opt) - end - end - if filepath then - local filename = path.filename(filepath) - local linkname = target.linkname(filename, {plat = opt.plat}) - return {kind = kind, filename = filename, linkdir = path.directory(filepath), link = linkname} - end - end + local directories = find_file._expand_paths(paths) + local suffixes = find_file._normalize_suffixes(opt.suffixes) + local kinds = table.wrap(opt.kind or {"static", "shared"}) + + if opt.async and xmake.in_main_thread() then + local result, _ = os._async_task().find_library(names, directories, kinds, {suffixes = suffixes, plat = opt.plat}) + return result end + + return sandbox_lib_detect_find_library._find_from_directories(names, directories, kinds, suffixes, {plat = opt.plat}) end -- return module diff --git a/xmake/core/sandbox/modules/import/lib/detect/find_path.lua b/xmake/core/sandbox/modules/import/lib/detect/find_path.lua index f6a9ec42631..cd7413fb7af 100644 --- a/xmake/core/sandbox/modules/import/lib/detect/find_path.lua +++ b/xmake/core/sandbox/modules/import/lib/detect/find_path.lua @@ -28,6 +28,43 @@ local table = require("base/table") local profiler = require("base/profiler") local raise = require("sandbox/modules/raise") local vformat = require("sandbox/modules/vformat") +local xmake = require("base/xmake") + +-- expand search paths +function sandbox_lib_detect_find_path._expand_paths(paths) + local results = {} + for _, _path in ipairs(table.wrap(paths)) do + if type(_path) == "function" then + local ok, result_or_errors = sandbox.load(_path) + if ok then + _path = result_or_errors or "" + else + raise(result_or_errors) + end + else + _path = vformat(_path) + end + for _, _s_path in ipairs(table.wrap(_path)) do + _s_path = tostring(_s_path) + if #_s_path > 0 then + table.insert(results, _s_path) + end + end + end + return results +end + +-- normalize suffixes +function sandbox_lib_detect_find_path._normalize_suffixes(suffixes) + local results = {} + for _, suffix in ipairs(table.wrap(suffixes)) do + suffix = tostring(suffix) + if #suffix > 0 then + table.insert(results, suffix) + end + end + return results +end -- find the given file path or directory function sandbox_lib_detect_find_path._find(filedir, name) @@ -52,6 +89,31 @@ function sandbox_lib_detect_find_path._find(filedir, name) end end +-- find from directories list +function sandbox_lib_detect_find_path._find_from_directories(name, directories, suffixes) + local results + suffixes = table.wrap(suffixes) + directories = table.wrap(directories) + if #suffixes > 0 then + for _, directory in ipairs(directories) do + for _, suffix in ipairs(suffixes) do + local filedir = path.join(directory, suffix) + results = sandbox_lib_detect_find_path._find(filedir, name) + if results then + return results + end + end + end + else + for _, directory in ipairs(directories) do + results = sandbox_lib_detect_find_path._find(directory, name) + if results then + return results + end + end + end +end + -- find path -- -- @param name the path name @@ -76,42 +138,17 @@ function sandbox_lib_detect_find_path.main(name, paths, opt) -- init options opt = opt or {} - -- find path - local results profiler:enter("find_path", name) - local suffixes = table.wrap(opt.suffixes) - for _, _path in ipairs(table.wrap(paths)) do + local suffixes = sandbox_lib_detect_find_path._normalize_suffixes(opt.suffixes) + local directories = sandbox_lib_detect_find_path._expand_paths(paths) - -- format path for builtin variables - if type(_path) == "function" then - local ok, result_or_errors = sandbox.load(_path) - if ok then - _path = result_or_errors or "" - else - raise(result_or_errors) - end - else - _path = vformat(_path) - end - - -- find file with suffixes - if #suffixes > 0 then - for _, suffix in ipairs(suffixes) do - local filedir = path.join(_path, suffix) - results = sandbox_lib_detect_find_path._find(filedir, name) - if results then - goto found - end - end - else - -- find file in the given path - results = sandbox_lib_detect_find_path._find(_path, name) - if results then - goto found - end - end + if opt.async and xmake.in_main_thread() then + local result, _ = os._async_task().find_path(name, directories, {suffixes = suffixes}) + profiler:leave("find_path", name) + return result end -::found:: + + local results = sandbox_lib_detect_find_path._find_from_directories(name, directories, suffixes) profiler:leave("find_path", name) return results end diff --git a/xmake/core/sandbox/modules/os.lua b/xmake/core/sandbox/modules/os.lua index 580e9e31dc8..aebb3cfb022 100644 --- a/xmake/core/sandbox/modules/os.lua +++ b/xmake/core/sandbox/modules/os.lua @@ -225,9 +225,9 @@ function sandbox_os.mkdir(dir) end -- remove directories -function sandbox_os.rmdir(dir) +function sandbox_os.rmdir(dir, opt) assert(dir) - local ok, errors = os.rmdir(vformat(dir)) + local ok, errors = os.rmdir(vformat(dir), opt) if not ok then os.raise(errors) end @@ -411,23 +411,23 @@ function sandbox_os.vexecv(program, argv, opt) end -- match files or directories -function sandbox_os.match(pattern, mode, callback) - return os.match(vformat(tostring(pattern)), mode, callback) +function sandbox_os.match(pattern, mode, opt) + return os.match(vformat(tostring(pattern)), mode, opt) end -- match directories -function sandbox_os.dirs(pattern, callback) - return (sandbox_os.match(pattern, 'd', callback)) +function sandbox_os.dirs(pattern, opt) + return os.dirs(vformat(tostring(pattern)), opt) end -- match files -function sandbox_os.files(pattern, callback) - return (sandbox_os.match(pattern, 'f', callback)) +function sandbox_os.files(pattern, opt) + return os.files(vformat(tostring(pattern)), opt) end -- match files and directories -function sandbox_os.filedirs(pattern, callback) - return (sandbox_os.match(pattern, 'a', callback)) +function sandbox_os.filedirs(pattern, opt) + return os.filedirs(vformat(tostring(pattern)), opt) end -- is directory? diff --git a/xmake/core/sandbox/modules/xmake.lua b/xmake/core/sandbox/modules/xmake.lua index 8c040a9a0d1..b83536f4a24 100644 --- a/xmake/core/sandbox/modules/xmake.lua +++ b/xmake/core/sandbox/modules/xmake.lua @@ -25,14 +25,15 @@ local xmake = require("base/xmake") local sandbox_xmake = sandbox_xmake or {} -- inherit some builtin interfaces -sandbox_xmake.arch = xmake.arch -sandbox_xmake.version = xmake.version -sandbox_xmake.branch = xmake.branch -sandbox_xmake.programdir = xmake.programdir -sandbox_xmake.programfile = xmake.programfile -sandbox_xmake.luajit = xmake.luajit -sandbox_xmake.is_embed = xmake.is_embed -sandbox_xmake.argv = xmake.argv +sandbox_xmake.arch = xmake.arch +sandbox_xmake.version = xmake.version +sandbox_xmake.branch = xmake.branch +sandbox_xmake.programdir = xmake.programdir +sandbox_xmake.programfile = xmake.programfile +sandbox_xmake.luajit = xmake.luajit +sandbox_xmake.is_embed = xmake.is_embed +sandbox_xmake.in_main_thread = xmake.in_main_thread +sandbox_xmake.argv = xmake.argv -- return module return sandbox_xmake diff --git a/xmake/core/sandbox/sandbox.lua b/xmake/core/sandbox/sandbox.lua index c3fa92a2ff4..758a4803021 100644 --- a/xmake/core/sandbox/sandbox.lua +++ b/xmake/core/sandbox/sandbox.lua @@ -245,6 +245,11 @@ function sandbox:namespace() return self._PRIVATE._NAMESPACE end +-- register api for builtin +function sandbox:api_register_builtin(name, func) + sandbox._api_register_builtin(self, name, func) +end + -- get current instance in the sandbox modules function sandbox.instance(script) diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index d3c0094d866..2e7e6be8964 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -339,7 +339,7 @@ function build(program, argv, opt) _g.cache_miss_total_time = (_g.cache_miss_total_time or 0) + (os.mclock() - cache_miss_start_time) end end - os.rm(cppinfo.cppfile) + os.tryrm(cppinfo.cppfile) else _g.preprocess_error_count = (_g.preprocess_error_count or 0) + 1 end diff --git a/xmake/modules/utils/run_script.lua b/xmake/modules/utils/run_script.lua index a5ab2fa5043..82fa8cb0d29 100644 --- a/xmake/modules/utils/run_script.lua +++ b/xmake/modules/utils/run_script.lua @@ -150,14 +150,19 @@ function _run(script, opt) option.set("quiet", true, {force = true}) end - local curdir = opt.curdir or os.workingdir() - local oldir = os.cd(curdir) + local oldir + if xmake.in_main_thread() then + local curdir = opt.curdir or os.workingdir() + oldir = os.cd(curdir) + end if opt.command then _run_commanad(script, _get_args(opt), opt) else _run_script(script, _get_args(opt), opt) end - os.cd(oldir) + if oldir then + os.cd(oldir) + end if opt.quiet then option.restore()