workaround for broken io.popen and os.execute in luajit

[rweichler]

On palera1n rootless, io.popen and os.execute are broken. I think a proper fix would require patching the source of luajit (replace the calls to system and popen with posix_spawn). But nobody's got time for that. So here's some FFI code. Examples at the bottom.

ffi = require 'ffi'
C = ffi.C

LazyCdef = function(s, lib)
    lib = lib or C
    local name = assert(s:match'^[^%s]+%s+([^(]+)%(')
    assert(not name:find'%s' and not name:find'%*', 'invalid cdef: '..s)
    local fn = function() return lib[name] end
    if not pcall(fn) then
        ffi.cdef(s)
    end
    return fn()
end

local pidPtr
Pspawn = function(args, callback)
    if type(args) == 'string' then
        args = args:find'%s'
            and {'sh', '-c', args}
            or {args}
    end
    if callback then
        assert(type(callback) == 'function', 'expected function')
        if not pcall(function() return ffi.typeof'struct pollfd' end) then
            ffi.cdef'struct pollfd { int fd; short events; short revents; }'
            ffi.typeof'struct pollfd'
        end
        LazyCdef'int poll(struct pollfd *, unsigned int, int);'
    end
    local bit_band = bit and bit.band or load'local a,b = ...; return a&b'
    -- pid_t is 32-bit on every platform that matters
    LazyCdef'int32_t waitpid(int32_t, int *, int);'
    LazyCdef'int posix_spawnp(int32_t *pid, const char *arg0, void *, void *, const char *argv[], const char *envp[]);'
    LazyCdef'int posix_spawn_file_actions_init(void **);'
    LazyCdef'int posix_spawn_file_actions_destroy(void **);'
    LazyCdef'int posix_spawn_file_actions_addclose(void **, int);'
    LazyCdef'int posix_spawn_file_actions_adddup2(void **, int, int);'
    LazyCdef'int pipe(int[2]);'
    LazyCdef'int close(int);'
    LazyCdef'ssize_t read(int, void *, size_t);'
    local STDIN_FILENO = 0
    local STDOUT_FILENO = 1
    local STDERR_FILENO = 2
    local EINTR = 4
    local POLLIN = 0x0001 -- in macOS arm64 and Linux x64
    local POLLERR = 0x0008 -- in macOS arm64 and Linux x64
    local POLLHUP = 0x0010 -- in macOS arm64 and Linux x64
    local POLLNVAL = 0x0020 -- in macOS arm64 (not in Linux x64)
    -- setup fa, outfd, errfd
    -- proxy for posix_spawn_file_actions_t;. made it huge just to be safe
    local fa = ffi.new'void *[8192]'
    local outfd = ffi.new('int[2]', -1, -1)
    local errfd = ffi.new('int[2]', -1, -1)
    local faInitted = false
    function rc(fn, ...)
        local rc = fn(...)
        if rc == 0 then return end
        if faInitted then C.posix_spawn_file_actions_destroy(fa) end
        if outfd[0] ~= -1 then C.close(outfd[0]) end
        if errfd[0] ~= -1 then C.close(errfd[0]) end
        if outfd[1] ~= -1 then C.close(outfd[1]) end
        if errfd[1] ~= -1 then C.close(errfd[1]) end
        error('rc is '..rc)
    end
    rc(C.posix_spawn_file_actions_init, fa)
    faInitted = true
    rc(C.pipe, outfd)
    rc(C.pipe, errfd)
    rc(C.posix_spawn_file_actions_adddup2, fa, outfd[1], STDOUT_FILENO)
    rc(C.posix_spawn_file_actions_adddup2, fa, errfd[1], STDERR_FILENO)
    rc(C.posix_spawn_file_actions_addclose, fa, outfd[0])
    rc(C.posix_spawn_file_actions_addclose, fa, errfd[0])
    rc(C.posix_spawn_file_actions_addclose, fa, outfd[1])
    rc(C.posix_spawn_file_actions_addclose, fa, errfd[1])
    -- setup other args for posix_spawn
    local argv = ffi.new('const char*[?]', #args + 1, args)
    argv[#args] = nil
    pidPtr = pidPtr or ffi.new('int32_t[1]', -1)
    local envp = ffi.cast('const char **', nil)
    -- critical function
    rc(C.posix_spawnp, pidPtr, argv[0], fa, nil, argv, envp)
    -- read from stdout / stderr
    C.posix_spawn_file_actions_destroy(fa)
    C.close(outfd[1])
    C.close(errfd[1])
    outfd[1] = -1
    errfd[1] = -1
    local buf = ffi.new'uint8_t[8192]'
    local function readfd(fd)
        local ans = {}
        while true do
            local n = C.read(fd, buf, ffi.sizeof(buf))
            if n > 0 then
                table.insert(ans, ffi.string(buf, n))
            elseif n == 0 or ffi.errno() ~= EINTR then
                -- eof or read error
                break
            end
        end
        C.close(fd)
        return table.concat(ans)
    end
    local stdout, stderr
    if callback then
        local pfds = ffi.new('struct pollfd[2]')
        pfds[0].fd = outfd[0]
        pfds[1].fd = errfd[0]
        pfds[0].events = POLLIN
        pfds[1].events = POLLIN
        while pfds[0].fd ~= -1 and pfds[1].fd ~= -1 do
            local n = C.poll(pfds, 2, -1)
            if n == 0 or (n < 0 and ffi.errno() == EINTR) then
                -- EAGAIN, i dont think this ever gets hit cuz theres no timeout
            elseif n < 0 then
                error'poll'
            else
                for i=0,1 do
                    if bit_band(pfds[i].revents, POLLERR+POLLHUP+POLLNVAL) ~= 0 then
                        -- eof probably
                        C.close(pfds[i].fd)
                        pfds[i].fd = -1
                    elseif bit_band(pfds[i].revents, POLLIN) ~= 0 then
                        local n = C.read(pfds[i].fd, buf, ffi.sizeof(buf))
                        local s = ffi.string(buf, n)
                        if i == 0 then
                            callback(s, nil)
                        else
                            callback(nil, s)
                        end
                    end
                end
            end
        end
    else
        stdout = readfd(outfd[0])
        stderr = readfd(errfd[0])
    end
    local statusPtr = ffi.new('int[1]')
    local w
    while not w or (w == -1 and ffi.errno() == EINTR) do
        w = C.waitpid(pidPtr[0], statusPtr, 0)
    end
    return bit_band(statusPtr[0], 0x7f) == 0 and 0 or statusPtr[0],
        stdout,
        stderr
end