Skip to content

Commit 6f86423

Browse files
jowayjoway
committed
perf: use block syscall enter for epoll_wait
1 parent 5719b53 commit 6f86423

File tree

6 files changed

+89
-88
lines changed

6 files changed

+89
-88
lines changed

‎poll_default_linux.go‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -97,13 +97,25 @@ func (p *defaultPoll) Wait() (err error) {
9797
if n == p.size && p.size < 128*1024 {
9898
p.Reset(p.size<<1, caps)
9999
}
100-
n, err = EpollWait(p.fd, p.events, msec)
100+
// msec: 0(raw) => 1ms(sched,raw) => -1(block_syscall)
101+
// poller's G will hold P at most 1ms
102+
if msec > 0 {
103+
n, err = EpollWaitRaw(p.fd, p.events, msec)
104+
} else if msec == 0 {
105+
n, err = EpollWaitRaw(p.fd, p.events, msec)
106+
} else { // < 0
107+
n, err = EpollWaitBlock(p.fd, p.events, msec)
108+
}
101109
if err != nil && err != syscall.EINTR {
102110
return err
103111
}
104112
if n <= 0 {
105-
msec = -1
106-
runtime.Gosched()
113+
if msec > 0 {
114+
msec = -1 // no need to sched, because we will use block syscall
115+
} else if msec == 0 {
116+
msec = 1
117+
runtime.Gosched()
118+
}
107119
continue
108120
}
109121
msec = 0

‎poll_default_linux_test.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"errors"
2222
"syscall"
2323
"testing"
24+
"unsafe"
2425

2526
"golang.org/x/sys/unix"
2627
)
@@ -331,7 +332,9 @@ func TestEpollConnectSameFD(t *testing.T) {
331332

332333
func epollWaitUntil(epfd int, events []epollevent, msec int) (n int, err error) {
333334
WAIT:
334-
n, err = EpollWait(epfd, events, msec)
335+
var eventptr = uintptr(unsafe.Pointer(&events[0]))
336+
var eventlen = uintptr(len(events))
337+
n, err = EpollWait(epfd, eventptr, eventlen, msec)
335338
if err == syscall.EINTR {
336339
goto WAIT
337340
}

‎sys_epoll_linux.go‎

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
//go:build !arm64 && !loong64
16-
// +build !arm64,!loong64
15+
//go:build linux
16+
// +build linux
1717

1818
package netpoll
1919

@@ -22,12 +22,11 @@ import (
2222
"unsafe"
2323
)
2424

25-
const EPOLLET = -syscall.EPOLLET
25+
//go:linkname entersyscallblock runtime.entersyscallblock
26+
func entersyscallblock()
2627

27-
type epollevent struct {
28-
events uint32
29-
data [8]byte // unaligned uintptr
30-
}
28+
//go:linkname exitsyscall runtime.exitsyscall
29+
func exitsyscall()
3130

3231
// EpollCreate implements epoll_create1.
3332
func EpollCreate(flag int) (fd int, err error) {
@@ -51,14 +50,36 @@ func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) {
5150
// EpollWait implements epoll_wait.
5251
func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) {
5352
var r0 uintptr
54-
var _p0 = unsafe.Pointer(&events[0])
55-
if msec == 0 {
56-
r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0)
57-
} else {
58-
r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
53+
r0, _, err = syscall.Syscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0)
54+
if err == syscall.Errno(0) {
55+
err = nil
5956
}
57+
return int(r0), err
58+
}
59+
60+
func EpollWaitRaw(epfd int, events []epollevent, msec int) (n int, err error) {
61+
var r0 uintptr
62+
r0, _, err = syscall.RawSyscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0)
6063
if err == syscall.Errno(0) {
6164
err = nil
6265
}
6366
return int(r0), err
6467
}
68+
69+
func EpollWaitBlock(epfd int, events []epollevent, msec int) (n int, err error) {
70+
r0, _, errno := BlockSyscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0)
71+
if errno == syscall.Errno(0) {
72+
err = nil
73+
} else {
74+
err = errno
75+
}
76+
return int(r0), err
77+
}
78+
79+
//go:nosplit
80+
func BlockSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) {
81+
entersyscallblock()
82+
r1, r2, err = syscall.RawSyscall6(trap, a1, a2, a3, a4, a5, a6)
83+
exitsyscall()
84+
return r1, r2, err
85+
}

‎sys_epoll_linux_amd64.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright 2022 CloudWeGo Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
//go:build !arm64 && !loong64
16+
// +build !arm64,!loong64
17+
18+
package netpoll
19+
20+
import (
21+
"syscall"
22+
)
23+
24+
const EPOLLET = -syscall.EPOLLET
25+
const SYS_EPOLL_WAIT = syscall.SYS_EPOLL_WAIT
26+
27+
type epollevent struct {
28+
events uint32
29+
data [8]byte // unaligned uintptr
30+
}

‎sys_epoll_linux_arm64.go‎

Lines changed: 4 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -12,51 +12,20 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15+
//go:build arm64
16+
// +build arm64
17+
1518
package netpoll
1619

1720
import (
1821
"syscall"
19-
"unsafe"
2022
)
2123

2224
const EPOLLET = syscall.EPOLLET
25+
const SYS_EPOLL_WAIT = syscall.SYS_EPOLL_PWAIT
2326

2427
type epollevent struct {
2528
events uint32
2629
_ int32
2730
data [8]byte // unaligned uintptr
2831
}
29-
30-
// EpollCreate implements epoll_create1.
31-
func EpollCreate(flag int) (fd int, err error) {
32-
var r0 uintptr
33-
r0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0)
34-
if err == syscall.Errno(0) {
35-
err = nil
36-
}
37-
return int(r0), err
38-
}
39-
40-
// EpollCtl implements epoll_ctl.
41-
func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) {
42-
_, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0)
43-
if err == syscall.Errno(0) {
44-
err = nil
45-
}
46-
return err
47-
}
48-
49-
// EpollWait implements epoll_wait.
50-
func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) {
51-
var r0 uintptr
52-
var _p0 = unsafe.Pointer(&events[0])
53-
if msec == 0 {
54-
r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0)
55-
} else {
56-
r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
57-
}
58-
if err == syscall.Errno(0) {
59-
err = nil
60-
}
61-
return int(r0), err
62-
}

‎sys_epoll_linux_loong64.go‎

Lines changed: 3 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -12,54 +12,20 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
//go:build linux && loong64
16-
// +build linux,loong64
15+
//go:build loong64
16+
// +build loong64
1717

1818
package netpoll
1919

2020
import (
2121
"syscall"
22-
"unsafe"
2322
)
2423

2524
const EPOLLET = syscall.EPOLLET
25+
const SYS_EPOLL_WAIT = syscall.SYS_EPOLL_PWAIT
2626

2727
type epollevent struct {
2828
events uint32
2929
_ int32
3030
data [8]byte // unaligned uintptr
3131
}
32-
33-
// EpollCreate implements epoll_create1.
34-
func EpollCreate(flag int) (fd int, err error) {
35-
var r0 uintptr
36-
r0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0)
37-
if err == syscall.Errno(0) {
38-
err = nil
39-
}
40-
return int(r0), err
41-
}
42-
43-
// EpollCtl implements epoll_ctl.
44-
func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) {
45-
_, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0)
46-
if err == syscall.Errno(0) {
47-
err = nil
48-
}
49-
return err
50-
}
51-
52-
// EpollWait implements epoll_wait.
53-
func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) {
54-
var r0 uintptr
55-
var _p0 = unsafe.Pointer(&events[0])
56-
if msec == 0 {
57-
r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0)
58-
} else {
59-
r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
60-
}
61-
if err == syscall.Errno(0) {
62-
err = nil
63-
}
64-
return int(r0), err
65-
}

0 commit comments

Comments
 (0)