Skip to content

Commit 940885a

Browse files
dwisiswant0dwisiswant0
authored
feat: generate CPU & PGO profiles (projectdiscovery#6058)
* feat: generate CPU profiles also adjust memory (heap) profiles ext to `.mem` Signed-off-by: Dwi Siswanto <[email protected]> * docs(DESIGN): add total samples for CPU profiles Signed-off-by: Dwi Siswanto <[email protected]> * chore(make): trimpath in go-build and append LDFLAGS ifneq "darwin" Signed-off-by: Dwi Siswanto <[email protected]> * chore: update goreleaser build * replace `go mod tidy` with `go mod download` and `go mod verify` * adjust indentations * add `-trimpath` flag * set `-pgo` flag to "`auto`" * add `ldflags` * quoting 386 GOARCH value Signed-off-by: Dwi Siswanto <[email protected]> * ci: add generate PGO workflow Signed-off-by: Dwi Siswanto <[email protected]> * chore(make): set CGO_ENABLED inline in go-build Signed-off-by: Dwi Siswanto <[email protected]> * refactor(main): streamline profile file creation Signed-off-by: Dwi Siswanto <[email protected]> * dummy: add PGO file (DO NOT MERGE) Signed-off-by: Dwi Siswanto <[email protected]> * feat: add main test (benchmark) Signed-off-by: Dwi Siswanto <[email protected]> * chore(make): add build-test Signed-off-by: Dwi Siswanto <[email protected]> * Revert "dummy: add PGO file (DO NOT MERGE)" This reverts commit ee87720. * test(main): set Output to /dev/null Signed-off-by: Dwi Siswanto <[email protected]> * feat(output): add option to disable stdout via env var Signed-off-by: Dwi Siswanto <[email protected]> * test(main): set `types.Options.Output` to empty Signed-off-by: Dwi Siswanto <[email protected]> * chore(generate-pgo): add TODO note Signed-off-by: Dwi Siswanto <[email protected]> * ci: add reusable perf regression workflow Signed-off-by: Dwi Siswanto <[email protected]> * ci(perf-regression): enabe `DISABLE_STDOUT` Signed-off-by: Dwi Siswanto <[email protected]> --------- Signed-off-by: Dwi Siswanto <[email protected]>
1 parent 047d49f commit 940885a

11 files changed

+301
-67
lines changed

‎.github/workflows/generate-pgo.yaml

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: 👤 Generate PGO
2+
3+
on:
4+
push:
5+
branches: ["dev"]
6+
paths:
7+
- '**.go'
8+
- '**.mod'
9+
workflow_dispatch:
10+
11+
concurrency:
12+
group: ${{ github.workflow }}-${{ github.ref }}
13+
cancel-in-progress: true
14+
15+
# TODO(dwisiswant0): https://go.dev/doc/pgo#merging-profiles
16+
17+
jobs:
18+
pgo:
19+
strategy:
20+
matrix:
21+
targets: [150]
22+
runs-on: ubuntu-latest-16-cores
23+
if: github.repository == 'projectdiscovery/nuclei'
24+
permissions:
25+
contents: write
26+
env:
27+
PGO_FILE: "cmd/nuclei/default.pgo"
28+
LIST_FILE: "/tmp/targets-${{ matrix.targets }}.txt"
29+
PROFILE_MEM: "/tmp/nuclei-profile-${{ matrix.targets }}-targets"
30+
steps:
31+
- uses: actions/checkout@v4
32+
- uses: projectdiscovery/actions/setup/git@v1
33+
- uses: projectdiscovery/actions/setup/go@v1
34+
- name: Generate list
35+
run: for i in {1..${{ matrix.targets }}}; do echo "https://honey.scanme.sh/?_=${i}" >> "${LIST_FILE}"; done
36+
# NOTE(dwisiswant0): use `-no-mhe` flag to get better samples.
37+
- run: go run . -l "${LIST_FILE}" -profile-mem="${PROFILE_MEM}" -no-mhe
38+
working-directory: cmd/nuclei/
39+
- run: mv "${PROFILE_MEM}.cpu" ${PGO_FILE}
40+
# NOTE(dwisiswant0): shall we prune $PGO_FILE git history?
41+
# if we prune it, this won't be linear since it requires a force-push.
42+
# if we don't, the git objects will just keep growing bigger.
43+
#
44+
# Ref:
45+
# - https://go.dev/blog/pgo#:~:text=We%20recommend%20committing%20default.pgo%20files%20to%20your%20repository
46+
# - https://gist.github.com/nottrobin/5758221
47+
- uses: projectdiscovery/actions/commit@v1
48+
with:
49+
files: "${PGO_FILE}"
50+
message: "build: update PGO profile :robot:"
51+
- run: git push origin $GITHUB_REF
52+
- uses: actions/upload-artifact@v4
53+
with:
54+
name: "pgo"
55+
path: "${{ env.PGO_FILE }}"

‎.github/workflows/perf-regression.yaml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: 🔨 Performance Regression
2+
3+
on:
4+
workflow_call:
5+
workflow_dispatch:
6+
7+
jobs:
8+
perf-regression:
9+
runs-on: ubuntu-latest-16-cores
10+
if: github.repository == 'projectdiscovery/nuclei'
11+
env:
12+
BENCH_OUT: "/tmp/bench.out"
13+
steps:
14+
- uses: actions/checkout@v4
15+
- uses: projectdiscovery/actions/setup/go@v1
16+
- run: make build-test
17+
- run: ./bin/nuclei.test -test.run - -test.bench=. -test.benchmem ./cmd/nuclei/ | tee $BENCH_OUT
18+
env:
19+
DISABLE_STDOUT: "1"
20+
- uses: actions/cache/restore@v4
21+
with:
22+
path: ./cache
23+
key: ${{ runner.os }}-benchmark
24+
- uses: benchmark-action/github-action-benchmark@v1
25+
with:
26+
name: 'RunEnumeration Benchmark'
27+
tool: 'go'
28+
output-file-path: ${{ env.BENCH_OUT }}
29+
external-data-json-path: ./cache/benchmark-data.json
30+
fail-on-alert: false
31+
github-token: ${{ secrets.GITHUB_TOKEN }}
32+
comment-on-alert: true
33+
summary-always: true
34+
- uses: actions/cache/save@v4
35+
if: github.event_name == 'push'
36+
with:
37+
path: ./cache
38+
key: ${{ runner.os }}-benchmark

‎.github/workflows/perf-test.yaml

+17-9
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,24 @@ jobs:
2020
- uses: projectdiscovery/actions/setup/go@v1
2121
- run: make verify
2222
- name: Generate list
23-
run: for i in {1..${{ matrix.count }}}; do echo "https://scanme.sh/?_=${i}" >> "${LIST_FILE}"; done
24-
- run: NUCLEI_ARGS=host-error-stats go run . -l "${LIST_FILE}" -profile-mem="${PROFILE_MEM}"
23+
run: for i in {1..${{ matrix.count }}}; do echo "https://honey.scanme.sh/?_=${i}" >> "${LIST_FILE}"; done
24+
- run: go run . -l "${LIST_FILE}" -profile-mem="${PROFILE_MEM}"
25+
env:
26+
NUCLEI_ARGS: host-error-stats
2527
working-directory: cmd/nuclei/
2628
- uses: projectdiscovery/actions/flamegraph@v1
27-
id: flamegraph
29+
id: flamegraph-cpu
2830
with:
29-
profile: "${{ env.PROFILE_MEM }}.prof"
30-
name: "nuclei-perf-test-${{ matrix.count }}"
31+
profile: "${{ env.PROFILE_MEM }}.cpu"
32+
name: "${{ env.FLAMEGRAPH_NAME }} CPU profiles"
3133
continue-on-error: true
32-
- if: ${{ steps.flamegraph.outputs.message == '' }}
33-
run: echo "::notice::${FLAMEGRAPH_URL}"
34-
env:
35-
FLAMEGRAPH_URL: ${{ steps.flamegraph.outputs.url }}
34+
- uses: projectdiscovery/actions/flamegraph@v1
35+
id: flamegraph-mem
36+
with:
37+
profile: "${{ env.PROFILE_MEM }}.mem"
38+
name: "${{ env.FLAMEGRAPH_NAME }} memory profiles"
39+
continue-on-error: true
40+
- if: ${{ steps.flamegraph-mem.outputs.message == '' }}
41+
run: |
42+
echo "::notice::CPU flamegraph: ${{ steps.flamegraph-cpu.outputs.url }}"
43+
echo "::notice::Memory (heap) flamegraph: ${{ steps.flamegraph-mem.outputs.url }}"

‎.github/workflows/tests.yaml

+18-8
Original file line numberDiff line numberDiff line change
@@ -156,13 +156,23 @@ jobs:
156156
echo "FLAMEGRAPH_NAME=nuclei (PR #${{ github.event.number }})" >> $GITHUB_ENV
157157
- run: ./bin/nuclei -silent -update-templates
158158
- run: ./bin/nuclei -silent -u "${TARGET_URL}" -profile-mem="${PROFILE_MEM}"
159-
- uses: projectdiscovery/actions/flamegraph@master
160-
id: flamegraph
159+
- uses: projectdiscovery/actions/flamegraph@v1
160+
id: flamegraph-cpu
161161
with:
162-
profile: "${{ env.PROFILE_MEM }}.prof"
163-
name: "${{ env.FLAMEGRAPH_NAME }}"
162+
profile: "${{ env.PROFILE_MEM }}.cpu"
163+
name: "${{ env.FLAMEGRAPH_NAME }} CPU profiles"
164164
continue-on-error: true
165-
- if: ${{ steps.flamegraph.outputs.message == '' }}
166-
run: echo "::notice::${FLAMEGRAPH_URL}"
167-
env:
168-
FLAMEGRAPH_URL: ${{ steps.flamegraph.outputs.url }}
165+
- uses: projectdiscovery/actions/flamegraph@v1
166+
id: flamegraph-mem
167+
with:
168+
profile: "${{ env.PROFILE_MEM }}.mem"
169+
name: "${{ env.FLAMEGRAPH_NAME }} memory profiles"
170+
continue-on-error: true
171+
- if: ${{ steps.flamegraph-mem.outputs.message == '' }}
172+
run: |
173+
echo "::notice::CPU flamegraph: ${{ steps.flamegraph-cpu.outputs.url }}"
174+
echo "::notice::Memory (heap) flamegraph: ${{ steps.flamegraph-mem.outputs.url }}"
175+
176+
perf-regression:
177+
needs: ["tests"]
178+
uses: ./.github/workflows/perf-regression.yaml

‎.gitignore

+3-1
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,6 @@ vendor
4848
# Profiling & tracing
4949
*.prof
5050
*.pprof
51-
*.trace
51+
*.trace
52+
*.mem
53+
*.cpu

‎.goreleaser.yml

+26-24
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,29 @@
11
before:
22
hooks:
3-
- go mod tidy
3+
- go mod download
4+
- go mod verify
45

56
builds:
6-
- main: cmd/nuclei/main.go
7-
binary: nuclei
8-
id: nuclei-cli
9-
10-
env:
11-
- CGO_ENABLED=0
12-
13-
goos: [windows,linux,darwin]
14-
goarch: [amd64,386,arm,arm64]
15-
ignore:
16-
- goos: darwin
17-
goarch: 386
18-
- goos: windows
19-
goarch: arm
20-
- goos: windows
21-
goarch: arm64
22-
23-
flags:
24-
- -trimpath
7+
- main: cmd/nuclei/main.go
8+
binary: nuclei
9+
id: nuclei-cli
10+
env:
11+
- CGO_ENABLED=0
12+
goos: [windows,linux,darwin]
13+
goarch: [amd64,'386',arm,arm64]
14+
ignore:
15+
- goos: darwin
16+
goarch: '386'
17+
- goos: windows
18+
goarch: arm
19+
- goos: windows
20+
goarch: arm64
21+
flags:
22+
- -trimpath
23+
- -pgo=auto
24+
ldflags:
25+
- -s
26+
- -w
2527

2628
#- main: cmd/tmc/main.go
2729
# binary: tmc
@@ -34,10 +36,10 @@ builds:
3436
# goarch: [amd64]
3537

3638
archives:
37-
- format: zip
38-
id: nuclei
39-
builds: [nuclei-cli]
40-
name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}'
39+
- format: zip
40+
id: nuclei
41+
builds: [nuclei-cli]
42+
name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}'
4143

4244
checksum:
4345
algorithm: sha256

‎DESIGN.md

+17-10
Original file line numberDiff line numberDiff line change
@@ -459,35 +459,42 @@ That's it, you've added a new protocol to Nuclei. The next good step would be to
459459

460460
## Profiling and Tracing
461461

462-
To analyze Nuclei's performance and resource usage, you can generate memory profiles and trace files using the `-profile-mem` flag:
462+
To analyze Nuclei's performance and resource usage, you can generate CPU & memory profiles and trace files using the `-profile-mem` flag:
463463

464464
```bash
465465
nuclei -t nuclei-templates/ -u https://example.com -profile-mem=nuclei-$(git describe --tags)
466466
```
467467

468-
This command creates two files:
468+
This command creates three files:
469469

470-
* `nuclei.prof`: Memory (heap) profile
470+
* `nuclei.cpu`: CPU profile
471+
* `nuclei.mem`: Memory (heap) profile
471472
* `nuclei.trace`: Execution trace
472473

473-
### Analyzing the Memory Profile
474+
### Analyzing the CPU/Memory Profiles
474475

475-
1. View the profile in the terminal:
476+
* View the profile in the terminal:
476477

477478
```bash
478-
go tool pprof nuclei.prof
479+
go tool pprof nuclei.{cpu,mem}
479480
```
480481

481-
2. Display top memory consumers:
482+
* Display overall CPU time for processing $$N$$ targets:
483+
484+
```
485+
go tool pprof -top nuclei.cpu | grep "Total samples"
486+
```
487+
488+
* Display top memory consumers:
482489

483490
```bash
484-
go tool pprof -top nuclei.prof | grep "$(go list -m)" | head -10
491+
go tool pprof -top nuclei.mem | grep "$(go list -m)" | head -10
485492
```
486493

487-
3. Visualize the profile in a web browser:
494+
* Visualize the profile in a web browser:
488495

489496
```bash
490-
go tool pprof -http=:$(shuf -i 1000-99999 -n 1) nuclei.prof
497+
go tool pprof -http=:$(shuf -i 1000-99999 -n 1) nuclei.{cpu,mem}
491498
```
492499

493500
### Analyzing the Trace File

‎Makefile

+11-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ GOFLAGS := -v
1111
LDFLAGS := -s -w
1212

1313
ifneq ($(shell go env GOOS),darwin)
14-
LDFLAGS = -extldflags "-static"
14+
LDFLAGS += -extldflags "-static"
1515
endif
1616

1717
.PHONY: all build build-stats clean devtools-all devtools-bindgen devtools-scrapefuncs
@@ -26,13 +26,22 @@ clean:
2626

2727
go-build: clean
2828
go-build:
29-
$(GOBUILD) $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
29+
CGO_ENABLED=0 $(GOBUILD) -trimpath $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
3030
-o '${GOBUILD_OUTPUT}' $(GOBUILD_PACKAGES)
3131

32+
build: GOFLAGS = -v -pgo=auto
3233
build: GOBUILD_OUTPUT = ./bin/nuclei
3334
build: GOBUILD_PACKAGES = cmd/nuclei/main.go
3435
build: go-build
3536

37+
build-test: GOFLAGS = -v -pgo=auto
38+
build-test: GOBUILD_OUTPUT = ./bin/nuclei.test
39+
build-test: GOBUILD_PACKAGES = ./cmd/nuclei/
40+
build-test: clean
41+
build-test:
42+
CGO_ENABLED=0 $(GOCMD) test -c -trimpath $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
43+
-o '${GOBUILD_OUTPUT}' ${GOBUILD_PACKAGES}
44+
3645
build-stats: GOBUILD_OUTPUT = ./bin/nuclei-stats
3746
build-stats: GOBUILD_PACKAGES = cmd/nuclei/main.go
3847
build-stats: GOBUILD_ADDITIONAL_ARGS = -tags=stats

‎cmd/nuclei/main.go

+24-13
Original file line numberDiff line numberDiff line change
@@ -106,18 +106,20 @@ func main() {
106106

107107
// Profiling & tracing related code
108108
if memProfile != "" {
109-
memProfile = strings.TrimSuffix(memProfile, filepath.Ext(memProfile)) + ".prof"
110-
memProfileFile, err := os.Create(memProfile)
111-
if err != nil {
112-
gologger.Fatal().Msgf("profile: could not create memory profile %q file: %v", memProfile, err)
113-
}
109+
memProfile = strings.TrimSuffix(memProfile, filepath.Ext(memProfile))
114110

115-
traceFilepath := strings.TrimSuffix(memProfile, filepath.Ext(memProfile)) + ".trace"
116-
traceFile, err := os.Create(traceFilepath)
117-
if err != nil {
118-
gologger.Fatal().Msgf("profile: could not create trace %q file: %v", traceFilepath, err)
111+
createProfileFile := func(ext, profileType string) *os.File {
112+
f, err := os.Create(memProfile + ext)
113+
if err != nil {
114+
gologger.Fatal().Msgf("profile: could not create %s profile %q file: %v", profileType, f.Name(), err)
115+
}
116+
return f
119117
}
120118

119+
memProfileFile := createProfileFile(".mem", "memory")
120+
cpuProfileFile := createProfileFile(".cpu", "CPU")
121+
traceFile := createProfileFile(".trace", "trace")
122+
121123
oldMemProfileRate := runtime.MemProfileRate
122124
runtime.MemProfileRate = 4096
123125

@@ -126,18 +128,27 @@ func main() {
126128
gologger.Fatal().Msgf("profile: could not start trace: %v", err)
127129
}
128130

131+
// Start CPU profiling
132+
if err := pprof.StartCPUProfile(cpuProfileFile); err != nil {
133+
gologger.Fatal().Msgf("profile: could not start CPU profile: %v", err)
134+
}
135+
129136
defer func() {
130-
// Start CPU profiling
137+
// Start heap memory snapshot
131138
if err := pprof.WriteHeapProfile(memProfileFile); err != nil {
132-
gologger.Fatal().Msgf("profile: could not start CPU profile: %v", err)
139+
gologger.Fatal().Msgf("profile: could not write memory profile: %v", err)
133140
}
141+
142+
pprof.StopCPUProfile()
134143
memProfileFile.Close()
135144
traceFile.Close()
136145
trace.Stop()
146+
137147
runtime.MemProfileRate = oldMemProfileRate
138148

139-
gologger.Info().Msgf("Memory profile saved at %q", memProfile)
140-
gologger.Info().Msgf("Traced at %q", traceFilepath)
149+
gologger.Info().Msgf("CPU profile saved at %q", cpuProfileFile.Name())
150+
gologger.Info().Msgf("Memory usage snapshot saved at %q", memProfileFile.Name())
151+
gologger.Info().Msgf("Traced at %q", traceFile.Name())
141152
}()
142153
}
143154

0 commit comments

Comments
 (0)