Skip to content

Commit

Permalink
pkg/corpus: move focus area configuration to the constructor
Browse files Browse the repository at this point in the history 
Set Corpus in HTTPServer dynamically.
Refactor syz-manager and syz-diff accordingly.
  • Loading branch information
@a-nogikh @tarasmadan
a-nogikh authored and tarasmadan committed Oct 25, 2024
1 parent f63b869 commit 42a1ab1
Show file tree
Hide file tree
Showing 5 changed files with 92 additions and 58 deletions.
41 changes: 18 additions & 23 deletions pkg/corpus/corpus.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ func NewCorpus(ctx context.Context) *Corpus {
}

func NewMonitoredCorpus(ctx context.Context, updates chan<- NewItemEvent) *Corpus {
return NewFocusedCorpus(ctx, updates, nil)
}

func NewFocusedCorpus(ctx context.Context, updates chan<- NewItemEvent, areas []FocusArea) *Corpus {
corpus := &Corpus{
ctx: ctx,
progsMap: make(map[string]*Item),
Expand All @@ -62,6 +66,20 @@ func NewMonitoredCorpus(ctx context.Context, updates chan<- NewItemEvent) *Corpu
stat.LenOf(&corpus.signal, &corpus.mu))
corpus.StatCover = stat.New("coverage", "Source coverage in the corpus", stat.Console,
stat.Link("/cover"), stat.Prometheus("syz_corpus_cover"), stat.LenOf(&corpus.cover, &corpus.mu))
for _, area := range areas {
obj := &ProgramsList{}
if len(areas) > 1 && area.Name != "" {
// Only show extra statistics if there's more than one area.
stat.New("corpus ["+area.Name+"]",
fmt.Sprintf("Corpus programs of the focus area %q", area.Name),
stat.Console, stat.Graph("corpus"),
stat.LenOf(&obj.progs, &corpus.mu))
}
corpus.focusAreas = append(corpus.focusAreas, &focusAreaState{
FocusArea: area,
ProgramsList: obj,
})
}
return corpus
}

Expand Down Expand Up @@ -106,29 +124,6 @@ type NewItemEvent struct {
NewCover []uint64
}

// SetFocusAreas can only be called once on an empty corpus.
func (corpus *Corpus) SetFocusAreas(areas []FocusArea) {
corpus.mu.Lock()
defer corpus.mu.Unlock()
if len(corpus.progsMap) > 0 {
panic("SetFocusAreas() is called on a non-empty corpus")
}
for _, area := range areas {
obj := &ProgramsList{}
if len(areas) > 1 && area.Name != "" {
// Only show extra statistics if there's more than one area.
stat.New("corpus ["+area.Name+"]",
fmt.Sprintf("Corpus programs of the focus area %q", area.Name),
stat.Console, stat.Graph("corpus"),
stat.LenOf(&obj.progs, &corpus.mu))
}
corpus.focusAreas = append(corpus.focusAreas, &focusAreaState{
FocusArea: area,
ProgramsList: obj,
})
}
}

func (corpus *Corpus) Save(inp NewInput) {
progData := inp.Prog.Serialize()
sig := hash.String(progData)
Expand Down
3 changes: 1 addition & 2 deletions pkg/corpus/prio_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,7 @@ func TestChooseProgram(t *testing.T) {

func TestFocusAreas(t *testing.T) {
target := getTarget(t, targets.TestOS, targets.TestArch64)
corpus := NewCorpus(context.Background())
corpus.SetFocusAreas([]FocusArea{
corpus := NewFocusedCorpus(context.Background(), nil, []FocusArea{
{
CoverPCs: map[uint64]struct{}{
0: {},
Expand Down
63 changes: 49 additions & 14 deletions pkg/manager/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ type HTTPServer struct {
// To be set once.
Cfg *mgrconfig.Config
StartTime time.Time
Corpus *corpus.Corpus
CrashStore *CrashStore
DiffStore *DiffFuzzerStore

// Set dynamically.
Corpus atomic.Pointer[corpus.Corpus]
Fuzzer atomic.Pointer[fuzzer.Fuzzer]
Cover atomic.Pointer[CoverageInfo]
ReproLoop atomic.Pointer[ReproLoop]
Expand Down Expand Up @@ -172,10 +172,12 @@ func (serv *HTTPServer) httpExpertMode(w http.ResponseWriter, r *http.Request) {

func (serv *HTTPServer) httpSyscalls(w http.ResponseWriter, r *http.Request) {
var calls map[string]*corpus.CallCov
if obj := serv.EnabledSyscalls.Load(); obj != nil {
calls = serv.Corpus.CallCover()
syscallsObj := serv.EnabledSyscalls.Load()
corpusObj := serv.Corpus.Load()
if corpusObj != nil && syscallsObj != nil {
calls = corpusObj.CallCover()
// Add enabled, but not yet covered calls.
for call := range obj.(map[*prog.Syscall]bool) {
for call := range syscallsObj.(map[*prog.Syscall]bool) {
if calls[call.Name] == nil {
calls[call.Name] = new(corpus.CallCov)
}
Expand Down Expand Up @@ -321,11 +323,16 @@ func (serv *HTTPServer) httpCrash(w http.ResponseWriter, r *http.Request) {
}

func (serv *HTTPServer) httpCorpus(w http.ResponseWriter, r *http.Request) {
corpus := serv.Corpus.Load()
if corpus == nil {
http.Error(w, "the corpus information is not yet available", http.StatusInternalServerError)
return
}
data := UICorpus{
Call: r.FormValue("call"),
RawCover: serv.Cfg.RawCover,
}
for _, inp := range serv.Corpus.Items() {
for _, inp := range corpus.Items() {
if data.Call != "" && data.Call != inp.StringCall() {
continue
}
Expand Down Expand Up @@ -416,6 +423,12 @@ func (serv *HTTPServer) httpCoverCover(w http.ResponseWriter, r *http.Request, f
return
}

corpus := serv.Corpus.Load()
if corpus == nil {
http.Error(w, "the corpus information is not yet available", http.StatusInternalServerError)
return
}

rg, err := coverInfo.ReportGenerator.Get()
if err != nil {
http.Error(w, fmt.Sprintf("failed to generate coverage profile: %v", err), http.StatusInternalServerError)
Expand All @@ -431,7 +444,7 @@ func (serv *HTTPServer) httpCoverCover(w http.ResponseWriter, r *http.Request, f

var progs []cover.Prog
if sig := r.FormValue("input"); sig != "" {
inp := serv.Corpus.Item(sig)
inp := corpus.Item(sig)
if inp == nil {
http.Error(w, "unknown input hash", http.StatusInternalServerError)
return
Expand All @@ -456,7 +469,7 @@ func (serv *HTTPServer) httpCoverCover(w http.ResponseWriter, r *http.Request, f
}
} else {
call := r.FormValue("call")
for _, inp := range serv.Corpus.Items() {
for _, inp := range corpus.Items() {
if call != "" && call != inp.StringCall() {
continue
}
Expand Down Expand Up @@ -511,8 +524,13 @@ func (serv *HTTPServer) httpCoverCover(w http.ResponseWriter, r *http.Request, f
}

func (serv *HTTPServer) httpCoverFallback(w http.ResponseWriter, r *http.Request) {
corpus := serv.Corpus.Load()
if corpus == nil {
http.Error(w, "the corpus information is not yet available", http.StatusInternalServerError)
return
}
calls := make(map[int][]int)
for s := range serv.Corpus.Signal() {
for s := range corpus.Signal() {
id, errno := prog.DecodeFallbackSignal(uint64(s))
calls[id] = append(calls[id], errno)
}
Expand Down Expand Up @@ -548,18 +566,25 @@ func (serv *HTTPServer) httpFileCover(w http.ResponseWriter, r *http.Request) {
}

func (serv *HTTPServer) httpPrio(w http.ResponseWriter, r *http.Request) {
corpus := serv.Corpus.Load()
if corpus == nil {
http.Error(w, "the corpus information is not yet available", http.StatusInternalServerError)
return
}

callName := r.FormValue("call")
call := serv.Cfg.Target.SyscallMap[callName]
if call == nil {
http.Error(w, fmt.Sprintf("unknown call: %v", callName), http.StatusInternalServerError)
return
}

var corpus []*prog.Prog
for _, inp := range serv.Corpus.Items() {
corpus = append(corpus, inp.Prog)
var progs []*prog.Prog
for _, inp := range corpus.Items() {
progs = append(progs, inp.Prog)
}
prios := serv.Cfg.Target.CalculatePriorities(corpus)

prios := serv.Cfg.Target.CalculatePriorities(progs)

data := &UIPrioData{Call: callName}
for i, p := range prios[call.ID] {
Expand Down Expand Up @@ -589,7 +614,12 @@ func (serv *HTTPServer) httpFile(w http.ResponseWriter, r *http.Request) {
}

func (serv *HTTPServer) httpInput(w http.ResponseWriter, r *http.Request) {
inp := serv.Corpus.Item(r.FormValue("sig"))
corpus := serv.Corpus.Load()
if corpus == nil {
http.Error(w, "the corpus information is not yet available", http.StatusInternalServerError)
return
}
inp := corpus.Item(r.FormValue("sig"))
if inp == nil {
http.Error(w, "can't find the input", http.StatusInternalServerError)
return
Expand All @@ -599,7 +629,12 @@ func (serv *HTTPServer) httpInput(w http.ResponseWriter, r *http.Request) {
}

func (serv *HTTPServer) httpDebugInput(w http.ResponseWriter, r *http.Request) {
inp := serv.Corpus.Item(r.FormValue("sig"))
corpus := serv.Corpus.Load()
if corpus == nil {
http.Error(w, "the corpus information is not yet available", http.StatusInternalServerError)
return
}
inp := corpus.Item(r.FormValue("sig"))
if inp == nil {
http.Error(w, "can't find the input", http.StatusInternalServerError)
return
Expand Down
33 changes: 20 additions & 13 deletions syz-manager/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,12 +208,10 @@ func RunManager(mode Mode, cfg *mgrconfig.Config) {
log.Fatalf("%v", err)
}

corpusUpdates := make(chan corpus.NewItemEvent, 128)
mgr := &Manager{
cfg: cfg,
mode: mode,
vmPool: vmPool,
corpus: corpus.NewMonitoredCorpus(context.Background(), corpusUpdates),
corpusPreload: make(chan []fuzzer.Candidate),
target: cfg.Target,
sysTarget: cfg.SysTarget,
Expand All @@ -235,7 +233,6 @@ func RunManager(mode Mode, cfg *mgrconfig.Config) {
mgr.http = &manager.HTTPServer{
Cfg: cfg,
StartTime: time.Now(),
Corpus: mgr.corpus,
CrashStore: mgr.crashStore,
}

Expand All @@ -246,7 +243,6 @@ func RunManager(mode Mode, cfg *mgrconfig.Config) {
close(mgr.corpusPreload)
}
go mgr.http.Serve()
go mgr.corpusInputHandler(corpusUpdates)
go mgr.trackUsedFiles()

// Create RPC server for fuzzers.
Expand Down Expand Up @@ -1031,11 +1027,16 @@ func (mgr *Manager) MachineChecked(features flatrpc.Feature, enabledSyscalls map
statSyscalls := stat.New("syscalls", "Number of enabled syscalls",
stat.Simple, stat.NoGraph, stat.Link("/syscalls"))
statSyscalls.Add(len(enabledSyscalls))
corpus := mgr.loadCorpus(enabledSyscalls)
candidates := mgr.loadCorpus(enabledSyscalls)
mgr.setPhaseLocked(phaseLoadedCorpus)
opts := fuzzer.DefaultExecOpts(mgr.cfg, features, *flagDebug)

if mgr.mode == ModeFuzzing {
corpusUpdates := make(chan corpus.NewItemEvent, 128)
mgr.corpus = corpus.NewFocusedCorpus(context.Background(),
corpusUpdates, mgr.coverFilters.Areas)
mgr.http.Corpus.Store(mgr.corpus)

rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
fuzzerObj := fuzzer.NewFuzzer(context.Background(), &fuzzer.Config{
Corpus: mgr.corpus,
Expand All @@ -1059,10 +1060,11 @@ func (mgr *Manager) MachineChecked(features flatrpc.Feature, enabledSyscalls map
return !mgr.saturatedCalls[call]
},
}, rnd, mgr.target)
fuzzerObj.AddCandidates(corpus)
fuzzerObj.AddCandidates(candidates)
mgr.fuzzer.Store(fuzzerObj)
mgr.http.Fuzzer.Store(fuzzerObj)

go mgr.corpusInputHandler(corpusUpdates)
go mgr.corpusMinimization()
go mgr.fuzzerLoop(fuzzerObj)
if mgr.dash != nil {
Expand All @@ -1085,7 +1087,7 @@ func (mgr *Manager) MachineChecked(features flatrpc.Feature, enabledSyscalls map
return source
} else if mgr.mode == ModeCorpusRun {
ctx := &corpusRunner{
candidates: corpus,
candidates: candidates,
rnd: rand.New(rand.NewSource(time.Now().UnixNano())),
}
return queue.DefaultOpts(ctx, opts)
Expand Down Expand Up @@ -1267,14 +1269,20 @@ func (mgr *Manager) dashboardReporter() {
var lastFuzzingTime time.Duration
var lastCrashes, lastSuppressedCrashes, lastExecs uint64
for range time.NewTicker(time.Minute).C {
mgr.mu.Lock()
corpus := mgr.corpus
mgr.mu.Unlock()
if corpus == nil {
continue
}
mgr.mu.Lock()
req := &dashapi.ManagerStatsReq{
Name: mgr.cfg.Name,
Addr: webAddr,
UpTime: time.Duration(mgr.statUptime.Val()) * time.Second,
Corpus: uint64(mgr.corpus.StatProgs.Val()),
PCs: uint64(mgr.corpus.StatCover.Val()),
Cover: uint64(mgr.corpus.StatSignal.Val()),
Corpus: uint64(corpus.StatProgs.Val()),
PCs: uint64(corpus.StatCover.Val()),
Cover: uint64(corpus.StatSignal.Val()),
CrashTypes: uint64(mgr.statCrashTypes.Val()),
FuzzingTime: time.Duration(mgr.statFuzzingTime.Val()) - lastFuzzingTime,
Crashes: uint64(mgr.statCrashes.Val()) - lastCrashes,
Expand All @@ -1283,8 +1291,8 @@ func (mgr *Manager) dashboardReporter() {
}
if mgr.phase >= phaseTriagedCorpus && !triageInfoSent {
triageInfoSent = true
req.TriagedCoverage = uint64(mgr.corpus.StatSignal.Val())
req.TriagedPCs = uint64(mgr.corpus.StatCover.Val())
req.TriagedCoverage = uint64(corpus.StatSignal.Val())
req.TriagedPCs = uint64(corpus.StatCover.Val())
}
mgr.mu.Unlock()

Expand Down Expand Up @@ -1337,7 +1345,6 @@ func (mgr *Manager) CoverageFilter(modules []*vminfo.KernelModule) []uint64 {
ReportGenerator: mgr.reportGenerator,
CoverFilter: filters.ExecutorFilter,
})
mgr.corpus.SetFocusAreas(filters.Areas)
var pcs []uint64
for pc := range filters.ExecutorFilter {
pcs = append(pcs, pc)
Expand Down
10 changes: 4 additions & 6 deletions tools/syz-diff/diff.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,6 @@ func main() {
http: &manager.HTTPServer{
Cfg: newCfg,
StartTime: time.Now(),
Corpus: new.corpus,
DiffStore: store,
},
}
Expand Down Expand Up @@ -232,7 +231,6 @@ type kernelContext struct {
ctx context.Context
cfg *mgrconfig.Config
reporter *report.Reporter
corpus *corpus.Corpus
fuzzer atomic.Pointer[fuzzer.Fuzzer]
serv rpcserver.Server
servStats rpcserver.Stats
Expand All @@ -256,7 +254,6 @@ func setup(ctx context.Context, name string, cfg *mgrconfig.Config) *kernelConte
name: name,
ctx: ctx,
cfg: cfg,
corpus: corpus.NewCorpus(ctx),
crashes: make(chan *report.Report, 128),
candidates: make(chan []fuzzer.Candidate),
servStats: rpcserver.NewNamedStats(name),
Expand Down Expand Up @@ -320,8 +317,9 @@ func (kc *kernelContext) MachineChecked(features flatrpc.Feature, syscalls map[*

func (kc *kernelContext) setupFuzzer(features flatrpc.Feature, syscalls map[*prog.Syscall]bool) queue.Source {
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
fuzzerObj := fuzzer.NewFuzzer(context.Background(), &fuzzer.Config{
Corpus: kc.corpus,
corpusObj := corpus.NewFocusedCorpus(kc.ctx, nil, kc.coverFilters.Areas)
fuzzerObj := fuzzer.NewFuzzer(kc.ctx, &fuzzer.Config{
Corpus: corpusObj,
Coverage: kc.cfg.Cover,
// TODO: it may be unstable between different revisions though.
// For now it's only kept true because it seems to increase repro chances in local runs (???).
Expand All @@ -343,6 +341,7 @@ func (kc *kernelContext) setupFuzzer(features flatrpc.Feature, syscalls map[*pro
if kc.http != nil {
kc.http.Fuzzer.Store(fuzzerObj)
kc.http.EnabledSyscalls.Store(syscalls)
kc.http.Corpus.Store(corpusObj)
}

filtered := manager.FilterCandidates(<-kc.candidates, syscalls, false).Candidates
Expand Down Expand Up @@ -376,7 +375,6 @@ func (kc *kernelContext) CoverageFilter(modules []*vminfo.KernelModule) []uint64
log.Fatalf("failed to init coverage filter: %v", err)
}
kc.coverFilters = filters
kc.corpus.SetFocusAreas(filters.Areas)
log.Logf(0, "cover filter size: %d", len(filters.ExecutorFilter))
if kc.http != nil {
kc.http.Cover.Store(&manager.CoverageInfo{
Expand Down

0 comments on commit 42a1ab1

Please sign in to comment.