✨ Add --skip-errors option to continue scans on check/probe failures

README.md

@@ -464,6 +464,22 @@ RESULTS
 
 For more information on available annotations or how to make annotations, see [the configuration doc](config/README.md).
 
+##### Continuing on Errors
+
+By default, if any check or probe fails with a runtime error (for example, a
+transient API failure while scanning), Scorecard exits with a non-zero status.
+Use the `--skip-errors` option to continue the scan and still report on every
+check or probe that succeeded:
+
+```
+./scorecard --repo=github.com/ossf-tests/scorecard-check-branch-protection-e2e --skip-errors
+```
+
+Failures are logged to stderr, results for the successful checks/probes are
+printed as usual, and the command exits with a zero status. This is useful when
+scanning many repositories (e.g. with `--repos` or `--org`) and you don't want a
+single failing check to abort the whole run.
+
 ##### Using a GitLab Repository
 
 To run Scorecard on a GitLab repository, you must create a [GitLab Access Token](https://gitlab.com/-/profile/personal_access_tokens) with the following permissions:

cmd/root.go

@@ -188,6 +188,9 @@ func rootCmd(o *options.Options) error {
 	if strings.EqualFold(o.FileMode, options.FileModeGit) {
 		opts = append(opts, scorecard.WithFileModeGit())
 	}
+	if o.SkipErrors {
+		opts = append(opts, scorecard.WithSkipErrors(true))
+	}
 
 	// Track whether any check produced a runtime error during scans. We want to
 	// continue scanning all repos but return a non-nil error at the end so the
@@ -212,7 +215,9 @@ func rootCmd(o *options.Options) error {
 		}
 	}
 
-	if sawRuntimeErr {
+	// When --skip-errors is set, per-check runtime errors are surfaced in the
+	// reports but must not fail the overall command, so we keep the exit code 0.
+	if sawRuntimeErr && !o.SkipErrors {
 		return errChecksFailed
 	}
 

options/flags.go

@@ -80,6 +80,10 @@ const (
 	FlagCommitDepth = "commit-depth"
 
 	FlagProbes = "probes"
+
+	// FlagSkipErrors is the flag name for continuing a scan when individual
+	// checks or probes fail at runtime.
+	FlagSkipErrors = "skip-errors"
 )
 
 // Command is an interface for handling options for command-line utilities.
@@ -208,6 +212,13 @@ func (o *Options) AddFlags(cmd *cobra.Command) {
 		"Probes to run.",
 	)
 
+	cmd.Flags().BoolVar(
+		&o.SkipErrors,
+		FlagSkipErrors,
+		o.SkipErrors,
+		"continue scanning and report on successful checks/probes even if one or more fail at runtime",
+	)
+
 	// TODO(options): Extract logic
 	allowedFormats := []string{
 		FormatDefault,

options/flags_test.go

@@ -178,6 +178,41 @@ func TestOptions_AddFlags_Format(t *testing.T) {
 	}
 }
 
+func TestOptions_AddFlags_SkipErrors(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		opts *Options
+		name string
+	}{
+		{
+			name: "Skip errors",
+			opts: &Options{
+				SkipErrors: true,
+			},
+		},
+		{
+			name: "Don't skip errors",
+			opts: &Options{
+				SkipErrors: false,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			cmd := &cobra.Command{}
+			tt.opts.AddFlags(cmd)
+			value, err := cmd.Flags().GetBool(FlagSkipErrors)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if tt.opts.SkipErrors != value {
+				t.Fatalf("expected FlagSkipErrors to be %t, got %t", tt.opts.SkipErrors, value)
+			}
+		})
+	}
+}
+
 func TestOptions_AddFlags_Annotations(t *testing.T) {
 	t.Parallel()
 	tests := []struct {

options/options.go

@@ -50,6 +50,9 @@ type Options struct {
 	CommitDepth     int
 	ShowDetails     bool
 	ShowAnnotations bool
+	// SkipErrors continues a scan when individual checks or probes fail at
+	// runtime, reporting on everything that succeeded instead of aborting.
+	SkipErrors bool
 	// Feature flags.
 	EnableSarif                 bool `env:"ENABLE_SARIF"`
 	EnableScorecardV6           bool `env:"SCORECARD_V6"`

pkg/scorecard/scorecard.go

@@ -96,6 +96,7 @@ func runScorecard(ctx context.Context,
 	ciiClient clients.CIIBestPracticesClient,
 	vulnsClient clients.VulnerabilitiesClient,
 	projectClient packageclient.ProjectPackageClient,
+	skipErrors bool,
 ) (Result, error) {
 	if err := repoClient.InitRepo(repo, commitSHA, commitDepth); err != nil {
 		// No need to call sce.WithMessage() since InitRepo will do that for us.
@@ -166,7 +167,7 @@ func runScorecard(ctx context.Context,
 
 	// If the user runs probes
 	if len(probesToRun) > 0 {
-		err = runEnabledProbes(request, probesToRun, &ret)
+		err = runEnabledProbes(request, probesToRun, &ret, skipErrors)
 		if err != nil {
 			return Result{}, err
 		}
@@ -222,9 +223,12 @@ func findConfigFile(rc clients.RepoClient) (io.ReadCloser, string) {
 func runEnabledProbes(request *checker.CheckRequest,
 	probesToRun []string,
 	ret *Result,
+	skipErrors bool,
 ) error {
+	logger := sclog.NewLogger(sclog.DefaultLevel)
+
 	// Add RawResults to request
-	err := populateRawResults(request, probesToRun, ret)
+	err := populateRawResults(request, probesToRun, ret, skipErrors)
 	if err != nil {
 		return err
 	}
@@ -243,6 +247,10 @@ func runEnabledProbes(request *checker.CheckRequest,
 			findings, _, err = probe.Implementation(&ret.RawResults)
 		}
 		if err != nil {
+			if skipErrors {
+				logger.Error(err, fmt.Sprintf("skipping probe %q due to error", probeName))
+				continue
+			}
 			return sce.WithMessage(sce.ErrScorecardInternal, "ending run")
 		}
 		probeFindings = append(probeFindings, findings...)
@@ -263,6 +271,7 @@ type runConfig struct {
 	probes        []string
 	commitDepth   int
 	gitMode       bool
+	skipErrors    bool
 }
 
 type Option func(*runConfig) error
@@ -358,6 +367,16 @@ func WithFileModeGit() Option {
 	}
 }
 
+// WithSkipErrors configures the analysis to continue when individual checks or
+// probes fail at runtime, returning results for everything that succeeded
+// instead of aborting the whole run on the first error.
+func WithSkipErrors(skip bool) Option {
+	return func(c *runConfig) error {
+		c.skipErrors = skip
+		return nil
+	}
+}
+
 // Run analyzes a given repository and returns the result. You can modify the
 // run behavior by passing in [Option] arguments. In the absence of a particular
 // option a default is used. Refer to the various Options for details.
@@ -431,5 +450,5 @@ func Run(ctx context.Context, repo clients.Repo, opts ...Option) (Result, error)
 	}
 
 	return runScorecard(ctx, repo, c.commit, c.commitDepth, checksToRun, c.probes,
-		c.client, c.ossfuzzClient, c.ciiClient, c.vulnClient, c.projectClient)
+		c.client, c.ossfuzzClient, c.ciiClient, c.vulnClient, c.projectClient, c.skipErrors)
 }

pkg/scorecard/scorecard_result.go

@@ -414,7 +414,8 @@ func assignRawData(probeCheckName string, request *checker.CheckRequest, ret *Re
 	return nil
 }
 
-func populateRawResults(request *checker.CheckRequest, probesToRun []string, ret *Result) error {
+func populateRawResults(request *checker.CheckRequest, probesToRun []string, ret *Result, skipErrors bool) error {
+	logger := log.NewLogger(log.DefaultLevel)
 	seen := map[string]bool{}
 	for _, probeName := range probesToRun {
 		p, err := proberegistration.Get(probeName)
@@ -425,6 +426,13 @@ func populateRawResults(request *checker.CheckRequest, probesToRun []string, ret
 			if !seen[checkName] {
 				err := assignRawData(checkName, request, ret)
 				if err != nil {
+					if skipErrors {
+						logger.Error(err, fmt.Sprintf("skipping raw data for %q due to error", checkName))
+						// Mark as seen so we don't retry the same failing collection
+						// for other probes that depend on it.
+						seen[checkName] = true
+						continue
+					}
 					return err
 				}
 				seen[checkName] = true

pkg/scorecard/scorecard_test.go

@@ -14,6 +14,7 @@
 package scorecard
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"slices"
@@ -34,6 +35,9 @@ import (
 	"github.com/ossf/scorecard/v5/probes/fuzzed"
 )
 
+// errTest is used to force a raw-data collection failure in tests.
+var errTest = errors.New("forced test error")
+
 func Test_getRepoCommitHash(t *testing.T) {
 	t.Parallel()
 	tests := []struct {
@@ -345,6 +349,134 @@ func TestRun_WithProbes(t *testing.T) {
 	}
 }
 
+func TestRun_WithProbes_SkipErrors(t *testing.T) {
+	t.Parallel()
+	versionInfo := version.GetVersionInfo()
+	type args struct {
+		uri        string
+		commitSHA  string
+		probes     []string
+		skipErrors bool
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    Result
+		wantErr bool
+	}{
+		{
+			// Without --skip-errors, a raw-data collection failure aborts the run.
+			name: "raw data error aborts run without skip-errors",
+			args: args{
+				uri:        "github.com/ossf/scorecard",
+				commitSHA:  "1a17bb812fb2ac23e9d09e86e122f8b67563aed7",
+				probes:     []string{fuzzed.Probe},
+				skipErrors: false,
+			},
+			want:    Result{},
+			wantErr: true,
+		},
+		{
+			// With --skip-errors, the failing raw data is skipped and the probe
+			// still runs (against empty data), so we get a result and no error.
+			name: "raw data error is skipped with skip-errors",
+			args: args{
+				uri:        "github.com/ossf/scorecard",
+				commitSHA:  "1a17bb812fb2ac23e9d09e86e122f8b67563aed7",
+				probes:     []string{fuzzed.Probe},
+				skipErrors: true,
+			},
+			want: Result{
+				Repo: RepoInfo{
+					Name:      "github.com/ossf/scorecard",
+					CommitSHA: "1a17bb812fb2ac23e9d09e86e122f8b67563aed7",
+				},
+				RawResults: checker.RawResults{
+					Metadata: checker.MetadataData{
+						Metadata: map[string]string{
+							"repository.defaultBranch": "main",
+							"repository.host":          "github.com",
+							"repository.name":          "ossf/scorecard",
+							"repository.sha1":          "1a17bb812fb2ac23e9d09e86e122f8b67563aed7",
+							"repository.uri":           "github.com/ossf/scorecard",
+							"localPath":                "test_path",
+						},
+					},
+				},
+				Scorecard: ScorecardInfo{
+					Version:   versionInfo.GitVersion,
+					CommitSHA: versionInfo.GitCommit,
+				},
+				Findings: []finding.Finding{
+					{
+						Probe:   fuzzed.Probe,
+						Outcome: finding.OutcomeFalse,
+						Message: "no fuzzer integrations found",
+						Remediation: &finding.Remediation{
+							Effort: finding.RemediationEffortHigh,
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			ctrl := gomock.NewController(t)
+			mockRepoClient := mockrepo.NewMockRepoClient(ctrl)
+			mockRepoClient.EXPECT().LocalPath().DoAndReturn(func() (string, error) {
+				return "test_path", nil
+			}).AnyTimes()
+			repo := mockrepo.NewMockRepo(ctrl)
+
+			repo.EXPECT().URI().Return(tt.args.uri).AnyTimes()
+			repo.EXPECT().Host().Return("github.com").AnyTimes()
+			repo.EXPECT().Type().Return(clients.RepoTypeGitHub).AnyTimes()
+
+			mockRepoClient.EXPECT().InitRepo(repo, tt.args.commitSHA, 0).Return(nil)
+			mockRepoClient.EXPECT().URI().Return(repo.URI()).AnyTimes()
+			mockRepoClient.EXPECT().Close().DoAndReturn(func() error {
+				return nil
+			})
+
+			mockRepoClient.EXPECT().ListCommits().DoAndReturn(func() ([]clients.Commit, error) {
+				return []clients.Commit{{SHA: tt.args.commitSHA}}, nil
+			})
+			mockRepoClient.EXPECT().ListFiles(gomock.Any()).Return(nil, nil).AnyTimes()
+
+			// Force the fuzzed probe's raw data collection (raw.Fuzzing) to fail
+			// by returning an error from ListProgrammingLanguages.
+			mockRepoClient.EXPECT().ListProgrammingLanguages().
+				Return(nil, errTest).AnyTimes()
+
+			mockRepoClient.EXPECT().GetDefaultBranchName().Return("main", nil).AnyTimes()
+			mockOSSFuzzClient := mockrepo.NewMockRepoClient(ctrl)
+			mockOSSFuzzClient.EXPECT().Search(gomock.Any()).Return(clients.SearchResponse{}, nil).AnyTimes()
+
+			got, err := Run(t.Context(), repo,
+				WithRepoClient(mockRepoClient),
+				WithOSSFuzzClient(mockOSSFuzzClient),
+				WithCommitSHA(tt.args.commitSHA),
+				WithProbes(tt.args.probes),
+				WithSkipErrors(tt.args.skipErrors),
+			)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			ignoreRemediationText := cmpopts.IgnoreFields(finding.Remediation{}, "Text", "Markdown")
+			ignoreDate := cmpopts.IgnoreFields(Result{}, "Date")
+			ignoreUnexported := cmpopts.IgnoreUnexported(finding.Finding{})
+			if !cmp.Equal(got, tt.want, ignoreDate, ignoreRemediationText, ignoreUnexported) {
+				t.Errorf("expected %v, got %v", got, cmp.Diff(tt.want, got, ignoreDate,
+					ignoreRemediationText, ignoreUnexported))
+			}
+		})
+	}
+}
+
 func Test_findConfigFile(t *testing.T) {
 	t.Parallel()