feat(output): redirect trace output to stderr

docs/debug.md

@@ -108,3 +108,17 @@ TRAC | Exit data.main.deny = _
 TRAC Redo data.main.deny = _
 TRAC | Redo data.main.deny = _
 ```
+
+## Using trace with other output formats
+
+You can use the `--trace` flag together with any output format. When using `--trace` with formats like `--output=table` or `--output=json`, the trace information will be written to stderr while the formatted output will be written to stdout. This allows you to capture trace information for debugging while still using your preferred output format.
+
+For example:
+
+```console
+# Output trace to stderr and table format to stdout
+$ conftest test --trace --output=table deployment.yaml
+
+# Capture trace output to a file while viewing table output
+$ conftest test --trace --output=table deployment.yaml 2>trace.log
+```

internal/commands/test.go

@@ -65,10 +65,18 @@ It expects one or more urls to fetch the latest policies from, e.g.:
 See the pull command for more details on supported protocols for fetching policies.
 
 When debugging policies it can be useful to use a more verbose policy evaluation output. By using the '--trace' flag
-the output will include a detailed trace of how the policy was evaluated. When both '--trace' and '--output' are specified,
-the output format takes priority and tracing will not be used.
+the output will include a detailed trace of how the policy was evaluated. The trace output will be written to stderr,
+while the regular output will be written to stdout. This allows you to use the '--trace' flag together with any output
+format, including table, JSON, etc.
 
+    # Trace output
 	$ conftest test --trace <input-file>
+
+	# Trace output with any non-standard output format
+	$ conftest test --trace --output=table <input-file>
+
+	# Redirect trace output to a file while viewing formatted output
+	$ conftest test --trace --output=json <input-file> 2>trace.log
 `
 
 // TestRun stores the compiler and store for a test run.

output/output.go

@@ -43,9 +43,34 @@ func Get(format string, options Options) Outputter {
 		options.File = os.Stdout
 	}
 
+	// If tracing is enabled, output trace to stderr first,
+	// then return the requested outputter
+	if options.Tracing {
+		traceHandler := &Standard{
+			Writer:  os.Stderr,
+			NoColor: options.NoColor,
+			Tracing: true,
+		}
+
+		// Return a trace outputter that handles both trace and regular output
+		return newTraceOutputter(traceHandler, newOutputter(format, options))
+	}
+
+	// If no tracing, return the regular outputter
+	return newOutputter(format, options)
+}
+
+// newOutputter creates an outputter based on the format and options
+func newOutputter(format string, options Options) Outputter {
 	switch format {
 	case OutputStandard:
-		return &Standard{Writer: options.File, NoColor: options.NoColor, SuppressExceptions: options.SuppressExceptions, Tracing: options.Tracing, ShowSkipped: options.ShowSkipped}
+		return &Standard{
+			Writer:             options.File,
+			NoColor:            options.NoColor,
+			SuppressExceptions: options.SuppressExceptions,
+			Tracing:            options.Tracing,
+			ShowSkipped:        options.ShowSkipped,
+		}
 	case OutputJSON:
 		return NewJSON(options.File)
 	case OutputTAP:
@@ -65,6 +90,36 @@ func Get(format string, options Options) Outputter {
 	}
 }
 
+// traceOutputter handles outputting trace to stderr while sending regular output to stdout
+type traceOutputter struct {
+	traceHandler  *Standard
+	mainOutputter Outputter
+}
+
+// newTraceOutputter creates a new traceOutputter with the given trace handler and main outputter
+func newTraceOutputter(traceHandler *Standard, mainOutputter Outputter) *traceOutputter {
+	return &traceOutputter{
+		traceHandler:  traceHandler,
+		mainOutputter: mainOutputter,
+	}
+}
+
+// Output outputs the results, handling trace separately
+func (t *traceOutputter) Output(results CheckResults) error {
+	// First, output trace to stderr
+	if err := t.traceHandler.outputTraceOnly(results); err != nil {
+		return err
+	}
+
+	// Then, output regular format to stdout
+	return t.mainOutputter.Output(results)
+}
+
+// Report passes through to the main outputter
+func (t *traceOutputter) Report(results []*tester.Result, flag string) error {
+	return t.mainOutputter.Report(results, flag)
+}
+
 // Outputs returns the available output formats.
 func Outputs() []string {
 	return []string{

output/output_test.go

@@ -1,64 +1,178 @@
 package output
 
 import (
+	"bytes"
+	"io"
 	"os"
 	"reflect"
+	"strings"
 	"testing"
 )
 
 func TestGetOutputter(t *testing.T) {
 	testCases := []struct {
 		input    string
 		expected Outputter
+		tracing  bool
 	}{
 		{
 			input:    OutputStandard,
 			expected: NewStandard(os.Stdout),
+			tracing:  false,
+		},
+		{
+			input:    OutputStandard,
+			expected: NewStandard(os.Stdout),
+			tracing:  true,
 		},
 		{
 			input:    OutputJSON,
 			expected: NewJSON(os.Stdout),
+			tracing:  false,
+		},
+		{
+			input:    OutputJSON,
+			expected: NewJSON(os.Stdout),
+			tracing:  true,
 		},
 		{
 			input:    OutputTAP,
 			expected: NewTAP(os.Stdout),
+			tracing:  false,
 		},
 		{
 			input:    OutputTable,
 			expected: NewTable(os.Stdout),
+			tracing:  false,
 		},
 		{
 			input:    OutputJUnit,
 			expected: NewJUnit(os.Stdout, false),
+			tracing:  false,
 		},
 		{
 			input:    OutputGitHub,
 			expected: NewGitHub(os.Stdout),
+			tracing:  false,
 		},
 		{
 			input:    OutputAzureDevOps,
 			expected: NewAzureDevOps(os.Stdout),
+			tracing:  false,
 		},
 		{
 			input:    OutputSARIF,
 			expected: NewSARIF(os.Stdout),
+			tracing:  false,
 		},
 		{
 			input:    "unknown_format",
 			expected: NewStandard(os.Stdout),
+			tracing:  false,
 		},
 	}
 
 	for _, testCase := range testCases {
 		t.Run(testCase.input, func(t *testing.T) {
-			actual := Get(testCase.input, Options{NoColor: true})
+			actual := Get(testCase.input, Options{NoColor: true, Tracing: testCase.tracing})
 
-			actualType := reflect.TypeOf(actual)
+			// If tracing is enabled, we expect a traceOutputter
+			if testCase.tracing {
+				if _, ok := actual.(*traceOutputter); !ok {
+					t.Errorf("Expected traceOutputter but got %T", actual)
+				}
+				return
+			}
 
+			actualType := reflect.TypeOf(actual)
 			expectedType := reflect.TypeOf(testCase.expected)
 			if expectedType != actualType {
 				t.Errorf("Unexpected outputter. expected %v actual %v", expectedType, actualType)
 			}
 		})
 	}
 }
+
+func TestTraceOutputter(t *testing.T) {
+	// Create a test result with trace information
+	results := CheckResults{
+		{
+			FileName:  "test.yaml",
+			Namespace: "test",
+			Failures:  []Result{{Message: "test failure"}},
+			Queries: []QueryResult{
+				{
+					Query: "data.main.deny",
+					Traces: []string{
+						"TRACE line 1",
+						"TRACE line 2",
+					},
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		name            string
+		createOutputter func(io.Writer) Outputter
+		expectedStdout  string
+		expectedTrace   []string
+	}{
+		{
+			name: "table format with trace",
+			createOutputter: func(w io.Writer) Outputter {
+				return NewTable(w)
+			},
+			expectedStdout: "test.yaml",
+			expectedTrace:  []string{"TRACE line 1", "TRACE line 2"},
+		},
+		{
+			name: "json format with trace",
+			createOutputter: func(w io.Writer) Outputter {
+				return NewJSON(w)
+			},
+			expectedStdout: "test.yaml",
+			expectedTrace:  []string{"TRACE line 1", "TRACE line 2"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Create buffers for stdout and stderr
+			stdoutBuf := new(bytes.Buffer)
+			stderrBuf := new(bytes.Buffer)
+
+			// Create the main outputter
+			mainOutputter := tt.createOutputter(stdoutBuf)
+
+			// Create the trace handler
+			traceHandler := &Standard{
+				Writer:  stderrBuf,
+				NoColor: true,
+				Tracing: true,
+			}
+
+			// Create the trace outputter
+			traceOut := newTraceOutputter(traceHandler, mainOutputter)
+
+			// Output the results
+			if err := traceOut.Output(results); err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			// Check that trace output went to stderr
+			stderrOutput := stderrBuf.String()
+			for _, trace := range tt.expectedTrace {
+				if !strings.Contains(stderrOutput, trace) {
+					t.Errorf("stderr missing expected trace: %q", trace)
+				}
+			}
+
+			// Check that formatted output went to stdout
+			stdoutOutput := stdoutBuf.String()
+			if !strings.Contains(stdoutOutput, tt.expectedStdout) {
+				t.Errorf("stdout missing expected content: %q", tt.expectedStdout)
+			}
+		})
+	}
+}

output/standard.go

@@ -175,6 +175,18 @@ func (s *Standard) outputTrace(results CheckResults, colorizer aurora.Aurora) {
 	}
 }
 
+// outputTraceOnly outputs only the trace information to the writer
+// This is used by the TraceOutputter to output trace to stderr
+func (s *Standard) outputTraceOnly(results CheckResults) error {
+	colorizer := aurora.NewAurora(true)
+	if s.NoColor {
+		colorizer = aurora.NewAurora(false)
+	}
+
+	s.outputTrace(results, colorizer)
+	return nil
+}
+
 // Report outputs results similar to OPA test output
 func (s *Standard) Report(results []*tester.Result, flag string) error {
 	reporter := tester.PrettyReporter{