Skip to content

Commit a282ce6

Browse files
authored
feat: Better error when panic happens in UnifiedDiff (#2217)
#### Summary We have a test in our ClickHouse destination that is flaky https://github.com/cloudquery/cloudquery/actions/runs/15976814323/job/45061355720#step:9:16 ``` FAIL: TestPlugin/TestInsert/All (2.80s) panic: json: error calling MarshalJSON for type *array.Struct: json: error calling MarshalJSON for type json.RawMessage: unexpected character '' [recovered] panic: json: error calling MarshalJSON for type *array.Struct: json: error calling MarshalJSON for type json.RawMessage: unexpected character '' goroutine 74 [running]: testing.tRunner.func1.2({0x10f42e0, 0xc00104ccf0}) /opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:1734 +0x3eb testing.tRunner.func1() /opt/hostedtoolcache/go/1.24.3/x64/src/testing/testing.go:1737 +0x696 panic({0x10f42e0?, 0xc00104ccf0?}) /opt/hostedtoolcache/go/1.24.3/x64/src/runtime/panic.go:792 +0x132 github.com/apache/arrow-go/v18/arrow/array.(*List).GetOneForMarshal(0xc00140be00, 0x0) /home/runner/go/pkg/mod/github.com/apache/arrow-go/[email protected]/arrow/array/list.go:110 +0x33b github.com/apache/arrow-go/v18/arrow/array.(*Struct).GetOneForMarshal(0xc000737b00, 0x0) /home/runner/go/pkg/mod/github.com/apache/arrow-go/[email protected]/arrow/array/struct.go:212 +0x2d6 github.com/apache/arrow-go/v18/arrow/array.(*Struct).MarshalJSON(0xc000737b00) /home/runner/go/pkg/mod/github.com/apache/arrow-go/[email protected]/arrow/array/struct.go:226 +0x189 github.com/goccy/go-json/internal/encoder.AppendMarshalJSON(0xc000179110, 0xc0009b9860, {0xc000ffdc00, 0x0, 0x400}, {0x119ea80, 0xc000737b00}) /home/runner/go/pkg/mod/github.com/goccy/[email protected]/internal/encoder/encoder.go:435 +0x472 github.com/goccy/go-json/internal/encoder/vm.appendMarshalJSON(0xc000179110, 0xc0009b9860, {0xc000ffdc00, 0x0, 0x400}, {0x119ea80, 0xc000737b00}) /home/runner/go/pkg/mod/github.com/goccy/[email protected]/internal/encoder/vm/util.go:152 +0x85 github.com/goccy/go-json/internal/encoder/vm.Run(0xc000179110, {0xc000ffdc00, 0x0, 0x400}, 0xc000df6cb0) /home/runner/go/pkg/mod/github.com/goccy/[email protected]/internal/encoder/vm/vm.go:271 +0x63c5 github.com/goccy/go-json.encodeRunCode(0xc000179110, {0xc000ffdc00, 0x0, 0x400}, 0xc000df6cb0) /home/runner/go/pkg/mod/github.com/goccy/[email protected]/encode.go:310 +0x235 github.com/goccy/go-json.encode(0xc000179110, {0x119ea80, 0xc000737b00}) /home/runner/go/pkg/mod/github.com/goccy/[email protected]/encode.go:235 +0x47d github.com/goccy/go-json.marshal({0x119ea80, 0xc000737b00}, {0x0, 0x0, 0x3?}) /home/runner/go/pkg/mod/github.com/goccy/[email protected]/encode.go:150 +0x16a github.com/goccy/go-json.MarshalWithOption(...) /home/runner/go/pkg/mod/github.com/goccy/[email protected]/json.go:185 github.com/goccy/go-json.Marshal({0x119ea80, 0xc000737b00}) /home/runner/go/pkg/mod/github.com/goccy/[email protected]/json.go:170 +0x3b github.com/apache/arrow-go/v18/internal/json.Marshal(...) /home/runner/go/pkg/mod/github.com/apache/arrow-go/[email protected]/internal/json/json.go:38 github.com/apache/arrow-go/v18/arrow/array.(*List).GetOneForMarshal(0xc00140b2c0, 0x0) /home/runner/go/pkg/mod/github.com/apache/arrow-go/[email protected]/arrow/array/list.go:108 +0x26c github.com/apache/arrow-go/v18/arrow/array.(*List).MarshalJSON(0xc00140b2c0) /home/runner/go/pkg/mod/github.com/apache/arrow-go/[email protected]/arrow/array/list.go:124 +0x189 github.com/apache/arrow-go/v18/arrow/array.stringAt({0x140ee40, 0xc00104d830}, 0x2) /home/runner/go/pkg/mod/github.com/apache/arrow-go/[email protected]/arrow/array/diff.go:110 +0xbdb github.com/apache/arrow-go/v18/arrow/array.Edits.UnifiedDiff({0xc000516000, 0x5, 0x140ee40?}, {0x140ee40, 0xc00104d830}, {0x140ee40, 0xc00104d800}) /home/runner/go/pkg/mod/github.com/apache/arrow-go/[email protected]/arrow/array/diff.go:72 +0x558 github.com/cloudquery/plugin-sdk/v4/plugin.TableDiff({0x1409f58, 0xc000298000}, {0x1409f58, 0xc000298030}) /home/runner/go/pkg/mod/github.com/cloudquery/plugin-sdk/[email protected]/plugin/diff.go:42 +0x5da ``` It's hard to debug the test since: 1. It's flaky 2. We don't have enough information when it fails This PR tries to error out with a bit more information on the data that was used while triggering the panic ---
1 parent c1375bb commit a282ce6

File tree

1 file changed

+20
-1
lines changed

1 file changed

+20
-1
lines changed

plugin/diff.go

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,25 @@ func RecordsDiff(sc *arrow.Schema, have, want []arrow.Record) string {
1313
return TableDiff(array.NewTableFromRecords(sc, have), array.NewTableFromRecords(sc, want))
1414
}
1515

16+
func getUnifiedDiff(edits array.Edits, wantCol, haveCol arrow.Array) string {
17+
defer func() {
18+
if r := recover(); r != nil {
19+
wantDataType := wantCol.DataType()
20+
wantData := make([]string, wantCol.Len())
21+
for i := 0; i < wantCol.Len(); i++ {
22+
wantData[i] = wantCol.ValueStr(i)
23+
}
24+
haveDataType := haveCol.DataType()
25+
haveData := make([]string, haveCol.Len())
26+
for i := 0; i < haveCol.Len(); i++ {
27+
haveData[i] = haveCol.ValueStr(i)
28+
}
29+
panic(fmt.Errorf("panic in getUnifiedDiff: %s, want: [%s], have: [%s], want type: %s, have type: %s", r, strings.Join(wantData, ", "), strings.Join(haveData, ", "), wantDataType, haveDataType))
30+
}
31+
}()
32+
return edits.UnifiedDiff(wantCol, haveCol)
33+
}
34+
1635
func TableDiff(have, want arrow.Table) string {
1736
if array.TableApproxEqual(have, want, array.WithUnorderedMapKeys(true)) {
1837
return ""
@@ -39,7 +58,7 @@ func TableDiff(have, want arrow.Table) string {
3958
if err != nil {
4059
panic(fmt.Errorf("want: %v, have: %v, error: %w", wantCol.DataType(), haveCol.DataType(), err))
4160
}
42-
diff := edits.UnifiedDiff(wantCol, haveCol)
61+
diff := getUnifiedDiff(edits, wantCol, haveCol)
4362
if diff != "" {
4463
sb.WriteString(have.Schema().Field(i).Name)
4564
sb.WriteString(": ")

0 commit comments

Comments
 (0)