Skip to content

Commit 874bfc2

Browse files
kyleconroykyleconroy
authored
build: Run MySQL and PostgreSQL locally on the runner (#3095)
- Use two new actions to setup / run PostgreSQL and MySQL without Docker. Container-based services only work on Linux runners - Add new sqltest/local package - Refactor TestValidSchema to use the local package - Mark some tests as having invalid schemas. We've never verified the MySQL schemas before this PR - Mark a few tests as failing on Windows
1 parent 1f24e10 commit 874bfc2

File tree

68 files changed

+457
-176
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+457
-176
lines changed

‎.github/workflows/ci.yml

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ on:
77
jobs:
88
test:
99
strategy:
10-
max-parallel: 2
1110
matrix:
1211
os: [ubuntu-latest, macos-latest, windows-latest]
1312
cgo: ['1', '0']
@@ -49,12 +48,25 @@ jobs:
4948
env:
5049
CGO_ENABLED: ${{ matrix.cgo }}
5150

51+
# Start a PostgreSQL server
52+
- uses: sqlc-dev/action-setup-postgres@master
53+
with:
54+
postgres-version: "16"
55+
id: postgres
56+
57+
# Start a MySQL server
58+
- uses: shogo82148/actions-setup-mysql@v1
59+
with:
60+
mysql-version: "8.1"
61+
5262
- name: test ./...
5363
run: gotestsum --junitfile junit.xml -- --tags=examples -timeout 20m ./...
5464
env:
5565
CI_SQLC_PROJECT_ID: ${{ secrets.CI_SQLC_PROJECT_ID }}
5666
CI_SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }}
5767
SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }}
68+
MYSQL_SERVER_URI: root:@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true
69+
POSTGRESQL_SERVER_URI: ${{ steps.postgres.outputs.connection-uri }}?sslmode=disable
5870
CGO_ENABLED: ${{ matrix.cgo }}
5971

6072
vuln_check:

‎examples/authors/postgresql/db_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@ import (
1010
"github.com/jackc/pgx/v5"
1111
"github.com/jackc/pgx/v5/pgtype"
1212

13-
"github.com/sqlc-dev/sqlc/internal/sqltest/hosted"
13+
"github.com/sqlc-dev/sqlc/internal/sqltest/local"
1414
)
1515

1616
func TestAuthors(t *testing.T) {
1717
ctx := context.Background()
18-
uri := hosted.PostgreSQL(t, []string{"schema.sql"})
18+
uri := local.PostgreSQL(t, []string{"schema.sql"})
1919
db, err := pgx.Connect(ctx, uri)
2020
if err != nil {
2121
t.Fatal(err)

‎examples/batch/postgresql/db_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,12 @@ import (
1010

1111
"github.com/jackc/pgx/v5"
1212
"github.com/jackc/pgx/v5/pgtype"
13-
"github.com/sqlc-dev/sqlc/internal/sqltest/hosted"
13+
14+
"github.com/sqlc-dev/sqlc/internal/sqltest/local"
1415
)
1516

1617
func TestBatchBooks(t *testing.T) {
17-
uri := hosted.PostgreSQL(t, []string{"schema.sql"})
18+
uri := local.PostgreSQL(t, []string{"schema.sql"})
1819

1920
ctx := context.Background()
2021

‎examples/booktest/postgresql/db_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,12 @@ import (
1111
"github.com/jackc/pgx/v5"
1212
"github.com/jackc/pgx/v5/pgtype"
1313

14-
"github.com/sqlc-dev/sqlc/internal/sqltest/hosted"
14+
"github.com/sqlc-dev/sqlc/internal/sqltest/local"
1515
)
1616

1717
func TestBooks(t *testing.T) {
1818
ctx := context.Background()
19-
uri := hosted.PostgreSQL(t, []string{"schema.sql"})
19+
uri := local.PostgreSQL(t, []string{"schema.sql"})
2020
db, err := pgx.Connect(ctx, uri)
2121
if err != nil {
2222
t.Fatal(err)

‎examples/ondeck/mysql/db_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
_ "github.com/go-sql-driver/mysql"
1313
"github.com/google/go-cmp/cmp"
1414

15-
"github.com/sqlc-dev/sqlc/internal/sqltest/hosted"
15+
"github.com/sqlc-dev/sqlc/internal/sqltest/local"
1616
)
1717

1818
func join(vals ...string) sql.NullString {
@@ -144,10 +144,10 @@ func runOnDeckQueries(t *testing.T, q *Queries) {
144144
func TestPrepared(t *testing.T) {
145145
t.Parallel()
146146

147-
uri := hosted.MySQL(t, []string{"schema"})
147+
uri := local.MySQL(t, []string{"schema"})
148148
db, err := sql.Open("mysql", uri)
149149
if err != nil {
150-
t.Fatal(err)
150+
t.Fatalf("%s: %s", uri, err)
151151
}
152152
defer db.Close()
153153

@@ -162,10 +162,10 @@ func TestPrepared(t *testing.T) {
162162
func TestQueries(t *testing.T) {
163163
t.Parallel()
164164

165-
uri := hosted.MySQL(t, []string{"schema"})
165+
uri := local.MySQL(t, []string{"schema"})
166166
db, err := sql.Open("mysql", uri)
167167
if err != nil {
168-
t.Fatal(err)
168+
t.Fatalf("%s: %s", uri, err)
169169
}
170170
defer db.Close()
171171

‎examples/ondeck/mysql/schema/0001_city.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
CREATE TABLE city (
22
slug varchar(255) PRIMARY KEY,
33
name text NOT NULL
4-
)
4+
);

‎examples/ondeck/postgresql/db_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
"github.com/google/go-cmp/cmp"
1212
_ "github.com/lib/pq"
1313

14-
"github.com/sqlc-dev/sqlc/internal/sqltest/hosted"
14+
"github.com/sqlc-dev/sqlc/internal/sqltest/local"
1515
)
1616

1717
func runOnDeckQueries(t *testing.T, q *Queries) {
@@ -126,10 +126,10 @@ func runOnDeckQueries(t *testing.T, q *Queries) {
126126
func TestPrepared(t *testing.T) {
127127
t.Parallel()
128128

129-
uri := hosted.PostgreSQL(t, []string{"schema"})
129+
uri := local.PostgreSQL(t, []string{"schema"})
130130
db, err := sql.Open("postgres", uri)
131131
if err != nil {
132-
t.Fatal(err)
132+
t.Fatalf("%s: %s", uri, err)
133133
}
134134
defer db.Close()
135135

@@ -144,10 +144,10 @@ func TestPrepared(t *testing.T) {
144144
func TestQueries(t *testing.T) {
145145
t.Parallel()
146146

147-
uri := hosted.PostgreSQL(t, []string{"schema"})
147+
uri := local.PostgreSQL(t, []string{"schema"})
148148
db, err := sql.Open("postgres", uri)
149149
if err != nil {
150-
t.Fatal(err)
150+
t.Fatalf("%s: %s", uri, err)
151151
}
152152
defer db.Close()
153153

‎internal/codegen/golang/result.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package golang
22

33
import (
4+
"bufio"
45
"fmt"
56
"sort"
67
"strings"
@@ -205,9 +206,14 @@ func buildQueries(req *plugin.GenerateRequest, options *opts.Options, structs []
205206
comments = append(comments, query.Name)
206207
}
207208
comments = append(comments, " ")
208-
for _, line := range strings.Split(query.Text, "\n") {
209+
scanner := bufio.NewScanner(strings.NewReader(query.Text))
210+
for scanner.Scan() {
211+
line := scanner.Text()
209212
comments = append(comments, " "+line)
210213
}
214+
if err := scanner.Err(); err != nil {
215+
return nil, err
216+
}
211217
}
212218

213219
gq := Query{

‎internal/endtoend/case_test.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,17 @@ type Testcase struct {
1717
Exec *Exec
1818
}
1919

20+
type ExecMeta struct {
21+
InvalidSchema bool `json:"invalid_schema"`
22+
}
23+
2024
type Exec struct {
2125
Command string `json:"command"`
2226
Contexts []string `json:"contexts"`
2327
Process string `json:"process"`
2428
OS []string `json:"os"`
2529
Env map[string]string `json:"env"`
30+
Meta ExecMeta `json:"meta"`
2631
}
2732

2833
func parseStderr(t *testing.T, dir, testctx string) []byte {
@@ -52,10 +57,10 @@ func parseExec(t *testing.T, dir string) *Exec {
5257
var e Exec
5358
blob, err := os.ReadFile(path)
5459
if err != nil {
55-
t.Fatal(err)
60+
t.Fatalf("%s: %s", path, err)
5661
}
5762
if err := json.Unmarshal(blob, &e); err != nil {
58-
t.Fatal(err)
63+
t.Fatalf("%s: %s", path, err)
5964
}
6065
if e.Command == "" {
6166
e.Command = "generate"

‎internal/endtoend/ddl_test.go

Lines changed: 14 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,21 @@
11
package main
22

33
import (
4-
"context"
54
"fmt"
65
"os"
76
"path/filepath"
8-
"runtime"
9-
"slices"
10-
"strings"
117
"testing"
128

13-
"github.com/jackc/pgx/v5"
14-
159
"github.com/sqlc-dev/sqlc/internal/config"
16-
"github.com/sqlc-dev/sqlc/internal/migrations"
17-
"github.com/sqlc-dev/sqlc/internal/quickdb"
18-
pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
19-
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
10+
"github.com/sqlc-dev/sqlc/internal/sqltest/local"
2011
)
2112

2213
func TestValidSchema(t *testing.T) {
23-
if os.Getenv("CI") != "" && runtime.GOOS != "linux" {
24-
t.Skipf("only run these tests in CI on linux: %s %s", os.Getenv("CI"), runtime.GOOS)
25-
}
26-
27-
ctx := context.Background()
28-
29-
projectID := os.Getenv("CI_SQLC_PROJECT_ID")
30-
authToken := os.Getenv("CI_SQLC_AUTH_TOKEN")
31-
if projectID == "" || authToken == "" {
32-
t.Skip("missing project id or auth token")
33-
}
34-
35-
client, err := quickdb.NewClient(projectID, authToken)
36-
if err != nil {
37-
t.Fatal(err)
38-
}
39-
40-
for _, replay := range FindTests(t, "testdata", "managed-db") {
14+
for _, replay := range FindTests(t, "testdata", "base") {
4115
replay := replay // https://golang.org/doc/faq#closures_and_goroutines
4216

43-
if len(replay.Stderr) > 0 {
44-
continue
45-
}
46-
4717
if replay.Exec != nil {
48-
if !slices.Contains(replay.Exec.Contexts, "managed-db") {
18+
if replay.Exec.Meta.InvalidSchema {
4919
continue
5020
}
5121
}
@@ -63,60 +33,28 @@ func TestValidSchema(t *testing.T) {
6333

6434
for j, pkg := range conf.SQL {
6535
j, pkg := j, pkg
66-
if pkg.Engine != config.EnginePostgreSQL {
36+
switch pkg.Engine {
37+
case config.EnginePostgreSQL:
38+
// pass
39+
case config.EngineMySQL:
40+
// pass
41+
default:
6742
continue
6843
}
6944
t.Run(fmt.Sprintf("endtoend-%s-%d", file, j), func(t *testing.T) {
7045
t.Parallel()
7146

72-
if strings.Contains(file, "pg_dump") {
73-
t.Skip("loading pg_dump not supported")
74-
}
75-
7647
var schema []string
7748
for _, path := range pkg.Schema {
7849
schema = append(schema, filepath.Join(filepath.Dir(file), path))
7950
}
8051

81-
files, err := sqlpath.Glob(schema)
82-
if err != nil {
83-
t.Fatal(err)
84-
}
85-
86-
var sqls []string
87-
for _, f := range files {
88-
contents, err := os.ReadFile(f)
89-
if err != nil {
90-
t.Fatalf("%s: %s", f, err)
91-
}
92-
// Support loading pg_dump SQL files
93-
before := strings.ReplaceAll(string(contents), "CREATE SCHEMA public;", "CREATE SCHEMA IF NOT EXISTS public;")
94-
sqls = append(sqls, migrations.RemoveRollbackStatements(before))
95-
}
96-
97-
resp, err := client.CreateEphemeralDatabase(ctx, &pb.CreateEphemeralDatabaseRequest{
98-
Engine: "postgresql",
99-
Region: quickdb.GetClosestRegion(),
100-
Migrations: sqls,
101-
})
102-
if err != nil {
103-
t.Fatalf("region %s: %s", quickdb.GetClosestRegion(), err)
104-
}
105-
106-
t.Cleanup(func() {
107-
_, err = client.DropEphemeralDatabase(ctx, &pb.DropEphemeralDatabaseRequest{
108-
DatabaseId: resp.DatabaseId,
109-
})
110-
if err != nil {
111-
t.Fatal(err)
112-
}
113-
})
114-
115-
conn, err := pgx.Connect(ctx, resp.Uri)
116-
if err != nil {
117-
t.Fatalf("connect %s: %s", resp.Uri, err)
52+
switch pkg.Engine {
53+
case config.EnginePostgreSQL:
54+
local.PostgreSQL(t, schema)
55+
case config.EngineMySQL:
56+
local.MySQL(t, schema)
11857
}
119-
defer conn.Close(ctx)
12058
})
12159
}
12260
}

0 commit comments

Comments
 (0)