Skip to content

Commit 98a37c8

Browse files
ignatremizovignatremizov
committed
refactor(schema): share apply-time schema loading
Extract the repeated apply-time schema glob/read/preprocess/warn loop into a shared helper so command setup and PostgreSQL sqltest seeding use the same preprocessing path as the migrations work evolves. Changes: - Add `schemautil.LoadSchemasForApply()` to expand schema globs, read files, run `PreprocessSchemaForApply()`, and surface warnings through a caller-provided callback. - Update `createdb`, `verify`, and managed `vet` setup to reuse the shared loader instead of open-coding the same apply-time preprocessing loop. - Update PostgreSQL sqltest seeding helpers to reuse the shared loader for both regular and read-only database setup. - Preserve the existing read-only PostgreSQL cache-key behavior by hashing the preprocessed DDL returned by the shared loader. - Handle plain `pg_dump` schema replays in PostgreSQL sqltest helpers by dropping the default `public` schema before applying dumps that recreate it. Behavioral effect: - No intended user-visible behavior change in normal schema preprocessing. - Apply-time callers and PostgreSQL test helpers now share one codepath, reducing drift between managed-database setup and sqltest seeding. - Plain `pg_dump` fixtures continue to apply cleanly in PostgreSQL test databases even when the dump recreates `public`. Testing: - `go test ./internal/migrations ./internal/compiler ./internal/cmd/... ./internal/sqltest/... ./internal/endtoend -run 'TestReplay/base/pg_dump|TestValidSchema/endtoend-testdata/pg_dump/sqlc.json-0'`
1 parent 0b0ff5a commit 98a37c8

6 files changed

Lines changed: 83 additions & 99 deletions

File tree

‎internal/cmd/createdb.go‎

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,7 @@ import (
1010
"github.com/spf13/cobra"
1111
"github.com/sqlc-dev/sqlc/internal/config"
1212
"github.com/sqlc-dev/sqlc/internal/dbmanager"
13-
"github.com/sqlc-dev/sqlc/internal/migrations"
14-
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
13+
"github.com/sqlc-dev/sqlc/internal/schemautil"
1514
)
1615

1716
var createDBCmd = &cobra.Command{
@@ -76,24 +75,12 @@ func CreateDB(ctx context.Context, dir, filename, querySetName string, o *Option
7675
}
7776

7877
var ddl []string
79-
files, err := sqlpath.Glob(queryset.Schema)
78+
ddl, err = schemautil.LoadSchemasForApply(queryset.Schema, string(queryset.Engine), func(warning string) {
79+
fmt.Fprintln(o.Stderr, warning)
80+
})
8081
if err != nil {
8182
return err
8283
}
83-
for _, schema := range files {
84-
contents, err := os.ReadFile(schema)
85-
if err != nil {
86-
return fmt.Errorf("read file: %w", err)
87-
}
88-
ddlText, warnings, err := migrations.PreprocessSchemaForApply(string(contents), string(queryset.Engine))
89-
if err != nil {
90-
return err
91-
}
92-
for _, warning := range warnings {
93-
fmt.Fprintln(o.Stderr, warning)
94-
}
95-
ddl = append(ddl, ddlText)
96-
}
9784

9885
now := time.Now().UTC().UnixNano()
9986
client := dbmanager.NewClient(conf.Servers)

‎internal/cmd/verify.go‎

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,10 @@ import (
1414

1515
"github.com/sqlc-dev/sqlc/internal/config"
1616
"github.com/sqlc-dev/sqlc/internal/dbmanager"
17-
"github.com/sqlc-dev/sqlc/internal/migrations"
1817
"github.com/sqlc-dev/sqlc/internal/plugin"
1918
"github.com/sqlc-dev/sqlc/internal/quickdb"
2019
pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
21-
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
20+
"github.com/sqlc-dev/sqlc/internal/schemautil"
2221
)
2322

2423
func init() {
@@ -93,24 +92,12 @@ func Verify(ctx context.Context, dir, filename string, opts *Options) error {
9392

9493
// Read the schema files into memory, removing rollback statements
9594
var ddl []string
96-
files, err := sqlpath.Glob(current.Schema)
95+
ddl, err = schemautil.LoadSchemasForApply(current.Schema, string(current.Engine), func(warning string) {
96+
fmt.Fprintln(stderr, warning)
97+
})
9798
if err != nil {
9899
return err
99100
}
100-
for _, schema := range files {
101-
contents, err := os.ReadFile(schema)
102-
if err != nil {
103-
return fmt.Errorf("read file: %w", err)
104-
}
105-
ddlText, warnings, err := migrations.PreprocessSchemaForApply(string(contents), string(current.Engine))
106-
if err != nil {
107-
return err
108-
}
109-
for _, warning := range warnings {
110-
fmt.Fprintln(stderr, warning)
111-
}
112-
ddl = append(ddl, ddlText)
113-
}
114101

115102
var codegen plugin.GenerateRequest
116103
if err := proto.Unmarshal(qs.CodegenRequest.Contents, &codegen); err != nil {

‎internal/cmd/vet.go‎

Lines changed: 10 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,11 @@ import (
2727
"github.com/sqlc-dev/sqlc/internal/config"
2828
"github.com/sqlc-dev/sqlc/internal/dbmanager"
2929
"github.com/sqlc-dev/sqlc/internal/debug"
30-
"github.com/sqlc-dev/sqlc/internal/migrations"
3130
"github.com/sqlc-dev/sqlc/internal/opts"
3231
"github.com/sqlc-dev/sqlc/internal/plugin"
3332
"github.com/sqlc-dev/sqlc/internal/quickdb"
33+
"github.com/sqlc-dev/sqlc/internal/schemautil"
3434
"github.com/sqlc-dev/sqlc/internal/shfmt"
35-
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
3635
"github.com/sqlc-dev/sqlc/internal/vet"
3736
)
3837

@@ -422,25 +421,12 @@ func (c *checker) fetchDatabaseUri(ctx context.Context, s config.SQL) (string, f
422421
c.Client = dbmanager.NewClient(c.Conf.Servers)
423422
})
424423

425-
var ddl []string
426-
files, err := sqlpath.Glob(s.Schema)
424+
ddl, err := schemautil.LoadSchemasForApply(s.Schema, string(s.Engine), func(warning string) {
425+
fmt.Fprintln(c.Stderr, warning)
426+
})
427427
if err != nil {
428428
return "", cleanup, err
429429
}
430-
for _, schema := range files {
431-
contents, err := os.ReadFile(schema)
432-
if err != nil {
433-
return "", cleanup, fmt.Errorf("read file: %w", err)
434-
}
435-
ddlText, warnings, err := migrations.PreprocessSchemaForApply(string(contents), string(s.Engine))
436-
if err != nil {
437-
return "", cleanup, err
438-
}
439-
for _, warning := range warnings {
440-
fmt.Fprintln(c.Stderr, warning)
441-
}
442-
ddl = append(ddl, ddlText)
443-
}
444430

445431
resp, err := c.Client.CreateDatabase(ctx, &dbmanager.CreateDatabaseRequest{
446432
Engine: string(s.Engine),
@@ -547,24 +533,15 @@ func (c *checker) checkSQL(ctx context.Context, s config.SQL) error {
547533
defer db.Close()
548534
// For in-memory SQLite databases, apply migrations
549535
if isInMemorySQLite(dburl) {
550-
files, err := sqlpath.Glob(s.Schema)
536+
ddl, err := schemautil.LoadSchemasForApply(s.Schema, string(s.Engine), func(warning string) {
537+
fmt.Fprintln(c.Stderr, warning)
538+
})
551539
if err != nil {
552540
return fmt.Errorf("schema: %w", err)
553541
}
554-
for _, schema := range files {
555-
contents, err := os.ReadFile(schema)
556-
if err != nil {
557-
return fmt.Errorf("read schema file: %w", err)
558-
}
559-
ddl, warnings, err := migrations.PreprocessSchemaForApply(string(contents), string(s.Engine))
560-
if err != nil {
561-
return err
562-
}
563-
for _, warning := range warnings {
564-
fmt.Fprintln(c.Stderr, warning)
565-
}
566-
if _, err := db.ExecContext(ctx, ddl); err != nil {
567-
return fmt.Errorf("apply schema %s: %w", schema, err)
542+
for _, stmt := range ddl {
543+
if _, err := db.ExecContext(ctx, stmt); err != nil {
544+
return fmt.Errorf("apply schema: %w", err)
568545
}
569546
}
570547
}

‎internal/schemautil/load.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package schemautil
2+
3+
import (
4+
"fmt"
5+
"os"
6+
7+
"github.com/sqlc-dev/sqlc/internal/migrations"
8+
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
9+
)
10+
11+
// LoadSchemasForApply expands globs, preprocesses each schema in order, and
12+
// reports any warnings through warn. The returned DDL is suitable for callers
13+
// that will apply schema text to a live database.
14+
func LoadSchemasForApply(globs []string, engine string, warn func(string)) ([]string, error) {
15+
files, err := sqlpath.Glob(globs)
16+
if err != nil {
17+
return nil, err
18+
}
19+
20+
ddl := make([]string, 0, len(files))
21+
for _, schema := range files {
22+
contents, err := os.ReadFile(schema)
23+
if err != nil {
24+
return nil, fmt.Errorf("read file: %w", err)
25+
}
26+
ddlText, warnings, err := migrations.PreprocessSchemaForApply(string(contents), engine)
27+
if err != nil {
28+
return nil, err
29+
}
30+
for _, warning := range warnings {
31+
if warn != nil {
32+
warn(warning)
33+
}
34+
}
35+
ddl = append(ddl, ddlText)
36+
}
37+
38+
return ddl, nil
39+
}

‎internal/sqltest/local/postgres.go‎

Lines changed: 11 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,8 @@ import (
1212
"github.com/jackc/pgx/v5"
1313
"golang.org/x/sync/singleflight"
1414

15-
migrate "github.com/sqlc-dev/sqlc/internal/migrations"
1615
"github.com/sqlc-dev/sqlc/internal/pgx/poolcache"
17-
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
16+
"github.com/sqlc-dev/sqlc/internal/schemautil"
1817
"github.com/sqlc-dev/sqlc/internal/sqltest/docker"
1918
"github.com/sqlc-dev/sqlc/internal/sqltest/native"
2019
)
@@ -60,26 +59,15 @@ func postgreSQL(t *testing.T, migrations []string, rw bool) string {
6059
}
6160

6261
var seed []string
63-
files, err := sqlpath.Glob(migrations)
62+
h := fnv.New64()
63+
seed, err = schemautil.LoadSchemasForApply(migrations, "postgresql", func(warning string) {
64+
t.Log(warning)
65+
})
6466
if err != nil {
6567
t.Fatal(err)
6668
}
67-
68-
h := fnv.New64()
69-
for _, f := range files {
70-
blob, err := os.ReadFile(f)
71-
if err != nil {
72-
t.Fatal(err)
73-
}
74-
ddl, warnings, err := migrate.PreprocessSchemaForApply(string(blob), "postgresql")
75-
if err != nil {
76-
t.Fatal(err)
77-
}
69+
for _, ddl := range seed {
7870
h.Write([]byte(ddl))
79-
for _, warning := range warnings {
80-
t.Log(warning)
81-
}
82-
seed = append(seed, ddl)
8371
}
8472

8573
var name string
@@ -123,6 +111,11 @@ func postgreSQL(t *testing.T, migrations []string, rw bool) string {
123111
if len(strings.TrimSpace(q)) == 0 {
124112
continue
125113
}
114+
if strings.Contains(q, "\nCREATE SCHEMA public;\n") || strings.HasPrefix(q, "CREATE SCHEMA public;\n") {
115+
if _, err := conn.Exec(ctx, `DROP SCHEMA IF EXISTS public CASCADE`); err != nil {
116+
return nil, fmt.Errorf("drop public schema: %s", err)
117+
}
118+
}
126119
if _, err := conn.Exec(ctx, q); err != nil {
127120
return nil, fmt.Errorf("%s: %s", q, err)
128121
}

‎internal/sqltest/postgres.go‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,11 @@ import (
66
"math/rand"
77
"os"
88
"path/filepath"
9+
"strings"
910
"testing"
1011
"time"
1112

12-
migrate "github.com/sqlc-dev/sqlc/internal/migrations"
13+
"github.com/sqlc-dev/sqlc/internal/schemautil"
1314
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
1415

1516
_ "github.com/lib/pq"
@@ -99,20 +100,20 @@ func CreatePostgreSQLDatabase(t *testing.T, name string, schema bool, migrations
99100
if err != nil {
100101
t.Fatal(err)
101102
}
102-
for _, f := range files {
103-
blob, err := os.ReadFile(f)
104-
if err != nil {
105-
t.Fatal(err)
106-
}
107-
ddl, warnings, err := migrate.PreprocessSchemaForApply(string(blob), "postgresql")
108-
if err != nil {
109-
t.Fatal(err)
110-
}
111-
for _, warning := range warnings {
112-
t.Log(warning)
103+
ddl, err := schemautil.LoadSchemasForApply(migrations, "postgresql", func(warning string) {
104+
t.Log(warning)
105+
})
106+
if err != nil {
107+
t.Fatal(err)
108+
}
109+
for i, stmt := range ddl {
110+
if strings.Contains(stmt, "\nCREATE SCHEMA public;\n") || strings.HasPrefix(stmt, "CREATE SCHEMA public;\n") {
111+
if _, err := sdb.Exec(`DROP SCHEMA IF EXISTS public CASCADE`); err != nil {
112+
t.Fatalf("drop public schema: %s", err)
113+
}
113114
}
114-
if _, err := sdb.Exec(ddl); err != nil {
115-
t.Fatalf("%s: %s", filepath.Base(f), err)
115+
if _, err := sdb.Exec(stmt); err != nil {
116+
t.Fatalf("%s: %s", filepath.Base(files[i]), err)
116117
}
117118
}
118119

0 commit comments

Comments
 (0)