From c0f312a23c39683f1d9621ad461fcfe085e09065 Mon Sep 17 00:00:00 2001 From: Seunghyun Hwang Date: Sat, 10 Oct 2026 17:31:37 +0900 Subject: [PATCH] dialect/sql/schema: a directory of what the schema cannot state, with a history of its own An exclusion constraint over a time range, or a foreign key that carries the tenant, is something a database holds and no ent schema can state. Written into the ent directory, it is what the next Plan proposes to drop. Written into a second directory applied with the same Apply, it shares schema_revisions, so the two directories are one history ordered by version: a file written on one branch before a plan made on another is refused as out of order once that plan has run. So Migrations can be a directory beside the ent one: - Revisions names the table its history is kept in, so the two are two histories (ExtraRevisionTable for the conventional one); - Atop applies it onto a database another directory already built, which atlas otherwise refuses as not clean; - Extra(dir) is both, with the ent directory's dialect; - Write puts a file written by hand into a directory and rehashes its sum, so OpenDir opens it without the Atlas CLI, and refuses a version that is already there; Rehash is for the file still being written; - Plan refuses a directory with no Tables (ErrNoSchema), which it would otherwise plan into a file that drops every table. Checked: TestExtra on SQLite -- a file older than the plan runs, the histories hold only their own, Atop is what lets it start, the next plan and Check leave it alone, there is nothing to plan; and the same on PostgreSQL 17 with a CHECK and an EXCLUDE (a throwaway program, since this module has no PostgreSQL driver). go test -race in dialect and in the integration module under Go 1.27.2; lint.sh on the three files. Reported as lesomnus/payday#33. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KxtnpRW533bdErnW8QPhsk --- .../sql/schema/integration/versioned_test.go | 133 +++++++++++++++ dialect/sql/schema/revision.go | 46 ++++-- dialect/sql/schema/versioned.go | 151 +++++++++++++++++- 3 files changed, 319 insertions(+), 11 deletions(-) diff --git a/dialect/sql/schema/integration/versioned_test.go b/dialect/sql/schema/integration/versioned_test.go index dadb0c51a1..c40eb0572e 100644 --- a/dialect/sql/schema/integration/versioned_test.go +++ b/dialect/sql/schema/integration/versioned_test.go @@ -479,3 +479,136 @@ func TestCheck(t *testing.T) { x.Empty(tableNames(ctx, t, db)) }) } + +// TestExtra is the directory beside the ent one: what the schema cannot state, +// written by hand, applied after the ent directory and with a history of its +// own. +// +// The case it exists for is the second one. Sharing the ent directory's +// history, a file written on one branch before a plan made on another is out of +// order once that plan has run, however unrelated the two are. Kept apart, it +// runs. +func TestExtra(t *testing.T) { + ctx := t.Context() + + dir, err := entschema.OpenDir(t.TempDir()) + require.NoError(t, err) + m := entschema.Migrations{Dir: dir, Dialect: dialect.SQLite, Tables: testSchema()} + + extraDir, err := entschema.OpenDir(t.TempDir()) + require.NoError(t, err) + extra := m.Extra(extraDir) + + db := open(t) + _, err = m.Plan(ctx, open(t), "init") + require.NoError(t, err) + _, err = m.Apply(ctx, db, dialect.SQLite) + require.NoError(t, err) + + // A partial unique index the ent schema here does not declare, which is + // the kind of thing this directory is for and one SQLite can hold. + const index = "CREATE UNIQUE INDEX `holder_one_per_tenant` ON `holder` (`holder_tenant`) WHERE `holder_tenant` IS NOT NULL;\n" + + versions := func(t *testing.T, table string) []string { + t.Helper() + + rows, err := db.QueryContext(ctx, `SELECT version FROM `+table+` ORDER BY version`) + require.NoError(t, err) + defer rows.Close() + + vs := []string{} + for rows.Next() { + var v string + require.NoError(t, rows.Scan(&v)) + vs = append(vs, v) + } + require.NoError(t, rows.Err()) + + return vs + } + + t.Run("a file written by hand is in the sum, so the directory opens", func(t *testing.T) { + x := require.New(t) + + // Older than the file planned above: written before that plan, on a + // branch of its own. + f, err := extra.Write("20000101000000", "holder_one_per_tenant", []byte(index)) + x.NoError(err) + x.Equal("20000101000000_holder_one_per_tenant.sql", f.Name()) + + _, err = entschema.OpenDir(extraDir.Path()) + x.NoError(err) + }) + + t.Run("a version that is already there is refused", func(t *testing.T) { + x := require.New(t) + + _, err := extra.Write("20000101000000", "again", []byte("SELECT 1;\n")) + x.ErrorContains(err, "cannot be ordered") + }) + + t.Run("it runs atop the ent schema, older than the plan though it is", func(t *testing.T) { + x := require.New(t) + + fs, err := extra.Apply(ctx, db, dialect.SQLite) + x.NoError(err) + x.Len(fs, 1) + + // Two histories, each holding only what it ran. + x.Len(versions(t, entschema.RevisionTable), 1) + x.Equal([]string{"20000101000000"}, versions(t, entschema.ExtraRevisionTable)) + + fs, err = m.Pending(ctx, db, dialect.SQLite) + x.NoError(err) + x.Empty(fs) + fs, err = extra.Pending(ctx, db, dialect.SQLite) + x.NoError(err) + x.Empty(fs) + }) + + t.Run("without Atop, a database that holds tables is refused", func(t *testing.T) { + x := require.New(t) + + // Which is what keeps the ent directory from being applied over a + // database somebody built another way -- and why this one says Atop. + other := open(t) + _, err := m.Apply(ctx, other, dialect.SQLite) + x.NoError(err) + + bare := extra + bare.Atop = false + _, err = bare.Apply(ctx, other, dialect.SQLite) + x.ErrorContains(err, "not clean") + }) + + t.Run("the next plan of the ent directory leaves what it made alone", func(t *testing.T) { + x := require.New(t) + + nextSecond(t) + + u := m + u.Tables = moved() + fs, err := u.Plan(ctx, open(t), "add_holder_email") + x.NoError(err) + x.Len(fs, 1) + x.NotContains(string(fs[0].Bytes()), "holder_one_per_tenant") + }) + + t.Run("and so does Check", func(t *testing.T) { + x := require.New(t) + + x.NoError(entschema.Check(ctx, db, dialect.SQLite, testSchema())) + }) + + t.Run("there is nothing to plan it from", func(t *testing.T) { + x := require.New(t) + + // Planned against no tables it would be a file that drops every one. + _, err := extra.Plan(ctx, open(t), "anything") + x.ErrorIs(err, entschema.ErrNoSchema) + + fs, err := extraDir.Files() + x.NoError(err) + x.Len(fs, 1) + }) +} diff --git a/dialect/sql/schema/revision.go b/dialect/sql/schema/revision.go index a062ce1095..cacd6f760f 100644 --- a/dialect/sql/schema/revision.go +++ b/dialect/sql/schema/revision.go @@ -9,6 +9,7 @@ import ( "database/sql" "encoding/json" "fmt" + "regexp" "time" "ariga.io/atlas/sql/migrate" @@ -19,8 +20,21 @@ import ( // RevisionTable is where a database records which migrations were applied to // it. It is not part of the ent schema; it belongs to the database, not to the // app. +// +// It is the ent directory's. A directory of its own beside it keeps its history +// in another table -- see [Migrations.Revisions] -- and [ExtraRevisionTable] is +// the one [Migrations.Extra] names. const RevisionTable = "schema_revisions" +// ExtraRevisionTable is where the directory beside the ent one keeps its +// history; see [Migrations.Extra]. +const ExtraRevisionTable = "schema_revisions_extra" + +// tableName is what a revision table may be called. It is written into SQL as +// it is, so it is held to the one shape every database reads the same way +// unquoted. +var tableName = regexp.MustCompile(`^[a-z_][a-z0-9_]*$`) + // revisionColumns is the order the columns are read and written in. var revisionColumns = []string{ "version", @@ -40,7 +54,8 @@ var revisionColumns = []string{ // revisionSchema holds no type that is spelled differently by the databases // this package migrates, so it is the same statement everywhere. Times are text // in RFC 3339 rather than a timestamp, which every database stores its own way. -const revisionSchema = `CREATE TABLE IF NOT EXISTS ` + RevisionTable + ` ( +func revisionSchema(table string) string { + return `CREATE TABLE IF NOT EXISTS ` + table + ` ( version TEXT NOT NULL PRIMARY KEY, description TEXT NOT NULL, type BIGINT NOT NULL, @@ -54,6 +69,7 @@ const revisionSchema = `CREATE TABLE IF NOT EXISTS ` + RevisionTable + ` ( partial_hashes TEXT NOT NULL, operator_version TEXT NOT NULL )` +} var _ migrate.RevisionReadWriter = (*Revisions)(nil) @@ -63,6 +79,10 @@ type Revisions struct { db *sql.DB dialect string + // table is where the history is kept: [RevisionTable] unless a directory + // said otherwise. + table string + // schema is where the table ended up, for the databases that have more // than one place to put it. schema string @@ -71,7 +91,15 @@ type Revisions struct { // NewRevisions reads the history of `db`, creating it if this is the first // time the database is migrated. func NewRevisions(ctx context.Context, db *sql.DB, d string) (*Revisions, error) { - v := &Revisions{db: db, dialect: d} + return newRevisions(ctx, db, d, RevisionTable) +} + +func newRevisions(ctx context.Context, db *sql.DB, d string, table string) (*Revisions, error) { + if !tableName.MatchString(table) { + return nil, fmt.Errorf("%q cannot name a revision table: it is written into SQL unquoted, so it is lowercase letters, digits and underscores", table) + } + + v := &Revisions{db: db, dialect: d, table: table} if d == dialect.Postgres { // The table is created wherever the search path points. Atlas has to @@ -86,21 +114,21 @@ func NewRevisions(ctx context.Context, db *sql.DB, d string) (*Revisions, error) v.schema = s.String } - if _, err := db.ExecContext(ctx, revisionSchema); err != nil { - return nil, fmt.Errorf("create %s: %w", RevisionTable, err) + if _, err := db.ExecContext(ctx, revisionSchema(table)); err != nil { + return nil, fmt.Errorf("create %s: %w", table, err) } return v, nil } func (r *Revisions) Ident() *migrate.TableIdent { - return &migrate.TableIdent{Name: RevisionTable, Schema: r.schema} + return &migrate.TableIdent{Name: r.table, Schema: r.schema} } func (r *Revisions) ReadRevisions(ctx context.Context) ([]*migrate.Revision, error) { q, args := entsql.Dialect(r.dialect). Select(revisionColumns...). - From(entsql.Table(RevisionTable)). + From(entsql.Table(r.table)). OrderBy(entsql.Asc("version")). Query() @@ -129,7 +157,7 @@ func (r *Revisions) ReadRevisions(ctx context.Context) ([]*migrate.Revision, err func (r *Revisions) ReadRevision(ctx context.Context, version string) (*migrate.Revision, error) { q, args := entsql.Dialect(r.dialect). Select(revisionColumns...). - From(entsql.Table(RevisionTable)). + From(entsql.Table(r.table)). Where(entsql.EQ("version", version)). Query() @@ -157,7 +185,7 @@ func (r *Revisions) WriteRevision(ctx context.Context, v *migrate.Revision) erro } q, args := entsql.Dialect(r.dialect). - Insert(RevisionTable). + Insert(r.table). Columns(revisionColumns...). Values( v.Version, @@ -190,7 +218,7 @@ func (r *Revisions) WriteRevision(ctx context.Context, v *migrate.Revision) erro func (r *Revisions) DeleteRevision(ctx context.Context, version string) error { q, args := entsql.Dialect(r.dialect). - Delete(RevisionTable). + Delete(r.table). Where(entsql.EQ("version", version)). Query() diff --git a/dialect/sql/schema/versioned.go b/dialect/sql/schema/versioned.go index 6a788766ef..75e01450f4 100644 --- a/dialect/sql/schema/versioned.go +++ b/dialect/sql/schema/versioned.go @@ -21,6 +21,22 @@ // Applying to a database that speaks something else runs statements meant for // another dialect: at best it fails halfway through a schema change, at worst // it succeeds and means something different. Migrations refuses it. +// +// # And what the schema cannot state is a directory of its own +// +// Some of what a database holds is not something an ent schema can say: an +// exclusion constraint over a time range, a foreign key that carries the tenant +// so a reference cannot cross one. Written into the ent directory, it is what +// the next [Migrations.Plan] proposes to drop, because planning is the +// directory catching up with the schema. Written into a second directory that +// shares the first one's history, it is ordered against the planned files by +// version, so a file written on one branch is refused as out of order once a +// plan made on another has run. +// +// So it is a second directory with a history of its own -- [Migrations.Extra] +// -- applied after the ent one, written by hand ([Migrations.Write]) and never +// planned. The order between the two is the order the app applies them in, +// which is the one order there is: what stands on the schema comes after it. package schema @@ -29,7 +45,9 @@ import ( "database/sql" "errors" "fmt" + "regexp" "strings" + "time" "ariga.io/atlas/sql/migrate" atpostgres "ariga.io/atlas/sql/postgres" @@ -43,6 +61,11 @@ import ( // package never reads it on its own. const DefaultDir = "migrations" +// DefaultExtraDir is where the files the ent schema cannot state are kept, +// beside the planned ones; see [Migrations.Extra]. Inside [DefaultDir] is +// safe: a directory reads the files in it and not the directories. +const DefaultExtraDir = "migrations/extra" + // ErrDialect is what a database speaking something other than the migrations // are written in is refused with. // @@ -79,7 +102,51 @@ type Migrations struct { // here that generated code owns, and this package does not know the name of // the package it is generated into. Only [Migrations.Plan] reads it -- applying // runs SQL that was already written and needs no schema. + // + // Empty for a directory written by hand, which there is nothing to plan + // from; [Migrations.Plan] refuses one rather than plan the database empty. Tables []*Table + + // Revisions is the table that records which of these files a database has + // run. Empty is [RevisionTable], the ent directory's. + // + // A directory beside the ent one keeps a table of its own, so the two are + // two histories: neither file is ordered against the other's, and each + // knows only what it ran. See the note on the package. + Revisions string + + // Atop says these files are applied onto a database another directory has + // already built -- the ent schema -- so the first of them meets tables it + // did not create, and that is expected rather than a reason to refuse. + // + // Off, a database that holds anything is refused before the first file, + // which is what keeps the ent directory from being applied over a database + // somebody built some other way. + Atop bool +} + +// Extra is the directory of what the ent schema cannot state, beside m: the +// same dialect, a history of its own ([ExtraRevisionTable]), applied atop the +// database m builds, and never planned. +// +// ent := entschema.Migrations{Dir: dir, Dialect: dialect.Postgres, Tables: entmigrate.Tables} +// extra := ent.Extra(extraDir) +// +// ent.Apply(ctx, db, d) // the schema first +// extra.Apply(ctx, db, d) // then what stands on it +func (m Migrations) Extra(dir migrate.Dir) Migrations { + return Migrations{Dir: dir, Dialect: m.Dialect, Revisions: ExtraRevisionTable, Atop: true} +} + +// ErrNoSchema is a directory with no schema asked to plan. +var ErrNoSchema = errors.New("these migrations are written by hand: there is no schema to plan them from") + +func (m Migrations) revisions() string { + if m.Revisions == "" { + return RevisionTable + } + + return m.Revisions } // OpenDir opens the directory of migration files at `path` and makes sure none @@ -110,6 +177,13 @@ func OpenDir(path string) (*migrate.LocalDir, error) { // announces itself: the atlas driver of the dialect reads the catalog the way // that database keeps it, and the wrong database has no such catalog. func (m Migrations) Plan(ctx context.Context, dev *sql.DB, name string) ([]migrate.File, error) { + // Planning against no tables is planning a database with nothing in it -- + // a file that drops every table. A directory written by hand has no schema, + // so it is refused here, before anything is written. + if len(m.Tables) == 0 { + return nil, ErrNoSchema + } + before, err := m.Dir.Files() if err != nil { return nil, fmt.Errorf("read the migration directory: %w", err) @@ -246,12 +320,12 @@ func (m Migrations) executor(ctx context.Context, db *sql.DB, d string) (*migrat return nil, err } - rrw, err := NewRevisions(ctx, db, d) + rrw, err := newRevisions(ctx, db, d, m.revisions()) if err != nil { return nil, err } - v, err := migrate.NewExecutor(drv, m.Dir, rrw) + v, err := migrate.NewExecutor(drv, m.Dir, rrw, migrate.WithAllowDirty(m.Atop)) if err != nil { return nil, fmt.Errorf("new executor: %w", err) } @@ -273,6 +347,79 @@ func atDriverFor(db *sql.DB, d string) (migrate.Driver, error) { } } +// Version is a migration's version for a file written at `t`: the second, in +// UTC, which is the version [Migrations.Plan] gives the files it writes. +func Version(t time.Time) string { + return t.UTC().Format("20060102150405") +} + +// fileName is what the name of a file written by hand may be. +var fileName = regexp.MustCompile(`^[a-z0-9_]+$`) + +// Write puts a file written by hand into the directory -- `_.sql` +// holding `stmts` -- and writes the directory's sum again, so that [OpenDir] +// opens it. +// +// For the directory beside the ent one, which nothing plans; see +// [Migrations.Extra]. A version that is already there is refused, for the reason +// [Migrations.Plan] refuses one: two files of one version cannot be ordered. +func (m Migrations) Write(version string, name string, stmts []byte) (migrate.File, error) { + if !fileName.MatchString(name) { + return nil, fmt.Errorf("%q cannot name a migration: lowercase letters, digits and underscores", name) + } + if _, err := time.Parse("20060102150405", version); err != nil { + return nil, fmt.Errorf("%q is not a version: it is a second, as Version writes one", version) + } + + fs, err := m.Dir.Files() + if err != nil { + return nil, fmt.Errorf("read the migration directory: %w", err) + } + for _, f := range fs { + if f.Version() == version { + return nil, fmt.Errorf("%s is already version %s, and two migrations of one version cannot be ordered", f.Name(), version) + } + } + + path := version + "_" + name + ".sql" + if err := m.Dir.WriteFile(path, stmts); err != nil { + return nil, fmt.Errorf("write %s: %w", path, err) + } + if err := m.Rehash(); err != nil { + return nil, err + } + + fs, err = m.Dir.Files() + if err != nil { + return nil, fmt.Errorf("read the migration directory: %w", err) + } + for _, f := range fs { + if f.Name() == path { + return f, nil + } + } + + return nil, fmt.Errorf("%s was written and is not in the directory", path) +} + +// Rehash writes the directory's sum again from the files as they are now. +// +// It is for a file being written by hand, which is edited until it is right. +// The sum is what stops a file being edited **after** a database ran it, so a +// file that has been applied anywhere is never edited and rehashed: what fixes +// it is a new file. +func (m Migrations) Rehash() error { + sum, err := m.Dir.Checksum() + if err != nil { + return fmt.Errorf("hash the migration directory: %w", err) + } + if err := migrate.WriteSumFile(m.Dir, sum); err != nil { + return fmt.Errorf("write the sum: %w", err) + } + + return nil +} + // ErrBehind is a database that has not run every migration the code ships // with. var ErrBehind = errors.New("the database is behind the migrations this binary carries")