diff --git a/dialect/sql/schema/integration/versioned_test.go b/dialect/sql/schema/integration/versioned_test.go index dadb0c51a1..c40eb0572e 100644 --- a/dialect/sql/schema/integration/versioned_test.go +++ b/dialect/sql/schema/integration/versioned_test.go @@ -479,3 +479,136 @@ func TestCheck(t *testing.T) { x.Empty(tableNames(ctx, t, db)) }) } + +// TestExtra is the directory beside the ent one: what the schema cannot state, +// written by hand, applied after the ent directory and with a history of its +// own. +// +// The case it exists for is the second one. Sharing the ent directory's +// history, a file written on one branch before a plan made on another is out of +// order once that plan has run, however unrelated the two are. Kept apart, it +// runs. +func TestExtra(t *testing.T) { + ctx := t.Context() + + dir, err := entschema.OpenDir(t.TempDir()) + require.NoError(t, err) + m := entschema.Migrations{Dir: dir, Dialect: dialect.SQLite, Tables: testSchema()} + + extraDir, err := entschema.OpenDir(t.TempDir()) + require.NoError(t, err) + extra := m.Extra(extraDir) + + db := open(t) + _, err = m.Plan(ctx, open(t), "init") + require.NoError(t, err) + _, err = m.Apply(ctx, db, dialect.SQLite) + require.NoError(t, err) + + // A partial unique index the ent schema here does not declare, which is + // the kind of thing this directory is for and one SQLite can hold. + const index = "CREATE UNIQUE INDEX `holder_one_per_tenant` ON `holder` (`holder_tenant`) WHERE `holder_tenant` IS NOT NULL;\n" + + versions := func(t *testing.T, table string) []string { + t.Helper() + + rows, err := db.QueryContext(ctx, `SELECT version FROM `+table+` ORDER BY version`) + require.NoError(t, err) + defer rows.Close() + + vs := []string{} + for rows.Next() { + var v string + require.NoError(t, rows.Scan(&v)) + vs = append(vs, v) + } + require.NoError(t, rows.Err()) + + return vs + } + + t.Run("a file written by hand is in the sum, so the directory opens", func(t *testing.T) { + x := require.New(t) + + // Older than the file planned above: written before that plan, on a + // branch of its own. + f, err := extra.Write("20000101000000", "holder_one_per_tenant", []byte(index)) + x.NoError(err) + x.Equal("20000101000000_holder_one_per_tenant.sql", f.Name()) + + _, err = entschema.OpenDir(extraDir.Path()) + x.NoError(err) + }) + + t.Run("a version that is already there is refused", func(t *testing.T) { + x := require.New(t) + + _, err := extra.Write("20000101000000", "again", []byte("SELECT 1;\n")) + x.ErrorContains(err, "cannot be ordered") + }) + + t.Run("it runs atop the ent schema, older than the plan though it is", func(t *testing.T) { + x := require.New(t) + + fs, err := extra.Apply(ctx, db, dialect.SQLite) + x.NoError(err) + x.Len(fs, 1) + + // Two histories, each holding only what it ran. + x.Len(versions(t, entschema.RevisionTable), 1) + x.Equal([]string{"20000101000000"}, versions(t, entschema.ExtraRevisionTable)) + + fs, err = m.Pending(ctx, db, dialect.SQLite) + x.NoError(err) + x.Empty(fs) + fs, err = extra.Pending(ctx, db, dialect.SQLite) + x.NoError(err) + x.Empty(fs) + }) + + t.Run("without Atop, a database that holds tables is refused", func(t *testing.T) { + x := require.New(t) + + // Which is what keeps the ent directory from being applied over a + // database somebody built another way -- and why this one says Atop. + other := open(t) + _, err := m.Apply(ctx, other, dialect.SQLite) + x.NoError(err) + + bare := extra + bare.Atop = false + _, err = bare.Apply(ctx, other, dialect.SQLite) + x.ErrorContains(err, "not clean") + }) + + t.Run("the next plan of the ent directory leaves what it made alone", func(t *testing.T) { + x := require.New(t) + + nextSecond(t) + + u := m + u.Tables = moved() + fs, err := u.Plan(ctx, open(t), "add_holder_email") + x.NoError(err) + x.Len(fs, 1) + x.NotContains(string(fs[0].Bytes()), "holder_one_per_tenant") + }) + + t.Run("and so does Check", func(t *testing.T) { + x := require.New(t) + + x.NoError(entschema.Check(ctx, db, dialect.SQLite, testSchema())) + }) + + t.Run("there is nothing to plan it from", func(t *testing.T) { + x := require.New(t) + + // Planned against no tables it would be a file that drops every one. + _, err := extra.Plan(ctx, open(t), "anything") + x.ErrorIs(err, entschema.ErrNoSchema) + + fs, err := extraDir.Files() + x.NoError(err) + x.Len(fs, 1) + }) +} diff --git a/dialect/sql/schema/revision.go b/dialect/sql/schema/revision.go index a062ce1095..cacd6f760f 100644 --- a/dialect/sql/schema/revision.go +++ b/dialect/sql/schema/revision.go @@ -9,6 +9,7 @@ import ( "database/sql" "encoding/json" "fmt" + "regexp" "time" "ariga.io/atlas/sql/migrate" @@ -19,8 +20,21 @@ import ( // RevisionTable is where a database records which migrations were applied to // it. It is not part of the ent schema; it belongs to the database, not to the // app. +// +// It is the ent directory's. A directory of its own beside it keeps its history +// in another table -- see [Migrations.Revisions] -- and [ExtraRevisionTable] is +// the one [Migrations.Extra] names. const RevisionTable = "schema_revisions" +// ExtraRevisionTable is where the directory beside the ent one keeps its +// history; see [Migrations.Extra]. +const ExtraRevisionTable = "schema_revisions_extra" + +// tableName is what a revision table may be called. It is written into SQL as +// it is, so it is held to the one shape every database reads the same way +// unquoted. +var tableName = regexp.MustCompile(`^[a-z_][a-z0-9_]*$`) + // revisionColumns is the order the columns are read and written in. var revisionColumns = []string{ "version", @@ -40,7 +54,8 @@ var revisionColumns = []string{ // revisionSchema holds no type that is spelled differently by the databases // this package migrates, so it is the same statement everywhere. Times are text // in RFC 3339 rather than a timestamp, which every database stores its own way. -const revisionSchema = `CREATE TABLE IF NOT EXISTS ` + RevisionTable + ` ( +func revisionSchema(table string) string { + return `CREATE TABLE IF NOT EXISTS ` + table + ` ( version TEXT NOT NULL PRIMARY KEY, description TEXT NOT NULL, type BIGINT NOT NULL, @@ -54,6 +69,7 @@ const revisionSchema = `CREATE TABLE IF NOT EXISTS ` + RevisionTable + ` ( partial_hashes TEXT NOT NULL, operator_version TEXT NOT NULL )` +} var _ migrate.RevisionReadWriter = (*Revisions)(nil) @@ -63,6 +79,10 @@ type Revisions struct { db *sql.DB dialect string + // table is where the history is kept: [RevisionTable] unless a directory + // said otherwise. + table string + // schema is where the table ended up, for the databases that have more // than one place to put it. schema string @@ -71,7 +91,15 @@ type Revisions struct { // NewRevisions reads the history of `db`, creating it if this is the first // time the database is migrated. func NewRevisions(ctx context.Context, db *sql.DB, d string) (*Revisions, error) { - v := &Revisions{db: db, dialect: d} + return newRevisions(ctx, db, d, RevisionTable) +} + +func newRevisions(ctx context.Context, db *sql.DB, d string, table string) (*Revisions, error) { + if !tableName.MatchString(table) { + return nil, fmt.Errorf("%q cannot name a revision table: it is written into SQL unquoted, so it is lowercase letters, digits and underscores", table) + } + + v := &Revisions{db: db, dialect: d, table: table} if d == dialect.Postgres { // The table is created wherever the search path points. Atlas has to @@ -86,21 +114,21 @@ func NewRevisions(ctx context.Context, db *sql.DB, d string) (*Revisions, error) v.schema = s.String } - if _, err := db.ExecContext(ctx, revisionSchema); err != nil { - return nil, fmt.Errorf("create %s: %w", RevisionTable, err) + if _, err := db.ExecContext(ctx, revisionSchema(table)); err != nil { + return nil, fmt.Errorf("create %s: %w", table, err) } return v, nil } func (r *Revisions) Ident() *migrate.TableIdent { - return &migrate.TableIdent{Name: RevisionTable, Schema: r.schema} + return &migrate.TableIdent{Name: r.table, Schema: r.schema} } func (r *Revisions) ReadRevisions(ctx context.Context) ([]*migrate.Revision, error) { q, args := entsql.Dialect(r.dialect). Select(revisionColumns...). - From(entsql.Table(RevisionTable)). + From(entsql.Table(r.table)). OrderBy(entsql.Asc("version")). Query() @@ -129,7 +157,7 @@ func (r *Revisions) ReadRevisions(ctx context.Context) ([]*migrate.Revision, err func (r *Revisions) ReadRevision(ctx context.Context, version string) (*migrate.Revision, error) { q, args := entsql.Dialect(r.dialect). Select(revisionColumns...). - From(entsql.Table(RevisionTable)). + From(entsql.Table(r.table)). Where(entsql.EQ("version", version)). Query() @@ -157,7 +185,7 @@ func (r *Revisions) WriteRevision(ctx context.Context, v *migrate.Revision) erro } q, args := entsql.Dialect(r.dialect). - Insert(RevisionTable). + Insert(r.table). Columns(revisionColumns...). Values( v.Version, @@ -190,7 +218,7 @@ func (r *Revisions) WriteRevision(ctx context.Context, v *migrate.Revision) erro func (r *Revisions) DeleteRevision(ctx context.Context, version string) error { q, args := entsql.Dialect(r.dialect). - Delete(RevisionTable). + Delete(r.table). Where(entsql.EQ("version", version)). Query() diff --git a/dialect/sql/schema/versioned.go b/dialect/sql/schema/versioned.go index 6a788766ef..75e01450f4 100644 --- a/dialect/sql/schema/versioned.go +++ b/dialect/sql/schema/versioned.go @@ -21,6 +21,22 @@ // Applying to a database that speaks something else runs statements meant for // another dialect: at best it fails halfway through a schema change, at worst // it succeeds and means something different. Migrations refuses it. +// +// # And what the schema cannot state is a directory of its own +// +// Some of what a database holds is not something an ent schema can say: an +// exclusion constraint over a time range, a foreign key that carries the tenant +// so a reference cannot cross one. Written into the ent directory, it is what +// the next [Migrations.Plan] proposes to drop, because planning is the +// directory catching up with the schema. Written into a second directory that +// shares the first one's history, it is ordered against the planned files by +// version, so a file written on one branch is refused as out of order once a +// plan made on another has run. +// +// So it is a second directory with a history of its own -- [Migrations.Extra] +// -- applied after the ent one, written by hand ([Migrations.Write]) and never +// planned. The order between the two is the order the app applies them in, +// which is the one order there is: what stands on the schema comes after it. package schema @@ -29,7 +45,9 @@ import ( "database/sql" "errors" "fmt" + "regexp" "strings" + "time" "ariga.io/atlas/sql/migrate" atpostgres "ariga.io/atlas/sql/postgres" @@ -43,6 +61,11 @@ import ( // package never reads it on its own. const DefaultDir = "migrations" +// DefaultExtraDir is where the files the ent schema cannot state are kept, +// beside the planned ones; see [Migrations.Extra]. Inside [DefaultDir] is +// safe: a directory reads the files in it and not the directories. +const DefaultExtraDir = "migrations/extra" + // ErrDialect is what a database speaking something other than the migrations // are written in is refused with. // @@ -79,7 +102,51 @@ type Migrations struct { // here that generated code owns, and this package does not know the name of // the package it is generated into. Only [Migrations.Plan] reads it -- applying // runs SQL that was already written and needs no schema. + // + // Empty for a directory written by hand, which there is nothing to plan + // from; [Migrations.Plan] refuses one rather than plan the database empty. Tables []*Table + + // Revisions is the table that records which of these files a database has + // run. Empty is [RevisionTable], the ent directory's. + // + // A directory beside the ent one keeps a table of its own, so the two are + // two histories: neither file is ordered against the other's, and each + // knows only what it ran. See the note on the package. + Revisions string + + // Atop says these files are applied onto a database another directory has + // already built -- the ent schema -- so the first of them meets tables it + // did not create, and that is expected rather than a reason to refuse. + // + // Off, a database that holds anything is refused before the first file, + // which is what keeps the ent directory from being applied over a database + // somebody built some other way. + Atop bool +} + +// Extra is the directory of what the ent schema cannot state, beside m: the +// same dialect, a history of its own ([ExtraRevisionTable]), applied atop the +// database m builds, and never planned. +// +// ent := entschema.Migrations{Dir: dir, Dialect: dialect.Postgres, Tables: entmigrate.Tables} +// extra := ent.Extra(extraDir) +// +// ent.Apply(ctx, db, d) // the schema first +// extra.Apply(ctx, db, d) // then what stands on it +func (m Migrations) Extra(dir migrate.Dir) Migrations { + return Migrations{Dir: dir, Dialect: m.Dialect, Revisions: ExtraRevisionTable, Atop: true} +} + +// ErrNoSchema is a directory with no schema asked to plan. +var ErrNoSchema = errors.New("these migrations are written by hand: there is no schema to plan them from") + +func (m Migrations) revisions() string { + if m.Revisions == "" { + return RevisionTable + } + + return m.Revisions } // OpenDir opens the directory of migration files at `path` and makes sure none @@ -110,6 +177,13 @@ func OpenDir(path string) (*migrate.LocalDir, error) { // announces itself: the atlas driver of the dialect reads the catalog the way // that database keeps it, and the wrong database has no such catalog. func (m Migrations) Plan(ctx context.Context, dev *sql.DB, name string) ([]migrate.File, error) { + // Planning against no tables is planning a database with nothing in it -- + // a file that drops every table. A directory written by hand has no schema, + // so it is refused here, before anything is written. + if len(m.Tables) == 0 { + return nil, ErrNoSchema + } + before, err := m.Dir.Files() if err != nil { return nil, fmt.Errorf("read the migration directory: %w", err) @@ -246,12 +320,12 @@ func (m Migrations) executor(ctx context.Context, db *sql.DB, d string) (*migrat return nil, err } - rrw, err := NewRevisions(ctx, db, d) + rrw, err := newRevisions(ctx, db, d, m.revisions()) if err != nil { return nil, err } - v, err := migrate.NewExecutor(drv, m.Dir, rrw) + v, err := migrate.NewExecutor(drv, m.Dir, rrw, migrate.WithAllowDirty(m.Atop)) if err != nil { return nil, fmt.Errorf("new executor: %w", err) } @@ -273,6 +347,79 @@ func atDriverFor(db *sql.DB, d string) (migrate.Driver, error) { } } +// Version is a migration's version for a file written at `t`: the second, in +// UTC, which is the version [Migrations.Plan] gives the files it writes. +func Version(t time.Time) string { + return t.UTC().Format("20060102150405") +} + +// fileName is what the name of a file written by hand may be. +var fileName = regexp.MustCompile(`^[a-z0-9_]+$`) + +// Write puts a file written by hand into the directory -- `_.sql` +// holding `stmts` -- and writes the directory's sum again, so that [OpenDir] +// opens it. +// +// For the directory beside the ent one, which nothing plans; see +// [Migrations.Extra]. A version that is already there is refused, for the reason +// [Migrations.Plan] refuses one: two files of one version cannot be ordered. +func (m Migrations) Write(version string, name string, stmts []byte) (migrate.File, error) { + if !fileName.MatchString(name) { + return nil, fmt.Errorf("%q cannot name a migration: lowercase letters, digits and underscores", name) + } + if _, err := time.Parse("20060102150405", version); err != nil { + return nil, fmt.Errorf("%q is not a version: it is a second, as Version writes one", version) + } + + fs, err := m.Dir.Files() + if err != nil { + return nil, fmt.Errorf("read the migration directory: %w", err) + } + for _, f := range fs { + if f.Version() == version { + return nil, fmt.Errorf("%s is already version %s, and two migrations of one version cannot be ordered", f.Name(), version) + } + } + + path := version + "_" + name + ".sql" + if err := m.Dir.WriteFile(path, stmts); err != nil { + return nil, fmt.Errorf("write %s: %w", path, err) + } + if err := m.Rehash(); err != nil { + return nil, err + } + + fs, err = m.Dir.Files() + if err != nil { + return nil, fmt.Errorf("read the migration directory: %w", err) + } + for _, f := range fs { + if f.Name() == path { + return f, nil + } + } + + return nil, fmt.Errorf("%s was written and is not in the directory", path) +} + +// Rehash writes the directory's sum again from the files as they are now. +// +// It is for a file being written by hand, which is edited until it is right. +// The sum is what stops a file being edited **after** a database ran it, so a +// file that has been applied anywhere is never edited and rehashed: what fixes +// it is a new file. +func (m Migrations) Rehash() error { + sum, err := m.Dir.Checksum() + if err != nil { + return fmt.Errorf("hash the migration directory: %w", err) + } + if err := migrate.WriteSumFile(m.Dir, sum); err != nil { + return fmt.Errorf("write the sum: %w", err) + } + + return nil +} + // ErrBehind is a database that has not run every migration the code ships // with. var ErrBehind = errors.New("the database is behind the migrations this binary carries")