feat(experimental): add collect migrations logic and new Provider opt…

…ions (#615)

Makefile

@@ -33,6 +33,9 @@ tools:
 test-packages:
  go test $(GO_TEST_FLAGS) $$(go list ./... | grep -v -e /tests -e /bin -e /cmd -e /examples)
 
+test-packages-short:
+ go test -test.short $(GO_TEST_FLAGS) $$(go list ./... | grep -v -e /tests -e /bin -e /cmd -e /examples)
+
 test-e2e: test-e2e-postgres test-e2e-mysql test-e2e-clickhouse test-e2e-vertica
 
 test-e2e-postgres:

create_test.go

@@ -11,6 +11,9 @@ import (
 
 func TestSequential(t *testing.T) {
  t.Parallel()
+ if testing.Short() {
+ t.Skip("skip long running test")
+ }
 
  dir := t.TempDir()
  defer os.Remove("./bin/create-goose") // clean up

fix_test.go

@@ -11,6 +11,9 @@ import (
 
 func TestFix(t *testing.T) {
  t.Parallel()
+ if testing.Short() {
+ t.Skip("skip long running test")
+ }
 
  dir := t.TempDir()
  defer os.Remove("./bin/fix-goose") // clean up

internal/migrationstats/migrationstats_test.go

@@ -8,6 +8,7 @@ import (
 )
 
 func TestParsingGoMigrations(t *testing.T) {
+ t.Parallel()
  tests := []struct {
  name string
  input string
@@ -38,6 +39,7 @@ func TestParsingGoMigrations(t *testing.T) {
 }
 
 func TestParsingGoMigrationsError(t *testing.T) {
+ t.Parallel()
  _, err := parseGoFile(strings.NewReader(emptyInit))
  check.HasError(t, err)
  check.Contains(t, err.Error(), "no registered goose functions")

internal/provider/collect.go

@@ -0,0 +1,176 @@
+package provider
+
+import (
+ "errors"
+ "fmt"
+ "io/fs"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/pressly/goose/v3"
+ "github.com/pressly/goose/v3/internal/migrate"
+)
+
+// fileSources represents a collection of migration files on the filesystem.
+type fileSources struct {
+ sqlSources []Source
+ goSources []Source
+}
+
+// collectFileSources scans the file system for migration files that have a numeric prefix (greater
+// than one) followed by an underscore and a file extension of either .go or .sql. fsys may be nil,
+// in which case an empty fileSources is returned.
+//
+// If strict is true, then any error parsing the numeric component of the filename will result in an
+// error. The file is skipped otherwise.
+//
+// This function DOES NOT parse SQL migrations or merge registered Go migrations. It only collects
+// migration sources from the filesystem.
+func collectFileSources(fsys fs.FS, strict bool, excludes map[string]bool) (*fileSources, error) {
+ if fsys == nil {
+ return new(fileSources), nil
+ }
+ sources := new(fileSources)
+ versionToBaseLookup := make(map[int64]string) // map[version]filepath.Base(fullpath)
+ for _, pattern := range []string{
+ "*.sql",
+ "*.go",
+ } {
+ files, err := fs.Glob(fsys, pattern)
+ if err != nil {
+ return nil, fmt.Errorf("failed to glob pattern %q: %w", pattern, err)
+ }
+ for _, fullpath := range files {
+ base := filepath.Base(fullpath)
+ // Skip explicit excludes or Go test files.
+ if excludes[base] || strings.HasSuffix(base, "_test.go") {
+ continue
+ }
+ // If the filename has a valid looking version of the form: NUMBER_.{sql,go}, then use
+ // that as the version. Otherwise, ignore it. This allows users to have arbitrary
+ // filenames, but still have versioned migrations within the same directory. For
+ // example, a user could have a helpers.go file which contains unexported helper
+ // functions for migrations.
+ version, err := goose.NumericComponent(base)
+ if err != nil {
+ if strict {
+ return nil, fmt.Errorf("failed to parse numeric component from %q: %w", base, err)
+ }
+ continue
+ }
+ // Ensure there are no duplicate versions.
+ if existing, ok := versionToBaseLookup[version]; ok {
+ return nil, fmt.Errorf("found duplicate migration version %d:\n\texisting:%v\n\tcurrent:%v",
+ version,
+ existing,
+ base,
+ )
+ }
+ switch filepath.Ext(base) {
+ case ".sql":
+ sources.sqlSources = append(sources.sqlSources, Source{
+ Fullpath: fullpath,
+ Version: version,
+ })
+ case ".go":
+ sources.goSources = append(sources.goSources, Source{
+ Fullpath: fullpath,
+ Version: version,
+ })
+ default:
+ // Should never happen since we already filtered out all other file types.
+ return nil, fmt.Errorf("unknown migration type: %s", base)
+ }
+ // Add the version to the lookup map.
+ versionToBaseLookup[version] = base
+ }
+ }
+ return sources, nil
+}
+
+func merge(sources *fileSources, registerd map[int64]*goose.Migration) ([]*migrate.Migration, error) {
+ var migrations []*migrate.Migration
+ migrationLookup := make(map[int64]*migrate.Migration)
+ // Add all SQL migrations to the list of migrations.
+ for _, s := range sources.sqlSources {
+ m := &migrate.Migration{
+ Type: migrate.TypeSQL,
+ Fullpath: s.Fullpath,
+ Version: s.Version,
+ SQLParsed: false,
+ }
+ migrations = append(migrations, m)
+ migrationLookup[s.Version] = m
+ }
+ // If there are no Go files in the filesystem and no registered Go migrations, return early.
+ if len(sources.goSources) == 0 && len(registerd) == 0 {
+ return migrations, nil
+ }
+ // Return an error if the given sources contain a versioned Go migration that has not been
+ // registered. This is a sanity check to ensure users didn't accidentally create a valid looking
+ // Go migration file on disk and forget to register it.
+ //
+ // This is almost always a user error.
+ var unregistered []string
+ for _, s := range sources.goSources {
+ if _, ok := registerd[s.Version]; !ok {
+ unregistered = append(unregistered, s.Fullpath)
+ }
+ }
+ if len(unregistered) > 0 {
+ return nil, unregisteredError(unregistered)
+ }
+ // Add all registered Go migrations to the list of migrations, checking for duplicate versions.
+ //
+ // Important, users can register Go migrations manually via goose.Add_ functions. These
+ // migrations may not have a corresponding file on disk. Which is fine! We include them
+ // wholesale as part of migrations. This allows users to build a custom binary that only embeds
+ // the SQL migration files.
+ for _, r := range registerd {
+ // Ensure there are no duplicate versions.
+ if existing, ok := migrationLookup[r.Version]; ok {
+ return nil, fmt.Errorf("found duplicate migration version %d:\n\texisting:%v\n\tcurrent:%v",
+ r.Version,
+ existing,
+ filepath.Base(r.Source),
+ )
+ }
+ m := &migrate.Migration{
+ Fullpath: r.Source, // May be empty if the migration was registered manually.
+ Version: r.Version,
+ Type: migrate.TypeGo,
+ Go: &migrate.Go{
+ UseTx: r.UseTx,
+ UpFn: r.UpFnContext,
+ UpFnNoTx: r.UpFnNoTxContext,
+ DownFn: r.DownFnContext,
+ DownFnNoTx: r.DownFnNoTxContext,
+ },
+ }
+ migrations = append(migrations, m)
+ migrationLookup[r.Version] = m
+ }
+ // Sort migrations by version in ascending order.
+ sort.Slice(migrations, func(i, j int) bool {
+ return migrations[i].Version < migrations[j].Version
+ })
+ return migrations, nil
+}
+
+func unregisteredError(unregistered []string) error {
+ f := "file"
+ if len(unregistered) > 1 {
+ f += "s"
+ }
+ var b strings.Builder
+
+ b.WriteString(fmt.Sprintf("error: detected %d unregistered Go %s:\n", len(unregistered), f))
+ for _, name := range unregistered {
+ b.WriteString("\t" + name + "\n")
+ }
+ b.WriteString("\n")
+ b.WriteString("go functions must be registered and built into a custom binary see:\nhttps:/pressly/goose/tree/master/examples/go-migrations")
+
+ return errors.New(b.String())
+}