migrate: add support for new Go io/fs.FS

This patch adds FromFS function and lets you use the new go:embed directive to add the migration files to the binary.
Also, migration tests use an in memory FS impl instead of working with tmp directories.

migrate/checksum.go

@@ -8,7 +8,7 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"io"
-	"os"
+	"io/fs"
 )
 
 var encode = hex.EncodeToString
@@ -18,15 +18,15 @@ func checksum(b []byte) string {
 	return encode(v[:])
 }
 
-func fileChecksum(path string) (string, error) {
-	f, err := os.Open(path)
+func fileChecksum(f fs.FS, path string) (string, error) {
+	file, err := f.Open(path)
 	if err != nil {
 		return "", nil
 	}
-	defer f.Close()
+	defer file.Close()
 
 	h := md5.New()
-	if _, err := io.Copy(h, f); err != nil {
+	if _, err := io.Copy(h, file); err != nil {
 		return "", err
 	}
 	v := h.Sum(nil)

migrate/checksum_test.go

@@ -4,10 +4,13 @@
 
 package migrate
 
-import "testing"
+import (
+	"os"
+	"testing"
+)
 
 func TestFileChecksum(t *testing.T) {
-	c, err := fileChecksum("testdata/file")
+	c, err := fileChecksum(os.DirFS("testdata"), "file")
 	if err != nil {
 		t.Fatal(err)
 	}

migrate/migrate.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -90,10 +91,20 @@ func ensureInfoTable(ctx context.Context, session gocqlx.Session) error {
 	return session.ContextQuery(ctx, infoSchema, nil).ExecRelease()
 }
 
-// Migrate reads the cql files from a directory and applies required migrations.
-// It also supports code based migrations, see Callback and CallbackFunc.
-// Any comment in form `-- CALL <name>;` will trigger an CallComment callback.
+// Migrate is a wrapper around FromFS.
+// It executes migrations from a directory on disk.
+//
+// Deprecated: use FromFS instead
 func Migrate(ctx context.Context, session gocqlx.Session, dir string) error {
+	return FromFS(ctx, session, os.DirFS(dir))
+}
+
+// FromFS executes new CQL files from a file system abstraction (io/fs.FS).
+// The provided FS has to be a flat directory containing *.cql files.
+//
+// It supports code based migrations, see Callback and CallbackFunc.
+// Any comment in form `-- CALL <name>;` will trigger an CallComment callback.
+func FromFS(ctx context.Context, session gocqlx.Session, f fs.FS) error {
 	// get database migrations
 	dbm, err := List(ctx, session)
 	if err != nil {
@@ -101,26 +112,26 @@ func Migrate(ctx context.Context, session gocqlx.Session, dir string) error {
 	}
 
 	// get file migrations
-	fm, err := filepath.Glob(filepath.Join(dir, "*.cql"))
+	fm, err := fs.Glob(f, "*.cql")
 	if err != nil {
-		return fmt.Errorf("list migrations in %q: %s", dir, err)
+		return fmt.Errorf("list migrations: %w", err)
 	}
 	if len(fm) == 0 {
-		return fmt.Errorf("no migration files found in %q", dir)
+		return fmt.Errorf("no migration files found")
 	}
 	sort.Strings(fm)
 
 	// verify migrations
 	if len(dbm) > len(fm) {
-		return fmt.Errorf("database is ahead of %q", dir)
+		return fmt.Errorf("database is ahead")
 	}
 
 	for i := 0; i < len(dbm); i++ {
-		if dbm[i].Name != filepath.Base(fm[i]) {
-			fmt.Println(dbm[i].Name, filepath.Base(fm[i]), i)
+		if dbm[i].Name != fm[i] {
+			fmt.Println(dbm[i].Name, fm[i], i)
 			return errors.New("inconsistent migrations")
 		}
-		c, err := fileChecksum(fm[i])
+		c, err := fileChecksum(f, fm[i])
 		if err != nil {
 			return fmt.Errorf("calculate checksum for %q: %s", fm[i], err)
 		}
@@ -132,13 +143,13 @@ func Migrate(ctx context.Context, session gocqlx.Session, dir string) error {
 	// apply migrations
 	if len(dbm) > 0 {
 		last := len(dbm) - 1
-		if err := applyMigration(ctx, session, fm[last], dbm[last].Done); err != nil {
+		if err := applyMigration(ctx, session, f, fm[last], dbm[last].Done); err != nil {
 			return fmt.Errorf("apply migration %q: %s", fm[last], err)
 		}
 	}
 
 	for i := len(dbm); i < len(fm); i++ {
-		if err := applyMigration(ctx, session, fm[i], 0); err != nil {
+		if err := applyMigration(ctx, session, f, fm[i], 0); err != nil {
 			return fmt.Errorf("apply migration %q: %s", fm[i], err)
 		}
 	}
@@ -150,14 +161,14 @@ func Migrate(ctx context.Context, session gocqlx.Session, dir string) error {
 	return nil
 }
 
-func applyMigration(ctx context.Context, session gocqlx.Session, path string, done int) error {
-	f, err := os.Open(path)
+func applyMigration(ctx context.Context, session gocqlx.Session, f fs.FS, path string, done int) error {
+	file, err := f.Open(path)
 	if err != nil {
 		return err
 	}
 
-	b, err := ioutil.ReadAll(f)
-	f.Close()
+	b, err := ioutil.ReadAll(file)
+	file.Close()
 	if err != nil {
 		return err
 	}

migrate/migrate_test.go

@@ -9,12 +9,11 @@ package migrate_test
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
+	"io/fs"
 	"strings"
 	"testing"
 
+	"github.com/psanford/memfs"
 	"github.com/scylladb/gocqlx/v2"
 	. "github.com/scylladb/gocqlx/v2/gocqlxtest"
 	"github.com/scylladb/gocqlx/v2/migrate"
@@ -52,10 +51,7 @@ func TestMigration(t *testing.T) {
 	ctx := context.Background()
 
 	t.Run("init", func(t *testing.T) {
-		dir := makeMigrationDir(t, 2)
-		defer os.Remove(dir)
-
-		if err := migrate.Migrate(ctx, session, dir); err != nil {
+		if err := migrate.FromFS(ctx, session, makeTestFS(2)); err != nil {
 			t.Fatal(err)
 		}
 		if c := countMigrations(t, session); c != 2 {
@@ -64,10 +60,7 @@ func TestMigration(t *testing.T) {
 	})
 
 	t.Run("update", func(t *testing.T) {
-		dir := makeMigrationDir(t, 4)
-		defer os.Remove(dir)
-
-		if err := migrate.Migrate(ctx, session, dir); err != nil {
+		if err := migrate.FromFS(ctx, session, makeTestFS(4)); err != nil {
 			t.Fatal(err)
 		}
 		if c := countMigrations(t, session); c != 4 {
@@ -76,10 +69,8 @@ func TestMigration(t *testing.T) {
 	})
 
 	t.Run("ahead", func(t *testing.T) {
-		dir := makeMigrationDir(t, 2)
-		defer os.Remove(dir)
-
-		if err := migrate.Migrate(ctx, session, dir); err == nil || !strings.Contains(err.Error(), "ahead") {
+		err := migrate.FromFS(ctx, session, makeTestFS(2))
+		if err == nil || !strings.Contains(err.Error(), "ahead") {
 			t.Fatal("expected error")
 		} else {
 			t.Log(err)
@@ -87,12 +78,10 @@ func TestMigration(t *testing.T) {
 	})
 
 	t.Run("tempered with file", func(t *testing.T) {
-		dir := makeMigrationDir(t, 4)
-		defer os.Remove(dir)
+		f := makeTestFS(4)
+		writeFile(f, 3, "SELECT * FROM bla;")
 
-		appendMigrationFile(t, dir, 3, "\nSELECT * FROM bla;\n")
-
-		if err := migrate.Migrate(ctx, session, dir); err == nil || !strings.Contains(err.Error(), "tempered") {
+		if err := migrate.FromFS(ctx, session, f); err == nil || !strings.Contains(err.Error(), "tempered") {
 			t.Fatal("expected error")
 		} else {
 			t.Log(err)
@@ -109,19 +98,11 @@ func TestMigrationNoSemicolon(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	f := makeTestFS(0)
+	f.WriteFile("0.cql", []byte(fmt.Sprintf(insertMigrate, 0)+";"+fmt.Sprintf(insertMigrate, 1)), fs.ModePerm)
+
 	ctx := context.Background()
-
-	dir := makeMigrationDir(t, 1)
-	defer os.Remove(dir)
-
-	f, err := os.OpenFile(filepath.Join(dir, "0.cql"), os.O_WRONLY|os.O_APPEND, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fmt.Fprintf(f, insertMigrate, 0) // note no ; at the end
-	f.Close()
-
-	if err := migrate.Migrate(ctx, session, dir); err != nil {
+	if err := migrate.FromFS(ctx, session, f); err != nil {
 		t.Fatal(err)
 	}
 	if c := countMigrations(t, session); c != 2 {
@@ -231,62 +212,38 @@ func TestMigrationCallback(t *testing.T) {
 	ctx := context.Background()
 
 	t.Run("init", func(t *testing.T) {
-		dir := makeMigrationDir(t, 2)
-		defer os.Remove(dir)
+		f := makeTestFS(2)
 		reset()
 
-		if err := migrate.Migrate(ctx, session, dir); err != nil {
+		if err := migrate.FromFS(ctx, session, f); err != nil {
 			t.Fatal(err)
 		}
 		assertCallbacks(t, 2, 2, 0)
 	})
 
 	t.Run("no duplicate calls", func(t *testing.T) {
-		dir := makeMigrationDir(t, 4)
-		defer os.Remove(dir)
+		f := makeTestFS(4)
 		reset()
 
-		if err := migrate.Migrate(ctx, session, dir); err != nil {
+		if err := migrate.FromFS(ctx, session, f); err != nil {
 			t.Fatal(err)
 		}
 		assertCallbacks(t, 2, 2, 0)
 	})
 
 	t.Run("in calls", func(t *testing.T) {
-		dir := makeMigrationDir(t, 4)
-		defer os.Remove(dir)
+		f := makeTestFS(4)
+		writeFile(f, 4, "\n-- CALL Foo;\n")
+		writeFile(f, 5, "\n-- CALL Bar;\n")
 		reset()
 
-		appendMigrationFile(t, dir, 4, "\n-- CALL Foo;\n")
-		appendMigrationFile(t, dir, 5, "\n-- CALL Bar;\n")
-
-		if err := migrate.Migrate(ctx, session, dir); err != nil {
+		if err := migrate.FromFS(ctx, session, f); err != nil {
 			t.Fatal(err)
 		}
 		assertCallbacks(t, 2, 2, 2)
 	})
 }
 
-func makeMigrationDir(tb testing.TB, n int) (dir string) {
-	tb.Helper()
-
-	dir, err := ioutil.TempDir("", "gocqlx_migrate")
-	if err != nil {
-		tb.Fatal(err)
-	}
-
-	for i := 0; i < n; i++ {
-		path := migrateFilePath(dir, i)
-		cql := []byte(fmt.Sprintf(insertMigrate, i) + ";")
-		if err := ioutil.WriteFile(path, cql, os.ModePerm); err != nil {
-			os.Remove(dir)
-			tb.Fatal(err)
-		}
-	}
-
-	return dir
-}
-
 func countMigrations(tb testing.TB, session gocqlx.Session) int {
 	tb.Helper()
 
@@ -297,17 +254,14 @@ func countMigrations(tb testing.TB, session gocqlx.Session) int {
 	return v
 }
 
-func appendMigrationFile(tb testing.TB, dir string, i int, text string) {
-	path := migrateFilePath(dir, i)
-	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.ModePerm)
-	if err != nil {
-		tb.Fatal(err)
-	}
-	if _, err := f.WriteString(text); err != nil {
-		tb.Fatal(err)
+func makeTestFS(n int) *memfs.FS {
+	f := memfs.New()
+	for i := 0; i < n; i++ {
+		writeFile(f, i, fmt.Sprintf(insertMigrate, i)+";")
 	}
+	return f
 }
 
-func migrateFilePath(dir string, i int) string {
-	return filepath.Join(dir, fmt.Sprint(i, ".cql"))
+func writeFile(f *memfs.FS, i int, text string) {
+	f.WriteFile(fmt.Sprint(i, ".cql"), []byte(text), fs.ModePerm)
 }