migrate: add support for new Go io/fs.FS

This patch adds FromFS function and lets you use the new go:embed directive to add the migration files to the binary.
Also, migration tests use an in memory FS impl instead of working with tmp directories.

go.mod

@@ -3,10 +3,10 @@ module github.com/scylladb/gocqlx/v2
 go 1.16
 
 require (
-	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537
 	github.com/golang/snappy v0.0.1 // indirect
-	github.com/google/go-cmp v0.2.0
+	github.com/google/go-cmp v0.5.4
+	github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef
 	github.com/scylladb/go-reflectx v1.0.1
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 	gopkg.in/inf.v0 v0.9.1

go.sum

@@ -2,18 +2,14 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE
 github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537 h1:NaMut1fdw76YYX/TPinSAbai4DShF5tPort3bHpET6g=
 github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY=
-github.com/golang/snappy v0.0.0-20170215233205-553a64147049 h1:K9KHZbXKpGydfDN0aZrsoHpLJlZsBrGMFWbgLDGnPZk=
 github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -21,14 +17,16 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef h1:NKxTG6GVGbfMXc2mIk+KphcH6hagbVXhcFkbTgYleTI=
+github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef/go.mod h1:tcaRap0jS3eifrEEllL6ZMd9dg8IlDpi2S1oARrQ+NI=
 github.com/scylladb/go-reflectx v1.0.1 h1:b917wZM7189pZdlND9PbIJ6NQxfDPfBvUaQ7cjj1iZQ=
 github.com/scylladb/go-reflectx v1.0.1/go.mod h1:rWnOfDIRWBGN0miMLIcoPt/Dhi2doCMZqwMCJ3KupFc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=

migrate/checksum.go

@@ -8,7 +8,7 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"io"
-	"os"
+	"io/fs"
 )
 
 var encode = hex.EncodeToString
@@ -18,15 +18,15 @@ func checksum(b []byte) string {
 	return encode(v[:])
 }
 
-func fileChecksum(path string) (string, error) {
+func fileChecksum(f fs.FS, path string) (string, error) {
-	f, err := os.Open(path)
+	file, err := f.Open(path)
 	if err != nil {
 		return "", nil
 	}
-	defer f.Close()
+	defer file.Close()
 
 	h := md5.New()
-	if _, err := io.Copy(h, f); err != nil {
+	if _, err := io.Copy(h, file); err != nil {
 		return "", err
 	}
 	v := h.Sum(nil)

migrate/checksum_test.go

@@ -4,10 +4,13 @@
 
 package migrate
 
-import "testing"
+import (
+	"os"
+	"testing"
+)
 
 func TestFileChecksum(t *testing.T) {
-	c, err := fileChecksum("testdata/file")
+	c, err := fileChecksum(os.DirFS("testdata"), "file")
 	if err != nil {
 		t.Fatal(err)
 	}

migrate/migrate.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -90,10 +91,20 @@ func ensureInfoTable(ctx context.Context, session gocqlx.Session) error {
 	return session.ContextQuery(ctx, infoSchema, nil).ExecRelease()
 }
 
-// Migrate reads the cql files from a directory and applies required migrations.
+// Migrate is a wrapper around FromFS.
-// It also supports code based migrations, see Callback and CallbackFunc.
+// It executes migrations from a directory on disk.
-// Any comment in form `-- CALL <name>;` will trigger an CallComment callback.
+//
+// Deprecated: use FromFS instead
 func Migrate(ctx context.Context, session gocqlx.Session, dir string) error {
+	return FromFS(ctx, session, os.DirFS(dir))
+}
+
+// FromFS executes new CQL files from a file system abstraction (io/fs.FS).
+// The provided FS has to be a flat directory containing *.cql files.
+//
+// It supports code based migrations, see Callback and CallbackFunc.
+// Any comment in form `-- CALL <name>;` will trigger an CallComment callback.
+func FromFS(ctx context.Context, session gocqlx.Session, f fs.FS) error {
 	// get database migrations
 	dbm, err := List(ctx, session)
 	if err != nil {
@@ -101,26 +112,26 @@ func Migrate(ctx context.Context, session gocqlx.Session, dir string) error {
 	}
 
 	// get file migrations
-	fm, err := filepath.Glob(filepath.Join(dir, "*.cql"))
+	fm, err := fs.Glob(f, "*.cql")
 	if err != nil {
-		return fmt.Errorf("list migrations in %q: %s", dir, err)
+		return fmt.Errorf("list migrations: %w", err)
 	}
 	if len(fm) == 0 {
-		return fmt.Errorf("no migration files found in %q", dir)
+		return fmt.Errorf("no migration files found")
 	}
 	sort.Strings(fm)
 
 	// verify migrations
 	if len(dbm) > len(fm) {
-		return fmt.Errorf("database is ahead of %q", dir)
+		return fmt.Errorf("database is ahead")
 	}
 
 	for i := 0; i < len(dbm); i++ {
-		if dbm[i].Name != filepath.Base(fm[i]) {
+		if dbm[i].Name != fm[i] {
-			fmt.Println(dbm[i].Name, filepath.Base(fm[i]), i)
+			fmt.Println(dbm[i].Name, fm[i], i)
 			return errors.New("inconsistent migrations")
 		}
-		c, err := fileChecksum(fm[i])
+		c, err := fileChecksum(f, fm[i])
 		if err != nil {
 			return fmt.Errorf("calculate checksum for %q: %s", fm[i], err)
 		}
@@ -132,13 +143,13 @@ func Migrate(ctx context.Context, session gocqlx.Session, dir string) error {
 	// apply migrations
 	if len(dbm) > 0 {
 		last := len(dbm) - 1
-		if err := applyMigration(ctx, session, fm[last], dbm[last].Done); err != nil {
+		if err := applyMigration(ctx, session, f, fm[last], dbm[last].Done); err != nil {
 			return fmt.Errorf("apply migration %q: %s", fm[last], err)
 		}
 	}
 
 	for i := len(dbm); i < len(fm); i++ {
-		if err := applyMigration(ctx, session, fm[i], 0); err != nil {
+		if err := applyMigration(ctx, session, f, fm[i], 0); err != nil {
 			return fmt.Errorf("apply migration %q: %s", fm[i], err)
 		}
 	}
@@ -150,14 +161,14 @@ func Migrate(ctx context.Context, session gocqlx.Session, dir string) error {
 	return nil
 }
 
-func applyMigration(ctx context.Context, session gocqlx.Session, path string, done int) error {
+func applyMigration(ctx context.Context, session gocqlx.Session, f fs.FS, path string, done int) error {
-	f, err := os.Open(path)
+	file, err := f.Open(path)
 	if err != nil {
 		return err
 	}
 
-	b, err := ioutil.ReadAll(f)
+	b, err := ioutil.ReadAll(file)
-	f.Close()
+	file.Close()
 	if err != nil {
 		return err
 	}

migrate/migrate_test.go

@@ -9,12 +9,11 @@ package migrate_test
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io/fs"
-	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
+	"github.com/psanford/memfs"
 	"github.com/scylladb/gocqlx/v2"
 	. "github.com/scylladb/gocqlx/v2/gocqlxtest"
 	"github.com/scylladb/gocqlx/v2/migrate"
@@ -52,10 +51,7 @@ func TestMigration(t *testing.T) {
 	ctx := context.Background()
 
 	t.Run("init", func(t *testing.T) {
-		dir := makeMigrationDir(t, 2)
+		if err := migrate.FromFS(ctx, session, makeTestFS(2)); err != nil {
-		defer os.Remove(dir)
-
-		if err := migrate.Migrate(ctx, session, dir); err != nil {
 			t.Fatal(err)
 		}
 		if c := countMigrations(t, session); c != 2 {
@@ -64,10 +60,7 @@ func TestMigration(t *testing.T) {
 	})
 
 	t.Run("update", func(t *testing.T) {
-		dir := makeMigrationDir(t, 4)
+		if err := migrate.FromFS(ctx, session, makeTestFS(4)); err != nil {
-		defer os.Remove(dir)
-
-		if err := migrate.Migrate(ctx, session, dir); err != nil {
 			t.Fatal(err)
 		}
 		if c := countMigrations(t, session); c != 4 {
@@ -76,10 +69,8 @@ func TestMigration(t *testing.T) {
 	})
 
 	t.Run("ahead", func(t *testing.T) {
-		dir := makeMigrationDir(t, 2)
+		err := migrate.FromFS(ctx, session, makeTestFS(2))
-		defer os.Remove(dir)
+		if err == nil || !strings.Contains(err.Error(), "ahead") {
-
-		if err := migrate.Migrate(ctx, session, dir); err == nil || !strings.Contains(err.Error(), "ahead") {
 			t.Fatal("expected error")
 		} else {
 			t.Log(err)
@@ -87,12 +78,10 @@ func TestMigration(t *testing.T) {
 	})
 
 	t.Run("tempered with file", func(t *testing.T) {
-		dir := makeMigrationDir(t, 4)
+		f := makeTestFS(4)
-		defer os.Remove(dir)
+		writeFile(f, 3, "SELECT * FROM bla;")
 
-		appendMigrationFile(t, dir, 3, "\nSELECT * FROM bla;\n")
+		if err := migrate.FromFS(ctx, session, f); err == nil || !strings.Contains(err.Error(), "tempered") {
-
-		if err := migrate.Migrate(ctx, session, dir); err == nil || !strings.Contains(err.Error(), "tempered") {
 			t.Fatal("expected error")
 		} else {
 			t.Log(err)
@@ -109,19 +98,11 @@ func TestMigrationNoSemicolon(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	f := makeTestFS(0)
+	f.WriteFile("0.cql", []byte(fmt.Sprintf(insertMigrate, 0)+";"+fmt.Sprintf(insertMigrate, 1)), fs.ModePerm)
+
 	ctx := context.Background()
-
+	if err := migrate.FromFS(ctx, session, f); err != nil {
-	dir := makeMigrationDir(t, 1)
-	defer os.Remove(dir)
-
-	f, err := os.OpenFile(filepath.Join(dir, "0.cql"), os.O_WRONLY|os.O_APPEND, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fmt.Fprintf(f, insertMigrate, 0) // note no ; at the end
-	f.Close()
-
-	if err := migrate.Migrate(ctx, session, dir); err != nil {
 		t.Fatal(err)
 	}
 	if c := countMigrations(t, session); c != 2 {
@@ -231,62 +212,38 @@ func TestMigrationCallback(t *testing.T) {
 	ctx := context.Background()
 
 	t.Run("init", func(t *testing.T) {
-		dir := makeMigrationDir(t, 2)
+		f := makeTestFS(2)
-		defer os.Remove(dir)
 		reset()
 
-		if err := migrate.Migrate(ctx, session, dir); err != nil {
+		if err := migrate.FromFS(ctx, session, f); err != nil {
 			t.Fatal(err)
 		}
 		assertCallbacks(t, 2, 2, 0)
 	})
 
 	t.Run("no duplicate calls", func(t *testing.T) {
-		dir := makeMigrationDir(t, 4)
+		f := makeTestFS(4)
-		defer os.Remove(dir)
 		reset()
 
-		if err := migrate.Migrate(ctx, session, dir); err != nil {
+		if err := migrate.FromFS(ctx, session, f); err != nil {
 			t.Fatal(err)
 		}
 		assertCallbacks(t, 2, 2, 0)
 	})
 
 	t.Run("in calls", func(t *testing.T) {
-		dir := makeMigrationDir(t, 4)
+		f := makeTestFS(4)
-		defer os.Remove(dir)
+		writeFile(f, 4, "\n-- CALL Foo;\n")
+		writeFile(f, 5, "\n-- CALL Bar;\n")
 		reset()
 
-		appendMigrationFile(t, dir, 4, "\n-- CALL Foo;\n")
+		if err := migrate.FromFS(ctx, session, f); err != nil {
-		appendMigrationFile(t, dir, 5, "\n-- CALL Bar;\n")
-
-		if err := migrate.Migrate(ctx, session, dir); err != nil {
 			t.Fatal(err)
 		}
 		assertCallbacks(t, 2, 2, 2)
 	})
 }
 
-func makeMigrationDir(tb testing.TB, n int) (dir string) {
-	tb.Helper()
-
-	dir, err := ioutil.TempDir("", "gocqlx_migrate")
-	if err != nil {
-		tb.Fatal(err)
-	}
-
-	for i := 0; i < n; i++ {
-		path := migrateFilePath(dir, i)
-		cql := []byte(fmt.Sprintf(insertMigrate, i) + ";")
-		if err := ioutil.WriteFile(path, cql, os.ModePerm); err != nil {
-			os.Remove(dir)
-			tb.Fatal(err)
-		}
-	}
-
-	return dir
-}
-
 func countMigrations(tb testing.TB, session gocqlx.Session) int {
 	tb.Helper()
 
@@ -297,17 +254,14 @@ func countMigrations(tb testing.TB, session gocqlx.Session) int {
 	return v
 }
 
-func appendMigrationFile(tb testing.TB, dir string, i int, text string) {
+func makeTestFS(n int) *memfs.FS {
-	path := migrateFilePath(dir, i)
+	f := memfs.New()
-	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.ModePerm)
+	for i := 0; i < n; i++ {
-	if err != nil {
+		writeFile(f, i, fmt.Sprintf(insertMigrate, i)+";")
-		tb.Fatal(err)
-	}
-	if _, err := f.WriteString(text); err != nil {
-		tb.Fatal(err)
 	}
+	return f
 }
 
-func migrateFilePath(dir string, i int) string {
+func writeFile(f *memfs.FS, i int, text string) {
-	return filepath.Join(dir, fmt.Sprint(i, ".cql"))
+	f.WriteFile(fmt.Sprint(i, ".cql"), []byte(text), fs.ModePerm)
 }