benchmark: Update benchmark

- Run on a single CPU
- Moved query initialisation outside the timed zone

Signed-off-by: Michał Matczuk <michal@scylladb.com>

.travis.yml

@@ -8,7 +8,7 @@ services:
 
 before_install:
   - docker pull scylladb/scylla:latest
-  - docker run --name scylla -p 9042:9042 -d scylladb/scylla
+  - docker run --name scylla -p 9042:9042 --cpus=0 -d scylladb/scylla
   - until docker exec scylla cqlsh -e "DESCRIBE SCHEMA"; do sleep 2; done
 
 install:

README.md

@@ -97,16 +97,16 @@ See more examples in [example_test.go](https://github.com/scylladb/gocqlx/blob/m
 
 ## Performance
 
-Gocqlx is fast, this is a benchmark result comparing `gocqlx` to raw `gocql` 
+With regards to performance `gocqlx` package is comparable to the raw `gocql` baseline.
-on a local machine, Intel(R) Core(TM) i7-7500U CPU @ 2.70GHz.
+Below benchmark results running on a local machine.
 
 ```
-BenchmarkE2EGocqlInsert            20000             86713 ns/op            2030 B/op         33 allocs/op
+BenchmarkBaseGocqlInsert            1840            586579 ns/op            7803 B/op         39 allocs/op
-BenchmarkE2EGocqlxInsert           20000             87882 ns/op            2030 B/op         33 allocs/op
+BenchmarkGocqlxInsert               2107            665080 ns/op            7803 B/op         39 allocs/op
-BenchmarkE2EGocqlGet               20000             94308 ns/op            1504 B/op         29 allocs/op
+BenchmarkBaseGocqlGet               2124            579242 ns/op            7311 B/op         35 allocs/op
-BenchmarkE2EGocqlxGet              20000             95722 ns/op            2128 B/op         33 allocs/op
+BenchmarkGocqlxGet                  2206            529855 ns/op            7647 B/op         38 allocs/op
-BenchmarkE2EGocqlSelect             1000           1792469 ns/op           43595 B/op        921 allocs/op
+BenchmarkBaseGocqlSelect             418           3051462 ns/op           49428 B/op        927 allocs/op
-BenchmarkE2EGocqlxSelect            1000           1839574 ns/op           36986 B/op        927 allocs/op
+BenchmarkGocqlxSelect                357           3492430 ns/op           42632 B/op        933 allocs/op
 ```
 
 See the benchmark in [benchmark_test.go](https://github.com/scylladb/gocqlx/blob/master/benchmark_test.go).

benchmark_test.go

@@ -39,31 +39,12 @@ CREATE TABLE IF NOT EXISTS gocqlx_test.bench_person (
 
 var benchPersonCols = []string{"id", "first_name", "last_name", "email", "gender", "ip_address"}
 
-func loadFixtures() []*benchPerson {
-	f, err := os.Open("testdata/people.json")
-	if err != nil {
-		panic(err)
-	}
-	defer func() {
-		if err := f.Close(); err != nil {
-			panic(err)
-		}
-	}()
-
-	var v []*benchPerson
-	if err := json.NewDecoder(f).Decode(&v); err != nil {
-		panic(err)
-	}
-
-	return v
-}
-
 //
 // Insert
 //
 
-// BenchmarkE2EGocqlInsert performs standard insert.
+// BenchmarkBaseGocqlInsert performs standard insert.
-func BenchmarkE2EGocqlInsert(b *testing.B) {
+func BenchmarkBaseGocqlInsert(b *testing.B) {
 	people := loadFixtures()
 	session := CreateSession(b)
 	defer session.Close()
@@ -85,8 +66,8 @@ func BenchmarkE2EGocqlInsert(b *testing.B) {
 	}
 }
 
-// BenchmarkE2EGocqlInsert performs insert with struct binding.
+// BenchmarkGocqlInsert performs insert with struct binding.
-func BenchmarkE2EGocqlxInsert(b *testing.B) {
+func BenchmarkGocqlxInsert(b *testing.B) {
 	people := loadFixtures()
 	session := CreateSession(b)
 	defer session.Close()
@@ -112,8 +93,8 @@ func BenchmarkE2EGocqlxInsert(b *testing.B) {
 // Get
 //
 
-// BenchmarkE2EGocqlGet performs standard scan.
+// BenchmarkBaseGocqlGet performs standard scan.
-func BenchmarkE2EGocqlGet(b *testing.B) {
+func BenchmarkBaseGocqlGet(b *testing.B) {
 	people := loadFixtures()
 	session := CreateSession(b)
 	defer session.Close()
@@ -121,24 +102,22 @@ func BenchmarkE2EGocqlGet(b *testing.B) {
 	initTable(b, session, people)
 
 	stmt, _ := qb.Select("gocqlx_test.bench_person").Columns(benchPersonCols...).Where(qb.Eq("id")).Limit(1).ToCql()
+	q := session.Query(stmt)
+	defer q.Release()
+
 	var p benchPerson
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		// prepare
-		q := session.Query(stmt)
 		q.Bind(people[i%len(people)].ID)
-		// scan
 		if err := q.Scan(&p.ID, &p.FirstName, &p.LastName, &p.Email, &p.Gender, &p.IPAddress); err != nil {
 			b.Fatal(err)
 		}
-		// release
-		q.Release()
 	}
 }
 
-// BenchmarkE2EGocqlxGet performs get.
+// BenchmarkGocqlxGet performs get.
-func BenchmarkE2EGocqlxGet(b *testing.B) {
+func BenchmarkGocqlxGet(b *testing.B) {
 	people := loadFixtures()
 	session := CreateSession(b)
 	defer session.Close()
@@ -146,14 +125,15 @@ func BenchmarkE2EGocqlxGet(b *testing.B) {
 	initTable(b, session, people)
 
 	stmt, _ := qb.Select("gocqlx_test.bench_person").Columns(benchPersonCols...).Where(qb.Eq("id")).Limit(1).ToCql()
+	q := session.Query(stmt)
+	defer q.Release()
+
 	var p benchPerson
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		// prepare
+		q.Bind(people[i%len(people)].ID)
-		q := session.Query(stmt).Bind(people[i%len(people)].ID)
+		if err := gocqlx.Query(q, nil).Get(&p); err != nil {
-		// get and release
-		if err := gocqlx.Query(q, nil).GetRelease(&p); err != nil {
 			b.Fatal(err)
 		}
 	}
@@ -163,8 +143,9 @@ func BenchmarkE2EGocqlxGet(b *testing.B) {
 // Select
 //
 
-// BenchmarkE2EGocqlSelect performs standard loop scan.
+// BenchmarkBaseGocqlSelect performs standard loop scan with a slice of
-func BenchmarkE2EGocqlSelect(b *testing.B) {
+// pointers.
+func BenchmarkBaseGocqlSelect(b *testing.B) {
 	people := loadFixtures()
 	session := CreateSession(b)
 	defer session.Close()
@@ -172,29 +153,26 @@ func BenchmarkE2EGocqlSelect(b *testing.B) {
 	initTable(b, session, people)
 
 	stmt, _ := qb.Select("gocqlx_test.bench_person").Columns(benchPersonCols...).Limit(100).ToCql()
+	q := session.Query(stmt)
+	defer q.Release()
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		// prepare
+		iter := q.Iter()
 		v := make([]*benchPerson, 100)
-		q := session.Query(stmt)
-		i := q.Iter()
-		// loop scan
 		p := new(benchPerson)
-		for i.Scan(&p.ID, &p.FirstName, &p.LastName, &p.Email, &p.Gender, &p.IPAddress) {
+		for iter.Scan(&p.ID, &p.FirstName, &p.LastName, &p.Email, &p.Gender, &p.IPAddress) {
 			v = append(v, p)
 			p = new(benchPerson)
 		}
-		if err := i.Close(); err != nil {
+		if err := iter.Close(); err != nil {
 			b.Fatal(err)
 		}
-		// release
-		q.Release()
 	}
 }
 
-// BenchmarkE2EGocqlSelect performs select.
+// BenchmarkGocqlSelect performs select to a slice pointers.
-func BenchmarkE2EGocqlxSelect(b *testing.B) {
+func BenchmarkGocqlxSelect(b *testing.B) {
 	people := loadFixtures()
 	session := CreateSession(b)
 	defer session.Close()
@@ -202,19 +180,37 @@ func BenchmarkE2EGocqlxSelect(b *testing.B) {
 	initTable(b, session, people)
 
 	stmt, _ := qb.Select("gocqlx_test.bench_person").Columns(benchPersonCols...).Limit(100).ToCql()
+	q := gocqlx.Query(session.Query(stmt), nil)
+	defer q.Release()
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		// prepare
-		q := session.Query(stmt)
 		var v []*benchPerson
-		// select and release
+		if err := q.Select(&v); err != nil {
-		if err := gocqlx.Query(q, nil).SelectRelease(&v); err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 
+func loadFixtures() []*benchPerson {
+	f, err := os.Open("testdata/people.json")
+	if err != nil {
+		panic(err)
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			panic(err)
+		}
+	}()
+
+	var v []*benchPerson
+	if err := json.NewDecoder(f).Decode(&v); err != nil {
+		panic(err)
+	}
+
+	return v
+}
+
 func initTable(b *testing.B, session *gocql.Session, people []*benchPerson) {
 	if err := ExecStmt(session, benchPersonSchema); err != nil {
 		b.Fatal(err)