From 8d47a6d133e14a662e77f5ab3827d9f2219caf51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Thu, 3 Aug 2017 17:02:22 +0200 Subject: [PATCH 1/3] qb: fixed benchmarks --- qb/delete_test.go | 2 +- qb/insert_test.go | 2 +- qb/select_test.go | 2 +- qb/update_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/qb/delete_test.go b/qb/delete_test.go index 01d6bf2..ff6de25 100644 --- a/qb/delete_test.go +++ b/qb/delete_test.go @@ -72,6 +72,6 @@ func TestDeleteBuilder(t *testing.T) { func BenchmarkDeleteBuilder(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - Delete("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").Where(Eq("id")) + Delete("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").Where(Eq("id")).ToCql() } } diff --git a/qb/insert_test.go b/qb/insert_test.go index 7a4d600..b561373 100644 --- a/qb/insert_test.go +++ b/qb/insert_test.go @@ -65,6 +65,6 @@ func TestInsertBuilder(t *testing.T) { func BenchmarkInsertBuilder(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - Insert("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars") + Insert("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").ToCql() } } diff --git a/qb/select_test.go b/qb/select_test.go index ee0371c..6787802 100644 --- a/qb/select_test.go +++ b/qb/select_test.go @@ -85,6 +85,6 @@ func TestSelectBuilder(t *testing.T) { func BenchmarkSelectBuilder(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - Select("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").Where(Eq("id")) + Select("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").Where(Eq("id")).ToCql() } } diff --git a/qb/update_test.go b/qb/update_test.go index 2a2447f..2577c72 100644 --- a/qb/update_test.go +++ b/qb/update_test.go @@ -78,6 +78,6 @@ func TestUpdateBuilder(t *testing.T) { func BenchmarkUpdateBuilder(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - Update("cycling.cyclist_name").Set("id", "user_uuid", "firstname", "stars").Where(Eq("id")) + Update("cycling.cyclist_name").Set("id", "user_uuid", "firstname", "stars").Where(Eq("id")).ToCql() } } From 804cfad37cdf6e5fc6acd1043fab88f732af2914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Thu, 3 Aug 2017 17:06:03 +0200 Subject: [PATCH 2/3] qb: batch builder --- example_test.go | 31 +++++++++++++++++ qb/batch.go | 91 ++++++++++++++++++++++++++++++++++++++++++++++++ qb/batch_test.go | 80 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 202 insertions(+) create mode 100644 qb/batch.go create mode 100644 qb/batch_test.go diff --git a/example_test.go b/example_test.go index 4eba8aa..627d7d9 100644 --- a/example_test.go +++ b/example_test.go @@ -155,6 +155,37 @@ func TestExample(t *testing.T) { } } + // Batch + { + i := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email") + + stmt, names := qb.Batch(). + Add("a.", i). + Add("b.", i). + ToCql() + q := gocqlx.Query(session.Query(stmt), names) + + b := struct { + A Person + B Person + }{ + A: Person{ + "Ian", + "Citizen", + []string{"ian.citzen@gocqlx_test.com"}, + }, + B: Person{ + "Igy", + "Citizen", + []string{"igy.citzen@gocqlx_test.com"}, + }, + } + + if err := q.BindStruct(&b).Exec(); err != nil { + t.Fatal(err) + } + } + // Select { stmt, names := qb.Select("gocqlx_test.person").Where(qb.In("first_name")).ToCql() diff --git a/qb/batch.go b/qb/batch.go new file mode 100644 index 0000000..bcfa7fc --- /dev/null +++ b/qb/batch.go @@ -0,0 +1,91 @@ +package qb + +import ( + "bytes" + "fmt" +) + +// BATCH reference: +// https://cassandra.apache.org/doc/latest/cql/dml.html#batch + +// builder is interface implemented by other builders. +type builder interface { + ToCql() (stmt string, names []string) +} + +// BatchBuilder builds CQL BATCH statements. +type BatchBuilder struct { + unlogged bool + counter bool + using using + stmts []string + names []string +} + +// Batch returns a new BatchBuilder. +func Batch() *BatchBuilder { + return &BatchBuilder{} +} + +// ToCql builds the query into a CQL string and named args. +func (b *BatchBuilder) ToCql() (stmt string, names []string) { + cql := bytes.Buffer{} + + cql.WriteString("BEGIN ") + if b.unlogged { + cql.WriteString("UNLOGGED ") + } + if b.counter { + cql.WriteString("COUNTER ") + } + cql.WriteString("BATCH ") + + names = append(names, b.using.writeCql(&cql)...) + + for _, stmt := range b.stmts { + cql.WriteString(stmt) + cql.WriteByte(';') + cql.WriteByte(' ') + } + names = append(names, b.names...) + + cql.WriteString("APPLY BATCH ") + + stmt = cql.String() + return +} + +// UnLogged sets a UNLOGGED BATCH clause on the query. +func (b *BatchBuilder) UnLogged() *BatchBuilder { + b.unlogged = true + return b +} + +// Counter sets a COUNTER BATCH clause on the query. +func (b *BatchBuilder) Counter() *BatchBuilder { + b.counter = true + return b +} + +// Timestamp sets a USING TIMESTAMP clause on the query. +func (b *BatchBuilder) Timestamp() *BatchBuilder { + b.using.timestamp = true + return b +} + +// TTL sets a USING TTL clause on the query. +func (b *BatchBuilder) TTL() *BatchBuilder { + b.using.ttl = true + return b +} + +// Add adds another batch statement from a builder. +func (b *BatchBuilder) Add(prefix string, builder Builder) *BatchBuilder { + stmt, names := builder.ToCql() + + b.stmts = append(b.stmts, stmt) + for _, name := range names { + b.names = append(b.names, fmt.Sprint(prefix, name)) + } + return b +} diff --git a/qb/batch_test.go b/qb/batch_test.go new file mode 100644 index 0000000..1168a71 --- /dev/null +++ b/qb/batch_test.go @@ -0,0 +1,80 @@ +package qb + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +type mockBuilder struct { + stmt string + names []string +} + +func (b mockBuilder) ToCql() (stmt string, names []string) { + return b.stmt, b.names +} + +func TestBatchBuilder(t *testing.T) { + m := mockBuilder{"INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ", []string{"id", "user_uuid", "firstname"}} + + table := []struct { + B *BatchBuilder + N []string + S string + }{ + // Basic test for Batch + { + B: Batch().Add("a.", m), + S: "BEGIN BATCH INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ; APPLY BATCH ", + N: []string{"a.id", "a.user_uuid", "a.firstname"}, + }, + // Add statement + { + B: Batch(). + Add("a.", m). + Add("b.", m), + S: "BEGIN BATCH INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ; INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ; APPLY BATCH ", + N: []string{"a.id", "a.user_uuid", "a.firstname", "b.id", "b.user_uuid", "b.firstname"}, + }, + // Add UNLOGGED + { + B: Batch().UnLogged(), + S: "BEGIN UNLOGGED BATCH APPLY BATCH ", + }, + // Add COUNTER + { + B: Batch().Counter(), + S: "BEGIN COUNTER BATCH APPLY BATCH ", + }, + // Add TTL + { + B: Batch().TTL(), + S: "BEGIN BATCH USING TTL ? APPLY BATCH ", + N: []string{"_ttl"}, + }, + // Add TIMESTAMP + { + B: Batch().Timestamp(), + S: "BEGIN BATCH USING TIMESTAMP ? APPLY BATCH ", + N: []string{"_ts"}, + }, + } + + for _, test := range table { + stmt, names := test.B.ToCql() + if diff := cmp.Diff(test.S, stmt); diff != "" { + t.Error(diff) + } + if diff := cmp.Diff(test.N, names); diff != "" { + t.Error(diff) + } + } +} + +func BenchmarkBatchBuilder(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + Batch().Add("", mockBuilder{"INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ", []string{"id", "user_uuid", "firstname"}}).ToCql() + } +} From bc289ada4513718475987518ed4ea9b19a968841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Thu, 3 Aug 2017 18:39:16 +0200 Subject: [PATCH 3/3] doc: example and readme fixes --- README.md | 113 +++++++++++------------ example_test.go | 241 +++++++++++++++++------------------------------- 2 files changed, 140 insertions(+), 214 deletions(-) diff --git a/README.md b/README.md index c26d78c..1d71ebd 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,17 @@ hood it uses `sqlx/reflectx` package so `sqlx` models will also work with `gocql ## Installation - go get github.com/scylladb/gocqlx + go get -u github.com/scylladb/gocqlx ## Features -Fast, boilerplate free and flexible `SELECTS`, `INSERTS`, `UPDATES` and `DELETES`. +* Flexible `SELECT`, `INSERT`, `UPDATE` `DELETE` and `BATCH` query building using a DSL +* Support for named parameters (:identifier) in queries +* Binding parameters form struct or map +* Scanning results into structs +* Fast! + +Example, see [full example here](https://github.com/scylladb/gocqlx/blob/master/example_test.go) ```go type Person struct { @@ -38,57 +44,69 @@ p := &Person{ } } -// Insert with TTL +// Batch { - stmt, names := qb.Insert("person").Columns("first_name", "last_name", "email").TTL().ToCql() - q := gocqlx.Query(session.Query(stmt), names) + i := qb.Insert("person").Columns("first_name", "last_name", "email") - if err := q.BindStructMap(p, qb.M{"_ttl": qb.TTL(86400 * time.Second)}).Exec(); err != nil { - log.Fatal(err) - } + stmt, names := qb.Batch(). + Add("a.", i). + Add("b.", i). + ToCql() + q := gocqlx.Query(session.Query(stmt), names) + + b := struct { + A Person + B Person + }{ + A: Person{ + "Igy", + "Citizen", + []string{"ian.citzen@gocqlx_test.com"}, + }, + B: Person{ + "Ian", + "Citizen", + []string{"igy.citzen@gocqlx_test.com"}, + }, + } + + if err := q.BindStruct(&b).Exec(); err != nil { + t.Fatal(err) + } } -// Update +// Get { - p.Email = append(p.Email, "patricia1.citzen@gocqlx_test.com") - - stmt, names := qb.Update("person").Set("email").Where(qb.Eq("first_name"), qb.Eq("last_name")).ToCql() - q := gocqlx.Query(session.Query(stmt), names) - - if err := q.BindStruct(p).Exec(); err != nil { - log.Fatal(err) - } + var p Person + if err := gocqlx.Get(&p, session.Query("SELECT * FROM gocqlx_test.person WHERE first_name=?", "Patricia")); err != nil { + t.Fatal("get:", err) + } + t.Log(p) // {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]} } // Select { - stmt, names := qb.Select("person").Where(qb.In("first_name")).ToCql() - q := gocqlx.Query(session.Query(stmt), names) + stmt, names := qb.Select("gocqlx_test.person").Where(qb.In("first_name")).ToCql() + q := gocqlx.Query(session.Query(stmt), names) - q.BindMap(qb.M{"first_name": []string{"Patricia", "John"}}) - if err := q.Err(); err != nil { - log.Fatal(err) - } + q.BindMap(qb.M{"first_name": []string{"Patricia", "Igy", "Ian"}}) + if err := q.Err(); err != nil { + t.Fatal(err) + } - var people []Person - if err := gocqlx.Select(&people, q.Query); err != nil { - log.Fatal("select:", err) - } - log.Println(people) - - // [{Patricia Citizen [patricia.citzen@com patricia1.citzen@com]} {John Doe [johndoeDNE@gmail.net]}] + var people []Person + if err := gocqlx.Select(&people, q.Query); err != nil { + t.Fatal("select:", err) + } + t.Log(people) // [{Ian Citizen [igy.citzen@gocqlx_test.com]} {Igy Citizen [ian.citzen@gocqlx_test.com]} {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]}] } ``` -For more details see [example test](https://github.com/scylladb/gocqlx/blob/master/example_test.go). - ## Performance -Gocqlx is fast, below is a benchmark result comparing `gocqlx` to raw `gocql` on -my machine, see the benchmark [here](https://github.com/scylladb/gocqlx/blob/master/benchmark_test.go). - -For query binding gocqlx is faster as it does not require parameter rewriting -while binding. For get and insert the performance is comparable. +Gocqlx is fast, this is a benchmark result comparing `gocqlx` to raw `gocql` +on a local machine. For query binding (insert) `gocqlx` is faster then `gocql` +thanks to smart caching, otherwise the performance is comparable. ``` BenchmarkE2EGocqlInsert-4 500000 258434 ns/op 2627 B/op 59 allocs/op @@ -99,23 +117,4 @@ BenchmarkE2EGocqlSelect-4 30000 2588562 ns/op 34605 BenchmarkE2EGocqlxSelect-4 30000 2637187 ns/op 27718 B/op 951 allocs/op ``` -Gocqlx comes with automatic snake case support for field names and does not -require manual tagging. This is also fast, below is a comparison to -`strings.ToLower` function (`sqlx` default). - -``` -BenchmarkSnakeCase-4 10000000 124 ns/op 32 B/op 2 allocs/op -BenchmarkToLower-4 100000000 57.9 ns/op 0 B/op 0 allocs/op -``` - -Building queries is fast and low on allocations too. - -``` -BenchmarkCmp-4 3000000 464 ns/op 112 B/op 3 allocs/op -BenchmarkDeleteBuilder-4 10000000 214 ns/op 112 B/op 2 allocs/op -BenchmarkInsertBuilder-4 20000000 103 ns/op 64 B/op 1 allocs/op -BenchmarkSelectBuilder-4 10000000 214 ns/op 112 B/op 2 allocs/op -BenchmarkUpdateBuilder-4 10000000 212 ns/op 112 B/op 2 allocs/op -``` - -Enyoy! +See the [benchmark here](https://github.com/scylladb/gocqlx/blob/master/benchmark_test.go). diff --git a/example_test.go b/example_test.go index 627d7d9..15288f5 100644 --- a/example_test.go +++ b/example_test.go @@ -6,7 +6,6 @@ import ( "testing" "time" - "github.com/gocql/gocql" "github.com/scylladb/gocqlx" "github.com/scylladb/gocqlx/qb" ) @@ -19,14 +18,6 @@ CREATE TABLE IF NOT EXISTS gocqlx_test.person ( PRIMARY KEY(first_name, last_name) )` -var placeSchema = ` -CREATE TABLE IF NOT EXISTS gocqlx_test.place ( - country text, - city text, - code int, - PRIMARY KEY(country, city) -)` - // Field names are converted to camel case by default, no need to add // `db:"first_name"`, if you want to disable a filed add `db:"-"` tag. type Person struct { @@ -35,175 +26,111 @@ type Person struct { Email []string } -type Place struct { - Country string - City string - TelCode int `db:"code"` -} - func TestExample(t *testing.T) { session := createSession(t) defer session.Close() - mustExec := func(q *gocql.Query) { - if err := q.Exec(); err != nil { - t.Fatal("query:", q, err) + if err := createTable(session, personSchema); err != nil { + t.Fatal("create table:", err) + } + + p := &Person{ + "Patricia", + "Citizen", + []string{"patricia.citzen@gocqlx_test.com"}, + } + + // Insert + { + stmt, names := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email").ToCql() + q := gocqlx.Query(session.Query(stmt), names) + + if err := q.BindStruct(p).Exec(); err != nil { + t.Fatal(err) } } - // Fill person table. + // Insert with TTL { - if err := createTable(session, personSchema); err != nil { - t.Fatal("create table:", err) - } + stmt, names := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email").TTL().ToCql() + q := gocqlx.Query(session.Query(stmt), names) - q := session.Query("INSERT INTO gocqlx_test.person (first_name, last_name, email) VALUES (?, ?, ?)") - mustExec(q.Bind("Jason", "Moiron", []string{"jmoiron@jmoiron.net"})) - mustExec(q.Bind("John", "Doe", []string{"johndoeDNE@gmail.net"})) - q.Release() + if err := q.BindStructMap(p, qb.M{"_ttl": qb.TTL(86400 * time.Second)}).Exec(); err != nil { + t.Fatal(err) + } } - // Fill place table. + // Update { - if err := createTable(session, placeSchema); err != nil { - t.Fatal("create table:", err) - } + p.Email = append(p.Email, "patricia1.citzen@gocqlx_test.com") - q := session.Query("INSERT INTO gocqlx_test.place (country, city, code) VALUES (?, ?, ?)") - mustExec(q.Bind("United States", "New York", 1)) - mustExec(q.Bind("Hong Kong", "", 852)) - mustExec(q.Bind("Singapore", "", 65)) - q.Release() + stmt, names := qb.Update("gocqlx_test.person").Set("email").Where(qb.Eq("first_name"), qb.Eq("last_name")).ToCql() + q := gocqlx.Query(session.Query(stmt), names) + + if err := q.BindStruct(p).Exec(); err != nil { + t.Fatal(err) + } } - // Query the database, storing results in a []Person (wrapped in []interface{}). + // Batch { + i := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email") + + stmt, names := qb.Batch(). + Add("a.", i). + Add("b.", i). + ToCql() + q := gocqlx.Query(session.Query(stmt), names) + + b := struct { + A Person + B Person + }{ + A: Person{ + "Igy", + "Citizen", + []string{"ian.citzen@gocqlx_test.com"}, + }, + B: Person{ + "Ian", + "Citizen", + []string{"igy.citzen@gocqlx_test.com"}, + }, + } + + if err := q.BindStruct(&b).Exec(); err != nil { + t.Fatal(err) + } + } + + // Get + { + var p Person + if err := gocqlx.Get(&p, session.Query("SELECT * FROM gocqlx_test.person WHERE first_name=?", "Patricia")); err != nil { + t.Fatal("get:", err) + } + t.Log(p) + + // {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]} + } + + // Select + { + stmt, names := qb.Select("gocqlx_test.person").Where(qb.In("first_name")).ToCql() + q := gocqlx.Query(session.Query(stmt), names) + + q.BindMap(qb.M{"first_name": []string{"Patricia", "Igy", "Ian"}}) + if err := q.Err(); err != nil { + t.Fatal(err) + } + var people []Person - if err := gocqlx.Select(&people, session.Query("SELECT * FROM gocqlx_test.person")); err != nil { + if err := gocqlx.Select(&people, q.Query); err != nil { t.Fatal("select:", err) } t.Log(people) - // [{John Doe [johndoeDNE@gmail.net]} {Jason Moiron [jmoiron@jmoiron.net]}] - } - - // Get a single result. - { - var jason Person - if err := gocqlx.Get(&jason, session.Query("SELECT * FROM gocqlx_test.person WHERE first_name=?", "Jason")); err != nil { - t.Fatal("get:", err) - } - t.Log(jason) - - // Jason Moiron [jmoiron@jmoiron.net]} - } - - // Loop through rows using only one struct. - { - var place Place - iter := gocqlx.Iter(session.Query("SELECT * FROM gocqlx_test.place")) - for iter.StructScan(&place) { - t.Log(place) - } - if err := iter.Close(); err != nil { - t.Fatal("iter:", err) - } - iter.ReleaseQuery() - - // {Hong Kong 852} - // {United States New York 1} - // {Singapore 65} - } - - // Query builder, using DSL to build queries, using `:name` as the bindvar. - { - p := &Person{ - "Patricia", - "Citizen", - []string{"patricia.citzen@gocqlx_test.com"}, - } - - // Insert - { - stmt, names := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email").ToCql() - q := gocqlx.Query(session.Query(stmt), names) - - if err := q.BindStruct(p).Exec(); err != nil { - t.Fatal(err) - } - } - - // Insert with TTL - { - stmt, names := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email").TTL().ToCql() - q := gocqlx.Query(session.Query(stmt), names) - - if err := q.BindStructMap(p, qb.M{"_ttl": qb.TTL(86400 * time.Second)}).Exec(); err != nil { - t.Fatal(err) - } - } - - // Update - { - p.Email = append(p.Email, "patricia1.citzen@gocqlx_test.com") - - stmt, names := qb.Update("gocqlx_test.person").Set("email").Where(qb.Eq("first_name"), qb.Eq("last_name")).ToCql() - q := gocqlx.Query(session.Query(stmt), names) - - if err := q.BindStruct(p).Exec(); err != nil { - t.Fatal(err) - } - } - - // Batch - { - i := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email") - - stmt, names := qb.Batch(). - Add("a.", i). - Add("b.", i). - ToCql() - q := gocqlx.Query(session.Query(stmt), names) - - b := struct { - A Person - B Person - }{ - A: Person{ - "Ian", - "Citizen", - []string{"ian.citzen@gocqlx_test.com"}, - }, - B: Person{ - "Igy", - "Citizen", - []string{"igy.citzen@gocqlx_test.com"}, - }, - } - - if err := q.BindStruct(&b).Exec(); err != nil { - t.Fatal(err) - } - } - - // Select - { - stmt, names := qb.Select("gocqlx_test.person").Where(qb.In("first_name")).ToCql() - q := gocqlx.Query(session.Query(stmt), names) - - q.BindMap(qb.M{"first_name": []string{"Patricia", "John"}}) - if err := q.Err(); err != nil { - t.Fatal(err) - } - - var people []Person - if err := gocqlx.Select(&people, q.Query); err != nil { - t.Fatal("select:", err) - } - t.Log(people) - - // [{Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]} {John Doe [johndoeDNE@gmail.net]}] - } + // [{Ian Citizen [igy.citzen@gocqlx_test.com]} {Igy Citizen [ian.citzen@gocqlx_test.com]} {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]}] } // Named queries, using `:name` as the bindvar.