pixeldrain/gocqlx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #8 from scylladb/mmt/batch

Browse Source 
Batch statement support
This commit is contained in:
Michał Matczuk
2017-08-03 18:58:51 +02:00
committed by GitHub
parent 91cb873025 bc289ada45
commit 4708de6653
8 changed files with 315 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
README.md
View File

@@ -9,11 +9,17 @@ hood it uses `sqlx/reflectx` package so `sqlx` models will also work with `gocql
## Installation ## Installation
go get github.com/scylladb/gocqlx go get -u github.com/scylladb/gocqlx
## Features ## Features
Fast, boilerplate free and flexible `SELECTS`, `INSERTS`, `UPDATES` and `DELETES`. * Flexible `SELECT`, `INSERT`, `UPDATE` `DELETE` and `BATCH` query building using a DSL
* Support for named parameters (:identifier) in queries
* Binding parameters form struct or map
* Scanning results into structs
* Fast!
Example, see [full example here](https://github.com/scylladb/gocqlx/blob/master/example_test.go)
```go ```go
type Person struct { type Person struct {
@@ -38,57 +44,69 @@ p := &Person{
} }
} }
// Insert with TTL // Batch
{ {
stmt, names := qb.Insert("person").Columns("first_name", "last_name", "email").TTL().ToCql() i := qb.Insert("person").Columns("first_name", "last_name", "email")
q := gocqlx.Query(session.Query(stmt), names)
if err := q.BindStructMap(p, qb.M{"_ttl": qb.TTL(86400 * time.Second)}).Exec(); err != nil { stmt, names := qb.Batch().
log.Fatal(err) Add("a.", i).
} Add("b.", i).
ToCql()
q := gocqlx.Query(session.Query(stmt), names)
b := struct {
A Person
B Person
}{
A: Person{
"Igy",
"Citizen",
[]string{"ian.citzen@gocqlx_test.com"},
},
B: Person{
"Ian",
"Citizen",
[]string{"igy.citzen@gocqlx_test.com"},
},
}
if err := q.BindStruct(&b).Exec(); err != nil {
t.Fatal(err)
}
} }
// Update // Get
{ {
p.Email = append(p.Email, "patricia1.citzen@gocqlx_test.com") var p Person
if err := gocqlx.Get(&p, session.Query("SELECT * FROM gocqlx_test.person WHERE first_name=?", "Patricia")); err != nil {
stmt, names := qb.Update("person").Set("email").Where(qb.Eq("first_name"), qb.Eq("last_name")).ToCql() t.Fatal("get:", err)
q := gocqlx.Query(session.Query(stmt), names) }
t.Log(p) // {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]}
if err := q.BindStruct(p).Exec(); err != nil {
log.Fatal(err)
}
} }
// Select // Select
{ {
stmt, names := qb.Select("person").Where(qb.In("first_name")).ToCql() stmt, names := qb.Select("gocqlx_test.person").Where(qb.In("first_name")).ToCql()
q := gocqlx.Query(session.Query(stmt), names) q := gocqlx.Query(session.Query(stmt), names)
q.BindMap(qb.M{"first_name": []string{"Patricia", "John"}}) q.BindMap(qb.M{"first_name": []string{"Patricia", "Igy", "Ian"}})
if err := q.Err(); err != nil { if err := q.Err(); err != nil {
log.Fatal(err) t.Fatal(err)
} }
var people []Person var people []Person
if err := gocqlx.Select(&people, q.Query); err != nil { if err := gocqlx.Select(&people, q.Query); err != nil {
log.Fatal("select:", err) t.Fatal("select:", err)
} }
log.Println(people) t.Log(people) // [{Ian Citizen [igy.citzen@gocqlx_test.com]} {Igy Citizen [ian.citzen@gocqlx_test.com]} {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]}]
// [{Patricia Citizen [patricia.citzen@com patricia1.citzen@com]} {John Doe [johndoeDNE@gmail.net]}]
} }
``` ```
For more details see [example test](https://github.com/scylladb/gocqlx/blob/master/example_test.go).
## Performance ## Performance
Gocqlx is fast, below is a benchmark result comparing `gocqlx` to raw `gocql` on Gocqlx is fast, this is a benchmark result comparing `gocqlx` to raw `gocql`
my machine, see the benchmark [here](https://github.com/scylladb/gocqlx/blob/master/benchmark_test.go). on a local machine. For query binding (insert) `gocqlx` is faster then `gocql`
thanks to smart caching, otherwise the performance is comparable.
For query binding gocqlx is faster as it does not require parameter rewriting
while binding. For get and insert the performance is comparable.
``` ```
BenchmarkE2EGocqlInsert-4 500000 258434 ns/op 2627 B/op 59 allocs/op BenchmarkE2EGocqlInsert-4 500000 258434 ns/op 2627 B/op 59 allocs/op
@@ -99,23 +117,4 @@ BenchmarkE2EGocqlSelect-4 30000 2588562 ns/op 34605
BenchmarkE2EGocqlxSelect-4 30000 2637187 ns/op 27718 B/op 951 allocs/op BenchmarkE2EGocqlxSelect-4 30000 2637187 ns/op 27718 B/op 951 allocs/op
``` ```
Gocqlx comes with automatic snake case support for field names and does not See the [benchmark here](https://github.com/scylladb/gocqlx/blob/master/benchmark_test.go).
require manual tagging. This is also fast, below is a comparison to
`strings.ToLower` function (`sqlx` default).
```
BenchmarkSnakeCase-4 10000000 124 ns/op 32 B/op 2 allocs/op
BenchmarkToLower-4 100000000 57.9 ns/op 0 B/op 0 allocs/op
```
Building queries is fast and low on allocations too.
```
BenchmarkCmp-4 3000000 464 ns/op 112 B/op 3 allocs/op
BenchmarkDeleteBuilder-4 10000000 214 ns/op 112 B/op 2 allocs/op
BenchmarkInsertBuilder-4 20000000 103 ns/op 64 B/op 1 allocs/op
BenchmarkSelectBuilder-4 10000000 214 ns/op 112 B/op 2 allocs/op
BenchmarkUpdateBuilder-4 10000000 212 ns/op 112 B/op 2 allocs/op
```
Enyoy!

210
example_test.go
View File

@@ -6,7 +6,6 @@ import (
"testing" "testing"
"time" "time"
"github.com/gocql/gocql"
"github.com/scylladb/gocqlx" "github.com/scylladb/gocqlx"
"github.com/scylladb/gocqlx/qb" "github.com/scylladb/gocqlx/qb"
) )
@@ -19,14 +18,6 @@ CREATE TABLE IF NOT EXISTS gocqlx_test.person (
PRIMARY KEY(first_name, last_name) PRIMARY KEY(first_name, last_name)
)` )`
var placeSchema = `
CREATE TABLE IF NOT EXISTS gocqlx_test.place (
country text,
city text,
code int,
PRIMARY KEY(country, city)
)`
// Field names are converted to camel case by default, no need to add // Field names are converted to camel case by default, no need to add
// `db:"first_name"`, if you want to disable a filed add `db:"-"` tag. // `db:"first_name"`, if you want to disable a filed add `db:"-"` tag.
type Person struct { type Person struct {
@@ -35,144 +26,111 @@ type Person struct {
Email []string Email []string
} }
type Place struct {
Country string
City string
TelCode int `db:"code"`
}
func TestExample(t *testing.T) { func TestExample(t *testing.T) {
session := createSession(t) session := createSession(t)
defer session.Close() defer session.Close()
mustExec := func(q *gocql.Query) { if err := createTable(session, personSchema); err != nil {
if err := q.Exec(); err != nil { t.Fatal("create table:", err)
t.Fatal("query:", q, err) }
p := &Person{
"Patricia",
"Citizen",
[]string{"patricia.citzen@gocqlx_test.com"},
}
// Insert
{
stmt, names := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email").ToCql()
q := gocqlx.Query(session.Query(stmt), names)
if err := q.BindStruct(p).Exec(); err != nil {
t.Fatal(err)
} }
} }
// Fill person table. // Insert with TTL
{ {
if err := createTable(session, personSchema); err != nil { stmt, names := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email").TTL().ToCql()
t.Fatal("create table:", err) q := gocqlx.Query(session.Query(stmt), names)
}
q := session.Query("INSERT INTO gocqlx_test.person (first_name, last_name, email) VALUES (?, ?, ?)") if err := q.BindStructMap(p, qb.M{"_ttl": qb.TTL(86400 * time.Second)}).Exec(); err != nil {
mustExec(q.Bind("Jason", "Moiron", []string{"jmoiron@jmoiron.net"})) t.Fatal(err)
mustExec(q.Bind("John", "Doe", []string{"johndoeDNE@gmail.net"})) }
q.Release()
} }
// Fill place table. // Update
{ {
if err := createTable(session, placeSchema); err != nil { p.Email = append(p.Email, "patricia1.citzen@gocqlx_test.com")
t.Fatal("create table:", err)
}
q := session.Query("INSERT INTO gocqlx_test.place (country, city, code) VALUES (?, ?, ?)") stmt, names := qb.Update("gocqlx_test.person").Set("email").Where(qb.Eq("first_name"), qb.Eq("last_name")).ToCql()
mustExec(q.Bind("United States", "New York", 1)) q := gocqlx.Query(session.Query(stmt), names)
mustExec(q.Bind("Hong Kong", "", 852))
mustExec(q.Bind("Singapore", "", 65)) if err := q.BindStruct(p).Exec(); err != nil {
q.Release() t.Fatal(err)
}
} }
// Query the database, storing results in a []Person (wrapped in []interface{}). // Batch
{ {
i := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email")
stmt, names := qb.Batch().
Add("a.", i).
Add("b.", i).
ToCql()
q := gocqlx.Query(session.Query(stmt), names)
b := struct {
A Person
B Person
}{
A: Person{
"Igy",
"Citizen",
[]string{"ian.citzen@gocqlx_test.com"},
},
B: Person{
"Ian",
"Citizen",
[]string{"igy.citzen@gocqlx_test.com"},
},
}
if err := q.BindStruct(&b).Exec(); err != nil {
t.Fatal(err)
}
}
// Get
{
var p Person
if err := gocqlx.Get(&p, session.Query("SELECT * FROM gocqlx_test.person WHERE first_name=?", "Patricia")); err != nil {
t.Fatal("get:", err)
}
t.Log(p)
// {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]}
}
// Select
{
stmt, names := qb.Select("gocqlx_test.person").Where(qb.In("first_name")).ToCql()
q := gocqlx.Query(session.Query(stmt), names)
q.BindMap(qb.M{"first_name": []string{"Patricia", "Igy", "Ian"}})
if err := q.Err(); err != nil {
t.Fatal(err)
}
var people []Person var people []Person
if err := gocqlx.Select(&people, session.Query("SELECT * FROM gocqlx_test.person")); err != nil { if err := gocqlx.Select(&people, q.Query); err != nil {
t.Fatal("select:", err) t.Fatal("select:", err)
} }
t.Log(people) t.Log(people)
// [{John Doe [johndoeDNE@gmail.net]} {Jason Moiron [jmoiron@jmoiron.net]}] // [{Ian Citizen [igy.citzen@gocqlx_test.com]} {Igy Citizen [ian.citzen@gocqlx_test.com]} {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]}]
}
// Get a single result.
{
var jason Person
if err := gocqlx.Get(&jason, session.Query("SELECT * FROM gocqlx_test.person WHERE first_name=?", "Jason")); err != nil {
t.Fatal("get:", err)
}
t.Log(jason)
// Jason Moiron [jmoiron@jmoiron.net]}
}
// Loop through rows using only one struct.
{
var place Place
iter := gocqlx.Iter(session.Query("SELECT * FROM gocqlx_test.place"))
for iter.StructScan(&place) {
t.Log(place)
}
if err := iter.Close(); err != nil {
t.Fatal("iter:", err)
}
iter.ReleaseQuery()
// {Hong Kong 852}
// {United States New York 1}
// {Singapore 65}
}
// Query builder, using DSL to build queries, using `:name` as the bindvar.
{
p := &Person{
"Patricia",
"Citizen",
[]string{"patricia.citzen@gocqlx_test.com"},
}
// Insert
{
stmt, names := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email").ToCql()
q := gocqlx.Query(session.Query(stmt), names)
if err := q.BindStruct(p).Exec(); err != nil {
t.Fatal(err)
}
}
// Insert with TTL
{
stmt, names := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email").TTL().ToCql()
q := gocqlx.Query(session.Query(stmt), names)
if err := q.BindStructMap(p, qb.M{"_ttl": qb.TTL(86400 * time.Second)}).Exec(); err != nil {
t.Fatal(err)
}
}
// Update
{
p.Email = append(p.Email, "patricia1.citzen@gocqlx_test.com")
stmt, names := qb.Update("gocqlx_test.person").Set("email").Where(qb.Eq("first_name"), qb.Eq("last_name")).ToCql()
q := gocqlx.Query(session.Query(stmt), names)
if err := q.BindStruct(p).Exec(); err != nil {
t.Fatal(err)
}
}
// Select
{
stmt, names := qb.Select("gocqlx_test.person").Where(qb.In("first_name")).ToCql()
q := gocqlx.Query(session.Query(stmt), names)
q.BindMap(qb.M{"first_name": []string{"Patricia", "John"}})
if err := q.Err(); err != nil {
t.Fatal(err)
}
var people []Person
if err := gocqlx.Select(&people, q.Query); err != nil {
t.Fatal("select:", err)
}
t.Log(people)
// [{Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]} {John Doe [johndoeDNE@gmail.net]}]
}
} }
// Named queries, using `:name` as the bindvar. // Named queries, using `:name` as the bindvar.

91
qb/batch.go Normal file
View File

@@ -0,0 +1,91 @@
package qb
import (
"bytes"
"fmt"
)
// BATCH reference:
// https://cassandra.apache.org/doc/latest/cql/dml.html#batch
// builder is interface implemented by other builders.
type builder interface {
ToCql() (stmt string, names []string)
}
// BatchBuilder builds CQL BATCH statements.
type BatchBuilder struct {
unlogged bool
counter bool
using using
stmts []string
names []string
}
// Batch returns a new BatchBuilder.
func Batch() *BatchBuilder {
return &BatchBuilder{}
}
// ToCql builds the query into a CQL string and named args.
func (b *BatchBuilder) ToCql() (stmt string, names []string) {
cql := bytes.Buffer{}
cql.WriteString("BEGIN ")
if b.unlogged {
cql.WriteString("UNLOGGED ")
}
if b.counter {
cql.WriteString("COUNTER ")
}
cql.WriteString("BATCH ")
names = append(names, b.using.writeCql(&cql)...)
for _, stmt := range b.stmts {
cql.WriteString(stmt)
cql.WriteByte(';')
cql.WriteByte(' ')
}
names = append(names, b.names...)
cql.WriteString("APPLY BATCH ")
stmt = cql.String()
return
}
// UnLogged sets a UNLOGGED BATCH clause on the query.
func (b *BatchBuilder) UnLogged() *BatchBuilder {
b.unlogged = true
return b
}
// Counter sets a COUNTER BATCH clause on the query.
func (b *BatchBuilder) Counter() *BatchBuilder {
b.counter = true
return b
}
// Timestamp sets a USING TIMESTAMP clause on the query.
func (b *BatchBuilder) Timestamp() *BatchBuilder {
b.using.timestamp = true
return b
}
// TTL sets a USING TTL clause on the query.
func (b *BatchBuilder) TTL() *BatchBuilder {
b.using.ttl = true
return b
}
// Add adds another batch statement from a builder.
func (b *BatchBuilder) Add(prefix string, builder Builder) *BatchBuilder {
stmt, names := builder.ToCql()
b.stmts = append(b.stmts, stmt)
for _, name := range names {
b.names = append(b.names, fmt.Sprint(prefix, name))
}
return b
}

80
qb/batch_test.go Normal file
View File

@@ -0,0 +1,80 @@
package qb
import (
"testing"
"github.com/google/go-cmp/cmp"
)
type mockBuilder struct {
stmt string
names []string
}
func (b mockBuilder) ToCql() (stmt string, names []string) {
return b.stmt, b.names
}
func TestBatchBuilder(t *testing.T) {
m := mockBuilder{"INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ", []string{"id", "user_uuid", "firstname"}}
table := []struct {
B *BatchBuilder
N []string
S string
}{
// Basic test for Batch
{
B: Batch().Add("a.", m),
S: "BEGIN BATCH INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ; APPLY BATCH ",
N: []string{"a.id", "a.user_uuid", "a.firstname"},
},
// Add statement
{
B: Batch().
Add("a.", m).
Add("b.", m),
S: "BEGIN BATCH INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ; INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ; APPLY BATCH ",
N: []string{"a.id", "a.user_uuid", "a.firstname", "b.id", "b.user_uuid", "b.firstname"},
},
// Add UNLOGGED
{
B: Batch().UnLogged(),
S: "BEGIN UNLOGGED BATCH APPLY BATCH ",
},
// Add COUNTER
{
B: Batch().Counter(),
S: "BEGIN COUNTER BATCH APPLY BATCH ",
},
// Add TTL
{
B: Batch().TTL(),
S: "BEGIN BATCH USING TTL ? APPLY BATCH ",
N: []string{"_ttl"},
},
// Add TIMESTAMP
{
B: Batch().Timestamp(),
S: "BEGIN BATCH USING TIMESTAMP ? APPLY BATCH ",
N: []string{"_ts"},
},
}
for _, test := range table {
stmt, names := test.B.ToCql()
if diff := cmp.Diff(test.S, stmt); diff != "" {
t.Error(diff)
}
if diff := cmp.Diff(test.N, names); diff != "" {
t.Error(diff)
}
}
}
func BenchmarkBatchBuilder(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
Batch().Add("", mockBuilder{"INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ", []string{"id", "user_uuid", "firstname"}}).ToCql()
}
}

2
qb/delete_test.go
View File

@@ -72,6 +72,6 @@ func TestDeleteBuilder(t *testing.T) {
func BenchmarkDeleteBuilder(b *testing.B) { func BenchmarkDeleteBuilder(b *testing.B) {
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
Delete("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").Where(Eq("id")) Delete("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").Where(Eq("id")).ToCql()
} }
} }

2
qb/insert_test.go
View File

@@ -65,6 +65,6 @@ func TestInsertBuilder(t *testing.T) {
func BenchmarkInsertBuilder(b *testing.B) { func BenchmarkInsertBuilder(b *testing.B) {
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
Insert("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars") Insert("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").ToCql()
} }
} }

2
qb/select_test.go
View File

@@ -85,6 +85,6 @@ func TestSelectBuilder(t *testing.T) {
func BenchmarkSelectBuilder(b *testing.B) { func BenchmarkSelectBuilder(b *testing.B) {
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
Select("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").Where(Eq("id")) Select("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").Where(Eq("id")).ToCql()
} }
} }

2
qb/update_test.go
View File

@@ -78,6 +78,6 @@ func TestUpdateBuilder(t *testing.T) {
func BenchmarkUpdateBuilder(b *testing.B) { func BenchmarkUpdateBuilder(b *testing.B) {
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
Update("cycling.cyclist_name").Set("id", "user_uuid", "firstname", "stars").Where(Eq("id")) Update("cycling.cyclist_name").Set("id", "user_uuid", "firstname", "stars").Where(Eq("id")).ToCql()
} }
} }