pixeldrain/gocqlx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #8 from scylladb/mmt/batch

Browse Source 
Batch statement support
This commit is contained in:
Michał Matczuk
2017-08-03 18:58:51 +02:00
committed by GitHub
parent 91cb873025 bc289ada45
commit 4708de6653
8 changed files with 315 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
README.md
View File

@@ -9,11 +9,17 @@ hood it uses `sqlx/reflectx` package so `sqlx` models will also work with `gocql
## Installation
go get github.com/scylladb/gocqlx
go get -u github.com/scylladb/gocqlx
## Features
Fast, boilerplate free and flexible `SELECTS`, `INSERTS`, `UPDATES` and `DELETES`.
* Flexible `SELECT`, `INSERT`, `UPDATE` `DELETE` and `BATCH` query building using a DSL
* Support for named parameters (:identifier) in queries
* Binding parameters form struct or map
* Scanning results into structs
* Fast!
Example, see [full example here](https://github.com/scylladb/gocqlx/blob/master/example_test.go)
```go
type Person struct {
@@ -38,57 +44,69 @@ p := &Person{
}
}
// Insert with TTL
// Batch
{
stmt, names := qb.Insert("person").Columns("first_name", "last_name", "email").TTL().ToCql()
i := qb.Insert("person").Columns("first_name", "last_name", "email")
stmt, names := qb.Batch().
Add("a.", i).
Add("b.", i).
ToCql()
q := gocqlx.Query(session.Query(stmt), names)
if err := q.BindStructMap(p, qb.M{"_ttl": qb.TTL(86400 * time.Second)}).Exec(); err != nil {
log.Fatal(err)
b := struct {
A Person
B Person
}{
A: Person{
"Igy",
"Citizen",
[]string{"ian.citzen@gocqlx_test.com"},
},
B: Person{
"Ian",
"Citizen",
[]string{"igy.citzen@gocqlx_test.com"},
},
}
if err := q.BindStruct(&b).Exec(); err != nil {
t.Fatal(err)
}
}
// Update
// Get
{
p.Email = append(p.Email, "patricia1.citzen@gocqlx_test.com")
stmt, names := qb.Update("person").Set("email").Where(qb.Eq("first_name"), qb.Eq("last_name")).ToCql()
q := gocqlx.Query(session.Query(stmt), names)
if err := q.BindStruct(p).Exec(); err != nil {
log.Fatal(err)
var p Person
if err := gocqlx.Get(&p, session.Query("SELECT * FROM gocqlx_test.person WHERE first_name=?", "Patricia")); err != nil {
t.Fatal("get:", err)
}
t.Log(p) // {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]}
}
// Select
{
stmt, names := qb.Select("person").Where(qb.In("first_name")).ToCql()
stmt, names := qb.Select("gocqlx_test.person").Where(qb.In("first_name")).ToCql()
q := gocqlx.Query(session.Query(stmt), names)
q.BindMap(qb.M{"first_name": []string{"Patricia", "John"}})
q.BindMap(qb.M{"first_name": []string{"Patricia", "Igy", "Ian"}})
if err := q.Err(); err != nil {
log.Fatal(err)
t.Fatal(err)
}
var people []Person
if err := gocqlx.Select(&people, q.Query); err != nil {
log.Fatal("select:", err)
t.Fatal("select:", err)
}
log.Println(people)
// [{Patricia Citizen [patricia.citzen@com patricia1.citzen@com]} {John Doe [johndoeDNE@gmail.net]}]
t.Log(people) // [{Ian Citizen [igy.citzen@gocqlx_test.com]} {Igy Citizen [ian.citzen@gocqlx_test.com]} {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]}]
}
```
For more details see [example test](https://github.com/scylladb/gocqlx/blob/master/example_test.go).
## Performance
Gocqlx is fast, below is a benchmark result comparing `gocqlx` to raw `gocql` on
my machine, see the benchmark [here](https://github.com/scylladb/gocqlx/blob/master/benchmark_test.go).
For query binding gocqlx is faster as it does not require parameter rewriting
while binding. For get and insert the performance is comparable.
Gocqlx is fast, this is a benchmark result comparing `gocqlx` to raw `gocql`
on a local machine. For query binding (insert) `gocqlx` is faster then `gocql`
thanks to smart caching, otherwise the performance is comparable.
```
BenchmarkE2EGocqlInsert-4 500000 258434 ns/op 2627 B/op 59 allocs/op
@@ -99,23 +117,4 @@ BenchmarkE2EGocqlSelect-4 30000 2588562 ns/op 34605
BenchmarkE2EGocqlxSelect-4 30000 2637187 ns/op 27718 B/op 951 allocs/op
```
Gocqlx comes with automatic snake case support for field names and does not
require manual tagging. This is also fast, below is a comparison to
`strings.ToLower` function (`sqlx` default).
```
BenchmarkSnakeCase-4 10000000 124 ns/op 32 B/op 2 allocs/op
BenchmarkToLower-4 100000000 57.9 ns/op 0 B/op 0 allocs/op
```
Building queries is fast and low on allocations too.
```
BenchmarkCmp-4 3000000 464 ns/op 112 B/op 3 allocs/op
BenchmarkDeleteBuilder-4 10000000 214 ns/op 112 B/op 2 allocs/op
BenchmarkInsertBuilder-4 20000000 103 ns/op 64 B/op 1 allocs/op
BenchmarkSelectBuilder-4 10000000 214 ns/op 112 B/op 2 allocs/op
BenchmarkUpdateBuilder-4 10000000 212 ns/op 112 B/op 2 allocs/op
```
Enyoy!
See the [benchmark here](https://github.com/scylladb/gocqlx/blob/master/benchmark_test.go).

130
example_test.go
View File

@@ -6,7 +6,6 @@ import (
"testing"
"time"
"github.com/gocql/gocql"
"github.com/scylladb/gocqlx"
"github.com/scylladb/gocqlx/qb"
)
@@ -19,14 +18,6 @@ CREATE TABLE IF NOT EXISTS gocqlx_test.person (
PRIMARY KEY(first_name, last_name)
)`
var placeSchema = `
CREATE TABLE IF NOT EXISTS gocqlx_test.place (
country text,
city text,
code int,
PRIMARY KEY(country, city)
)`
// Field names are converted to camel case by default, no need to add
// `db:"first_name"`, if you want to disable a filed add `db:"-"` tag.
type Person struct {
@@ -35,88 +26,14 @@ type Person struct {
Email []string
}
type Place struct {
Country string
City string
TelCode int `db:"code"`
}
func TestExample(t *testing.T) {
session := createSession(t)
defer session.Close()
mustExec := func(q *gocql.Query) {
if err := q.Exec(); err != nil {
t.Fatal("query:", q, err)
}
}
// Fill person table.
{
if err := createTable(session, personSchema); err != nil {
t.Fatal("create table:", err)
}
q := session.Query("INSERT INTO gocqlx_test.person (first_name, last_name, email) VALUES (?, ?, ?)")
mustExec(q.Bind("Jason", "Moiron", []string{"jmoiron@jmoiron.net"}))
mustExec(q.Bind("John", "Doe", []string{"johndoeDNE@gmail.net"}))
q.Release()
}
// Fill place table.
{
if err := createTable(session, placeSchema); err != nil {
t.Fatal("create table:", err)
}
q := session.Query("INSERT INTO gocqlx_test.place (country, city, code) VALUES (?, ?, ?)")
mustExec(q.Bind("United States", "New York", 1))
mustExec(q.Bind("Hong Kong", "", 852))
mustExec(q.Bind("Singapore", "", 65))
q.Release()
}
// Query the database, storing results in a []Person (wrapped in []interface{}).
{
var people []Person
if err := gocqlx.Select(&people, session.Query("SELECT * FROM gocqlx_test.person")); err != nil {
t.Fatal("select:", err)
}
t.Log(people)
// [{John Doe [johndoeDNE@gmail.net]} {Jason Moiron [jmoiron@jmoiron.net]}]
}
// Get a single result.
{
var jason Person
if err := gocqlx.Get(&jason, session.Query("SELECT * FROM gocqlx_test.person WHERE first_name=?", "Jason")); err != nil {
t.Fatal("get:", err)
}
t.Log(jason)
// Jason Moiron [jmoiron@jmoiron.net]}
}
// Loop through rows using only one struct.
{
var place Place
iter := gocqlx.Iter(session.Query("SELECT * FROM gocqlx_test.place"))
for iter.StructScan(&place) {
t.Log(place)
}
if err := iter.Close(); err != nil {
t.Fatal("iter:", err)
}
iter.ReleaseQuery()
// {Hong Kong 852}
// {United States New York 1}
// {Singapore 65}
}
// Query builder, using DSL to build queries, using `:name` as the bindvar.
{
p := &Person{
"Patricia",
"Citizen",
@@ -155,12 +72,54 @@ func TestExample(t *testing.T) {
}
}
// Batch
{
i := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email")
stmt, names := qb.Batch().
Add("a.", i).
Add("b.", i).
ToCql()
q := gocqlx.Query(session.Query(stmt), names)
b := struct {
A Person
B Person
}{
A: Person{
"Igy",
"Citizen",
[]string{"ian.citzen@gocqlx_test.com"},
},
B: Person{
"Ian",
"Citizen",
[]string{"igy.citzen@gocqlx_test.com"},
},
}
if err := q.BindStruct(&b).Exec(); err != nil {
t.Fatal(err)
}
}
// Get
{
var p Person
if err := gocqlx.Get(&p, session.Query("SELECT * FROM gocqlx_test.person WHERE first_name=?", "Patricia")); err != nil {
t.Fatal("get:", err)
}
t.Log(p)
// {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]}
}
// Select
{
stmt, names := qb.Select("gocqlx_test.person").Where(qb.In("first_name")).ToCql()
q := gocqlx.Query(session.Query(stmt), names)
q.BindMap(qb.M{"first_name": []string{"Patricia", "John"}})
q.BindMap(qb.M{"first_name": []string{"Patricia", "Igy", "Ian"}})
if err := q.Err(); err != nil {
t.Fatal(err)
}
@@ -171,8 +130,7 @@ func TestExample(t *testing.T) {
}
t.Log(people)
// [{Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]} {John Doe [johndoeDNE@gmail.net]}]
}
// [{Ian Citizen [igy.citzen@gocqlx_test.com]} {Igy Citizen [ian.citzen@gocqlx_test.com]} {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com]}]
}
// Named queries, using `:name` as the bindvar.

91
qb/batch.go Normal file
View File

@@ -0,0 +1,91 @@
package qb
import (
"bytes"
"fmt"
)
// BATCH reference:
// https://cassandra.apache.org/doc/latest/cql/dml.html#batch
// builder is interface implemented by other builders.
type builder interface {
ToCql() (stmt string, names []string)
}
// BatchBuilder builds CQL BATCH statements.
type BatchBuilder struct {
unlogged bool
counter bool
using using
stmts []string
names []string
}
// Batch returns a new BatchBuilder.
func Batch() *BatchBuilder {
return &BatchBuilder{}
}
// ToCql builds the query into a CQL string and named args.
func (b *BatchBuilder) ToCql() (stmt string, names []string) {
cql := bytes.Buffer{}
cql.WriteString("BEGIN ")
if b.unlogged {
cql.WriteString("UNLOGGED ")
}
if b.counter {
cql.WriteString("COUNTER ")
}
cql.WriteString("BATCH ")
names = append(names, b.using.writeCql(&cql)...)
for _, stmt := range b.stmts {
cql.WriteString(stmt)
cql.WriteByte(';')
cql.WriteByte(' ')
}
names = append(names, b.names...)
cql.WriteString("APPLY BATCH ")
stmt = cql.String()
return
}
// UnLogged sets a UNLOGGED BATCH clause on the query.
func (b *BatchBuilder) UnLogged() *BatchBuilder {
b.unlogged = true
return b
}
// Counter sets a COUNTER BATCH clause on the query.
func (b *BatchBuilder) Counter() *BatchBuilder {
b.counter = true
return b
}
// Timestamp sets a USING TIMESTAMP clause on the query.
func (b *BatchBuilder) Timestamp() *BatchBuilder {
b.using.timestamp = true
return b
}
// TTL sets a USING TTL clause on the query.
func (b *BatchBuilder) TTL() *BatchBuilder {
b.using.ttl = true
return b
}
// Add adds another batch statement from a builder.
func (b *BatchBuilder) Add(prefix string, builder Builder) *BatchBuilder {
stmt, names := builder.ToCql()
b.stmts = append(b.stmts, stmt)
for _, name := range names {
b.names = append(b.names, fmt.Sprint(prefix, name))
}
return b
}

80
qb/batch_test.go Normal file
View File

@@ -0,0 +1,80 @@
package qb
import (
"testing"
"github.com/google/go-cmp/cmp"
)
type mockBuilder struct {
stmt string
names []string
}
func (b mockBuilder) ToCql() (stmt string, names []string) {
return b.stmt, b.names
}
func TestBatchBuilder(t *testing.T) {
m := mockBuilder{"INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ", []string{"id", "user_uuid", "firstname"}}
table := []struct {
B *BatchBuilder
N []string
S string
}{
// Basic test for Batch
{
B: Batch().Add("a.", m),
S: "BEGIN BATCH INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ; APPLY BATCH ",
N: []string{"a.id", "a.user_uuid", "a.firstname"},
},
// Add statement
{
B: Batch().
Add("a.", m).
Add("b.", m),
S: "BEGIN BATCH INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ; INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ; APPLY BATCH ",
N: []string{"a.id", "a.user_uuid", "a.firstname", "b.id", "b.user_uuid", "b.firstname"},
},
// Add UNLOGGED
{
B: Batch().UnLogged(),
S: "BEGIN UNLOGGED BATCH APPLY BATCH ",
},
// Add COUNTER
{
B: Batch().Counter(),
S: "BEGIN COUNTER BATCH APPLY BATCH ",
},
// Add TTL
{
B: Batch().TTL(),
S: "BEGIN BATCH USING TTL ? APPLY BATCH ",
N: []string{"_ttl"},
},
// Add TIMESTAMP
{
B: Batch().Timestamp(),
S: "BEGIN BATCH USING TIMESTAMP ? APPLY BATCH ",
N: []string{"_ts"},
},
}
for _, test := range table {
stmt, names := test.B.ToCql()
if diff := cmp.Diff(test.S, stmt); diff != "" {
t.Error(diff)
}
if diff := cmp.Diff(test.N, names); diff != "" {
t.Error(diff)
}
}
}
func BenchmarkBatchBuilder(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
Batch().Add("", mockBuilder{"INSERT INTO cycling.cyclist_name (id,user_uuid,firstname) VALUES (?,?,?) ", []string{"id", "user_uuid", "firstname"}}).ToCql()
}
}

2
qb/delete_test.go
View File

@@ -72,6 +72,6 @@ func TestDeleteBuilder(t *testing.T) {
func BenchmarkDeleteBuilder(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
Delete("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").Where(Eq("id"))
Delete("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").Where(Eq("id")).ToCql()
}
}

2
qb/insert_test.go
View File

@@ -65,6 +65,6 @@ func TestInsertBuilder(t *testing.T) {
func BenchmarkInsertBuilder(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
Insert("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars")
Insert("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").ToCql()
}
}

2
qb/select_test.go
View File

@@ -85,6 +85,6 @@ func TestSelectBuilder(t *testing.T) {
func BenchmarkSelectBuilder(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
Select("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").Where(Eq("id"))
Select("cycling.cyclist_name").Columns("id", "user_uuid", "firstname", "stars").Where(Eq("id")).ToCql()
}
}

2
qb/update_test.go
View File

@@ -78,6 +78,6 @@ func TestUpdateBuilder(t *testing.T) {
func BenchmarkUpdateBuilder(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
Update("cycling.cyclist_name").Set("id", "user_uuid", "firstname", "stars").Where(Eq("id"))
Update("cycling.cyclist_name").Set("id", "user_uuid", "firstname", "stars").Where(Eq("id")).ToCql()
}
}