From 77a024a6bc78de18a49d30cf35fe42f3d8c69731 Mon Sep 17 00:00:00 2001 From: Ilya Sinelnikov Date: Thu, 9 Apr 2020 16:00:41 +0300 Subject: [PATCH] lectures: 07-sql rc1 --- go.mod | 3 + go.sum | 9 +++ lectures/07-sql/clickhouse/clickhouse.go | 29 ++++++++++ lectures/07-sql/query/query.go | 2 +- lectures/07-sql/redis/redis.go | 31 ++++++++++ lectures/07-sql/resources/conn.go | 16 ++++++ lectures/07-sql/resources/queries.go | 72 ++++++++++++++++++++++++ lectures/07-sql/resources/rows.go | 24 ++++++++ lectures/07-sql/resources/tx.go | 35 ++++++++++++ lectures/07-sql/sql.slide | 56 +++++++++++------- lectures/07-sql/sqlmock/sqlmock_test.go | 35 ++++++++++++ 11 files changed, 291 insertions(+), 21 deletions(-) create mode 100644 lectures/07-sql/clickhouse/clickhouse.go create mode 100644 lectures/07-sql/redis/redis.go create mode 100644 lectures/07-sql/resources/conn.go create mode 100644 lectures/07-sql/resources/queries.go create mode 100644 lectures/07-sql/resources/rows.go create mode 100644 lectures/07-sql/resources/tx.go create mode 100644 lectures/07-sql/sqlmock/sqlmock_test.go diff --git a/go.mod b/go.mod index 6869796..3d698f4 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,9 @@ module gitlab.com/slon/shad-go go 1.13 require ( + github.com/ClickHouse/clickhouse-go v1.4.0 // indirect + github.com/DATA-DOG/go-sqlmock v1.4.1 + github.com/go-redis/redis v6.15.7+incompatible // indirect github.com/go-resty/resty/v2 v2.1.0 github.com/gofrs/uuid v3.2.0+incompatible github.com/golang/mock v1.4.1 diff --git a/go.sum b/go.sum index 0081b44..9d99c56 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,16 @@ cloud.google.com/go v0.0.0-20170206221025-ce650573d812/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/ClickHouse/clickhouse-go v1.4.0 h1:cC1DEZ1TL74QviZY4svlwow84X5r7/BGd78kf18swhI= +github.com/ClickHouse/clickhouse-go v1.4.0/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM= +github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190129172621-c8b1d7a94ddf/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo= github.com/aclements/go-gg v0.0.0-20170118225347-6dbb4e4fefb0/go.mod h1:55qNq4vcpkIuHowELi5C8e+1yUHtoLoOUR9QU5j7Tes= github.com/aclements/go-moremath v0.0.0-20161014184102-0ff62e0875ff/go.mod h1:idZL3yvz4kzx1dsBOAC+oYv6L92P1oFEhUXUB1A/lwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -17,6 +23,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U= +github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-resty/resty/v2 v2.1.0 h1:Z6IefCpUMfnvItVJaJXWv/pMiiD11So35QgwEELsldE= github.com/go-resty/resty/v2 v2.1.0/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -113,6 +121,7 @@ github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOq github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/lectures/07-sql/clickhouse/clickhouse.go b/lectures/07-sql/clickhouse/clickhouse.go new file mode 100644 index 0000000..55be4e5 --- /dev/null +++ b/lectures/07-sql/clickhouse/clickhouse.go @@ -0,0 +1,29 @@ +package clickhouse + +import ( + "context" + "database/sql" + + _ "github.com/ClickHouse/clickhouse-go" +) + +func Example(ctx context.Context) { + db, _ := sql.Open("clickhouse", "tcp://127.0.0.1:9000?debug=true") + defer db.Close() + + // Начало батча + tx, _ := db.BeginTx(ctx, nil) + defer tx.Rollback() + + // Описание батча + stmt, _ := tx.PrepareContext(ctx, "INSERT INTO example (id) VALUES (?)") + defer stmt.Close() + + // Добавление данных + for i := 0; i < 100; i++ { + _, _ = stmt.ExecContext(ctx, i) + } + + // Отправка батча в ClickHouse + _ = tx.Commit() +} diff --git a/lectures/07-sql/query/query.go b/lectures/07-sql/query/query.go index 7dac48b..8e42774 100644 --- a/lectures/07-sql/query/query.go +++ b/lectures/07-sql/query/query.go @@ -7,7 +7,7 @@ import ( ) func Query(ctx context.Context, db *sql.DB) { - rows, err := db.QueryContext(ctx, "SELECT id, name FROM users WHERE id = $1", 1) + rows, err := db.QueryContext(ctx, "SELECT id, name FROM users") if err != nil { log.Fatal(err) } diff --git a/lectures/07-sql/redis/redis.go b/lectures/07-sql/redis/redis.go new file mode 100644 index 0000000..d20643e --- /dev/null +++ b/lectures/07-sql/redis/redis.go @@ -0,0 +1,31 @@ +package redis + +import ( + "log" + "time" + + "github.com/go-redis/redis" +) + +func Example() { + rdb := redis.NewUniversalClient(&redis.UniversalOptions{ + MasterName: "master", + Addrs: []string{":26379"}, + }) + defer rdb.Close() + + if err := rdb.Ping(); err != nil { + log.Fatal(err) + } + + if err := rdb.Set("key", "value", time.Hour).Err(); err != nil { + log.Fatal(err) + } + + value, err := rdb.Get("key").Result() + if err != nil { + log.Fatal(err) + } + + log.Println(value) +} diff --git a/lectures/07-sql/resources/conn.go b/lectures/07-sql/resources/conn.go new file mode 100644 index 0000000..c854606 --- /dev/null +++ b/lectures/07-sql/resources/conn.go @@ -0,0 +1,16 @@ +package resources + +import ( + "context" + "database/sql" + "log" +) + +func ConnExhaust(ctx context.Context, db *sql.DB) { + c, err := db.Conn(ctx) + if err != nil { + log.Fatal(err) + } + + _ = c.PingContext(ctx) +} diff --git a/lectures/07-sql/resources/queries.go b/lectures/07-sql/resources/queries.go new file mode 100644 index 0000000..6d26293 --- /dev/null +++ b/lectures/07-sql/resources/queries.go @@ -0,0 +1,72 @@ +package resources + +import ( + "context" + "database/sql" + "log" +) + +func QueryDeadlock(ctx context.Context, db *sql.DB) { + rows, _ := db.QueryContext(ctx, "SELECT id, name FROM users") + defer rows.Close() + + for rows.Next() { + var id int + var name string + _ = rows.Scan(&id, &name) + + rowsAddrs, _ := db.QueryContext( + ctx, + "SELECT address FROM addresses WHERE user_id = $1", + id, + ) + defer rowsAddrs.Close() + + var addr string + _ = rowsAddrs.Scan(&addr) + + log.Println(id, name, addr) + } +} + +func QueryDeadlockFixOne(ctx context.Context, db *sql.DB) { + type Res struct { + ID int + Name string + Addr string + } + var values []Res + rows, _ := db.QueryContext(ctx, "SELECT id, name FROM users") + + for rows.Next() { + var res Res + _ = rows.Scan(&res.ID, &res.Name) + values = append(values, res) + } + rows.Close() + + for _, v := range values { + _ = db.QueryRowContext( + ctx, + "SELECT address FROM addresses WHERE user_id = $1", + v.ID, + ).Scan(&v.Addr) + log.Println(v) + } +} + +func QueryDeadlockFixTwo(ctx context.Context, db *sql.DB) { + rows, _ := db.QueryContext( + ctx, + "SELECT u.id, u.name, a.address FROM users AS u, addresses as a WHERE u.id == a.user_id", + ) + defer rows.Close() + + for rows.Next() { + var id int + var name string + var addr string + _ = rows.Scan(&id, &name, &addr) + log.Println(id, name, addr) + } +} diff --git a/lectures/07-sql/resources/rows.go b/lectures/07-sql/resources/rows.go new file mode 100644 index 0000000..0538604 --- /dev/null +++ b/lectures/07-sql/resources/rows.go @@ -0,0 +1,24 @@ +package resources + +import ( + "context" + "database/sql" + "log" +) + +func RowsExhaust(ctx context.Context, db *sql.DB) { + rows, err := db.QueryContext(ctx, "SELECT id, name FROM users") + if err != nil { + log.Fatal(err) + } + + if rows.Next() { + var id int + var name string + if err := rows.Scan(&id, &name); err != nil { + log.Fatal(err) + } + + log.Println(id, name) + } +} diff --git a/lectures/07-sql/resources/tx.go b/lectures/07-sql/resources/tx.go new file mode 100644 index 0000000..0607ba8 --- /dev/null +++ b/lectures/07-sql/resources/tx.go @@ -0,0 +1,35 @@ +package resources + +import ( + "context" + "database/sql" + "log" +) + +func TxExhaust(ctx context.Context, db *sql.DB) { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + log.Println(err) + return + } + + if _, err = tx.ExecContext(ctx, `UPDATE users SET name = "Surl/Tesh-echer" WHERE id = 1`); err != nil { + log.Println(err) + return + } + + if err = tx.Commit(); err != nil { + log.Println(err) + } +} + +func TxDeadlock(ctx context.Context, db *sql.DB) { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + log.Fatal(err) + } + defer tx.Rollback() + + _, _ = db.QueryContext(ctx, "SELECT id, name FROM users") + _, _ = db.QueryContext(ctx, "SELECT id, name FROM users") +} diff --git a/lectures/07-sql/sql.slide b/lectures/07-sql/sql.slide index 8489d7b..df3a159 100644 --- a/lectures/07-sql/sql.slide +++ b/lectures/07-sql/sql.slide @@ -101,19 +101,33 @@ database/sql * Типовые ошибки и подводные камни -* Deadlock +* Запросы - deadlock -- ПРИМЕР много SQL вызовов подряд без вычитки данных +.code resources/queries.go /^func QueryDeadlock/,/^}/ -* Исчерпание ресурсов +* Запросы - deadlock - fix 1 -- ПРИМЕР незакрытие транзакций -- ПРИМЕР незакрытие rows -- ПРИМЕР неотдача коннектов +.code resources/queries.go /^func QueryDeadlockFixOne/,/^}/ -* Рефлексия +* Запросы - deadlock - fix 2 -ПРИМЕР ColumnType +.code resources/queries.go /^func QueryDeadlockFixTwo/,/^}/ + +* Запросы - исчерпание ресурсов + +.code resources/rows.go /^func RowsExhaust/,/^}/ + +* Транзакции - исчерпание ресурсов + +.code resources/tx.go /^func TxExhaust/,/^}/ + +* Транзакции - deadlock + +.code resources/tx.go /^func TxDeadlock/,/^}/ + +* Коннекты - исчерпание ресурсов + +.code resources/conn.go /^func ConnExhaust/,/^}/ * Удобства и расширения @@ -160,22 +174,24 @@ database/sql .code sqlx/named.go /^func Insert/,/^}/ -* GORM - -- ORM которая умеет всё -- ORM которая не умеет ничего - -* Моки - -Здесь будет пара слайдов о github.com/DATA-DOG/go-sqlmock - * Нетипичные драйверы -- драйвер к кликхаусу использует транзакции как батчинг (ЗДЕСЬ БУДЕТ ПРИМЕР) +* Нетипичные драйверы - github.com/DATA-DOG/go-sqlmock -* Пример не-sql драйвера +.play sqlmock/sqlmock_test.go /^func TestSelect/,/^}/ -ПРИМЕР go-redis +* Нетипичные драйверы - github.com/ClickHouse/clickhouse-go + +.code clickhouse/clickhouse.go /^func Example/,/^}/ + +* Не-SQL драйверы - Redis + +.code redis/redis.go /^func Example/,/^}/ + +* ORM + +- github.com/jinzhu/gorm +- github.com/go-pg/pg * Популярные базы данных и их драйверы diff --git a/lectures/07-sql/sqlmock/sqlmock_test.go b/lectures/07-sql/sqlmock/sqlmock_test.go new file mode 100644 index 0000000..a9e1c81 --- /dev/null +++ b/lectures/07-sql/sqlmock/sqlmock_test.go @@ -0,0 +1,35 @@ +package sqlmock + +import ( + "database/sql" + "log" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/require" +) + +func TestSelect(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + log.Fatal(err) + } + defer db.Close() + + mock.ExpectBegin() + mock.ExpectExec("SELECT name FROM users WHERE id = ?"). + WithArgs(1). + WillReturnError(sql.ErrNoRows) + mock.ExpectCommit() + + tx, err := db.Begin() + require.NoError(t, err) + + _, err = db.Exec("SELECT name FROM users WHERE id = ?", 1) + require.NotNil(t, err) + require.Equal(t, err, sql.ErrNoRows) + + require.NoError(t, tx.Commit()) + + require.NoError(t, mock.ExpectationsWereMet()) +}