From ca3a20d492443ef0c769279abb9346b7b0fe0b2a Mon Sep 17 00:00:00 2001 From: Fedor Korotkiy Date: Tue, 18 Apr 2023 14:48:06 +0400 Subject: [PATCH] Add consistenthash task --- consistenthash/README.md | 16 ++++ consistenthash/consistenthash.go | 27 +++++++ consistenthash/hash_test.go | 132 +++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 + 5 files changed, 178 insertions(+) create mode 100644 consistenthash/README.md create mode 100644 consistenthash/consistenthash.go create mode 100644 consistenthash/hash_test.go diff --git a/consistenthash/README.md b/consistenthash/README.md new file mode 100644 index 0000000..bbebf47 --- /dev/null +++ b/consistenthash/README.md @@ -0,0 +1,16 @@ +# consistenthash + +Реализуйте consistent hashing. + +Consistent hashing - это хеш функция, которая отображает множество ключей на +множество нод (процессов в кластере). + +Пусть у вас есть `N` процессов. Можно было бы использовать функцию `hash(key) % N`, +где `hash` - какая-то "хорошая" хеш функция. Но если в систему добавить еще один процесс, +почти все ключи "переедут". Потому что `hash(key) % N != hash(key) % (N + 1)`. + +Такие "переезды" в распределённой системе могут вызывать недоступность или +просто создавать большую нагрузку. Эту проблему решает consistent hashing. +Он гарантирует, что при добавлении новой ноды, "переедут" только `~ 1/N` ключей. + +Для реализации используйте кольцо с виртуальными нодами, которое описано в [CS168: Introduction and Consistent Hashing](https://web.stanford.edu/class/cs168/l/l1.pdf) \ No newline at end of file diff --git a/consistenthash/consistenthash.go b/consistenthash/consistenthash.go new file mode 100644 index 0000000..f83ceb1 --- /dev/null +++ b/consistenthash/consistenthash.go @@ -0,0 +1,27 @@ +//go:build !solution + +package consistenthash + +type Node interface { + // ID is some persistent and unique identifier + ID() string +} + +type ConsistentHash[N Node] struct { +} + +func New[N Node]() *ConsistentHash[N] { + panic("implement me") +} + +func (h *ConsistentHash[N]) AddNode(n *N) { + panic("implement me") +} + +func (h *ConsistentHash[N]) RemoveNode(n *N) { + panic("implement me") +} + +func (h *ConsistentHash[N]) GetNode(key string) *N { + panic("implement me") +} diff --git a/consistenthash/hash_test.go b/consistenthash/hash_test.go new file mode 100644 index 0000000..aa17a7d --- /dev/null +++ b/consistenthash/hash_test.go @@ -0,0 +1,132 @@ +package consistenthash + +import ( + "fmt" + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/exp/maps" +) + +type node string + +func (n node) ID() string { return string(n) } + +func TestHash_SingleNode(t *testing.T) { + h := New[node]() + + n1 := node("1") + h.AddNode(&n1) + + require.Equal(t, &n1, h.GetNode("key0")) +} + +func TestHash_TwoNodes(t *testing.T) { + h := New[node]() + + n1 := node("1") + h.AddNode(&n1) + + n2 := node("2") + h.AddNode(&n2) + + n := h.GetNode("key0") + require.True(t, n == &n1 || n == &n2) + for i := 0; i < 32; i++ { + require.Equal(t, n, h.GetNode("key0")) + } + + differs := false + for i := 0; i < 32; i++ { + other := h.GetNode(fmt.Sprintf("key%d", i)) + if other != n { + differs = true + } + } + require.True(t, differs) +} + +func TestHash_EvenDistribution(t *testing.T) { + h := New[node]() + + const K = 32 + for i := 0; i < K; i++ { + n := node(fmt.Sprint(i)) + h.AddNode(&n) + } + + counts := map[*node]int{} + const N = 1 << 16 + for i := 0; i < N; i++ { + counts[h.GetNode(fmt.Sprintf("key%d", i))] += 1 + } + + const P = 1 / float64(K) + const variance = N * (P) * (1 - P) + stddev := math.Sqrt(variance) + t.Logf("P = %v, var = %v, stddev = %v", P, variance, stddev) + t.Logf("counts = %v", maps.Values(counts)) + + for _, count := range counts { + require.Greater(t, count, int(N/K-stddev*10)) + require.Less(t, count, int(N/K+stddev*10)) + } +} + +func TestHash_ConsistentDistribution(t *testing.T) { + h := New[node]() + + const K = 32 + for i := 0; i < K; i++ { + n := node(fmt.Sprint(i)) + h.AddNode(&n) + } + + nodes := map[string]*node{} + + const N = 1 << 16 + for i := 0; i < N; i++ { + key := fmt.Sprintf("key%d", i) + nodes[key] = h.GetNode(key) + } + + newNode := node("new_node") + h.AddNode(&newNode) + + changed := 0 + movedToNewNode := 0 + + for key, oldNode := range nodes { + n := h.GetNode(key) + if n != oldNode { + changed++ + } + + if n == &newNode { + movedToNewNode++ + } + } + + t.Logf("changed = %d, movedToNewNode = %d", changed, movedToNewNode) + assert.Less(t, changed, N/K*2) + assert.Equal(t, movedToNewNode, changed) +} + +func BenchmarkHashSpeed(b *testing.B) { + for _, K := range []int{32, 1024, 4096} { + h := New[node]() + + for i := 0; i < K; i++ { + n := node(fmt.Sprint(i)) + h.AddNode(&n) + } + + b.Run(fmt.Sprintf("K=%d", K), func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = h.GetNode(fmt.Sprintf("key%d", i)) + } + }) + } +} diff --git a/go.mod b/go.mod index 9cc1338..f62a7b7 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/go-playground/validator/v10 v10.11.2 // indirect github.com/goccy/go-json v0.10.0 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/btree v1.1.2 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.5.0 // indirect diff --git a/go.sum b/go.sum index 0daecfe..85e57bb 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,8 @@ github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9/go.mod h1:XA3DeT6rxh2 github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9/go.mod h1:0EXg4mc1CNP0HCqCz+K4ts155PXIlUywf0wqN+GfPZw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=