Add consistenthash task
This commit is contained in:
parent
91fdec209b
commit
ca3a20d492
5 changed files with 178 additions and 0 deletions
16
consistenthash/README.md
Normal file
16
consistenthash/README.md
Normal file
|
@ -0,0 +1,16 @@
|
|||
# consistenthash
|
||||
|
||||
Реализуйте consistent hashing.
|
||||
|
||||
Consistent hashing - это хеш функция, которая отображает множество ключей на
|
||||
множество нод (процессов в кластере).
|
||||
|
||||
Пусть у вас есть `N` процессов. Можно было бы использовать функцию `hash(key) % N`,
|
||||
где `hash` - какая-то "хорошая" хеш функция. Но если в систему добавить еще один процесс,
|
||||
почти все ключи "переедут". Потому что `hash(key) % N != hash(key) % (N + 1)`.
|
||||
|
||||
Такие "переезды" в распределённой системе могут вызывать недоступность или
|
||||
просто создавать большую нагрузку. Эту проблему решает consistent hashing.
|
||||
Он гарантирует, что при добавлении новой ноды, "переедут" только `~ 1/N` ключей.
|
||||
|
||||
Для реализации используйте кольцо с виртуальными нодами, которое описано в [CS168: Introduction and Consistent Hashing](https://web.stanford.edu/class/cs168/l/l1.pdf)
|
27
consistenthash/consistenthash.go
Normal file
27
consistenthash/consistenthash.go
Normal file
|
@ -0,0 +1,27 @@
|
|||
//go:build !solution
|
||||
|
||||
package consistenthash
|
||||
|
||||
type Node interface {
|
||||
// ID is some persistent and unique identifier
|
||||
ID() string
|
||||
}
|
||||
|
||||
type ConsistentHash[N Node] struct {
|
||||
}
|
||||
|
||||
func New[N Node]() *ConsistentHash[N] {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (h *ConsistentHash[N]) AddNode(n *N) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (h *ConsistentHash[N]) RemoveNode(n *N) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (h *ConsistentHash[N]) GetNode(key string) *N {
|
||||
panic("implement me")
|
||||
}
|
132
consistenthash/hash_test.go
Normal file
132
consistenthash/hash_test.go
Normal file
|
@ -0,0 +1,132 @@
|
|||
package consistenthash
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/exp/maps"
|
||||
)
|
||||
|
||||
type node string
|
||||
|
||||
func (n node) ID() string { return string(n) }
|
||||
|
||||
func TestHash_SingleNode(t *testing.T) {
|
||||
h := New[node]()
|
||||
|
||||
n1 := node("1")
|
||||
h.AddNode(&n1)
|
||||
|
||||
require.Equal(t, &n1, h.GetNode("key0"))
|
||||
}
|
||||
|
||||
func TestHash_TwoNodes(t *testing.T) {
|
||||
h := New[node]()
|
||||
|
||||
n1 := node("1")
|
||||
h.AddNode(&n1)
|
||||
|
||||
n2 := node("2")
|
||||
h.AddNode(&n2)
|
||||
|
||||
n := h.GetNode("key0")
|
||||
require.True(t, n == &n1 || n == &n2)
|
||||
for i := 0; i < 32; i++ {
|
||||
require.Equal(t, n, h.GetNode("key0"))
|
||||
}
|
||||
|
||||
differs := false
|
||||
for i := 0; i < 32; i++ {
|
||||
other := h.GetNode(fmt.Sprintf("key%d", i))
|
||||
if other != n {
|
||||
differs = true
|
||||
}
|
||||
}
|
||||
require.True(t, differs)
|
||||
}
|
||||
|
||||
func TestHash_EvenDistribution(t *testing.T) {
|
||||
h := New[node]()
|
||||
|
||||
const K = 32
|
||||
for i := 0; i < K; i++ {
|
||||
n := node(fmt.Sprint(i))
|
||||
h.AddNode(&n)
|
||||
}
|
||||
|
||||
counts := map[*node]int{}
|
||||
const N = 1 << 16
|
||||
for i := 0; i < N; i++ {
|
||||
counts[h.GetNode(fmt.Sprintf("key%d", i))] += 1
|
||||
}
|
||||
|
||||
const P = 1 / float64(K)
|
||||
const variance = N * (P) * (1 - P)
|
||||
stddev := math.Sqrt(variance)
|
||||
t.Logf("P = %v, var = %v, stddev = %v", P, variance, stddev)
|
||||
t.Logf("counts = %v", maps.Values(counts))
|
||||
|
||||
for _, count := range counts {
|
||||
require.Greater(t, count, int(N/K-stddev*10))
|
||||
require.Less(t, count, int(N/K+stddev*10))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHash_ConsistentDistribution(t *testing.T) {
|
||||
h := New[node]()
|
||||
|
||||
const K = 32
|
||||
for i := 0; i < K; i++ {
|
||||
n := node(fmt.Sprint(i))
|
||||
h.AddNode(&n)
|
||||
}
|
||||
|
||||
nodes := map[string]*node{}
|
||||
|
||||
const N = 1 << 16
|
||||
for i := 0; i < N; i++ {
|
||||
key := fmt.Sprintf("key%d", i)
|
||||
nodes[key] = h.GetNode(key)
|
||||
}
|
||||
|
||||
newNode := node("new_node")
|
||||
h.AddNode(&newNode)
|
||||
|
||||
changed := 0
|
||||
movedToNewNode := 0
|
||||
|
||||
for key, oldNode := range nodes {
|
||||
n := h.GetNode(key)
|
||||
if n != oldNode {
|
||||
changed++
|
||||
}
|
||||
|
||||
if n == &newNode {
|
||||
movedToNewNode++
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("changed = %d, movedToNewNode = %d", changed, movedToNewNode)
|
||||
assert.Less(t, changed, N/K*2)
|
||||
assert.Equal(t, movedToNewNode, changed)
|
||||
}
|
||||
|
||||
func BenchmarkHashSpeed(b *testing.B) {
|
||||
for _, K := range []int{32, 1024, 4096} {
|
||||
h := New[node]()
|
||||
|
||||
for i := 0; i < K; i++ {
|
||||
n := node(fmt.Sprint(i))
|
||||
h.AddNode(&n)
|
||||
}
|
||||
|
||||
b.Run(fmt.Sprintf("K=%d", K), func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = h.GetNode(fmt.Sprintf("key%d", i))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
1
go.mod
1
go.mod
|
@ -44,6 +44,7 @@ require (
|
|||
github.com/go-playground/validator/v10 v10.11.2 // indirect
|
||||
github.com/goccy/go-json v0.10.0 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/google/btree v1.1.2 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
|
||||
github.com/jackc/pgconn v1.5.0 // indirect
|
||||
|
|
2
go.sum
2
go.sum
|
@ -136,6 +136,8 @@ github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9/go.mod h1:XA3DeT6rxh2
|
|||
github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9/go.mod h1:0EXg4mc1CNP0HCqCz+K4ts155PXIlUywf0wqN+GfPZw=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
|
||||
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
|
|
Loading…
Reference in a new issue