Skip to content

Commit 3ee79b2

Browse files
authored
fix: keydb consistent hashing (#6403)
# Description Updating keydb to use new client with consistent hashing. ## Linear Ticket Fixes [PIPE-2437](https://linear.app/rudderstack/issue/PIPE-2437/keydb-fix-flaky-test) ## Security - [x] The code changed/added as part of this pull request won't create any security issues with how the software is being used.
1 parent 7628efa commit 3ee79b2

File tree

4 files changed

+9
-5
lines changed

4 files changed

+9
-5
lines changed

go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ require (
8787
github.com/rudderlabs/analytics-go v3.3.3+incompatible
8888
github.com/rudderlabs/bing-ads-go-sdk v0.2.3
8989
github.com/rudderlabs/compose-test v0.1.3
90-
github.com/rudderlabs/keydb v1.1.0
90+
github.com/rudderlabs/keydb v1.2.0
9191
github.com/rudderlabs/rudder-go-kit v0.62.0
9292
github.com/rudderlabs/rudder-observability-kit v0.0.5
9393
github.com/rudderlabs/rudder-schemas v0.7.0
@@ -183,6 +183,7 @@ require (
183183
github.com/beorn7/perks v1.0.1 // indirect
184184
github.com/bitfield/gotestdox v0.2.2 // indirect
185185
github.com/bits-and-blooms/bitset v1.14.3 // indirect
186+
github.com/buraksezer/consistent v0.10.0 // indirect
186187
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
187188
github.com/cespare/xxhash/v2 v2.3.0 // indirect
188189
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect

go.sum

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,8 @@ github.com/bufbuild/httplb v0.4.1 h1:f8dMp7tx2aJfMX2UcOId1A58QDiBag7Dv6BA1OtV/YA
373373
github.com/bufbuild/httplb v0.4.1/go.mod h1:9XDjl/3UvlkOQUKthLlKn92C1/1SuZ3UCiekxZbenck=
374374
github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY=
375375
github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE=
376+
github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW25rrZlU=
377+
github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw=
376378
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
377379
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
378380
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
@@ -1195,8 +1197,8 @@ github.com/rudderlabs/compose-test v0.1.3 h1:uyep6jDCIF737sfv4zIaMsKRQKX95IDz5Xb
11951197
github.com/rudderlabs/compose-test v0.1.3/go.mod h1:tuvS1eQdSfwOYv1qwyVAcpdJxPLQXJgy5xGDd/9XmMg=
11961198
github.com/rudderlabs/goqu/v10 v10.3.1 h1:rnfX+b4EwBWQ2UQfIGeEW299JBBkK5biEbnf7Kq4/Gg=
11971199
github.com/rudderlabs/goqu/v10 v10.3.1/go.mod h1:LH2vI5gGHBxEQuESqFyk5ZA2anGINc8o25hbidDWOYw=
1198-
github.com/rudderlabs/keydb v1.1.0 h1:ycuQOZVxqzKrkXq4CoRhPBGlYks2WDNg7LaQqlS7FHM=
1199-
github.com/rudderlabs/keydb v1.1.0/go.mod h1:BXBXBGS06wr1twtd+mTyQJCxlR4kMq3CzkzUShkgkuU=
1200+
github.com/rudderlabs/keydb v1.2.0 h1:LMwWezUh3C+xheNirHNMhUlcb09ZH0Alo6bDDwwMaNQ=
1201+
github.com/rudderlabs/keydb v1.2.0/go.mod h1:ZYouneft71uF85OTUGS5cI9DoVdsfKDItgUTGNli9Mk=
12001202
github.com/rudderlabs/parquet-go v0.0.3 h1:/zgRj929pGKHsthc0kw8stVEcFu1JUcpxDRlhxjSLic=
12011203
github.com/rudderlabs/parquet-go v0.0.3/go.mod h1:WmwBOdvwpXl2aZGRk3NxxgzC/DaWGfax3jrCRhKhtSo=
12021204
github.com/rudderlabs/rudder-go-kit v0.62.0 h1:7wieb60KcKTmsMW0Ag7UwYeAEaG4pzPP6/21VhY5/es=

services/dedup/dedup_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ func Test_KeyDB(t *testing.T) {
169169
found, err := d.Allowed(kvs...)
170170
require.NoError(t, err)
171171
for _, kv := range kvs {
172-
require.True(t, found[kv])
172+
require.True(t, found[kv], kv)
173173
}
174174
err = d.Commit([]string{"e", "f", "g"})
175175
require.NoError(t, err)
@@ -570,6 +570,7 @@ func startKeydb(t testing.TB, conf *config.Config) {
570570
address := "localhost:" + strconv.Itoa(freePort)
571571
conf.Set("KeyDB.Dedup.Addresses", address)
572572
conf.Set("KeyDB.Dedup.RetryCount", 3)
573+
conf.Set("BadgerDB.Dedup.Path", t.TempDir())
573574

574575
nodeConfig := keydb.Config{
575576
NodeID: 0,

services/dedup/keydb/keydb.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func NewKeyDB(conf *config.Config, stat stats.Stats, log logger.Logger) (types.D
3333

3434
clientConfig := client.Config{
3535
Addresses: strings.Split(nodeAddresses, ","),
36-
TotalHashRanges: uint32(conf.GetInt("KeyDB.Dedup.TotalHashRanges", 128)),
36+
TotalHashRanges: uint32(conf.GetInt("KeyDB.Dedup.TotalHashRanges", int(client.DefaultTotalHashRanges))),
3737
RetryPolicy: client.RetryPolicy{
3838
Disabled: conf.GetBool("KeyDB.Dedup.RetryPolicy.Disabled", false),
3939
InitialInterval: conf.GetDuration("KeyDB.Dedup.RetryPolicy.InitialInterval", 100, time.Millisecond),

0 commit comments

Comments
 (0)