From e99cb2d12f84d6bac12b03db5736d8a5210cde4d Mon Sep 17 00:00:00 2001 From: Tass Date: Wed, 21 Aug 2024 17:17:14 +0700 Subject: [PATCH] fix(lib): bug redis-go-cluster cannot convert a string type Signed-off-by: anhdt9 --- src/full_check/client/client.go | 79 +++++++++++++++++++++++--------- src/full_check/common/cluster.go | 14 ++++-- src/full_check/go.mod | 15 +++--- src/full_check/go.sum | 21 +++++++++ src/full_check/main.go | 65 +++++++++++++++++++++----- 5 files changed, 147 insertions(+), 47 deletions(-) diff --git a/src/full_check/client/client.go b/src/full_check/client/client.go index 84444a1..49c1b49 100644 --- a/src/full_check/client/client.go +++ b/src/full_check/client/client.go @@ -1,19 +1,19 @@ package client import ( + "errors" "fmt" "io" "net" + "reflect" "strconv" "strings" "time" - "errors" "full_check/common" "github.com/gomodule/redigo/redis" - redigoCluster "github.com/najoast/redis-go-cluster" - "reflect" + redigoCluster "github.com/tasszz2k/redis-go-cluster" ) var ( @@ -94,8 +94,13 @@ func (p *RedisClient) Connect() error { if p.redisHost.TimeoutMs == 0 { p.conn, err = redis.Dial("tcp", p.redisHost.Addr[0]) } else { - p.conn, err = redis.DialTimeout("tcp", p.redisHost.Addr[0], time.Millisecond*time.Duration(p.redisHost.TimeoutMs), - time.Millisecond*time.Duration(p.redisHost.TimeoutMs), time.Millisecond*time.Duration(p.redisHost.TimeoutMs)) + p.conn, err = redis.DialTimeout( + "tcp", + p.redisHost.Addr[0], + time.Millisecond*time.Duration(p.redisHost.TimeoutMs), + time.Millisecond*time.Duration(p.redisHost.TimeoutMs), + time.Millisecond*time.Duration(p.redisHost.TimeoutMs), + ) } } else { // cluster @@ -108,7 +113,8 @@ func (p *RedisClient) Connect() error { KeepAlive: 16, AliveTime: 60 * time.Second, Password: p.redisHost.Password, - }) + }, + ) if err == nil { p.conn = common.NewClusterConn(cluster, 0) } @@ -122,6 +128,10 @@ func (p *RedisClient) Connect() error { for _, arg := range strings.Split(p.redisHost.Password, ":") { args = append(args, arg) } + if p.conn == nil { + + return fmt.Errorf("connect host[%v] failed: unknown", p.redisHost.Addr) + } if _, err := p.conn.Do(p.redisHost.Authtype, args...); err != nil { return err } @@ -179,7 +189,7 @@ type combine struct { } func (c combine) String() string { - all := make([]string, 0, len(c.params) + 1) + all := make([]string, 0, len(c.params)+1) all = append(all, c.command) for _, ele := range c.params { all = append(all, string(ele.([]byte))) @@ -187,7 +197,10 @@ func (c combine) String() string { return strings.Join(all, " ") } -func (p *RedisClient) PipeRawCommand(commands []combine, specialErrorPrefix string) ([]interface{}, error) { +func (p *RedisClient) PipeRawCommand(commands []combine, specialErrorPrefix string) ( + []interface{}, + error, +) { if len(commands) == 0 { common.Logger.Warnf("input commands length is 0") return nil, emptyError @@ -270,8 +283,10 @@ func (p *RedisClient) PipeTypeCommand(keyInfo []*common.Key) ([]string, error) { if v, ok := ele.(string); ok { result[i] = v } else { - err := fmt.Errorf("run PipeRawCommand with commands[%s] return element[%v] isn't type string[%v]", - printCombinList(commands), ele, reflect.TypeOf(ele)) + err := fmt.Errorf( + "run PipeRawCommand with commands[%s] return element[%v] isn't type string[%v]", + printCombinList(commands), ele, reflect.TypeOf(ele), + ) common.Logger.Error(err) return nil, err } @@ -299,8 +314,10 @@ func (p *RedisClient) PipeExistsCommand(keyInfo []*common.Key) ([]int64, error) if v, ok := ele.(int64); ok { result[i] = v } else { - err := fmt.Errorf("run PipeRawCommand with commands[%s] return element[%v] isn't type int64[%v]", - printCombinList(commands), ele, reflect.TypeOf(ele)) + err := fmt.Errorf( + "run PipeRawCommand with commands[%s] return element[%v] isn't type int64[%v]", + printCombinList(commands), ele, reflect.TypeOf(ele), + ) common.Logger.Error(err) return nil, err } @@ -328,8 +345,10 @@ func (p *RedisClient) PipeLenCommand(keyInfo []*common.Key) ([]int64, error) { if v, ok := ele.(int64); ok { result[i] = v } else { - err := fmt.Errorf("run PipeRawCommand with commands[%s] return element[%v] isn't type int64[%v]", - printCombinList(commands), ele, reflect.TypeOf(ele)) + err := fmt.Errorf( + "run PipeRawCommand with commands[%s] return element[%v] isn't type int64[%v]", + printCombinList(commands), ele, reflect.TypeOf(ele), + ) common.Logger.Error(err) return nil, err } @@ -357,8 +376,10 @@ func (p *RedisClient) PipeTTLCommand(keyInfo []*common.Key) ([]bool, error) { if v, ok := ele.(int64); ok { result[i] = v == 0 } else { - err := fmt.Errorf("run PipeRawCommand with commands[%s] return element[%v] isn't type int64[%v]", - printCombinList(commands), ele, reflect.TypeOf(ele)) + err := fmt.Errorf( + "run PipeRawCommand with commands[%s] return element[%v] isn't type int64[%v]", + printCombinList(commands), ele, reflect.TypeOf(ele), + ) common.Logger.Error(err) return nil, err } @@ -443,7 +464,10 @@ func (p *RedisClient) PipeZscoreCommand(key []byte, field [][]byte) ([]interface } } -func (p *RedisClient) FetchValueUseScan_Hash_Set_SortedSet(oneKeyInfo *common.Key, onceScanCount int) (map[string][]byte, error) { +func (p *RedisClient) FetchValueUseScan_Hash_Set_SortedSet( + oneKeyInfo *common.Key, + onceScanCount int, +) (map[string][]byte, error) { var scanCmd string switch oneKeyInfo.Tp { case common.HashKeyType: @@ -465,14 +489,18 @@ func (p *RedisClient) FetchValueUseScan_Hash_Set_SortedSet(oneKeyInfo *common.Ke replyList, ok := reply.([]interface{}) if ok == false || len(replyList) != 2 { - return nil, fmt.Errorf("%s %s %d count %d failed, result: %+v", scanCmd, string(oneKeyInfo.Key), - cursor, onceScanCount, reply) + return nil, fmt.Errorf( + "%s %s %d count %d failed, result: %+v", scanCmd, string(oneKeyInfo.Key), + cursor, onceScanCount, reply, + ) } cursorBytes, ok := replyList[0].([]byte) if ok == false { - return nil, fmt.Errorf("%s %s %d count %d failed, result: %+v", scanCmd, string(oneKeyInfo.Key), - cursor, onceScanCount, reply) + return nil, fmt.Errorf( + "%s %s %d count %d failed, result: %+v", scanCmd, string(oneKeyInfo.Key), + cursor, onceScanCount, reply, + ) } cursor, err = strconv.Atoi(string(cursorBytes)) @@ -482,7 +510,14 @@ func (p *RedisClient) FetchValueUseScan_Hash_Set_SortedSet(oneKeyInfo *common.Ke keylist, ok := replyList[1].([]interface{}) if ok == false { - panic(common.Logger.Criticalf("%s %s failed, result: %+v", scanCmd, string(oneKeyInfo.Key), reply)) + panic( + common.Logger.Criticalf( + "%s %s failed, result: %+v", + scanCmd, + string(oneKeyInfo.Key), + reply, + ), + ) } switch oneKeyInfo.Tp { case common.HashKeyType: diff --git a/src/full_check/common/cluster.go b/src/full_check/common/cluster.go index 620cce4..edc844c 100644 --- a/src/full_check/common/cluster.go +++ b/src/full_check/common/cluster.go @@ -1,11 +1,11 @@ package common import ( - redigoCluster "github.com/najoast/redis-go-cluster" redigo "github.com/gomodule/redigo/redis" + redigoCluster "github.com/tasszz2k/redis-go-cluster" ) -const( +const ( RecvChanSize = 4096 ) @@ -80,7 +80,11 @@ func (cc *ClusterConn) Flush() error { retLength := len(ret) availableSize := cap(cc.recvChan) - len(cc.recvChan) if availableSize < retLength { - Logger.Warnf("available channel size[%v] less than current returned batch size[%v]", availableSize, retLength) + Logger.Warnf( + "available channel size[%v] less than current returned batch size[%v]", + availableSize, + retLength, + ) } // Logger.Debugf("cluster flush batch with size[%v], return replies size[%v]", cc.batcher.GetBatchSize(), retLength) @@ -96,6 +100,6 @@ func (cc *ClusterConn) Flush() error { // read recvChan func (cc *ClusterConn) Receive() (reply interface{}, err error) { - ret := <- cc.recvChan + ret := <-cc.recvChan return ret.answer, ret.err -} \ No newline at end of file +} diff --git a/src/full_check/go.mod b/src/full_check/go.mod index d3cc12a..fab88b2 100644 --- a/src/full_check/go.mod +++ b/src/full_check/go.mod @@ -4,19 +4,18 @@ go 1.17 require ( github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 - github.com/gomodule/redigo v1.8.9 + github.com/gomodule/redigo v1.9.2 github.com/gugemichael/nimo4go v0.0.0-20210413043712-ccb2ff0d7b40 - github.com/jessevdk/go-flags v1.5.0 - github.com/jinzhu/copier v0.3.5 - github.com/mattn/go-sqlite3 v1.14.16 - github.com/najoast/redis-go-cluster v1.0.0 - github.com/stretchr/testify v1.8.1 - github.com/vinllen/redis-go-cluster v1.0.0 + github.com/jessevdk/go-flags v1.6.1 + github.com/jinzhu/copier v0.4.0 + github.com/mattn/go-sqlite3 v1.14.22 + github.com/stretchr/testify v1.9.0 + github.com/tasszz2k/redis-go-cluster v1.0.1 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 // indirect + golang.org/x/sys v0.24.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/src/full_check/go.sum b/src/full_check/go.sum index 23529b9..a93537f 100644 --- a/src/full_check/go.sum +++ b/src/full_check/go.sum @@ -1,19 +1,29 @@ github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= +github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s= +github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw= github.com/gugemichael/nimo4go v0.0.0-20210413043712-ccb2ff0d7b40 h1:6TWAiHVyKs75ZHEn7XtVv7SO7M4rHwvY/5Tf7xdJBkc= github.com/gugemichael/nimo4go v0.0.0-20210413043712-ccb2ff0d7b40/go.mod h1:ibO7uKpO8fOH/bKD4trmwm5tHhHKiAjC0u288Rd+GnI= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= +github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= +github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/najoast/redis-go-cluster v1.0.0 h1:GJhtiwitgaQ0Kc9ZcRE9FJCcu1GLCIIW7u7vpRrgE6k= github.com/najoast/redis-go-cluster v1.0.0/go.mod h1:lGMMsVLZW+0gAuA+oo1YrFTZjjaIhkmhR6cA77/etiw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -21,10 +31,21 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tasszz2k/redis-go-cluster v0.0.0-20240821095154-d37e4abd9000 h1:/Wyh4ul9BuLAFVHR6q0IShdqCLXar4P9laq7o/Jg97Y= +github.com/tasszz2k/redis-go-cluster v0.0.0-20240821095154-d37e4abd9000/go.mod h1:7xhUY5NIwXz7HLYC6uBToiTBbuO9vhjaelWeK6P1/58= +github.com/tasszz2k/redis-go-cluster v1.0.1 h1:250/pJBRLZde3qREvTPcTWqYZVjr/9ypQqx4vSNEHJc= +github.com/tasszz2k/redis-go-cluster v1.0.1/go.mod h1:7xhUY5NIwXz7HLYC6uBToiTBbuO9vhjaelWeK6P1/58= github.com/vinllen/redis-go-cluster v1.0.0/go.mod h1:xig5hQAOZX1K+KNUVDqAbhTRzMTPcb257nJl7OCHrI4= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/full_check/main.go b/src/full_check/main.go index 1b9ec3a..27666b8 100644 --- a/src/full_check/main.go +++ b/src/full_check/main.go @@ -6,14 +6,14 @@ import ( "strconv" "strings" - "full_check/configure" - "full_check/full_check" "full_check/checker" "full_check/client" "full_check/common" + "full_check/configure" + "full_check/full_check" - "github.com/jessevdk/go-flags" "github.com/gugemichael/nimo4go" + "github.com/jessevdk/go-flags" ) var VERSION = "$" @@ -66,28 +66,58 @@ func main() { compareCount, err := strconv.Atoi(conf.Opts.CompareTimes) if err != nil || compareCount < 1 { - panic(common.Logger.Errorf("invalid option cmpcount %s, expect int >=1", conf.Opts.CompareTimes)) + panic( + common.Logger.Errorf( + "invalid option cmpcount %s, expect int >=1", + conf.Opts.CompareTimes, + ), + ) } if conf.Opts.Interval < 0 { - panic(common.Logger.Errorf("invalid option interval %d, expect int >=0", conf.Opts.Interval)) + panic( + common.Logger.Errorf( + "invalid option interval %d, expect int >=0", + conf.Opts.Interval, + ), + ) } batchCount, err := strconv.Atoi(conf.Opts.BatchCount) if err != nil || batchCount < 1 || batchCount > 10000 { - panic(common.Logger.Errorf("invalid option batchcount %s, expect int 1<=batchcount<=10000", conf.Opts.BatchCount)) + panic( + common.Logger.Errorf( + "invalid option batchcount %s, expect int 1<=batchcount<=10000", + conf.Opts.BatchCount, + ), + ) } parallel := conf.Opts.Parallel if parallel < 1 || parallel > 100 { - panic(common.Logger.Errorf("invalid option parallel %d, expect 1<=parallel<=100", conf.Opts.Parallel)) + panic( + common.Logger.Errorf( + "invalid option parallel %d, expect 1<=parallel<=100", + conf.Opts.Parallel, + ), + ) } qps := conf.Opts.Qps if qps < 1 || qps > 5000000 { panic(common.Logger.Errorf("invalid option qps %d, expect 1<=qps<=5000000", conf.Opts.Qps)) } if conf.Opts.SourceAuthType != "auth" && conf.Opts.SourceAuthType != "adminauth" { - panic(common.Logger.Errorf("invalid sourceauthtype %s, expect auth/adminauth", conf.Opts.SourceAuthType)) + panic( + common.Logger.Errorf( + "invalid sourceauthtype %s, expect auth/adminauth", + conf.Opts.SourceAuthType, + ), + ) } if conf.Opts.TargetAuthType != "auth" && conf.Opts.TargetAuthType != "adminauth" { - panic(common.Logger.Errorf("invalid targetauthtype %s, expect auth/adminauth", conf.Opts.TargetAuthType)) + panic( + common.Logger.Errorf( + "invalid targetauthtype %s, expect auth/adminauth", + conf.Opts.TargetAuthType, + ), + ) } if conf.Opts.CompareMode < full_check.FullValue || conf.Opts.CompareMode > full_check.FullValueWithOutline { panic(common.Logger.Errorf("invalid compare mode %d", conf.Opts.CompareMode)) @@ -100,7 +130,11 @@ func main() { common.BigKeyThreshold = conf.Opts.BigKeyThreshold } - sourceAddressList, err := client.HandleAddress(conf.Opts.SourceAddr, conf.Opts.SourcePassword, conf.Opts.SourceAuthType) + sourceAddressList, err := client.HandleAddress( + conf.Opts.SourceAddr, + conf.Opts.SourcePassword, + conf.Opts.SourceAuthType, + ) if err != nil { panic(common.Logger.Errorf("source address[%v] illegal[%v]", conf.Opts.SourceAddr, err)) } else if len(sourceAddressList) > 1 && conf.Opts.SourceDBType != 1 { @@ -109,7 +143,11 @@ func main() { panic(common.Logger.Errorf("input source address is empty")) } - targetAddressList, err := client.HandleAddress(conf.Opts.TargetAddr, conf.Opts.TargetPassword, conf.Opts.TargetAuthType) + targetAddressList, err := client.HandleAddress( + conf.Opts.TargetAddr, + conf.Opts.TargetPassword, + conf.Opts.TargetAuthType, + ) if err != nil { panic(common.Logger.Errorf("target address[%v] illegal[%v]", conf.Opts.TargetAddr, err)) } else if len(targetAddressList) > 1 && conf.Opts.TargetDBType != 1 { @@ -167,6 +205,9 @@ func main() { common.Logger.Info("configuration: ", conf.Opts) common.Logger.Info("---------") - fullCheck := full_check.NewFullCheck(fullCheckParameter, full_check.CheckType(conf.Opts.CompareMode)) + fullCheck := full_check.NewFullCheck( + fullCheckParameter, + full_check.CheckType(conf.Opts.CompareMode), + ) fullCheck.Start() }