Skip to content

Commit 857bb84

Browse files
Dmitry Shmulevichcsmarchbanks
Dmitry Shmulevich
authored andcommitted
Add Redis support (#1612)
* adding support for Redis cache Signed-off-by: Dmitry Shmulevich <[email protected]> * fix lint errors Signed-off-by: Dmitry Shmulevich <[email protected]> * minor bugfix Signed-off-by: Dmitry Shmulevich <[email protected]> * proper use of pool connections Signed-off-by: Dmitry Shmulevich <[email protected]> * fix lint errors Signed-off-by: Dmitry Shmulevich <[email protected]> * addressed comments added unit test Signed-off-by: Dmitry Shmulevich <[email protected]> * update unit tests Signed-off-by: Dmitry Shmulevich <[email protected]> * reuse context in request timeout Signed-off-by: Dmitry Shmulevich <[email protected]> * use correct function names in logs Signed-off-by: Dmitry Shmulevich <[email protected]> * use common config for cache storage Signed-off-by: Dmitry Shmulevich <[email protected]> * added missing module dependency Signed-off-by: Dmitry Shmulevich <[email protected]> * delete extra newline Signed-off-by: Dmitry Shmulevich <[email protected]> * optimize redis SET Signed-off-by: Dmitry Shmulevich <[email protected]> * adding support for Redis cache Signed-off-by: Dmitry Shmulevich <[email protected]> * fix lint errors Signed-off-by: Dmitry Shmulevich <[email protected]> * minor bugfix Signed-off-by: Dmitry Shmulevich <[email protected]> * proper use of pool connections Signed-off-by: Dmitry Shmulevich <[email protected]> * fix lint errors Signed-off-by: Dmitry Shmulevich <[email protected]> * addressed comments added unit test Signed-off-by: Dmitry Shmulevich <[email protected]> * update unit tests Signed-off-by: Dmitry Shmulevich <[email protected]> * reuse context in request timeout Signed-off-by: Dmitry Shmulevich <[email protected]> * use correct function names in logs Signed-off-by: Dmitry Shmulevich <[email protected]> * use common config for cache storage Signed-off-by: Dmitry Shmulevich <[email protected]> * added missing module dependency Signed-off-by: Dmitry Shmulevich <[email protected]> * delete extra newline Signed-off-by: Dmitry Shmulevich <[email protected]> * optimize redis SET Signed-off-by: Dmitry Shmulevich <[email protected]> * undo common config for storage systems Signed-off-by: Dmitry Shmulevich <[email protected]> * restore changes in CHANGELOG.md Signed-off-by: Dmitry Shmulevich <[email protected]> * fixed lint error Signed-off-by: Dmitry Shmulevich <[email protected]> * fixed mod-check error Signed-off-by: Dmitry Shmulevich <[email protected]> * fix metric labels Signed-off-by: Dmitry Shmulevich <[email protected]> * addressed comments Signed-off-by: Dmitry Shmulevich <[email protected]>
1 parent 0be2dd5 commit 857bb84

34 files changed

+4858
-0
lines changed

CHANGELOG.md

+2
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22

33
* [CHANGE] In table-manager, default DynamoDB capacity was reduced from 3,000 units to 1,000 units. We recommend you do not run with the defaults: find out what figures are needed for your environment and set that via `-dynamodb.periodic-table.write-throughput` and `-dynamodb.chunk-table.write-throughput`.
44
* [CHANGE] `--alertmanager.configs.auto-slack-root` flag was dropped as auto Slack root is not supported anymore. #1597
5+
* [FEATURE] Add Redis support for caching #1612
56
* [ENHANCEMENT] Upgraded Prometheus to 2.12.0 and Alertmanager to 0.19.0. #1597
67

8+
79
## 0.2.0 / 2019-09-05
810

911
This release has several exciting features, the most notable of them being setting `-ingester.spread-flushes` to potentially reduce your storage space by upto 50%.

docs/arguments.md

+4
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,10 @@ The ingester query API was improved over time, but defaults to the old behaviour
8383

8484
Use these flags to specify the location and timeout of the memcached cluster used to cache query results.
8585

86+
- `-redis.{endpoint, timeout}`
87+
88+
Use these flags to specify the location and timeout of the Redis service used to cache query results.
89+
8690
## Distributor
8791

8892
- `-distributor.shard-by-all-labels`

go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ require (
3131
github.com/gogo/status v1.0.3
3232
github.com/golang/protobuf v1.3.2
3333
github.com/golang/snappy v0.0.1
34+
github.com/gomodule/redigo v2.0.0+incompatible
3435
github.com/gorilla/context v1.1.1 // indirect
3536
github.com/gorilla/mux v1.6.2
3637
github.com/gorilla/websocket v1.4.0 // indirect
@@ -56,6 +57,7 @@ require (
5657
github.com/prometheus/client_golang v1.1.0
5758
github.com/prometheus/common v0.7.0
5859
github.com/prometheus/prometheus v1.8.2-0.20190918104050-8744afdd1ea0
60+
github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1
5961
github.com/satori/go.uuid v1.2.0 // indirect
6062
github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e
6163
github.com/sercand/kuberesolver v2.1.0+incompatible // indirect

go.sum

+4
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,8 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
241241
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
242242
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
243243
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
244+
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
245+
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
244246
github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
245247
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
246248
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -495,6 +497,8 @@ github.com/prometheus/prometheus v0.0.0-20190818123050-43acd0e2e93f h1:7C9G4yUog
495497
github.com/prometheus/prometheus v0.0.0-20190818123050-43acd0e2e93f/go.mod h1:rMTlmxGCvukf2KMu3fClMDKLLoJ5hl61MhcJ7xKakf0=
496498
github.com/prometheus/prometheus v1.8.2-0.20190918104050-8744afdd1ea0 h1:W4dTblzSVIBNfDimJhh70OpZQQMwLVpwK50scXdH94w=
497499
github.com/prometheus/prometheus v1.8.2-0.20190918104050-8744afdd1ea0/go.mod h1:elNqjVbwD3sCZJqKzyN7uEuwGcCpeJvv67D6BrHsDbw=
500+
github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 h1:+kGqA4dNN5hn7WwvKdzHl0rdN5AEkbNZd0VjRltAiZg=
501+
github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY=
498502
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
499503
github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI=
500504
github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=

pkg/chunk/cache/cache.go

+16
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package cache
22

33
import (
44
"context"
5+
"errors"
56
"flag"
67
"time"
78
)
@@ -28,6 +29,7 @@ type Config struct {
2829
Background BackgroundConfig `yaml:"background,omitempty"`
2930
Memcache MemcachedConfig `yaml:"memcached,omitempty"`
3031
MemcacheClient MemcachedClientConfig `yaml:"memcached_client,omitempty"`
32+
Redis RedisConfig `yaml:"redis,omitempty"`
3133
Fifocache FifoCacheConfig `yaml:"fifocache,omitempty"`
3234

3335
// This is to name the cache metrics properly.
@@ -42,6 +44,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, description string, f
4244
cfg.Background.RegisterFlagsWithPrefix(prefix, description, f)
4345
cfg.Memcache.RegisterFlagsWithPrefix(prefix, description, f)
4446
cfg.MemcacheClient.RegisterFlagsWithPrefix(prefix, description, f)
47+
cfg.Redis.RegisterFlagsWithPrefix(prefix, description, f)
4548
cfg.Fifocache.RegisterFlagsWithPrefix(prefix, description, f)
4649

4750
f.BoolVar(&cfg.EnableFifoCache, prefix+"cache.enable-fifocache", false, description+"Enable in-memory cache.")
@@ -67,6 +70,10 @@ func New(cfg Config) (Cache, error) {
6770
caches = append(caches, Instrument(cfg.Prefix+"fifocache", cache))
6871
}
6972

73+
if cfg.MemcacheClient.Host != "" && cfg.Redis.Endpoint != "" {
74+
return nil, errors.New("use of multiple cache storage systems is not supported")
75+
}
76+
7077
if cfg.MemcacheClient.Host != "" {
7178
if cfg.Memcache.Expiration == 0 && cfg.DefaultValidity != 0 {
7279
cfg.Memcache.Expiration = cfg.DefaultValidity
@@ -79,6 +86,15 @@ func New(cfg Config) (Cache, error) {
7986
caches = append(caches, NewBackground(cacheName, cfg.Background, Instrument(cacheName, cache)))
8087
}
8188

89+
if cfg.Redis.Endpoint != "" {
90+
if cfg.Redis.Expiration == 0 && cfg.DefaultValidity != 0 {
91+
cfg.Redis.Expiration = cfg.DefaultValidity
92+
}
93+
cacheName := cfg.Prefix + "redis"
94+
cache := NewRedisCache(cfg.Redis, cacheName, nil)
95+
caches = append(caches, NewBackground(cacheName, cfg.Background, Instrument(cacheName, cache)))
96+
}
97+
8298
cache := NewTiered(caches)
8399
if len(caches) > 1 {
84100
cache = Instrument(cfg.Prefix+"tiered", cache)

pkg/chunk/cache/redis_cache.go

+154
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
package cache
2+
3+
import (
4+
"context"
5+
"flag"
6+
"time"
7+
8+
"github.com/cortexproject/cortex/pkg/util"
9+
"github.com/go-kit/kit/log/level"
10+
"github.com/gomodule/redigo/redis"
11+
)
12+
13+
// RedisCache type caches chunks in redis
14+
type RedisCache struct {
15+
name string
16+
expiration int
17+
timeout time.Duration
18+
pool *redis.Pool
19+
}
20+
21+
// RedisConfig defines how a RedisCache should be constructed.
22+
type RedisConfig struct {
23+
Endpoint string `yaml:"endpoint,omitempty"`
24+
Timeout time.Duration `yaml:"timeout,omitempty"`
25+
Expiration time.Duration `yaml:"expiration,omitempty"`
26+
MaxIdleConns int `yaml:"max_idle_conns,omitempty"`
27+
MaxActiveConns int `yaml:"max_active_conns,omitempty"`
28+
}
29+
30+
// RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet
31+
func (cfg *RedisConfig) RegisterFlagsWithPrefix(prefix, description string, f *flag.FlagSet) {
32+
f.StringVar(&cfg.Endpoint, prefix+"redis.endpoint", "", description+"Redis service endpoint to use when caching chunks. If empty, no redis will be used.")
33+
f.DurationVar(&cfg.Timeout, prefix+"redis.timeout", 100*time.Millisecond, description+"Maximum time to wait before giving up on redis requests.")
34+
f.DurationVar(&cfg.Expiration, prefix+"redis.expiration", 0, description+"How long keys stay in the redis.")
35+
f.IntVar(&cfg.MaxIdleConns, prefix+"redis.max-idle-conns", 80, description+"Maximum number of idle connections in pool.")
36+
f.IntVar(&cfg.MaxActiveConns, prefix+"redis.max-active-conns", 0, description+"Maximum number of active connections in pool.")
37+
}
38+
39+
// NewRedisCache creates a new RedisCache
40+
func NewRedisCache(cfg RedisConfig, name string, pool *redis.Pool) *RedisCache {
41+
// pool != nil only in unit tests
42+
if pool == nil {
43+
pool = &redis.Pool{
44+
MaxIdle: cfg.MaxIdleConns,
45+
MaxActive: cfg.MaxActiveConns,
46+
Dial: func() (redis.Conn, error) {
47+
c, err := redis.Dial("tcp", cfg.Endpoint)
48+
if err != nil {
49+
return nil, err
50+
}
51+
return c, err
52+
},
53+
}
54+
}
55+
56+
cache := &RedisCache{
57+
expiration: int(cfg.Expiration.Seconds()),
58+
timeout: cfg.Timeout,
59+
name: name,
60+
pool: pool,
61+
}
62+
63+
if err := cache.ping(context.Background()); err != nil {
64+
level.Error(util.Logger).Log("msg", "error connecting to redis", "endpoint", cfg.Endpoint, "err", err)
65+
}
66+
67+
return cache
68+
}
69+
70+
// Fetch gets keys from the cache. The keys that are found must be in the order of the keys requested.
71+
func (c *RedisCache) Fetch(ctx context.Context, keys []string) (found []string, bufs [][]byte, missed []string) {
72+
data, err := c.mget(ctx, keys)
73+
74+
if err != nil {
75+
level.Error(util.Logger).Log("msg", "failed to get from redis", "name", c.name, "err", err)
76+
missed = make([]string, len(keys))
77+
copy(missed, keys)
78+
return
79+
}
80+
for i, key := range keys {
81+
if data[i] != nil {
82+
found = append(found, key)
83+
bufs = append(bufs, data[i])
84+
} else {
85+
missed = append(missed, key)
86+
}
87+
}
88+
return
89+
}
90+
91+
// Store stores the key in the cache.
92+
func (c *RedisCache) Store(ctx context.Context, keys []string, bufs [][]byte) {
93+
err := c.mset(ctx, keys, bufs, c.expiration)
94+
if err != nil {
95+
level.Error(util.Logger).Log("msg", "failed to put to redis", "name", c.name, "err", err)
96+
}
97+
}
98+
99+
// Stop stops the redis client.
100+
func (c *RedisCache) Stop() error {
101+
return c.pool.Close()
102+
}
103+
104+
// mset adds key-value pairs to the cache.
105+
func (c *RedisCache) mset(ctx context.Context, keys []string, bufs [][]byte, ttl int) error {
106+
conn := c.pool.Get()
107+
defer conn.Close()
108+
109+
if err := conn.Send("MULTI"); err != nil {
110+
return err
111+
}
112+
for i := range keys {
113+
if err := conn.Send("SETEX", keys[i], ttl, bufs[i]); err != nil {
114+
return err
115+
}
116+
}
117+
_, err := redis.DoWithTimeout(conn, c.timeout, "EXEC")
118+
return err
119+
}
120+
121+
// mget retrieves values from the cache.
122+
func (c *RedisCache) mget(ctx context.Context, keys []string) ([][]byte, error) {
123+
intf := make([]interface{}, len(keys))
124+
for i, key := range keys {
125+
intf[i] = key
126+
}
127+
128+
conn := c.pool.Get()
129+
defer conn.Close()
130+
131+
return redis.ByteSlices(redis.DoWithTimeout(conn, c.timeout, "MGET", intf...))
132+
}
133+
134+
func (c *RedisCache) ping(ctx context.Context) error {
135+
conn := c.pool.Get()
136+
defer conn.Close()
137+
138+
pong, err := redis.DoWithTimeout(conn, c.timeout, "PING")
139+
if err == nil {
140+
_, err = redis.String(pong, err)
141+
}
142+
return err
143+
}
144+
145+
func redisStatusCode(err error) string {
146+
switch err {
147+
case nil:
148+
return "200"
149+
case redis.ErrNil:
150+
return "404"
151+
default:
152+
return "500"
153+
}
154+
}

pkg/chunk/cache/redis_cache_test.go

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
package cache_test
2+
3+
import (
4+
"context"
5+
"testing"
6+
"time"
7+
8+
"github.com/cortexproject/cortex/pkg/chunk/cache"
9+
"github.com/gomodule/redigo/redis"
10+
"github.com/rafaeljusto/redigomock"
11+
"github.com/stretchr/testify/require"
12+
)
13+
14+
func TestRedisCache(t *testing.T) {
15+
cfg := cache.RedisConfig{
16+
Timeout: 10 * time.Millisecond,
17+
}
18+
19+
conn := redigomock.NewConn()
20+
conn.Clear()
21+
pool := redis.NewPool(func() (redis.Conn, error) {
22+
return conn, nil
23+
}, 10)
24+
25+
keys := []string{"key1", "key2", "key3"}
26+
bufs := [][]byte{[]byte("data1"), []byte("data2"), []byte("data3")}
27+
miss := []string{"miss1", "miss2"}
28+
29+
// ensure input correctness
30+
nHit := len(keys)
31+
require.Len(t, bufs, nHit)
32+
33+
// mock Redis Store
34+
mockRedisStore(conn, keys, bufs)
35+
36+
//mock cache hit
37+
keyIntf := make([]interface{}, nHit)
38+
bufIntf := make([]interface{}, nHit)
39+
40+
for i := 0; i < nHit; i++ {
41+
keyIntf[i] = keys[i]
42+
bufIntf[i] = bufs[i]
43+
}
44+
conn.Command("MGET", keyIntf...).Expect(bufIntf)
45+
46+
// mock cache miss
47+
nMiss := len(miss)
48+
missIntf := make([]interface{}, nMiss)
49+
for i, s := range miss {
50+
missIntf[i] = s
51+
}
52+
conn.Command("MGET", missIntf...).ExpectError(nil)
53+
54+
// mock the cache
55+
c := cache.NewRedisCache(cfg, "mock", pool)
56+
ctx := context.Background()
57+
58+
c.Store(ctx, keys, bufs)
59+
60+
// test hits
61+
found, data, missed := c.Fetch(ctx, keys)
62+
63+
require.Len(t, found, nHit)
64+
require.Len(t, missed, 0)
65+
for i := 0; i < nHit; i++ {
66+
require.Equal(t, keys[i], found[i])
67+
require.Equal(t, bufs[i], data[i])
68+
}
69+
70+
// test misses
71+
found, _, missed = c.Fetch(ctx, miss)
72+
73+
require.Len(t, found, 0)
74+
require.Len(t, missed, nMiss)
75+
for i := 0; i < nMiss; i++ {
76+
require.Equal(t, miss[i], missed[i])
77+
}
78+
}
79+
80+
func mockRedisStore(conn *redigomock.Conn, keys []string, bufs [][]byte) {
81+
conn.Command("MULTI")
82+
ret := []interface{}{}
83+
for i := range keys {
84+
conn.Command("SETEX", keys[i], 0, bufs[i])
85+
ret = append(ret, "OK")
86+
}
87+
conn.Command("EXEC").Expect(ret)
88+
}

0 commit comments

Comments
 (0)