diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c432a131..561f07bca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## jackal - main / unreleased * [ENHANCEMENT] Added memory ballast. #198 +* [ENHANCEMENT] Added support for Redis cached repository. #202 * [CHANGE] Introduced measured repository transaction type. #200 * [CHANGE] Use PgSQL locker. #201 * [BUGFIX] Fix S2S db key check when nop KV is used. #199 diff --git a/config/example.config.yaml b/config/example.config.yaml index df002539e..2fc26f28b 100644 --- a/config/example.config.yaml +++ b/config/example.config.yaml @@ -32,6 +32,11 @@ storage: database: jackal max_open_conns: 16 + cache: + type: redis + redis: + addr: localhost:6379 + cluster: etcd: endpoints: diff --git a/go.mod b/go.mod index 7608e624c..8e76156ae 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/cockroachdb/errors v1.8.4 github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 github.com/go-kit/log v0.2.0 + github.com/go-redis/redis/v8 v8.11.4 github.com/go-sql-driver/mysql v1.5.0 // indirect github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.1.2 @@ -31,12 +32,13 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect github.com/cockroachdb/redact v1.0.8 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -57,7 +59,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.17.0 // indirect - golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d // indirect + golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 // indirect golang.org/x/text v0.3.6 // indirect google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect diff --git a/go.sum b/go.sum index 7356e24ae..57534e794 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,9 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -84,6 +85,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= @@ -98,6 +101,7 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -118,9 +122,12 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= @@ -163,8 +170,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -297,12 +305,18 @@ github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= @@ -476,8 +490,9 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d h1:BgJvlyh+UqCUaPlscHJ+PN8GcpfrFdr7NHjd1JL0+Gs= golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -520,9 +535,11 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -562,6 +579,7 @@ golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -627,6 +645,7 @@ gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/R gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/storage/cached/cached.go b/pkg/storage/cached/cached.go new file mode 100644 index 000000000..8d57cc171 --- /dev/null +++ b/pkg/storage/cached/cached.go @@ -0,0 +1,123 @@ +// Copyright 2021 The jackal Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cachedrepository + +import ( + "context" + "fmt" + + kitlog "github.com/go-kit/log" + "github.com/go-kit/log/level" + rediscache "github.com/ortuman/jackal/pkg/storage/cached/redis" + "github.com/ortuman/jackal/pkg/storage/repository" +) + +// Config contains cached repository configuration. +type Config struct { + Type string + Redis rediscache.Config +} + +// Cache defines cache store interface. +type Cache interface { + // Type identifies underlying cache store type. + Type() string + + // Get retrieves k value from the cache store. + // If not present nil will be returned. + Get(ctx context.Context, k string) ([]byte, error) + + // Put stores a value into the cache store. + Put(ctx context.Context, k string, val []byte) error + + // Del removes k value from the cache store. + Del(ctx context.Context, k string) error + + // HasKey tells whether k is present in the cache store. + HasKey(ctx context.Context, k string) (bool, error) + + // Start starts Cache component. + Start(ctx context.Context) error + + // Stop stops Cache component. + Stop(ctx context.Context) error +} + +// CachedRepository is cached Repository implementation. +type CachedRepository struct { + repository.User + repository.Last + repository.Capabilities + repository.Offline + repository.BlockList + repository.Private + repository.Roster + repository.VCard + repository.Locker + + rep repository.Repository + + cache Cache + logger kitlog.Logger +} + +// New returns a new initialized CachedRepository instance. +func New(cfg Config, rep repository.Repository, logger kitlog.Logger) (repository.Repository, error) { + if cfg.Type != rediscache.Type { + return nil, fmt.Errorf("unrecognized repository cache type: %s", cfg.Type) + } + c := rediscache.New(cfg.Redis) + + return &CachedRepository{ + User: &cachedUserRep{c: c, rep: rep}, + Last: rep, + Capabilities: rep, + Offline: rep, + BlockList: rep, + Private: rep, + Roster: rep, + VCard: rep, + Locker: rep, + rep: rep, + cache: c, + logger: logger, + }, nil +} + +// InTransaction generates a repository transaction and completes it after it's being used by f function. +// In case f returns no error tx transaction will be committed. +func (c *CachedRepository) InTransaction(ctx context.Context, f func(ctx context.Context, tx repository.Transaction) error) error { + return c.rep.InTransaction(ctx, func(ctx context.Context, tx repository.Transaction) error { + return f(ctx, newCacheTx(c.cache, tx)) + }) +} + +// Start starts cached repository component. +func (c *CachedRepository) Start(ctx context.Context) error { + if err := c.cache.Start(ctx); err != nil { + return err + } + level.Info(c.logger).Log("msg", "started cached repository", "type", c.cache.Type()) + return c.rep.Start(ctx) +} + +// Stop stops cached repository component. +func (c *CachedRepository) Stop(ctx context.Context) error { + if err := c.cache.Stop(ctx); err != nil { + return err + } + level.Info(c.logger).Log("msg", "stopped cached repository", "type", c.cache.Type()) + return c.rep.Stop(ctx) +} diff --git a/pkg/storage/cached/interface.go b/pkg/storage/cached/interface.go new file mode 100644 index 000000000..1101db817 --- /dev/null +++ b/pkg/storage/cached/interface.go @@ -0,0 +1,32 @@ +// Copyright 2021 The jackal Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cachedrepository + +import "github.com/ortuman/jackal/pkg/storage/repository" + +//go:generate moq -out cache.mock_test.go . cache:cacheMock +type cache interface { + Cache +} + +//go:generate moq -out codec.mock_test.go . cacheCodec:codecMock +type cacheCodec interface { + codec +} + +//go:generate moq -out repository.mock_test.go . globalRepository:repositoryMock +type globalRepository interface { + repository.Repository +} diff --git a/pkg/storage/cached/op.go b/pkg/storage/cached/op.go new file mode 100644 index 000000000..8ced0fb07 --- /dev/null +++ b/pkg/storage/cached/op.go @@ -0,0 +1,87 @@ +// Copyright 2021 The jackal Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cachedrepository + +import ( + "context" +) + +type codec interface { + encode(i interface{}) ([]byte, error) + decode([]byte) error + value() interface{} +} + +type existsOp struct { + c Cache + key string + missFn func(context.Context) (bool, error) +} + +func (op existsOp) do(ctx context.Context) (bool, error) { + ok, err := op.c.HasKey(ctx, op.key) + if err != nil { + return false, err + } + if ok { + return true, nil + } + return op.missFn(ctx) +} + +type updateOp struct { + c Cache + key string + updateFn func(context.Context) error +} + +func (op updateOp) do(ctx context.Context) error { + if err := op.c.Del(ctx, op.key); err != nil { + return err + } + return op.updateFn(ctx) +} + +type fetchOp struct { + c Cache + key string + codec codec + missFn func(context.Context) (interface{}, error) +} + +func (op fetchOp) do(ctx context.Context) (interface{}, error) { + b, err := op.c.Get(ctx, op.key) + if err != nil { + return nil, err + } + if b == nil { + obj, err := op.missFn(ctx) + if err != nil { + return nil, err + } + b, err = op.codec.encode(obj) + if err != nil { + return nil, err + } + if err := op.c.Put(ctx, op.key, b); err != nil { + return nil, err + } + return obj, nil + } + if err := op.codec.decode(b); err != nil { + return nil, err + } + return op.codec.value(), nil +} diff --git a/pkg/storage/cached/op_test.go b/pkg/storage/cached/op_test.go new file mode 100644 index 000000000..0d2a768b5 --- /dev/null +++ b/pkg/storage/cached/op_test.go @@ -0,0 +1,136 @@ +// Copyright 2021 The jackal Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cachedrepository + +import ( + "context" + "reflect" + "testing" + + usermodel "github.com/ortuman/jackal/pkg/model/user" + "github.com/stretchr/testify/require" +) + +func TestCachedRepository_ExistsOp(t *testing.T) { + // given + cacheMock := &cacheMock{} + cacheMock.HasKeyFunc = func(ctx context.Context, k string) (bool, error) { + if k == "k0" { + return true, nil + } + return false, nil + } + missFn := func(context.Context) (bool, error) { + return false, nil + } + + // when + op0 := existsOp{c: cacheMock, key: "k0", missFn: missFn} + op1 := existsOp{c: cacheMock, key: "k1", missFn: missFn} + + ok0, _ := op0.do(context.Background()) + ok1, _ := op1.do(context.Background()) + + // then + require.True(t, ok0) + require.False(t, ok1) +} + +func TestCachedRepository_UpdateOp(t *testing.T) { + // given + var output string + + cacheMock := &cacheMock{} + cacheMock.DelFunc = func(ctx context.Context, k string) error { + output += "del" + return nil + } + updateFn := func(context.Context) error { + output += ":rep_update" + return nil + } + + // when + op := updateOp{c: cacheMock, key: "k0", updateFn: updateFn} + _ = op.do(context.Background()) + + // then + require.Equal(t, "del:rep_update", output) // ensure proper order +} + +func TestCachedRepository_FetchOpHit(t *testing.T) { + // given + cacheMock := &cacheMock{} + cacheMock.GetFunc = func(ctx context.Context, k string) ([]byte, error) { + return []byte{255}, nil + } + + c := &codecMock{} + c.decodeFunc = func(bytes []byte) error { + return nil + } + var usr usermodel.User + c.valueFunc = func() interface{} { + return &usr + } + + var missed bool + op := fetchOp{ + c: cacheMock, + codec: c, + missFn: func(ctx context.Context) (interface{}, error) { + missed = true + return nil, nil + }, + } + + // when + v, _ := op.do(context.Background()) + + // then + require.False(t, missed) + require.True(t, reflect.DeepEqual(v, &usr)) +} + +func TestCachedRepository_FetchOpMiss(t *testing.T) { + // given + cacheMock := &cacheMock{} + cacheMock.GetFunc = func(ctx context.Context, k string) ([]byte, error) { + return nil, nil + } + cacheMock.PutFunc = func(ctx context.Context, k string, val []byte) error { + return nil + } + + c := &codecMock{} + c.encodeFunc = func(i interface{}) ([]byte, error) { + return []byte{255}, nil + } + + var usr usermodel.User + op := fetchOp{ + c: cacheMock, + codec: c, + missFn: func(ctx context.Context) (interface{}, error) { + return &usr, nil + }, + } + + // when + v, _ := op.do(context.Background()) + + // then + require.True(t, reflect.DeepEqual(v, &usr)) +} diff --git a/pkg/storage/cached/redis/cache.go b/pkg/storage/cached/redis/cache.go new file mode 100644 index 000000000..0ed99eb8a --- /dev/null +++ b/pkg/storage/cached/redis/cache.go @@ -0,0 +1,105 @@ +// Copyright 2021 The jackal Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rediscache + +import ( + "context" + "errors" + "time" + + "github.com/go-redis/redis/v8" +) + +// Type is redis type identifier. +const Type = "redis" + +// Config contains Redis cache configuration. +type Config struct { + Addr string `fig:"addr"` + Username string `fig:"username"` + Password string `fig:"password"` + DB int `fig:"db"` + DialTimeout time.Duration `fig:"dial_timeout" default:"3s"` + ReadTimeout time.Duration `fig:"read_timeout" default:"5s"` + WriteTimeout time.Duration `fig:"write_timeout" default:"5s"` + TTL time.Duration `fig:"ttl" default:"24h"` +} + +// Cache is Redis cache implementation. +type Cache struct { + rdb *redis.Client + ttl time.Duration +} + +// New creates and returns an initialized Redis Cache instance. +func New(cfg Config) *Cache { + rdb := redis.NewClient(&redis.Options{ + Addr: cfg.Addr, + Username: cfg.Username, + Password: cfg.Password, + DB: cfg.DB, + DialTimeout: cfg.DialTimeout, + ReadTimeout: cfg.ReadTimeout, + WriteTimeout: cfg.WriteTimeout, + }) + return &Cache{ + rdb: rdb, + ttl: cfg.TTL, + } +} + +// Type satisfies Cache interface. +func (c *Cache) Type() string { return Type } + +// Get satisfies Cache interface. +func (c *Cache) Get(ctx context.Context, k string) ([]byte, error) { + val, err := c.rdb.Get(ctx, k).Result() + if err != nil { + if errors.Is(err, redis.Nil) { + return nil, nil + } + return nil, err + } + return []byte(val), nil +} + +// Put satisfies Cache interface. +func (c *Cache) Put(ctx context.Context, k string, val []byte) error { + return c.rdb.Set(ctx, k, val, c.ttl).Err() +} + +// Del satisfies Cache interface. +func (c *Cache) Del(ctx context.Context, k string) error { + return c.rdb.Del(ctx, k).Err() +} + +// HasKey satisfies Cache interface. +func (c *Cache) HasKey(ctx context.Context, k string) (bool, error) { + v, err := c.rdb.Exists(ctx, k).Result() + if err != nil { + return false, err + } + return v == 1, nil +} + +// Start satisfies Cache interface. +func (c *Cache) Start(ctx context.Context) error { + return c.rdb.Ping(ctx).Err() +} + +// Stop satisfies Cache interface. +func (c *Cache) Stop(_ context.Context) error { + return c.rdb.Close() +} diff --git a/pkg/storage/cached/tx.go b/pkg/storage/cached/tx.go new file mode 100644 index 000000000..b70d141e7 --- /dev/null +++ b/pkg/storage/cached/tx.go @@ -0,0 +1,45 @@ +// Copyright 2021 The jackal Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cachedrepository + +import ( + "github.com/ortuman/jackal/pkg/storage/repository" +) + +type cachedTx struct { + repository.User + repository.Last + repository.Capabilities + repository.Offline + repository.BlockList + repository.Private + repository.Roster + repository.VCard + repository.Locker +} + +func newCacheTx(c Cache, tx repository.Transaction) *cachedTx { + return &cachedTx{ + User: &cachedUserRep{c: c, rep: tx}, + Last: tx, + Capabilities: tx, + Offline: tx, + BlockList: tx, + Private: tx, + Roster: tx, + VCard: tx, + Locker: tx, + } +} diff --git a/pkg/storage/cached/user.go b/pkg/storage/cached/user.go new file mode 100644 index 000000000..8fc32e3b4 --- /dev/null +++ b/pkg/storage/cached/user.go @@ -0,0 +1,103 @@ +// Copyright 2021 The jackal Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cachedrepository + +import ( + "context" + "fmt" + + "github.com/golang/protobuf/proto" + usermodel "github.com/ortuman/jackal/pkg/model/user" + "github.com/ortuman/jackal/pkg/storage/repository" +) + +type userCodec struct { + val *usermodel.User +} + +func (c *userCodec) encode(i interface{}) ([]byte, error) { + return proto.Marshal(i.(*usermodel.User)) +} + +func (c *userCodec) decode(b []byte) error { + var usr usermodel.User + if err := proto.Unmarshal(b, &usr); err != nil { + return err + } + c.val = &usr + return nil +} + +func (c *userCodec) value() interface{} { + return c.val +} + +type cachedUserRep struct { + c Cache + rep repository.User +} + +func (c *cachedUserRep) UpsertUser(ctx context.Context, user *usermodel.User) error { + op := updateOp{ + c: c.c, + key: userKey(user.Username), + updateFn: func(ctx context.Context) error { + return c.rep.UpsertUser(ctx, user) + }, + } + return op.do(ctx) +} + +func (c *cachedUserRep) DeleteUser(ctx context.Context, username string) error { + op := updateOp{ + c: c.c, + key: userKey(username), + updateFn: func(ctx context.Context) error { + return c.rep.DeleteUser(ctx, username) + }, + } + return op.do(ctx) +} + +func (c *cachedUserRep) FetchUser(ctx context.Context, username string) (*usermodel.User, error) { + op := fetchOp{ + c: c.c, + key: userKey(username), + codec: &userCodec{}, + missFn: func(ctx context.Context) (interface{}, error) { + return c.rep.FetchUser(ctx, username) + }, + } + v, err := op.do(ctx) + if err != nil { + return nil, err + } + return v.(*usermodel.User), nil +} + +func (c *cachedUserRep) UserExists(ctx context.Context, username string) (bool, error) { + op := existsOp{ + c: c.c, + key: userKey(username), + missFn: func(ctx context.Context) (bool, error) { + return c.rep.UserExists(ctx, username) + }, + } + return op.do(ctx) +} + +func userKey(username string) string { + return fmt.Sprintf("usr:%s", username) +} diff --git a/pkg/storage/cached/user_test.go b/pkg/storage/cached/user_test.go new file mode 100644 index 000000000..fcc69b0df --- /dev/null +++ b/pkg/storage/cached/user_test.go @@ -0,0 +1,149 @@ +// Copyright 2021 The jackal Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cachedrepository + +import ( + "context" + "testing" + + usermodel "github.com/ortuman/jackal/pkg/model/user" + "github.com/stretchr/testify/require" +) + +func TestCachedUserRep_UpsertUser(t *testing.T) { + // given + var cacheKey string + + cacheMock := &cacheMock{} + cacheMock.DelFunc = func(ctx context.Context, k string) error { + cacheKey = k + return nil + } + + repMock := &repositoryMock{} + repMock.UpsertUserFunc = func(ctx context.Context, user *usermodel.User) error { + return nil + } + + // when + rep := cachedUserRep{ + c: cacheMock, + rep: repMock, + } + err := rep.UpsertUser(context.Background(), &usermodel.User{Username: "u1"}) + + // then + require.NoError(t, err) + require.Equal(t, userKey("u1"), cacheKey) + require.Len(t, repMock.UpsertUserCalls(), 1) +} + +func TestCachedUserRep_DeleteUser(t *testing.T) { + // given + var cacheKey string + + cacheMock := &cacheMock{} + cacheMock.DelFunc = func(ctx context.Context, k string) error { + cacheKey = k + return nil + } + + repMock := &repositoryMock{} + repMock.DeleteUserFunc = func(ctx context.Context, username string) error { + return nil + } + + // when + rep := cachedUserRep{ + c: cacheMock, + rep: repMock, + } + err := rep.DeleteUser(context.Background(), "u1") + + // then + require.NoError(t, err) + require.Equal(t, userKey("u1"), cacheKey) + require.Len(t, repMock.DeleteUserCalls(), 1) +} + +func TestCachedUserRep_FetchUser(t *testing.T) { + // given + cacheMock := &cacheMock{} + cacheMock.GetFunc = func(ctx context.Context, k string) ([]byte, error) { + return nil, nil + } + cacheMock.PutFunc = func(ctx context.Context, k string, val []byte) error { + return nil + } + + repMock := &repositoryMock{} + repMock.FetchUserFunc = func(ctx context.Context, username string) (*usermodel.User, error) { + return &usermodel.User{Username: "u1"}, nil + } + + // when + rep := cachedUserRep{ + c: cacheMock, + rep: repMock, + } + usr, err := rep.FetchUser(context.Background(), "u1") + + // then + require.NotNil(t, usr) + require.NoError(t, err) + + require.Equal(t, "u1", usr.Username) + + require.Len(t, cacheMock.GetCalls(), 1) + require.Len(t, cacheMock.PutCalls(), 1) + require.Len(t, repMock.FetchUserCalls(), 1) +} + +func TestCachedUserRep_UserExists(t *testing.T) { + // given + cacheMock := &cacheMock{} + cacheMock.HasKeyFunc = func(ctx context.Context, k string) (bool, error) { + if k == userKey("u1") { + return true, nil + } + return false, nil + } + + repMock := &repositoryMock{} + repMock.UserExistsFunc = func(ctx context.Context, username string) (bool, error) { + return username == "u2", nil + } + + // when + rep := cachedUserRep{ + c: cacheMock, + rep: repMock, + } + ok1, err1 := rep.UserExists(context.Background(), "u1") + ok2, err2 := rep.UserExists(context.Background(), "u2") + ok3, err3 := rep.UserExists(context.Background(), "u3") + + // then + require.True(t, ok1) + require.NoError(t, err1) + + require.True(t, ok2) + require.NoError(t, err2) + + require.False(t, ok3) + require.NoError(t, err3) + + require.Len(t, repMock.UserExistsCalls(), 2) +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index b2ab96c3c..e07860001 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -18,10 +18,9 @@ import ( "fmt" kitlog "github.com/go-kit/log" - + cachedrepository "github.com/ortuman/jackal/pkg/storage/cached" measuredrepository "github.com/ortuman/jackal/pkg/storage/measured" pgsqlrepository "github.com/ortuman/jackal/pkg/storage/pgsql" - "github.com/ortuman/jackal/pkg/storage/repository" ) @@ -29,8 +28,9 @@ const pgSQLRepositoryType = "pgsql" // Config contains generic storage configuration. type Config struct { - Type string `fig:"type" default:"pgsql"` - PgSQL pgsqlrepository.Config `fig:"pgsql"` + Type string `fig:"type" default:"pgsql"` + PgSQL pgsqlrepository.Config `fig:"pgsql"` + Cache cachedrepository.Config `fig:"cache"` } // New returns an initialized repository.Repository derived from cfg configuration. @@ -38,6 +38,15 @@ func New(cfg Config, logger kitlog.Logger) (repository.Repository, error) { if cfg.Type != pgSQLRepositoryType { return nil, fmt.Errorf("unrecognized repository type: %s", cfg.Type) } - rep := pgsqlrepository.New(cfg.PgSQL, logger) + var rep repository.Repository + + rep = pgsqlrepository.New(cfg.PgSQL, logger) + if len(cfg.Cache.Type) > 0 { + var err error + rep, err = cachedrepository.New(cfg.Cache, rep, logger) + if err != nil { + return nil, err + } + } return measuredrepository.New(rep), nil }