diff --git a/make/harbor.yml.tmpl b/make/harbor.yml.tmpl index 7c953b9c61a..264fa4dd1bc 100644 --- a/make/harbor.yml.tmpl +++ b/make/harbor.yml.tmpl @@ -174,6 +174,17 @@ _version: 2.7.0 # password: notary_server_db_password # ssl_mode: disable +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 + # Uncomment external_redis if using external Redis server # external_redis: # # support redis, redis+sentinel @@ -190,6 +201,10 @@ _version: 2.7.0 # chartmuseum_db_index: 3 # trivy_db_index: 5 # idle_timeout_seconds: 30 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. # uaa: diff --git a/make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja b/make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja index 3be0173c1c6..44aa37d902f 100644 --- a/make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja +++ b/make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja @@ -371,6 +371,49 @@ external_database: # ssl_mode: disable {% endif %} +{% if redis is defined %} +redis: +# # db_index 0 is for core, it's unchangeable +{% if redis.registry_db_index is defined %} + registry_db_index: {{ redis.registry_db_index }} +{% else %} +# # registry_db_index: 1 +{% endif %} +{% if redis.jobservice_db_index is defined %} + jobservice_db_index: {{ redis.jobservice_db_index }} +{% else %} +# # jobservice_db_index: 2 +{% endif %} +{% if redis.trivy_db_index is defined %} + trivy_db_index: {{ redis.trivy_db_index }} +{% else %} +# # trivy_db_index: 5 +{% endif %} +{% if redis.harbor_db_index is defined %} + harbor_db_index: {{ redis.harbor_db_index }} +{% else %} +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +{% endif %} +{% if redis.cache_layer_db_index is defined %} + cache_layer_db_index: {{ redis.cache_layer_db_index }} +{% else %} +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 +{% endif %} +{% else %} +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 +{% endif %} + {% if external_redis is defined %} external_redis: # support redis, redis+sentinel @@ -387,6 +430,18 @@ external_redis: chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }} trivy_db_index: 5 idle_timeout_seconds: 30 + {% if external_redis.harbor_db_index is defined %} + harbor_db_index: {{ redis.harbor_db_index }} + {% else %} +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 + {% endif %} + {% if external_redis.cache_layer_db_index is defined %} + cache_layer_db_index: {{ redis.cache_layer_db_index }} + {% else %} +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 + {% endif %} {% else %} # Umcomments external_redis if using external Redis server # external_redis: @@ -404,6 +459,10 @@ external_redis: # chartmuseum_db_index: 3 # trivy_db_index: 5 # idle_timeout_seconds: 30 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 {% endif %} {% if uaa is defined %} diff --git a/make/photon/prepare/templates/core/env.jinja b/make/photon/prepare/templates/core/env.jinja index 5ea5d3734bf..2023f6f74ca 100644 --- a/make/photon/prepare/templates/core/env.jinja +++ b/make/photon/prepare/templates/core/env.jinja @@ -1,6 +1,9 @@ CONFIG_PATH=/etc/core/app.conf UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem _REDIS_URL_CORE={{redis_url_core}} +{% if redis_url_harbor %} +_REDIS_URL_HARBOR={{redis_url_harbor}} +{% endif %} SYNC_QUOTA=true CHART_CACHE_DRIVER={{chart_cache_driver}} _REDIS_URL_REG={{redis_url_reg}} @@ -88,6 +91,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }} {% endif %} {% if cache.enabled %} +{% if redis_url_cache_layer %} +_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}} +{% endif %} CACHE_ENABLED=true CACHE_EXPIRE_HOURS={{ cache.expire_hours }} {% endif %} diff --git a/make/photon/prepare/templates/jobservice/env.jinja b/make/photon/prepare/templates/jobservice/env.jinja index 31e039ff233..24f835841fc 100644 --- a/make/photon/prepare/templates/jobservice/env.jinja +++ b/make/photon/prepare/templates/jobservice/env.jinja @@ -50,6 +50,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }} {% if cache.enabled %} _REDIS_URL_CORE={{redis_url_core}} +{% if redis_url_cache_layer %} +_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}} +{% endif %} CACHE_ENABLED=true CACHE_EXPIRE_HOURS={{ cache.expire_hours }} {% endif %} \ No newline at end of file diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index d0052b8966c..5e53533aec0 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -312,7 +312,7 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu config_dict['external_database'] = False # update redis configs - config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy)) + config_dict.update(get_redis_configs(configs.get("redis", None), configs.get("external_redis", None), with_trivy)) # auto generated secret string for core config_dict['core_secret'] = generate_random_string(16) @@ -407,7 +407,7 @@ def get_redis_url_param(redis=None): return "" -def get_redis_configs(external_redis=None, with_trivy=True): +def get_redis_configs(internal_redis=None, external_redis=None, with_trivy=True): """Returns configs for redis >>> get_redis_configs()['external_redis'] @@ -440,6 +440,8 @@ def get_redis_configs(external_redis=None, with_trivy=True): >>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False) True """ + + internal_redis = internal_redis or {} external_redis = external_redis or {} configs = dict(external_redis=bool(external_redis)) @@ -455,14 +457,23 @@ def get_redis_configs(external_redis=None, with_trivy=True): 'idle_timeout_seconds': 30, } - # overwriting existing keys by external_redis - redis.update({key: value for (key, value) in external_redis.items() if value}) + if len(internal_redis) > 0: + # overwriting existing keys by internal_redis + redis.update({key: value for (key, value) in internal_redis.items() if value}) + else: + # overwriting existing keys by external_redis + redis.update({key: value for (key, value) in external_redis.items() if value}) configs['redis_url_core'] = get_redis_url(0, redis) configs['redis_url_chart'] = get_redis_url(redis['chartmuseum_db_index'], redis) configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis) configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis) + if redis.get('harbor_db_index'): + configs['redis_url_harbor'] = get_redis_url(redis['harbor_db_index'], redis) + if redis.get('cache_layer_db_index'): + configs['redis_url_cache_layer'] = get_redis_url(redis['cache_layer_db_index'], redis) + if with_trivy: configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis) diff --git a/src/core/main.go b/src/core/main.go index d8e68f8852c..4f904e0c32e 100755 --- a/src/core/main.go +++ b/src/core/main.go @@ -124,25 +124,35 @@ func main() { web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName web.BConfig.MaxMemory = 1 << 35 // (32GB) web.BConfig.MaxUploadSize = 1 << 35 // (32GB) - - redisURL := os.Getenv("_REDIS_URL_CORE") - if len(redisURL) > 0 { - u, err := url.Parse(redisURL) + // the core db used for beego session + redisCoreURL := os.Getenv("_REDIS_URL_CORE") + if len(redisCoreURL) > 0 { + _, err := url.Parse(redisCoreURL) if err != nil { - panic("bad _REDIS_URL") + panic("bad _REDIS_URL_CORE") } - + // configure the beego session redis web.BConfig.WebConfig.Session.SessionProvider = session.HarborProviderName - web.BConfig.WebConfig.Session.SessionProviderConfig = redisURL + web.BConfig.WebConfig.Session.SessionProviderConfig = redisCoreURL + } - log.Info("initializing cache ...") - if err := cache.Initialize(u.Scheme, redisURL); err != nil { - log.Fatalf("failed to initialize cache: %v", err) - } - // when config/db init function is called, the cache is not ready, - // enable config cache explicitly when the cache is ready - dbCfg.EnableConfigCache() + log.Info("initializing cache ...") + // the harbor db used for harbor business, use core db if not specified + redisHarborURL := os.Getenv("_REDIS_URL_HARBOR") + if redisHarborURL == "" { + redisHarborURL = redisCoreURL } + u, err := url.Parse(redisHarborURL) + if err != nil { + panic("bad _REDIS_URL_HARBOR") + } + if err := cache.Initialize(u.Scheme, redisHarborURL); err != nil { + log.Fatalf("failed to initialize cache: %v", err) + } + // when config/db init function is called, the cache is not ready, + // enable config cache explicitly when the cache is ready + dbCfg.EnableConfigCache() + web.AddTemplateExt("htm") log.Info("initializing configurations...") diff --git a/src/jobservice/main.go b/src/jobservice/main.go index 318d1129e1f..092190b5027 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -19,8 +19,6 @@ import ( "errors" "flag" "fmt" - "net/url" - "os" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/jobservice/common/utils" @@ -29,7 +27,6 @@ import ( "github.com/goharbor/harbor/src/jobservice/job/impl" "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/jobservice/runtime" - "github.com/goharbor/harbor/src/lib/cache" cfgLib "github.com/goharbor/harbor/src/lib/config" tracelib "github.com/goharbor/harbor/src/lib/trace" _ "github.com/goharbor/harbor/src/pkg/accessory/model/base" @@ -44,21 +41,6 @@ func main() { panic(fmt.Sprintf("failed to load configuration, error: %v", err)) } - // init cache if cache layer enabled - // gc needs to delete artifact by artifact manager, but the artifact cache store in - // core redis db so here require core redis url and init default cache. - if cfgLib.CacheEnabled() { - cacheURL := os.Getenv("_REDIS_URL_CORE") - u, err := url.Parse(cacheURL) - if err != nil { - panic("bad _REDIS_URL_CORE") - } - - if err = cache.Initialize(u.Scheme, cacheURL); err != nil { - panic(fmt.Sprintf("failed to initialize cache: %v", err)) - } - } - // Get parameters configPath := flag.String("c", "", "Specify the yaml config file path") flag.Parse() diff --git a/src/lib/cache/cache.go b/src/lib/cache/cache.go index 1e0dae56cc6..02a7b09264c 100644 --- a/src/lib/cache/cache.go +++ b/src/lib/cache/cache.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "net/url" + "os" "sync" "time" @@ -137,3 +138,37 @@ func Initialize(typ, addr string) error { func Default() Cache { return cache } + +var ( + // cacheLayer is the global cache layer cache instance. + cacheLayer Cache + // cacheLayerOnce is the once condition for initializing instance. + cacheLayerOnce sync.Once +) + +// LayerCache is the global cache instance for cache layer. +func LayerCache() Cache { + // parse the redis url for cache layer, use the default cache if not specify + redisCacheURL := os.Getenv("_REDIS_URL_CACHE_LAYER") + if redisCacheURL == "" { + if cache != nil { + return cache + } + // use the core url if cache layer url not found + redisCacheURL = os.Getenv("_REDIS_URL_CORE") + } + + u, err := url.Parse(redisCacheURL) + if err != nil { + log.Fatal("failed to parse the redis url for cache layer, bad _REDIS_URL_CACHE_LAYER") + } + + cacheLayerOnce.Do(func() { + cacheLayer, err = New(u.Scheme, Address(redisCacheURL), Prefix("cache:")) + if err != nil { + log.Fatalf("failed to initialize cache for cache layer, err: %v", err) + } + }) + + return cacheLayer +} diff --git a/src/pkg/cached/base_manager.go b/src/pkg/cached/base_manager.go index bc810372849..d7bf1caddbc 100644 --- a/src/pkg/cached/base_manager.go +++ b/src/pkg/cached/base_manager.go @@ -24,27 +24,27 @@ import ( ) // innerCache is the default cache client, -// actually it is a wrapper for cache.Default(). +// actually it is a wrapper for cache.LayerCache(). var innerCache cache.Cache = &cacheClient{} -// cacheClient is a interceptor for cache.Default, in order to implement specific +// cacheClient is a interceptor for cache.CacheLayer, in order to implement specific // case for cache layer. type cacheClient struct{} func (*cacheClient) Contains(ctx context.Context, key string) bool { - return cache.Default().Contains(ctx, key) + return cache.LayerCache().Contains(ctx, key) } func (*cacheClient) Delete(ctx context.Context, key string) error { - return cache.Default().Delete(ctx, key) + return cache.LayerCache().Delete(ctx, key) } func (*cacheClient) Fetch(ctx context.Context, key string, value interface{}) error { - return cache.Default().Fetch(ctx, key, value) + return cache.LayerCache().Fetch(ctx, key, value) } func (*cacheClient) Ping(ctx context.Context) error { - return cache.Default().Ping(ctx) + return cache.LayerCache().Ping(ctx) } func (*cacheClient) Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error { @@ -57,11 +57,11 @@ func (*cacheClient) Save(ctx context.Context, key string, value interface{}, exp return nil } - return cache.Default().Save(ctx, key, value, expiration...) + return cache.LayerCache().Save(ctx, key, value, expiration...) } func (*cacheClient) Scan(ctx context.Context, match string) (cache.Iterator, error) { - return cache.Default().Scan(ctx, match) + return cache.LayerCache().Scan(ctx, match) } var _ Manager = &BaseManager{}