Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

storage-metadata -> storage-system #3667

Merged
merged 1 commit into from
May 4, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .drone.star
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ config = {
"extensions/proxy",
"extensions/settings",
"extensions/sharing",
"extensions/storage-metadata",
"extensions/storage-system",
"extensions/storage-publiclink",
"extensions/storage-shares",
"extensions/storage-users",
Expand Down Expand Up @@ -1646,7 +1646,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
"STORAGE_USERS_DRIVER": "%s" % (storage),
"STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root",
"STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users",
"STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata",
"STORAGE_SYSTEM_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata",
"SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json",
"PROXY_ENABLE_BASIC_AUTH": True,
"WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
Expand Down Expand Up @@ -1721,7 +1721,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
"SHARING_USER_OWNCLOUDSQL_DB_NAME": "owncloud",
# General oCIS config
# OCIS_RUN_EXTENSIONS specifies to start all extensions except glauth, idp and accounts. These are replaced by external services
"OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,frontend,gateway,user,group,auth-basic,auth-bearer,auth-machine,storage-users,storage-shares,storage-publiclink,app-provider,sharing,proxy,nats,ocdav",
"OCIS_RUN_EXTENSIONS": "settings,storage-system,graph,graph-explorer,ocs,store,thumbnails,web,webdav,frontend,gateway,user,group,auth-basic,auth-bearer,auth-machine,storage-users,storage-shares,storage-publiclink,app-provider,sharing,proxy,nats,ocdav",
"OCIS_LOG_LEVEL": "info",
"OCIS_URL": OCIS_URL,
"OCIS_BASE_DATA_PATH": "/mnt/data/ocis",
Expand Down
2 changes: 1 addition & 1 deletion .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
// metadata storage
"METADATA_USER_ID": "some-metadata-user-id"
// OCIS_RUN_EXTENSIONS allows to start a subset of extensions even in the supervised mode
//"OCIS_RUN_EXTENSIONS": "settings,storage-metadata,glauth,graph,graph-explorer,idp,ocs,store,thumbnails,web,webdav,frontend,gateway,user,group,auth-basic,auth-bearer,storage-authmachine,storage-users,storage-shares,storage-publiclink,app-provider,sharing,accounts,proxy,ocdav",
//"OCIS_RUN_EXTENSIONS": "settings,storage-system,glauth,graph,graph-explorer,idp,ocs,store,thumbnails,web,webdav,frontend,gateway,user,group,auth-basic,auth-bearer,storage-authmachine,storage-users,storage-shares,storage-publiclink,app-provider,sharing,accounts,proxy,ocdav",
}
}
]
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ OCIS_MODULES = \
extensions/proxy \
extensions/settings \
extensions/sharing \
extensions/storage-metadata \
extensions/storage-system \
extensions/storage-publiclink \
extensions/storage-shares \
extensions/storage-users \
Expand Down
4 changes: 2 additions & 2 deletions deployments/examples/oc10_ocis_parallel/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ services:
STORAGE_LDAP_LOGIN_ATTRIBUTES: "uid,mail"
# ownCloudSQL storage driver
STORAGE_USERS_DRIVER: owncloudsql
STORAGE_METADATA_DRIVER: ocis # keep metadata on ocis storage since this are only small files atm
STORAGE_SYSTEM_DRIVER: ocis # keep system data on ocis storage since this are only small files atm
STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR: /mnt/data/files
STORAGE_USERS_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR: /tmp
STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER: "/Shares"
Expand All @@ -108,7 +108,7 @@ services:
OCIS_STORAGE_READ_ONLY: "false" # TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303
# General oCIS config
# OCIS_RUN_EXTENSIONS specifies to start all extensions except glauth, idp and accounts. These are replaced by external services
OCIS_RUN_EXTENSIONS: settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,app-provider,storage-sharing,proxy,nats
OCIS_RUN_EXTENSIONS: settings,storage-system,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,app-provider,storage-sharing,proxy,nats
OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-error} # make oCIS less verbose
OCIS_URL: https://${CLOUD_DOMAIN:-cloud.owncloud.test}
PROXY_TLS: "false" # do not use SSL between Traefik and oCIS
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ services:
STORAGE_FRONTEND_DEBUG_ADDR: 0.0.0.0:9141
STORAGE_GATEWAY_DEBUG_ADDR: 0.0.0.0:9143
STORAGE_GROUPPROVIDER_DEBUG_ADDR: 0.0.0.0:9161
STORAGE_METADATA_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_SYSTEM_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_PUBLIC_LINK_DEBUG_ADDR: 0.0.0.0:9179
STORAGE_USERPROVIDER_DEBUG_ADDR: 0.0.0.0:9145
STORAGE_USERS_DEBUG_ADDR: 0.0.0.0:9159
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ services:
STORAGE_FRONTEND_DEBUG_ADDR: 0.0.0.0:9141
STORAGE_GATEWAY_DEBUG_ADDR: 0.0.0.0:9143
STORAGE_GROUPPROVIDER_DEBUG_ADDR: 0.0.0.0:9161
STORAGE_METADATA_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_SYSTEM_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_PUBLIC_LINK_DEBUG_ADDR: 0.0.0.0:9179
STORAGE_USERPROVIDER_DEBUG_ADDR: 0.0.0.0:9145
STORAGE_USERS_DEBUG_ADDR: 0.0.0.0:9159
Expand Down
18 changes: 9 additions & 9 deletions deployments/examples/ocis_individual_services/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ services:
ACCOUNTS_GRPC_ADDR: 0.0.0.0:9180

ACCOUNTS_STORAGE_BACKEND: cs3
ACCOUNTS_STORAGE_CS3_PROVIDER_ADDR: storage-metadata:9215
ACCOUNTS_STORAGE_CS3_PROVIDER_ADDR: storage-system:9215

IDP_LDAP_BIND_PASSWORD: ${IDP_LDAP_BIND_PASSWORD:-idp}
STORAGE_LDAP_BIND_PASSWORD: ${STORAGE_LDAP_BIND_PASSWORD:-reva}
Expand Down Expand Up @@ -366,15 +366,15 @@ services:
driver: "local"
restart: always

storage-metadata:
storage-system:
image: owncloud/ocis:${OCIS_DOCKER_TAG:-latest}
deploy:
replicas: 1
networks:
ocis-net:
entrypoint:
- ocis
- storage-metadata
- storage-system
- server
environment:
OCIS_LOG_LEVEL: "${OCIS_LOG_LEVEL:-error}"
Expand All @@ -383,18 +383,18 @@ services:

STORAGE_HOME_DATAPROVIDER_INSECURE: "${INSECURE:-false}"

STORAGE_METADATA_GRPC_ADDR: 0.0.0.0:9215
STORAGE_METADATA_HTTP_ADDR: 0.0.0.0:9216
STORAGE_METADATA_DATA_SERVER_URL: http://storage-metadata:9216/data
STORAGE_SYSTEM_GRPC_ADDR: 0.0.0.0:9215
STORAGE_SYSTEM_HTTP_ADDR: 0.0.0.0:9216
STORAGE_SYSTEM_DATA_SERVER_URL: http://storage-system:9216/data

STORAGE_JWT_SECRET: ${OCIS_METADATA_JWT_SECRET:-Pive-Fumkiu4}
STORAGE_TRANSFER_SECRET: ${STORAGE_TRANSFER_SECRET:-replace-me-with-a-transfer-secret}

STORAGE_METADATA_DRIVER: ocis
STORAGE_SYSTEM_DRIVER: ocis

REVA_GATEWAY: storage-gateway:9142
volumes:
- "ocis-storage-metadata:/var/lib/ocis"
- "ocis-storage-system:/var/lib/ocis"
logging:
driver: "local"
restart: always
Expand Down Expand Up @@ -726,7 +726,7 @@ volumes:
traefik-certs: null
ocis-settings: null
ocis-store: null
ocis-storage-metadata: null
ocis-storage-system: null
ocis-storage-users: null
ocis-storage-sharing: null

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,14 +99,14 @@ services:
# OCIS_TRACING_ENDPOINT: jaeger-agent:6831
# # metrics
# GRAPH_EXPLORER_DEBUG_ADDR: 0.0.0.0:9136
storage-metadata:
storage-system:
environment:
# tracing
OCIS_TRACING_ENABLED: "true"
OCIS_TRACING_TYPE: jaeger
OCIS_TRACING_ENDPOINT: jaeger-agent:6831
# metrics
STORAGE_METADATA_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_SYSTEM_DEBUG_ADDR: 0.0.0.0:9217
storage-authbasic:
environment:
# tracing
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ services:
STORAGE_FRONTEND_DEBUG_ADDR: 0.0.0.0:9141
STORAGE_GATEWAY_DEBUG_ADDR: 0.0.0.0:9143
STORAGE_GROUPPROVIDER_DEBUG_ADDR: 0.0.0.0:9161
STORAGE_METADATA_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_SYSTEM_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_PUBLIC_LINK_DEBUG_ADDR: 0.0.0.0:9179
STORAGE_USERPROVIDER_DEBUG_ADDR: 0.0.0.0:9145
STORAGE_USERS_DEBUG_ADDR: 0.0.0.0:9159
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ services:
STORAGE_FRONTEND_DEBUG_ADDR: 0.0.0.0:9141
STORAGE_GATEWAY_DEBUG_ADDR: 0.0.0.0:9143
STORAGE_GROUPPROVIDER_DEBUG_ADDR: 0.0.0.0:9161
STORAGE_METADATA_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_SYSTEM_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_PUBLIC_LINK_DEBUG_ADDR: 0.0.0.0:9179
STORAGE_USERPROVIDER_DEBUG_ADDR: 0.0.0.0:9145
STORAGE_USERS_DEBUG_ADDR: 0.0.0.0:9159
Expand Down
2 changes: 1 addition & 1 deletion deployments/examples/ocis_s3/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ services:
OCIS_MACHINE_AUTH_API_KEY: ${OCIS_MACHINE_AUTH_API_KEY:-change-me-please}
# activate s3ng storage driver
STORAGE_USERS_DRIVER: s3ng
STORAGE_METADATA_DRIVER: ocis # keep metadata on ocis storage since this are only small files atm
STORAGE_SYSTEM_DRIVER: ocis # keep system data on ocis storage since this are only small files atm
# s3ng specific settings
STORAGE_USERS_DRIVER_S3NG_ENDPOINT: http://minio:9000
STORAGE_USERS_DRIVER_S3NG_REGION: default
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ services:
STORAGE_FRONTEND_DEBUG_ADDR: 0.0.0.0:9141
STORAGE_GATEWAY_DEBUG_ADDR: 0.0.0.0:9143
STORAGE_GROUPPROVIDER_DEBUG_ADDR: 0.0.0.0:9161
STORAGE_METADATA_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_SYSTEM_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_PUBLIC_LINK_DEBUG_ADDR: 0.0.0.0:9179
STORAGE_USERPROVIDER_DEBUG_ADDR: 0.0.0.0:9145
STORAGE_USERS_DEBUG_ADDR: 0.0.0.0:9159
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ services:
STORAGE_FRONTEND_DEBUG_ADDR: 0.0.0.0:9141
STORAGE_GATEWAY_DEBUG_ADDR: 0.0.0.0:9143
STORAGE_GROUPPROVIDER_DEBUG_ADDR: 0.0.0.0:9161
STORAGE_METADATA_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_SYSTEM_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_PUBLIC_LINK_DEBUG_ADDR: 0.0.0.0:9179
STORAGE_USERPROVIDER_DEBUG_ADDR: 0.0.0.0:9145
STORAGE_USERS_DEBUG_ADDR: 0.0.0.0:9159
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ services:
STORAGE_FRONTEND_DEBUG_ADDR: 0.0.0.0:9141
STORAGE_GATEWAY_DEBUG_ADDR: 0.0.0.0:9143
STORAGE_GROUPPROVIDER_DEBUG_ADDR: 0.0.0.0:9161
STORAGE_METADATA_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_SYSTEM_DEBUG_ADDR: 0.0.0.0:9217
STORAGE_PUBLIC_LINK_DEBUG_ADDR: 0.0.0.0:9179
STORAGE_USERPROVIDER_DEBUG_ADDR: 0.0.0.0:9145
STORAGE_USERS_DEBUG_ADDR: 0.0.0.0:9159
Expand Down
2 changes: 1 addition & 1 deletion docs/extensions/storage-metadata/_index.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ title: Metadata Storage
date: 2022-03-02T00:00:00+00:00
weight: 20
geekdocRepo: https://github.com/owncloud/ocis
geekdocEditPath: edit/master/docs/extensions/storage-metadata
geekdocEditPath: edit/master/docs/extensions/storage-system
geekdocFilePath: _index.md
geekdocCollapseSection: true
---
Expand Down
6 changes: 3 additions & 3 deletions docs/extensions/storage-metadata/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@ title: Service Configuration
date: 2018-05-02T00:00:00+00:00
weight: 20
geekdocRepo: https://github.com/owncloud/ocis
geekdocEditPath: edit/master/docs/extensions/storage-metadata
geekdocEditPath: edit/master/docs/extensions/storage-system
geekdocFilePath: configuration.md
geekdocCollapseSection: true
---

## Example YAML Config

{{< include file="extensions/_includes/storage-metadata-config-example.yaml" language="yaml" >}}
{{< include file="extensions/_includes/storage-system-config-example.yaml" language="yaml" >}}

{{< include file="extensions/_includes/storage-metadata_configvars.md" >}}
{{< include file="extensions/_includes/storage-system_configvars.md" >}}
2 changes: 1 addition & 1 deletion docs/ocis/release_notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,7 @@ Our recommended update strategy to oCIS 1.6.0 is:
1. let users recreate their shares

If you want to use oCIS 1.6.0 without following our recommended update strategy, you can also keep the pre 1.6.0 behaviour by setting this environment variable:
`export STORAGE_METADATA_ROOT=/var/tmp/ocis/storage/users`
`export STORAGE_SYSTEM_ROOT=/var/tmp/ocis/storage/users`
This may lead to faulty behaviour since both the metadata and user storage driver will be storing their data in the same filesystem path.

## ownCloud Infinite Scale 1.5.0 Technology Preview
Expand Down
4 changes: 2 additions & 2 deletions docs/ocis/storage-backends/eos.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ The uid and gid `2` are referencing the user `daemon` inside the ocis container.
Kill the metadata storage. By default it uses the `ocis` storage driver. We need to switch it to the `eos` driver:

```
docker-compose exec ocis ./bin/ocis kill storage-metadata
docker-compose exec -e STORAGE_METADATA_DRIVER=eos -e STORAGE_METADATA_ROOT=/eos/dockertest/ocis/metadata ocis ./bin/ocis run storage-metadata
docker-compose exec ocis ./bin/ocis kill storage-system
docker-compose exec -e STORAGE_SYSTEM_DRIVER=eos -e STORAGE_SYSTEM_ROOT=/eos/dockertest/ocis/metadata ocis ./bin/ocis run storage-system
```


Expand Down
2 changes: 1 addition & 1 deletion extensions/accounts/pkg/service/v0/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func New(opts ...Option) (s *Service, err error) {
r := oreg.GetRegistry()
if cfg.Repo.Backend == "cs3" {
if _, err := r.GetService("com.owncloud.storage.metadata"); err != nil {
logger.Error().Err(err).Msg("index: storage-metadata service not present")
logger.Error().Err(err).Msg("index: storage-system service not present")
return nil, err
}
}
Expand Down
14 changes: 0 additions & 14 deletions extensions/storage-metadata/cmd/storage-metadata/main.go

This file was deleted.

79 changes: 0 additions & 79 deletions extensions/storage-metadata/pkg/config/config.go

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
SHELL := bash
NAME := storage-metadata
NAME := storage-system

include ../../.make/recursion.mk

Expand Down
14 changes: 14 additions & 0 deletions extensions/storage-system/cmd/storage-system/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package main

import (
"os"

"github.com/owncloud/ocis/extensions/storage-system/pkg/command"
"github.com/owncloud/ocis/extensions/storage-system/pkg/config/defaults"
)

func main() {
if err := command.Execute(defaults.DefaultConfig()); err != nil {
os.Exit(1)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ import (
"fmt"
"net/http"

"github.com/owncloud/ocis/extensions/storage-metadata/pkg/config"
"github.com/owncloud/ocis/extensions/storage-metadata/pkg/config/parser"
"github.com/owncloud/ocis/extensions/storage-metadata/pkg/logging"
"github.com/owncloud/ocis/extensions/storage-system/pkg/config"
"github.com/owncloud/ocis/extensions/storage-system/pkg/config/parser"
"github.com/owncloud/ocis/extensions/storage-system/pkg/logging"
"github.com/urfave/cli/v2"
)

Expand Down
Loading