diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 14e47df394..3a796c8ac0 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -10,6 +10,8 @@ pipeline { DOCKER_REGISTRY_SECRET = 'secret/observability-team/ci/docker-registry/prod' PIPELINE_LOG_LEVEL='INFO' GO_VERSION = '1.13.12' + DOCKER_IMG = "${env.DOCKER_REGISTRY}/package-registry/distribution" + DOCKER_IMG_PR = "${env.DOCKER_REGISTRY}/observability-ci/package-registry/distribution" } options { timeout(time: 1, unit: 'HOURS') diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..c701421575 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.DS_Store +build +testing +vendor \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..6ce2394efb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +# Here the version of the registry is specified this storage branch uses. +# It should always be a specific version to make sure builds are reproducible. +ARG PACKAGE_REGISTRY=57d417a84fa038cfb50a0cd661372c194199b3fd +FROM docker.elastic.co/package-registry/package-registry:${PACKAGE_REGISTRY} + +LABEL package-registry=${PACKAGE_REGISTRY} + +# Adds specific config and packages +COPY deployment/package-registry.yml /registry/config.yml +COPY packages /packages + +# TODO: Find way to also copy in packages from staging and production + +# Sanity check on the packages. If packages are not valid, container does not even build. +RUN ./package-registry -dry-run \ No newline at end of file diff --git a/deployment/package-registry.yml b/deployment/package-registry.yml new file mode 100644 index 0000000000..cc48bc95a3 --- /dev/null +++ b/deployment/package-registry.yml @@ -0,0 +1,6 @@ +package_paths: + - /packages + +cache_time.search: 10s +cache_time.categories: 10s +cache_time.catch_all: 10s diff --git a/packages/base/0.1.0/docs/README.md b/packages/base/0.1.0/docs/README.md deleted file mode 100644 index 152200ea6d..0000000000 --- a/packages/base/0.1.0/docs/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Base package - -This is installed in the background by EPM to setup the Elastic Stack. diff --git a/packages/base/0.1.0/elasticsearch/component-template/events-mappings.json b/packages/base/0.1.0/elasticsearch/component-template/events-mappings.json deleted file mode 100644 index 35488413ae..0000000000 --- a/packages/base/0.1.0/elasticsearch/component-template/events-mappings.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "template": { - "mappings": { - "_meta": { - }, - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "date_detection": false, - "properties": { - "@timestamp": { - "type": "date" - }, - "stream": { - "properties": { - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - }, - "namespace": { - "type": "constant_keyword" - } - } - }, - "agent": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "message": { - "type": "text" - } - } - }, - "aliases": {} - } -} diff --git a/packages/base/0.1.0/elasticsearch/component-template/events-settings.json b/packages/base/0.1.0/elasticsearch/component-template/events-settings.json deleted file mode 100644 index 3cd594414e..0000000000 --- a/packages/base/0.1.0/elasticsearch/component-template/events-settings.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "template": { - "settings": { - "index": { - "lifecycle": { - "name": "events-default" - }, - "codec": "best_compression", - "refresh_interval": "5s", - "number_of_shards": "1", - "query": { - "default_field": [ - "message" - ] - }, - "number_of_routing_shards": "30" - } - }, - "aliases": {} - } -} diff --git a/packages/base/0.1.0/elasticsearch/component-template/logs-mappings.json b/packages/base/0.1.0/elasticsearch/component-template/logs-mappings.json deleted file mode 100644 index 35488413ae..0000000000 --- a/packages/base/0.1.0/elasticsearch/component-template/logs-mappings.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "template": { - "mappings": { - "_meta": { - }, - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "date_detection": false, - "properties": { - "@timestamp": { - "type": "date" - }, - "stream": { - "properties": { - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - }, - "namespace": { - "type": "constant_keyword" - } - } - }, - "agent": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "message": { - "type": "text" - } - } - }, - "aliases": {} - } -} diff --git a/packages/base/0.1.0/elasticsearch/component-template/logs-settings.json b/packages/base/0.1.0/elasticsearch/component-template/logs-settings.json deleted file mode 100644 index eaaa60a9a3..0000000000 --- a/packages/base/0.1.0/elasticsearch/component-template/logs-settings.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs-default" - }, - "codec": "best_compression", - "refresh_interval": "5s", - "number_of_shards": "1", - "query": { - "default_field": [ - "message" - ] - }, - "number_of_routing_shards": "30" - } - }, - "aliases": {} - } -} diff --git a/packages/base/0.1.0/elasticsearch/component-template/metrics-mappings.json b/packages/base/0.1.0/elasticsearch/component-template/metrics-mappings.json deleted file mode 100644 index 29f7d8bd10..0000000000 --- a/packages/base/0.1.0/elasticsearch/component-template/metrics-mappings.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "template": { - "mappings": { - "_meta": { - }, - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "date_detection": false, - "properties": { - "@timestamp": { - "type": "date" - }, - "stream": { - "properties": { - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - }, - "namespace": { - "type": "constant_keyword" - } - } - }, - "agent": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "aliases": {} - } -} diff --git a/packages/base/0.1.0/elasticsearch/component-template/metrics-settings.json b/packages/base/0.1.0/elasticsearch/component-template/metrics-settings.json deleted file mode 100644 index 539f79ed9d..0000000000 --- a/packages/base/0.1.0/elasticsearch/component-template/metrics-settings.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "template": { - "settings": { - "index": { - "lifecycle": { - "name": "metrics-default" - }, - "codec": "best_compression", - "refresh_interval": "5s", - "number_of_shards": "1", - "query": { - "default_field": [ - "message" - ] - }, - "number_of_routing_shards": "30" - } - }, - "aliases": {} - } -} diff --git a/packages/base/0.1.0/elasticsearch/ilm-policy/events-default.json b/packages/base/0.1.0/elasticsearch/ilm-policy/events-default.json deleted file mode 100644 index 26115b3a31..0000000000 --- a/packages/base/0.1.0/elasticsearch/ilm-policy/events-default.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "policy": { - "phases": { - "hot": { - "min_age": "0ms", - "actions": { - "rollover": { - "max_size": "50gb", - "max_age": "30d" - } - } - } - } - } -} diff --git a/packages/base/0.1.0/elasticsearch/ilm-policy/logs-default.json b/packages/base/0.1.0/elasticsearch/ilm-policy/logs-default.json deleted file mode 100644 index 8d8db418e7..0000000000 --- a/packages/base/0.1.0/elasticsearch/ilm-policy/logs-default.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "policy": { - "phases": { - "hot": { - "min_age": "0ms", - "actions": { - "rollover": { - "max_size": "50gb", - "max_age": "30d" - } - } - } - } - } -} diff --git a/packages/base/0.1.0/elasticsearch/ilm-policy/metrics-default.json b/packages/base/0.1.0/elasticsearch/ilm-policy/metrics-default.json deleted file mode 100644 index 26115b3a31..0000000000 --- a/packages/base/0.1.0/elasticsearch/ilm-policy/metrics-default.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "policy": { - "phases": { - "hot": { - "min_age": "0ms", - "actions": { - "rollover": { - "max_size": "50gb", - "max_age": "30d" - } - } - } - } - } -} diff --git a/packages/base/0.1.0/elasticsearch/index-template/events.json b/packages/base/0.1.0/elasticsearch/index-template/events.json deleted file mode 100644 index 2d0a06c087..0000000000 --- a/packages/base/0.1.0/elasticsearch/index-template/events.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "index_patterns": [ - "events-*-*" - ], - "priority": 0, - "composed_of": [ - "events-mappings", - "events-settings" - ] -} \ No newline at end of file diff --git a/packages/base/0.1.0/elasticsearch/index-template/logs.json b/packages/base/0.1.0/elasticsearch/index-template/logs.json deleted file mode 100644 index ac4e5f69a1..0000000000 --- a/packages/base/0.1.0/elasticsearch/index-template/logs.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "index_patterns": [ - "logs-*-*" - ], - "priority": 0, - "composed_of": [ - "logs-mappings", - "logs-settings" - ] -} \ No newline at end of file diff --git a/packages/base/0.1.0/elasticsearch/index-template/metrics.json b/packages/base/0.1.0/elasticsearch/index-template/metrics.json deleted file mode 100644 index c07ef16aff..0000000000 --- a/packages/base/0.1.0/elasticsearch/index-template/metrics.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "index_patterns": [ - "metrics-*-*" - ], - "priority": 0, - "composed_of": [ - "metrics-mappings", - "metrics-settings" - ] -} \ No newline at end of file diff --git a/packages/base/0.1.0/manifest.yml b/packages/base/0.1.0/manifest.yml deleted file mode 100644 index 7fc5d09679..0000000000 --- a/packages/base/0.1.0/manifest.yml +++ /dev/null @@ -1,30 +0,0 @@ -format_version: 1.0.0 - -name: base -title: Base package -description: > - The base package contains assets which are needed for the basic setup of the stack. - - It contains the default ILM policies. -version: 0.1.0 -categories: [] -release: ga - -# The base package cannot be removed -removable: false - -# The user should not see this package and not be able to install it -internal: true - -license: basic -# This is called type integration because it is required for all the integration packages -type: integration - -requirement: - elasticsearch: - # Requires ILM which was released in 6.6. - versions: ">6.6.0" - -# No icons -icons: - diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 0be9ead70b..82cca3d4d5 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -34,10 +34,9 @@ services: - ./kibana.config.yml:/usr/share/kibana/config/kibana.yml package-registry: - image: docker.elastic.co/package-registry/package-registry:master + build: ../../ healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080"] retries: 300 interval: 1s - volumes: - - ../../build/public:/registry/public +