From d9380eaac950c669864c0af60fd99eae281d2438 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Wed, 12 May 2021 05:03:50 -0400 Subject: [PATCH] Add a target to find dead link in our documentation. (#3713) This is not part of the CI because it can be throttle and it's not super fast ~2min. But still usefull to check that no link are down if we make some big structure changes. Signed-off-by: Cyril Tovena --- .drone/drone.yml | 8 +- .lychee.toml | 98 +++++++++++++++++++ Makefile | 17 +++- docs/sources/architecture/_index.md | 2 +- .../sources/clients/promtail/configuration.md | 2 +- docs/sources/overview/_index.md | 2 +- loki-build-image/Dockerfile | 9 ++ 7 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 .lychee.toml diff --git a/.drone/drone.yml b/.drone/drone.yml index 1309865825aec..d56879625ef66 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -12,28 +12,28 @@ workspace: steps: - name: test - image: grafana/loki-build-image:0.13.0 + image: grafana/loki-build-image:0.14.0 commands: - make BUILD_IN_CONTAINER=false test depends_on: - clone - name: lint - image: grafana/loki-build-image:0.13.0 + image: grafana/loki-build-image:0.14.0 commands: - make BUILD_IN_CONTAINER=false lint depends_on: - clone - name: check-generated-files - image: grafana/loki-build-image:0.13.0 + image: grafana/loki-build-image:0.14.0 commands: - make BUILD_IN_CONTAINER=false check-generated-files depends_on: - clone - name: check-mod - image: grafana/loki-build-image:0.13.0 + image: grafana/loki-build-image:0.14.0 commands: - make BUILD_IN_CONTAINER=false check-mod depends_on: diff --git a/.lychee.toml b/.lychee.toml new file mode 100644 index 0000000000000..f6fe723fc8cf6 --- /dev/null +++ b/.lychee.toml @@ -0,0 +1,98 @@ +### +### Display +### +# Verbose program output +verbose = false + +# Show progress +progress = false + + +### +### Runtime +### +# Number of threads to utilize. +# Defaults to number of cores available to the system if omitted. +#threads = 2 + +# Maximum number of allowed redirects +max_redirects = 10 + + +### +### Requests +### +# User agent to send with each request +user_agent = "curl/7.71.1" + +# Website timeout from connect to response finished +timeout = 9 + +# Comma-separated list of accepted status codes for valid links. +# Omit to accept all response types. +#accept = "text/html" + +# Proceed for server connections considered insecure (invalid TLS) +insecure = true + +# Only test links with the given scheme (e.g. https) +# Omit to check links with any scheme +#scheme = "https" + +# Request method +method = "get" + +# Custom request headers +headers = [] + + +### +### Exclusions +### +# Exclude URLs from checking (supports regex) +exclude = [ + # "github", todo github token + "file://.*", + "host.docker.internal:3100", + "fileb://.*", + "ec2-user@.*.compute.amazonaws.com", + "querier", + "loki:3100", + "ip_or_hostname_where_loki_run:3100", + "dynamodb://.*", + "s3://.*", + "ec2-13-59-62-37", + "promtail.default", + "loki_addr:3100", + "localhost", + "example.com", + "HelloAkkaHttpServer", + "https://loki/", + "http://loki/", + "logs-prod-us-central1.grafana.net", + "ip_or_hostname_where_loki_runs", + "inmemory://", + "myloki.domain:3100", + "grafana/cortex-rules-action@v0.4.0", + "loki.git", + "https://github.com/grafana/website", + "https://github.com/settings/keys", +] + +include = [] + +# Exclude all private IPs from checking +# Equivalent to setting `exclude_private`, `exclude_link_local`, and `exclude_loopback` to true +exclude_all_private = false + +# Exclude private IP address ranges from checking +exclude_private = false + +# Exclude link-local IP address range from checking +exclude_link_local = false + +# Exclude loopback IP address range from checking +exclude_loopback = true + +# Exclude all mail addresses from checking +exclude_mail = false diff --git a/Makefile b/Makefile index 40ba1b91b8454..5d4f8f9b62cee 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ .PHONY: push-images push-latest save-images load-images promtail-image loki-image build-image .PHONY: bigtable-backup, push-bigtable-backup .PHONY: benchmark-store, drone, check-mod -.PHONY: migrate migrate-image +.PHONY: migrate migrate-image lint-markdown SHELL = /usr/bin/env bash @@ -38,7 +38,7 @@ DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) # make BUILD_IN_CONTAINER=false target # or you can override this with an environment variable BUILD_IN_CONTAINER ?= true -BUILD_IMAGE_VERSION := 0.13.0 +BUILD_IMAGE_VERSION := 0.14.0 # Docker image info IMAGE_PREFIX ?= grafana @@ -576,6 +576,19 @@ fmt-jsonnet: @find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \ xargs -n 1 -- jsonnetfmt -i +# search for dead link in our documentation. +# To avoid being rate limited by Github you can use an env variable GITHUB_TOKEN to pass a github token API. +# see https://github.com/settings/tokens +lint-markdown: +ifeq ($(BUILD_IN_CONTAINER),true) + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; +else + lychee --verbose --config .lychee.toml ./*.md ./docs/**/*.md ./production/**/*.md ./cmd/**/*.md ./clients/**/*.md ./tools/**/*.md +endif + + # usage: FUZZ_TESTCASE_PATH=/tmp/testcase make test-fuzz # this will run the fuzzing using /tmp/testcase and save benchmark locally. test-fuzz: diff --git a/docs/sources/architecture/_index.md b/docs/sources/architecture/_index.md index 4fbb5ca471442..061162a79ae8e 100644 --- a/docs/sources/architecture/_index.md +++ b/docs/sources/architecture/_index.md @@ -103,7 +103,7 @@ Since all distributors share access to the same hash ring, write requests can be sent to any distributor. To ensure consistent query results, Loki uses -[Dynamo-style](https://www.allthingsdistributed.com/files/amazon-dynamo-sosp2007.pdf) +[Dynamo-style](https://www.cs.princeton.edu/courses/archive/fall15/cos518/studpres/dynamo.pdf) quorum consistency on reads and writes. This means that the distributor will wait for a positive response of at least one half plus one of the ingesters to send the sample to before responding to the client that initiated the send. diff --git a/docs/sources/clients/promtail/configuration.md b/docs/sources/clients/promtail/configuration.md index 049504350da6d..6ccd80492ef6b 100644 --- a/docs/sources/clients/promtail/configuration.md +++ b/docs/sources/clients/promtail/configuration.md @@ -1281,7 +1281,7 @@ positions: filename: /tmp/positions.yaml clients: - - url: http://ip_or_hostname_where_loki_runns:3100/loki/api/v1/push + - url: http://ip_or_hostname_where_loki_runs:3100/loki/api/v1/push scrape_configs: - job_name: journal diff --git a/docs/sources/overview/_index.md b/docs/sources/overview/_index.md index 7732e66e56446..322bf4cce1d09 100644 --- a/docs/sources/overview/_index.md +++ b/docs/sources/overview/_index.md @@ -62,7 +62,7 @@ Since all distributors share access to the same hash ring, write requests can be sent to any distributor. To ensure consistent query results, Loki uses -[Dynamo-style](https://www.allthingsdistributed.com/files/amazon-dynamo-sosp2007.pdf) +[Dynamo-style](https://www.cs.princeton.edu/courses/archive/fall15/cos518/studpres/dynamo.pdf) quorum consistency on reads and writes. This means that the distributor will wait for a positive response of at least one half plus one of the ingesters to send the sample to before responding to the user. diff --git a/loki-build-image/Dockerfile b/loki-build-image/Dockerfile index 18c3f789634ac..eb96b56a46140 100644 --- a/loki-build-image/Dockerfile +++ b/loki-build-image/Dockerfile @@ -7,6 +7,14 @@ RUN apk add --no-cache curl && \ mv /tmp/linux-amd64/helm /usr/bin/helm && \ rm -rf /tmp/linux-amd64 /tmp/helm-$HELM_VER.tgz +FROM alpine as lychee +ARG LYCHEE_VER="0.7.0" +RUN apk add --no-cache curl && \ + curl -L -o /tmp/lychee-$LYCHEE_VER.tgz https://github.com/lycheeverse/lychee/releases/download/${LYCHEE_VER}/lychee-${LYCHEE_VER}-x86_64-unknown-linux-gnu.tar.gz && \ + tar -xz -C /tmp -f /tmp/lychee-$LYCHEE_VER.tgz && \ + mv /tmp/lychee /usr/bin/lychee && \ + rm -rf /tmp/linux-amd64 /tmp/lychee-$LYCHEE_VER.tgz + FROM alpine as golangci RUN apk add --no-cache curl && \ cd / && \ @@ -40,6 +48,7 @@ RUN apt-get update && \ COPY --from=docker /usr/bin/docker /usr/bin/docker COPY --from=helm /usr/bin/helm /usr/bin/helm +COPY --from=lychee /usr/bin/lychee /usr/bin/lychee COPY --from=golangci /bin/golangci-lint /usr/local/bin COPY --from=drone /go/bin/drone /usr/bin/drone COPY --from=faillint /go/bin/faillint /usr/bin/faillint