Skip to content

Commit

Permalink
Add prometheus module and config
Browse files Browse the repository at this point in the history
  • Loading branch information
iknite committed Feb 4, 2019
1 parent a034cf3 commit 3b4f382
Show file tree
Hide file tree
Showing 7 changed files with 288 additions and 94 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,4 @@ config.yml
deploy/aws/config_files/*
!deploy/aws/config_files/README.md
deploy/aws/modules/inmemory_storage/data
deploy/aws/modules/prometheus/data
6 changes: 3 additions & 3 deletions deploy/aws/config_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ node_path=${pub}/node_exporter
cd ${tdir}

if [ ! -f ${node_path} ]; then (
mkdir -p ./node_exporter
version=0.17.0
link=https://github.com/prometheus/node_exporter/releases/download/v${version}/node_exporter-${version}.linux-amd64.tar.gz
folder=node_exporter-${version}.linux-amd64
link=https://github.com/prometheus/node_exporter/releases/download/v${version}/${folder}.tar.gz
wget -qO- ${link} | tar xvz -C ./
cp node_exporter-${version}.linux-amd64/node_exporter ${node_path}
cp ${folder}/node_exporter ${node_path}
) fi

if [ ! -f ${sign_path} ]; then
Expand Down
227 changes: 136 additions & 91 deletions deploy/aws/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -100,104 +100,149 @@ module "follower-2" {
CONFIG
}

module "inmemory-storage" {
source = "./modules/inmemory_storage"

name = "inmemory-storage"
instance_type = "t2.2xlarge"
volume_size = "20"
vpc_security_group_ids = "${module.security_group.this_security_group_id}"
subnet_id = "${element(data.aws_subnet_ids.all.ids, 0)}"
key_name = "${aws_key_pair.qed.key_name}"
}

module "agent-publisher" {
source = "./modules/qed"

name = "agent-publisher"
instance_type = "t2.2xlarge"
volume_size = "20"
vpc_security_group_ids = "${module.security_group.this_security_group_id}"
subnet_id = "${element(data.aws_subnet_ids.all.ids, 0)}"
key_name = "${aws_key_pair.qed.key_name}"

command="agent publisher"
config = <<-CONFIG
---
api_key: "terraform_qed"
path: "/var/tmp/qed/"
agent:
node: "publisher"
bind: ":9300"
advertise: ""
join:
- "${module.leader.private_ip}:9100"
server_urls:
- "${module.leader.private_ip}:8080"
alert_urls:
- "${module.inmemory-storage.private_ip}:8888"
snapshots_store_urls:
- "${module.inmemory-storage.private_ip}:8888"
CONFIG
}


module "agent-monitor" {
source = "./modules/qed"

name = "agent-monitor"
instance_type = "t2.2xlarge"
volume_size = "20"
vpc_security_group_ids = "${module.security_group.this_security_group_id}"
subnet_id = "${element(data.aws_subnet_ids.all.ids, 0)}"
key_name = "${aws_key_pair.qed.key_name}"

command="agent monitor"
config = <<-CONFIG
---
api_key: "terraform_qed"
path: "/var/tmp/qed/"
agent:
node: "monitor"
bind: ":9200"
advertise: ""
join:
- "${module.leader.private_ip}:9100"
server_urls:
- "${module.leader.private_ip}:8080"
alert_urls:
- "${module.inmemory-storage.private_ip}:8888"
snapshots_store_urls:
- "${module.inmemory-storage.private_ip}:8888"
CONFIG
}

module "agent-auditor" {
source = "./modules/qed"
# module "inmemory-storage" {
# source = "./modules/inmemory_storage"
#
# name = "inmemory-storage"
# instance_type = "t2.2xlarge"
# volume_size = "20"
# vpc_security_group_ids = "${module.security_group.this_security_group_id}"
# subnet_id = "${element(data.aws_subnet_ids.all.ids, 0)}"
# key_name = "${aws_key_pair.qed.key_name}"
# }

# module "agent-publisher" {
# source = "./modules/qed"
#
# name = "agent-publisher"
# instance_type = "t2.2xlarge"
# volume_size = "20"
# vpc_security_group_ids = "${module.security_group.this_security_group_id}"
# subnet_id = "${element(data.aws_subnet_ids.all.ids, 0)}"
# key_name = "${aws_key_pair.qed.key_name}"
#
# command="agent publisher"
# config = <<-CONFIG
# ---
# api_key: "terraform_qed"
# path: "/var/tmp/qed/"
# agent:
# node: "publisher"
# bind: ":9300"
# advertise: ""
# join:
# - "${module.leader.private_ip}:9100"
# server_urls:
# - "${module.leader.private_ip}:8080"
# alert_urls:
# - "${module.inmemory-storage.private_ip}:8888"
# snapshots_store_urls:
# - "${module.inmemory-storage.private_ip}:8888"
# CONFIG
# }


# module "agent-monitor" {
# source = "./modules/qed"
#
# name = "agent-monitor"
# instance_type = "t2.2xlarge"
# volume_size = "20"
# vpc_security_group_ids = "${module.security_group.this_security_group_id}"
# subnet_id = "${element(data.aws_subnet_ids.all.ids, 0)}"
# key_name = "${aws_key_pair.qed.key_name}"
#
# command="agent monitor"
# config = <<-CONFIG
# ---
# api_key: "terraform_qed"
# path: "/var/tmp/qed/"
# agent:
# node: "monitor"
# bind: ":9200"
# advertise: ""
# join:
# - "${module.leader.private_ip}:9100"
# server_urls:
# - "${module.leader.private_ip}:8080"
# alert_urls:
# - "${module.inmemory-storage.private_ip}:8888"
# snapshots_store_urls:
# - "${module.inmemory-storage.private_ip}:8888"
# CONFIG
# }

# module "agent-auditor" {
# source = "./modules/qed"
#
# name = "agent-auditor"
# instance_type = "t2.2xlarge"
# volume_size = "20"
# vpc_security_group_ids = "${module.security_group.this_security_group_id}"
# subnet_id = "${element(data.aws_subnet_ids.all.ids, 0)}"
# key_name = "${aws_key_pair.qed.key_name}"
#
# command="agent auditor"
# config = <<-CONFIG
# ---
# api_key: "terraform_qed"
# path: "/var/tmp/qed/"
# agent:
# node: "auditor"
# bind: ":9100"
# advertise: ""
# join:
# - "${module.leader.private_ip}:9100"
# server_urls:
# - "${module.leader.private_ip}:8080"
# alert_urls:
# - "${module.inmemory-storage.private_ip}:8888"
# snapshots_store_urls:
# - "${module.inmemory-storage.private_ip}:8888"
# CONFIG
# }

module "prometheus" {
source = "./modules/prometheus"

name = "agent-auditor"
instance_type = "t2.2xlarge"
volume_size = "20"
vpc_security_group_ids = "${module.security_group.this_security_group_id}"
subnet_id = "${element(data.aws_subnet_ids.all.ids, 0)}"
key_name = "${aws_key_pair.qed.key_name}"

command="agent auditor"
config = <<-CONFIG
---
api_key: "terraform_qed"
path: "/var/tmp/qed/"
agent:
node: "auditor"
bind: ":9100"
advertise: ""
join:
- "${module.leader.private_ip}:9100"
server_urls:
- "${module.leader.private_ip}:8080"
alert_urls:
- "${module.inmemory-storage.private_ip}:8888"
snapshots_store_urls:
- "${module.inmemory-storage.private_ip}:8888"
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
- job_name: 'prometheus'
scrape_interval: 5s
static_configs:
- targets: ['localhost:9090']
- job_name: 'Qed-00-Host'
scrape_interval: 10s
static_configs:
- targets: ['${module.leader.private_ip}:9100', ]
- job_name: 'Qed-00-Service'
scrape_interval: 10s
static_configs:
- targets: ['${module.leader.private_ip}:9990']
- job_name: 'Qed-01-Host'
scrape_interval: 10s
static_configs:
- targets: ['${module.follower-1.private_ip}:9100']
- job_name: 'Qed-01-Service'
scrape_interval: 10s
static_configs:
- targets: ['${module.follower-1.private_ip}:9990']
- job_name: 'Qed-02-Host'
scrape_interval: 10s
static_configs:
- targets: ['${module.follower-2.private_ip}:9100']
- job_name: 'Qed-02-Service'
scrape_interval: 10s
static_configs:
- targets: ['${module.follower-2.private_ip}:9990']
CONFIG
}
30 changes: 30 additions & 0 deletions deploy/aws/modules/prometheus/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/bash
set -x
set -e

function _readlink() { (
# INFO: readlink does not exists on OSX ¯\_(ツ)_/¯
cd $(dirname $1)
echo $PWD/$(basename $1)
) }

pub=$(_readlink ./data)
tdir=$(mktemp -d /tmp/prometheus.XXX)

app_path=${pub}/prometheus

mkdir -p ${pub}

(
cd ${tdir}

if [ ! -f ${app_path} ]; then (
version=2.7.0
folder=prometheus-${version}.linux-amd64
link=https://github.com/prometheus/prometheus/releases/download/v${version}/${folder}.tar.gz
wget -qO- ${link} | tar xvz -C ./
cp ${folder}/prometheus ${app_path}
) fi

)
rm -rf ${tdir}
78 changes: 78 additions & 0 deletions deploy/aws/modules/prometheus/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

data "aws_ami" "amazon_linux" {
most_recent = true

filter {
name = "name"
values = ["amzn-ami-hvm-*-x86_64-gp2"]
}

filter {
name = "owner-alias"
values = ["amazon"]
}
}

resource "aws_instance" "prometheus" {
count = "1"
ami = "${data.aws_ami.amazon_linux.id}"
instance_type = "${var.instance_type}"

vpc_security_group_ids = ["${var.vpc_security_group_ids}"]
subnet_id = "${var.subnet_id}"
associate_public_ip_address = true
key_name = "${var.key_name}"

root_block_device = [{
volume_type = "gp2"
volume_size = "${var.volume_size}"
}]

tags {
Name = "qed-prometheus"
}


provisioner "file" {
source = "${path.module}/data"
destination = "${var.path}"

connection {
user = "ec2-user"
}
}

provisioner "file" {
content = "${var.config}"
destination = "${var.path}/prometheus.yml"

connection {
user = "ec2-user"
}
}

user_data = <<-DATA
#!/bin/bash
while [ ! -f ${var.path}/prometheus ]; do
sleep 1 # INFO: wait until binary exists
done
sleep 1
chmod +x ${var.path}/prometheus
${var.path}/prometheus --config-file=${var.path}/prometheus.yml
DATA
}
7 changes: 7 additions & 0 deletions deploy/aws/modules/prometheus/outputs.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
output "private_ip" {
value = "${aws_instance.prometheus.private_ip}"
}

output "public_ip" {
value = "${aws_instance.prometheus.public_ip}"
}
Loading

0 comments on commit 3b4f382

Please sign in to comment.