From 9161b617c6705a4f216f87569b8cebd226f5dd74 Mon Sep 17 00:00:00 2001 From: Lucas Santos Date: Thu, 24 Feb 2022 10:09:13 -0300 Subject: [PATCH 1/9] Add Ansible files --- ansible-files/install-dependencies.sh | 16 ++++++++++++++++ ansible-files/log-export-container.service | 12 ++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 ansible-files/install-dependencies.sh create mode 100644 ansible-files/log-export-container.service diff --git a/ansible-files/install-dependencies.sh b/ansible-files/install-dependencies.sh new file mode 100644 index 0000000..ff17c30 --- /dev/null +++ b/ansible-files/install-dependencies.sh @@ -0,0 +1,16 @@ +apt update +ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime +export DEBIAN_FRONTEND=noninteractive +apt-get install -y tzdata +dpkg-reconfigure --frontend noninteractive tzdata +apt install --yes build-essential ruby-dev zlib1g +gem install bundler -v '~> 2.3.3' +bundle install + +apt update +apt install --yes curl +curl -J -O -L https://app.strongdm.com/releases/cli/linux + +unzip -x sdm*.zip +rm sdm*.zip +mv sdm /home/fluent \ No newline at end of file diff --git a/ansible-files/log-export-container.service b/ansible-files/log-export-container.service new file mode 100644 index 0000000..8587019 --- /dev/null +++ b/ansible-files/log-export-container.service @@ -0,0 +1,12 @@ +[Unit] +Description=log export container daemon +After=network.target + +[Service] +User=ubuntu +EnvironmentFile=-/etc/sysconfig/log-export-container +WorkingDirectory=/opt/log-export-container +ExecStart=/opt/log-export-container/start.sh + +[Install] +WantedBy=multi-user.target \ No newline at end of file From 5287f385069b4d31881abadc060be7786014774f Mon Sep 17 00:00:00 2001 From: Lucas Santos Date: Thu, 24 Feb 2022 14:53:28 -0300 Subject: [PATCH 2/9] Add Fluentd Gem and refactor install-dependencies.sh --- Gemfile | 1 + ansible-files/install-dependencies.sh | 16 ---------------- .../install-fluentd-dependencies-ubuntu.sh | 1 + build-utils/install-ruby-dependencies.sh | 3 +++ build-utils/install-sdm-cli.sh | 7 +++++++ .../log-export-container.service | 0 build-utils/setup-os-ubuntu.sh | 5 +++++ 7 files changed, 17 insertions(+), 16 deletions(-) delete mode 100644 ansible-files/install-dependencies.sh create mode 100644 build-utils/install-fluentd-dependencies-ubuntu.sh create mode 100644 build-utils/install-ruby-dependencies.sh create mode 100644 build-utils/install-sdm-cli.sh rename {ansible-files => build-utils}/log-export-container.service (100%) create mode 100644 build-utils/setup-os-ubuntu.sh diff --git a/Gemfile b/Gemfile index f380e0b..4825f28 100644 --- a/Gemfile +++ b/Gemfile @@ -5,6 +5,7 @@ source 'https://rubygems.org' git_source(:github) {|repo_name| "https://github.com/#{repo_name}" } gem 'fluent' +gem 'fluentd' gem 'fluent-plugin-rewrite-tag-filter' gem 'fluent-plugin-s3' gem 'fluent-plugin-cloudwatch-logs' diff --git a/ansible-files/install-dependencies.sh b/ansible-files/install-dependencies.sh deleted file mode 100644 index ff17c30..0000000 --- a/ansible-files/install-dependencies.sh +++ /dev/null @@ -1,16 +0,0 @@ -apt update -ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime -export DEBIAN_FRONTEND=noninteractive -apt-get install -y tzdata -dpkg-reconfigure --frontend noninteractive tzdata -apt install --yes build-essential ruby-dev zlib1g -gem install bundler -v '~> 2.3.3' -bundle install - -apt update -apt install --yes curl -curl -J -O -L https://app.strongdm.com/releases/cli/linux - -unzip -x sdm*.zip -rm sdm*.zip -mv sdm /home/fluent \ No newline at end of file diff --git a/build-utils/install-fluentd-dependencies-ubuntu.sh b/build-utils/install-fluentd-dependencies-ubuntu.sh new file mode 100644 index 0000000..3d08a5c --- /dev/null +++ b/build-utils/install-fluentd-dependencies-ubuntu.sh @@ -0,0 +1 @@ +apt install --yes build-essential zlib1g \ No newline at end of file diff --git a/build-utils/install-ruby-dependencies.sh b/build-utils/install-ruby-dependencies.sh new file mode 100644 index 0000000..4ca0199 --- /dev/null +++ b/build-utils/install-ruby-dependencies.sh @@ -0,0 +1,3 @@ +apt install --yes ruby-dev +gem install bundler -v '~> 2.3.3' +bundle install \ No newline at end of file diff --git a/build-utils/install-sdm-cli.sh b/build-utils/install-sdm-cli.sh new file mode 100644 index 0000000..ecb8c5e --- /dev/null +++ b/build-utils/install-sdm-cli.sh @@ -0,0 +1,7 @@ +apt update +apt install --yes curl +curl -J -O -L https://app.strongdm.com/releases/cli/linux + +unzip -x sdm*.zip +rm sdm*.zip +mv sdm /home/fluent \ No newline at end of file diff --git a/ansible-files/log-export-container.service b/build-utils/log-export-container.service similarity index 100% rename from ansible-files/log-export-container.service rename to build-utils/log-export-container.service diff --git a/build-utils/setup-os-ubuntu.sh b/build-utils/setup-os-ubuntu.sh new file mode 100644 index 0000000..dc2194b --- /dev/null +++ b/build-utils/setup-os-ubuntu.sh @@ -0,0 +1,5 @@ +apt update +ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime +export DEBIAN_FRONTEND=noninteractive +apt-get install -y tzdata +dpkg-reconfigure --frontend noninteractive tzdata From cba7986b399efccb7085dc3e1e18253e3be50be8 Mon Sep 17 00:00:00 2001 From: Lucas Santos Date: Fri, 25 Feb 2022 08:44:11 -0300 Subject: [PATCH 3/9] remove user from log-export-container.service --- build-utils/log-export-container.service | 1 - 1 file changed, 1 deletion(-) diff --git a/build-utils/log-export-container.service b/build-utils/log-export-container.service index 8587019..d3559b4 100644 --- a/build-utils/log-export-container.service +++ b/build-utils/log-export-container.service @@ -3,7 +3,6 @@ Description=log export container daemon After=network.target [Service] -User=ubuntu EnvironmentFile=-/etc/sysconfig/log-export-container WorkingDirectory=/opt/log-export-container ExecStart=/opt/log-export-container/start.sh From 2858560e04a3dd1aa307887c50016f231124032a Mon Sep 17 00:00:00 2001 From: Lucas Santos Date: Fri, 25 Feb 2022 10:17:18 -0300 Subject: [PATCH 4/9] Add env-file.example, Ansible setup script and playbook --- env-file.example | 3 ++ lec-playbook.yml | 65 ++++++++++++++++++++++++++++++++++++++++ setup-ansible-and-lec.sh | 5 ++++ 3 files changed, 73 insertions(+) create mode 100644 env-file.example create mode 100644 lec-playbook.yml create mode 100644 setup-ansible-and-lec.sh diff --git a/env-file.example b/env-file.example new file mode 100644 index 0000000..b1939c0 --- /dev/null +++ b/env-file.example @@ -0,0 +1,3 @@ +FLUENTD_DIR=fluentd +LOG_EXPORT_CONTAINER_INPUT=syslog-json +LOG_EXPORT_CONTAINER_OUTPUT=stdout diff --git a/lec-playbook.yml b/lec-playbook.yml new file mode 100644 index 0000000..8fde15b --- /dev/null +++ b/lec-playbook.yml @@ -0,0 +1,65 @@ +--- +- name: "Installing Log Export Container with Ansible" + hosts: localhost + connection: local + +tasks: + +- name: Install Git + shell: apt install --yes git + +- name: Clone Log Export Container + shell: git clone https://github.com/devlucassantos/log-export-container.git /opt/log-export-container/ + +- name: REMOVEME + shell: git fetch && git checkout feat/ansible + args: + chdir: /opt/log-export-container + +- name: Update PATH variable + shell: export PATH=/home/fluent:$PATH + +- name: Modify install-dependencies.sh permissions + file: dest=/opt/log-export-container/build-utils owner=root group=root mode=777 recurse=yes + +- name: Configure Ubuntu + shell: /opt/log-export-container/build-utils/setup-os-ubuntu.sh + +- name: Install Ubuntu packages + shell: /opt/log-export-container/build-utils/install-fluentd-dependencies-ubuntu.sh + +- name: Install Ruby and Gem dependencies + shell: /opt/log-export-container/build-utils/install-ruby-dependencies.sh + args: + chdir: /opt/log-export-container + +- name: Install StrongDM CLI + shell: /opt/log-export-container/build-utils/install-sdm-cli.sh + +- name: Modify start.sh permissions + ansible.builtin.file: + path: /opt/log-export-container/start.sh + owner: root + group: root + mode: '777' + +- name: Create /etc/sysconfig directory if it does not exist + ansible.builtin.file: + path: /etc/sysconfig + state: directory + mode: '777' + +- name: Copy Log Export Container env file + ansible.builtin.copy: + src: env-file + dest: /etc/sysconfig/log-export-container + mode: '777' + +- name: Copy Log Export Container service file + ansible.builtin.copy: + src: /opt/log-export-container/build-utils/log-export-container.service + dest: /etc/systemd/system/ + mode: '777' + +- name: Enable and start Log Export Container service + shell: systemctl enable log-export-container.service && systemctl start log-export-container.service \ No newline at end of file diff --git a/setup-ansible-and-lec.sh b/setup-ansible-and-lec.sh new file mode 100644 index 0000000..0c3b345 --- /dev/null +++ b/setup-ansible-and-lec.sh @@ -0,0 +1,5 @@ +apt update +sudo apt install software-properties-common +add-apt-repository --yes --update ppa:ansible/ansible +apt install --yes ansible +ansible-playbook lec-playbook.yml \ No newline at end of file From a28b06e4a0edb30643df6fc33f19aa8a0c3628fe Mon Sep 17 00:00:00 2001 From: Lucas Santos Date: Fri, 25 Feb 2022 14:18:47 -0300 Subject: [PATCH 5/9] Fix setup-ansible-and-lec.sh and lec-playbook.yml --- lec-playbook.yml | 98 ++++++++++++++++++++-------------------- setup-ansible-and-lec.sh | 2 +- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/lec-playbook.yml b/lec-playbook.yml index 8fde15b..0166c6f 100644 --- a/lec-playbook.yml +++ b/lec-playbook.yml @@ -1,65 +1,65 @@ --- -- name: "Installing Log Export Container with Ansible" - hosts: localhost - connection: local + - name: "Installing Log Export Container with Ansible" + hosts: localhost + connection: local -tasks: + tasks: -- name: Install Git - shell: apt install --yes git + - name: Install Git + shell: apt install --yes git -- name: Clone Log Export Container - shell: git clone https://github.com/devlucassantos/log-export-container.git /opt/log-export-container/ + - name: Clone Log Export Container + shell: git clone https://github.com/devlucassantos/log-export-container.git /opt/log-export-container/ -- name: REMOVEME - shell: git fetch && git checkout feat/ansible - args: - chdir: /opt/log-export-container + - name: REMOVEME + shell: git fetch && git checkout feat/ansible + args: + chdir: /opt/log-export-container -- name: Update PATH variable - shell: export PATH=/home/fluent:$PATH + - name: Update PATH variable + shell: export PATH=/home/fluent:$PATH -- name: Modify install-dependencies.sh permissions - file: dest=/opt/log-export-container/build-utils owner=root group=root mode=777 recurse=yes + - name: Modify install-dependencies.sh permissions + file: dest=/opt/log-export-container/build-utils owner=root group=root mode=777 recurse=yes -- name: Configure Ubuntu - shell: /opt/log-export-container/build-utils/setup-os-ubuntu.sh + - name: Configure Ubuntu + shell: /opt/log-export-container/build-utils/setup-os-ubuntu.sh -- name: Install Ubuntu packages - shell: /opt/log-export-container/build-utils/install-fluentd-dependencies-ubuntu.sh + - name: Install Ubuntu packages + shell: /opt/log-export-container/build-utils/install-fluentd-dependencies-ubuntu.sh -- name: Install Ruby and Gem dependencies - shell: /opt/log-export-container/build-utils/install-ruby-dependencies.sh - args: - chdir: /opt/log-export-container + - name: Install Ruby and Gem dependencies + shell: /opt/log-export-container/build-utils/install-ruby-dependencies.sh + args: + chdir: /opt/log-export-container -- name: Install StrongDM CLI - shell: /opt/log-export-container/build-utils/install-sdm-cli.sh + - name: Install StrongDM CLI + shell: /opt/log-export-container/build-utils/install-sdm-cli.sh -- name: Modify start.sh permissions - ansible.builtin.file: - path: /opt/log-export-container/start.sh - owner: root - group: root - mode: '777' + - name: Modify start.sh permissions + ansible.builtin.file: + path: /opt/log-export-container/start.sh + owner: root + group: root + mode: '777' -- name: Create /etc/sysconfig directory if it does not exist - ansible.builtin.file: - path: /etc/sysconfig - state: directory - mode: '777' + - name: Create /etc/sysconfig directory if it does not exist + ansible.builtin.file: + path: /etc/sysconfig + state: directory + mode: '777' -- name: Copy Log Export Container env file - ansible.builtin.copy: - src: env-file - dest: /etc/sysconfig/log-export-container - mode: '777' + - name: Copy Log Export Container env file + ansible.builtin.copy: + src: env-file + dest: /etc/sysconfig/log-export-container + mode: '777' -- name: Copy Log Export Container service file - ansible.builtin.copy: - src: /opt/log-export-container/build-utils/log-export-container.service - dest: /etc/systemd/system/ - mode: '777' + - name: Copy Log Export Container service file + ansible.builtin.copy: + src: /opt/log-export-container/build-utils/log-export-container.service + dest: /etc/systemd/system/ + mode: '777' -- name: Enable and start Log Export Container service - shell: systemctl enable log-export-container.service && systemctl start log-export-container.service \ No newline at end of file + - name: Enable and start Log Export Container service + shell: systemctl enable log-export-container.service && systemctl start log-export-container.service diff --git a/setup-ansible-and-lec.sh b/setup-ansible-and-lec.sh index 0c3b345..b0bed5b 100644 --- a/setup-ansible-and-lec.sh +++ b/setup-ansible-and-lec.sh @@ -1,5 +1,5 @@ apt update -sudo apt install software-properties-common +apt install --yes software-properties-common add-apt-repository --yes --update ppa:ansible/ansible apt install --yes ansible ansible-playbook lec-playbook.yml \ No newline at end of file From 937187a4ae371f0891f5087ce2d5f0bf8d4336b2 Mon Sep 17 00:00:00 2001 From: Lucas Santos Date: Fri, 25 Feb 2022 16:05:04 -0300 Subject: [PATCH 6/9] Add AMI User Data script and fix install-sdm-cli.sh --- ami-cloud-init-user-data.sh | 7 +++++++ build-utils/install-sdm-cli.sh | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 ami-cloud-init-user-data.sh diff --git a/ami-cloud-init-user-data.sh b/ami-cloud-init-user-data.sh new file mode 100644 index 0000000..ec5982c --- /dev/null +++ b/ami-cloud-init-user-data.sh @@ -0,0 +1,7 @@ +#!/bin/bash +apt install --yes curl +curl -O https://raw.githubusercontent.com/devlucassantos/log-export-container/feat/ansible/setup-ansible-and-lec.sh +chmod +x setup-ansible-and-lec.sh +curl -O https://raw.githubusercontent.com/devlucassantos/log-export-container/feat/ansible/lec-playbook.yml +curl -o env-file https://raw.githubusercontent.com/devlucassantos/log-export-container/feat/ansible/env-file.example +./setup-ansible-and-lec.sh diff --git a/build-utils/install-sdm-cli.sh b/build-utils/install-sdm-cli.sh index ecb8c5e..ca28aa8 100644 --- a/build-utils/install-sdm-cli.sh +++ b/build-utils/install-sdm-cli.sh @@ -4,4 +4,5 @@ curl -J -O -L https://app.strongdm.com/releases/cli/linux unzip -x sdm*.zip rm sdm*.zip -mv sdm /home/fluent \ No newline at end of file +mkdir /home/fluent +mv sdm /home/fluent/ \ No newline at end of file From 7cba3b0c07877dcb5814ea1d39fd08ef7e70cc6b Mon Sep 17 00:00:00 2001 From: Lucas Santos <89457923+devlucassantos@users.noreply.github.com> Date: Thu, 3 Mar 2022 11:19:19 -0300 Subject: [PATCH 7/9] Create ami-variables-user-data.sh.example --- ami-variables-user-data.sh.example | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ami-variables-user-data.sh.example diff --git a/ami-variables-user-data.sh.example b/ami-variables-user-data.sh.example new file mode 100644 index 0000000..6051f7b --- /dev/null +++ b/ami-variables-user-data.sh.example @@ -0,0 +1,7 @@ +#!/bin/bash + +echo "LOG_EXPORT_CONTAINER_OUTPUT=mongo" >> /etc/sysconfig/log-export-container +echo "MONGO_URI=mongodb://username:password@hostname.com" >> /etc/sysconfig/log-export-container + +# Needed for applying the new variables configured above +systemctl restart log-export-container From 91df17ff41141886a11873668f154f72e50cdc0c Mon Sep 17 00:00:00 2001 From: vassalo Date: Mon, 7 Mar 2022 12:17:18 -0300 Subject: [PATCH 8/9] Update deeplinks Co-authored-by: devlucassantos Co-authored-by: gaabrieljesuss --- ami-cloud-init-user-data.sh | 6 +++--- lec-playbook.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ami-cloud-init-user-data.sh b/ami-cloud-init-user-data.sh index ec5982c..e3f9f41 100644 --- a/ami-cloud-init-user-data.sh +++ b/ami-cloud-init-user-data.sh @@ -1,7 +1,7 @@ #!/bin/bash apt install --yes curl -curl -O https://raw.githubusercontent.com/devlucassantos/log-export-container/feat/ansible/setup-ansible-and-lec.sh +curl -O https://raw.githubusercontent.com/strongdm/log-export-container/feat/ansible/setup-ansible-and-lec.sh chmod +x setup-ansible-and-lec.sh -curl -O https://raw.githubusercontent.com/devlucassantos/log-export-container/feat/ansible/lec-playbook.yml -curl -o env-file https://raw.githubusercontent.com/devlucassantos/log-export-container/feat/ansible/env-file.example +curl -O https://raw.githubusercontent.com/strongdm/log-export-container/feat/ansible/lec-playbook.yml +curl -o env-file https://raw.githubusercontent.com/strongdm/log-export-container/feat/ansible/env-file.example ./setup-ansible-and-lec.sh diff --git a/lec-playbook.yml b/lec-playbook.yml index 0166c6f..d2dfaf3 100644 --- a/lec-playbook.yml +++ b/lec-playbook.yml @@ -9,7 +9,7 @@ shell: apt install --yes git - name: Clone Log Export Container - shell: git clone https://github.com/devlucassantos/log-export-container.git /opt/log-export-container/ + shell: git clone https://github.com/strongdm/log-export-container.git /opt/log-export-container/ - name: REMOVEME shell: git fetch && git checkout feat/ansible From de7569e4a53f817301161015a00e843041fa1e66 Mon Sep 17 00:00:00 2001 From: Lucas Santos Date: Thu, 10 Mar 2022 12:36:52 -0300 Subject: [PATCH 9/9] Add clean-up-credentials.sh (#56) Co-authored-by: vassalo Co-authored-by: gaabrieljesuss --- build-utils/clean-up-credentials.sh | 5 +++++ lec-playbook.yml | 11 ++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 build-utils/clean-up-credentials.sh diff --git a/build-utils/clean-up-credentials.sh b/build-utils/clean-up-credentials.sh new file mode 100644 index 0000000..df892f1 --- /dev/null +++ b/build-utils/clean-up-credentials.sh @@ -0,0 +1,5 @@ +sed -i 's/#PermitRootLogin.\+/PermitRootLogin without-password/g' /etc/ssh/sshd_config +passwd -l root +shred -u /etc/ssh/*_key /etc/ssh/*_key.pub +sed -i 's/#UseDNS.\+/UseDNS no/g' /etc/ssh/sshd_config +rm /root/.ssh/authorized_keys diff --git a/lec-playbook.yml b/lec-playbook.yml index d2dfaf3..30a0909 100644 --- a/lec-playbook.yml +++ b/lec-playbook.yml @@ -11,11 +11,6 @@ - name: Clone Log Export Container shell: git clone https://github.com/strongdm/log-export-container.git /opt/log-export-container/ - - name: REMOVEME - shell: git fetch && git checkout feat/ansible - args: - chdir: /opt/log-export-container - - name: Update PATH variable shell: export PATH=/home/fluent:$PATH @@ -63,3 +58,9 @@ - name: Enable and start Log Export Container service shell: systemctl enable log-export-container.service && systemctl start log-export-container.service + + - name: Copy clean-up-credentials.sh script + ansible.builtin.copy: + src: /opt/log-export-container/build-utils/clean-up-credentials.sh + dest: / + mode: '777'