Playbook for backing up WordPress servers Assumes WordPress is installed at /var/www/wordpress

- hosts: all
  become: true
  vars_files:
    - $PATH_TO_VARS_FILE

  tasks:
    - name: Update repositories
      apt: update_cache=yes force_apt_get=yes cache_valid_time=3600

    - name: Check if server is a WordPress host
      stat:
        path: /var/www/wordpress
      register: wp_check

    - name: Install prerequisites python3-pip
      apt:
        name: python3-pip
        state: present
      when: wp_check.stat.exists

    - name: Install prerequisites boto3
      pip:
        name:
          - boto3
          - botocore

    - name: Create temporary folder for backups on source
      file:
        path: "/tmp/{{ ansible_host }}"
        state: directory
        mode: '0755'
      tags: [ backup ]

    - name: Copy WordPress wp-content folder to backup folder
      copy:
        src: /var/www/wordpress/wp-content
        dest: "/tmp/{{ ansible_host }}"
        mode: '0755'
        remote_src: yes
      tags: [ backup ]

    - name: Do a MySQL dump to copy the database
      mysql_db:
        name: wordpress
        state: dump
        login_user: root
        login_password: "{{ mysql_root_pw }}"
        target: "/tmp/{{ ansible_host }}/backup.sql"

    - name: Compress backup directory
      archive:
        path: "/tmp/{{ ansible_host }}"
        dest: "/tmp/{{ ansible_host }}.{{ ansible_date_time.date }}.tar.gz"

    - name: Copy backup file to storage
      aws_s3:
        aws_access_key: "{{ do_access_key }}"
        aws_secret_key: "{{ do_secret_key }}"
        region: sfo3
        s3_url: "$DO_SPACES_URL_GOES_HERE"
        bucket: "backups"
        src: "/tmp/{{ ansible_host }}.{{ ansible_date_time.date }}.tar.gz"
        object: "{{ ansible_date_time.date }}/{{ ansible_host }}.tar.gz"
        mode: put

    - name: Cleanup temporary files on source
      shell: rm -rf /tmp/{{ ansible_host }} ; rm -rf /tmp/{{ ansible_host }}.*
      args:
        warn: false

Show active Vhost configuration in Apache

apache2ctl -S

Enable vhost in apache

a2ensite $VHOST.conf

Disable vhost in apache

a2dissite $VHOST.conf

Enable SSL in apache

a3enmod ssl

Check for errors in apache

apache2ctl configtest

Note This will NOT work while using wordpress with .htaccess file Install Apache utils package

apt-get update
apt-get install apache2-utils

create password file and user

htpasswd -c /etc/apache2/.htpasswd $USERNAME

LEAVE OFF -c for additional users

htpasswd /etc/apache2/.htpasswd $USERNAME2

Must be added to the ‘Directory’ section of the Apache VirtualHost file

  AuthType Basic
  AuthName "Restricted Content"
  AuthUserFile /etc/apache2/.htpasswd
  Require valid-user

check apache configuration

apache2ctl configtest

restart apache

systemctl restart apache

if seeing misconfiguration errors, may need to update ownership of .htpasswd

chown www-data:www-data /etc/apache2/.htpasswd

install java

sudo apt install default-jre default-jdk

Install Elasticsearch
get public GPG key

curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -

add elastic to sources list

echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list

check for updates

sudo apt update

install elasticsearch

sudo apt install elasticsearch

edit elasticsearch.yml config file

sudo vim /etc/elasticsearch/elasticsearch.yml

update network.host to read localhost

network.host: localhost

start elasticsearch service and then enable

sudo systemctl start elasticsearch
sudo systemctl enable elasticsearch

IF the service fails to start due to ‘Timeout’ you may need to increase the timeout time

Make a folder

sudo mkdir /etc/systemd/system/elasticsearch.service.d

Now create a file and add the content to increase timeout time to 900 seconds

echo -e "[Service]\nTimeoutStartSec=900" | sudo tee /etc/systemd/system/elasticsearch.service.d/startup-timeout.conf

test that its working

curl -X GET "localhost:9200"

Install Kibana

sudo apt install kibana

start and enable

sudo systemctl enable kibana
sudo systemctl start kibana

Install and configure Apache w/ ACL

Install Logstash

sudo apt install logstash

create files to process input

sudo vim /etc/logstash/conf.d/02-beats-input.conf

copy below into that file

input {
  beats {
    port => 5044
  }
}

create output file

sudo vim /etc/logstash/conf.d/30-elasticsearch-output.conf

copy below into that file

output {
  if [@metadata][pipeline] {
    elasticsearch {
    hosts => ["localhost:9200"]
    manage_template => false
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
    pipeline => "%{[@metadata][pipeline]}"
    }
  } else {
    elasticsearch {
    hosts => ["localhost:9200"]
    manage_template => false
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
    }
  }
}

test logstash

sudo -u logstash /usr/share/logstash/bin/logstash --path.settings /etc/logstash -t

need to see - Config Validation Result: OK. Exiting Logstash

if ok, start and enable

sudo systemctl start logstash
sudo systemctl enable logstash

Installing Filebeat

sudo apt install filebeat

configure yml file

sudo vim /etc/filebeat/filebeat.yml

from here we need to disable the elasticsearch output

...
#output.elasticsearch:
  # Array of hosts to connect to.
  #hosts: ["localhost:9200"]
...

and then enable the logstash output

output.logstash:
  # The Logstash hosts
  hosts: ["localhost:5044"]

list filebeat modules

sudo filebeat modules list

enable filebeat modules

sudo filebeat modules enable system apache

create filebeat ingest pipeline

sudo filebeat setup --pipelines --modules system

load template index into elasticsearch

sudo filebeat setup --index-management -E output.logstash.enabled=false -E 'output.elasticsearch.hosts=["localhost:9200"]'

need to see - Index setup finished.

load filebeat dashboards into Kibana

sudo filebeat setup -E output.logstash.enabled=false -E output.elasticsearch.hosts=['localhost:9200'] -E setup.kibana.host=localhost:5601

start and enable filebeat

sudo systemctl start filebeat
sudo systemctl enable filebeat

Adding filebeat to other sources

curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
sudo apt update
sudo apt install filebeat

vim /etc/filebeat/filebeat.yml

hashout the Elasticsearch output section and then fill in the logstash output section with the correct remote IP

sudo filebeat modules enable system apache
sudo systemctl start filebeat.service
sudo systemctl enable filebeat.service

create folder for exporter apps

sudo mkdir /usr/local/bin/metrics
sudo mkdir /etc/metrics
sudo useradd -rs /bin/false exporter
sudo chmod -R 777 /usr/local/bin/metrics
sudo chmod -R 777 /etc/metrics
sudo chown -R exporter:exporter /usr/local/bin/metrics
sudo chown -R exporter:exporter /etc/metrics

Install node_exporter

curl -O -L https://github.com/prometheus/node_exporter/releases/download/v1.2.2/node_exporter-1.2.2.linux-amd64.tar.gz
tar xzvf node_exporter-*.*-amd64.tar.gz
sudo mv node_exporter-*.*-amd64 /usr/local/bin/node_exporter

create service

sudo vim /etc/systemd/system/node_exporter.service

[Unit]
Description=Node exporter for Prometheus
After=network.target

[Service]
User=exporter
Type=simple
ExecStart=/usr/local/bin/node_exporter/node_exporter

[Install]
WantedBy=multi-user.target

start and enable service

sudo systemctl start node_exporter.service
sudo systemctl status node_exporter.service
sudo systemctl enable node_exporter.service

open firewall on server

sudo ufw allow from $PANOPTICON proto tcp to any port 9100

Install Promtail from https://github.com/grafana/loki/releases/

curl -O -L https://github.com/grafana/loki/releases/download/v2.3.0/promtail-linux-amd64.zip
sudo apt install unzip
unzip promtail-linux-amd64.zip
sudo mv promtail-linux-amd64 /usr/local/bin/promtail

make conf file

sudo vim /etc/metrics/promtail.yml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /etc/metrics/positions.yaml

clients:
  - url: http://137.184.43.210:3100/loki/api/v1/push

scrape_configs:
- job_name: system
  static_configs:
  - targets:
      - localhost
    labels:
      job: system logs
      host: $HOSTNAME
      __path__: /var/log/*log

create service

sudo vim /etc/systemd/system/promtail.service

[Unit]
Description=Promtail service
After=network.target

[Service]
Type=simple
User=root
ExecStart=/usr/local/bin/promtail -config.file /etc/metrics/promtail.yml

[Install]
WantedBy=multi-user.target

start and enable service

sudo systemctl start promtail.service
sudo sytemctl status promtail.service
sudo systemctl enable promtail.service

open firewall on LOKI server

sudo ufw allow from $REMOTE_IP proto tcp to any port 3100

Installing BlackBox Exporter

curl -O -L https://github.com/prometheus/blackbox_exporter/releases/download/v0.19.0/blackbox_exporter-0.19.0.linux-amd64.tar.gz
tar xzf blackbox_exporter-*.*.linux-amd64.tar.gz
sudo mv /blackbox_exporter /usr/local/bin/metrics/blackbox_exporter

make conf file

sudo vim /etc/metrics/blackbox.yml

modules:
  http_prometheus:
     prober: http
     timeout: 5s
     http:
       valid_http_versions: ["HTTP/1.1", "HTTP/2"]
       method: GET
       fail_if_ssl: false
       fail_if_not_ssl: true
       tls_config:
         insecure_skip_verify: true
  http_2xx:
    prober: http
    http:
     preferred_ip_protocol: ip4
  http_post_2xx:
    prober: http
    http:
      method: POST
  tcp_connect:
    prober: tcp
  pop3s_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^+OK"
      tls: true
      tls_config:
        insecure_skip_verify: false
  ssh_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^SSH-2.0-"
      - send: "SSH-2.0-blackbox-ssh-check"
  irc_banner:
    prober: tcp
    tcp:
      query_response:
      - send: "NICK prober"
      - send: "USER prober prober prober :prober"
      - expect: "PING :([^ ]+)"
        send: "PONG ${1}"
      - expect: "^:[^ ]+ 001"
  icmp:

create blackbox service

sudo vim /etc/systemd/system/blackbox_exporter.service

[Unit]
Description=Blackbox exporter for Prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=exporter
Type=simple
ExecStart=/usr/local/bin/metrics/blackbox/blackbox_exporter \
 --config.file=/etc/metrics/blackbox.yml \
 --web.listen-address=":9115"


[Install]
WantedBy=multi-user.target

enable service

sudo systemctl start blackbox_exporter.service
sudo systemctl status blackbox_exporter.service
sudo systemctl enable blackbox_exporter.service

open UFW port on server

sudo ufw allow from 137.184.43.210 proto tcp to any port 9115

add target to prometheus.yml file on panopticon server

<VirtualHost *:80>
  ServerAdmin  $EMAIL
  ServerName   $FQDN
  ServerAlias  www.$FQDN
  DocumentRoot /var/www/$HOSTNAME
  <Directory /var/www/$HOSTNAME/>
    Options FollowSymLinks
    AllowOverride All
    Require all granted
  </Directory>
ErrorLog  ${APACHE_LOG_DIR}/error.log
CustomLog ${APACHE_LOG_DIR}/access.log combined
</VirtualHost>

<VirtualHost *:80>
  ServerName  $FQDN
  ServerAlias www.$FQDN
  Redirect permanent / https://$FQDN/
</VirtualHost>

<VirtualHost *:443>
  ServerAdmin $EMAIL
  DocumentRoot /var/www/$HOSTNAME
  ServerName  $FQDN
  ServerAlias www.$FQDN
  SSLEngine on
  SSLCertificateFile    $PATH_TO_CERTIFICATE
  SSLCertificateKeyFile $PATH_TO_PRIVATE_KEY
  <Directory /var/www/$HOSTNAME/>
    Options FollowSymLinks
    AllowOverride All
    Require all granted
  </Directory>
ErrorLog  ${APACHE_LOG_DIR}/error.log
CustomLog ${APACHE_LOG_DIR}/access.log combined
</VirtualHost>

Will need to enable the Apache SSL Mod for HTTPS to work

sudo a2enmod ssl

This VHost file will accept requests at $FQDN and then redirect them to the address listed in Proxy/ProxyPass and then applies the $FQDN in the address bar.

<VirtualHost *:80>
  ServerName $FQDN
  ProxyRequests Off
  <Location />
    ProxyPreserveHost On
    ProxyPass         http://137.184.11.237:8080/
    ProxyPassReverse  http://137.184.11.237:8080/
  </Location>
ErrorLog  ${APACHE_LOG_DIR}/error.log
CustomLog ${APACHE_LOG_DIR}/access.log combined
</VirtualHost>

Will need to enable the Apache Proxy Mod for this to work

sudo a2enmod proxy proxy_http

This works similarly to above except it will appply a SSL certificate as well.

<VirtualHost *:80>
ServerName storage.casat.org
Redirect Permanent / https://storage.casat.org/
</VirtualHost>

<VirtualHost *:443>
  ServerName storage.casat.org
  SSLEngine On
  SSLCertificateFile    /etc/letsencrypt/live/storage.casat.org/fullchain.pem
  SSLCertificateKeyFile /etc/letsencrypt/live/storage.casat.org/privkey.pem
  ProxyRequests Off
  <Location />
    ProxyPreserveHost On
    ProxyPass         http://137.184.11.237:8080/
    ProxyPassReverse  http://137.184.11.237:8080/
  </Location>
  # uncomment for SSL
  SSLProxyEngine On  
ErrorLog  ${APACHE_LOG_DIR}/error.log
CustomLog ${APACHE_LOG_DIR}/access.log combined
</VirtualHost>

This requires a few Apache Mods to be enabled to work

sudo a2enmod ssl proxy proxy_http

Note This will NOT work while using wordpress with .htaccess file Install Apache utils package

apt-get update
apt-get install apache2-utils

create password file and user

htpasswd -c /etc/apache2/htpasswd.users $USERNAME

LEAVE OFF -c for additional users

htpasswd /etc/apache2/htpasswd.users $USERNAME2

Must be added to the ‘Directory’ section of the Apache VirtualHost file

  AuthType Basic
  AuthName "Restricted Content"
  AuthUserFile /etc/apache2/htpasswd.users
  Require valid-user

check apache configuration

apache2ctl configtest

restart apache

systemctl restart apache

if seeing misconfiguration errors, may need to update ownership of .htpasswd

chown www-data:www-data /etc/apache2/htpasswd.users