Technical Note: Disaster recovery configuration automation

[yujames]

Description

This article describes how to automatically generate and apply a DR configuration from a fresh install of FortiSIEM DR instance.

Scope

FortiSIEM 5.2.1

1. A running FortiSIEM instance (primary FortiSIEM)
2. A fresh install of FortiSIEM with default root password (Secondary FortiSIEM)
3. DNS server

Prerequisite:

1. DNS configuration: both instances must have a valid fqdn, a third fqdn is required for a seamless failover and should point to the primary (CNAME) when the cluster is in normal working condition. TTL must be reasonably small (few minutes) for the change to occur quickly after a failover.
2. The DR instance must have the default root password (it is used by the automation script)
3. The DR instance has to be prepared (/opt/vmware/share/vami/vami_config_net) and licensed
4. Make sure the storage mode is the same for both instances (Local or NFS. ElasticSearch is out of this scope)

Solution

1. Copy the script (dr_automation.sh) to the Primary FortiSIEM instance
2. make it executable (chmod +x dr_automation.sh)
3. ./dr_automation 192.0.2.72 hq.fortielab.com dr.fortielab.com
1. Where:
1. 192.0.2.72: is the IP address of the secondary FortiSIEM instance
2. hq.fortielab.com: primary FortiSIEM Instance FQDN
3. dr.fortielab.com: secondary FortiSIEM Instance FQDN
4. if the secondary FSM is reachable and has the default root password and licensed the output should look like:

5. (Optional) to verify the keys have been created and installed properly:
1. su – admin
2. ssh 192.0.2.72 (use your secondary FortiSIEM IP)
3. It should open an ssh session without authentication nor authenticity check (known_hosts)
4. From the secondary instance ssh back to the primary:
5. ssh 192.0.2.71 (use your primary FortiSIEM IP)
6. It should open an ssh session without authentication nor authenticity check
6. Copy the JSON output (from lines [ to ]) and save it to a file. or simply copy the dr.json file you will find in the same folder where the script was executed.
7. Open the primary FortiSIEM and navigate to Admin => Settings => Replicate
8. Click: “Enable Replication” then click on the Import button, sleect the JSON config file you saved earlier, the config then should be imported:
9. 
10. Click Apply
11. It would typically take a some time when the task progress reaches 40%
12. Open the secondary FortiSIEM web GUI and navigate to: Admin => Settings => Replicate
13. Perform the same config import than the one done on the primary (click Enable replication => import button => select the JSON file saved from the script output => click Apply)
14. After a reboot open the secondary FortiSIEM instance and make sure:
1. Incidents (GUI) are replicated
2. Events : (after 10 minutes or so the size of the eventdb on the secondary should be similar to the primary “du -sh /data/eventdb/”)
3. CMDB (GUI) objects are replicated
4. SVN (device configurations : du -sh /svn/)

Automation Script:

dr_automation.sh:

#!/bin/bash

# Creates a key pair in preparation for FSM DR setup, it assumes the default passwords are unchanged on both instances

# the script takes as arguments: FSM_DR_IP_ADDRESS HO_HOSTNAME DR_HOSTNAME

#Vars

if [ "$#" -ne 3 ]; then

    echo "Usage: $0 FSM_DR_IP_ADDRESS HO_HOSTNAME DR_HOSTNAME" && exit

fi

EXP="/tmp/expector.exp"

DR_IP="$1"

HO_IP=$(hostname -I|tr -d ' ')

CMD_RES="/tmp/cmd_res"

DR_KEY="/tmp/dr_key"

JSON="dr.json"

HO_HOSTNAME="$2"

DR_HOSTNAME="$3"

#Functions

function valid_ip()

{

    local  ip=$1

    local  stat=1

    if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then

        OIFS=$IFS

        IFS='.'

        ip=($ip)

        IFS=$OIFS

        [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \

            && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]

        stat=$?

    fi

    return $stat

}

#Checking Arguments

if ! valid_ip $DR_IP

then

    echo "$DR_IP is not a valid IP address" && exit

else

    if ! ping -c1 $1 &>/dev/null

    then

        echo "FSM DR Host $1 Unreachable" && exit

    fi

fi

# prepare the expect sub-script

cat << 'EOF' > $EXP && chmod 755 $EXP

#!/usr/bin/expect -f

log_user 0

exp_internal 0

match_max -d 50000

if {[llength $argv] != 7} {

    puts "scriptName {ssh|telnet} hostIP hostPort user password finalCmd ouputFileName"

    exit

}

set method [lindex $argv 0]

set sshMethod "ssh"

set telnetMethod "telnet"

set host [lindex $argv 1]

set port [lindex $argv 2]

set user [lindex $argv 3]

set passwd [lindex $argv 4]

set finalCmd [lindex $argv 5]

set outFile [lindex $argv 6]

set fh [open $outFile w]

set timeout 9

set sentUser 0

set sentPasswd 0

set sentYes 0

set env(TERM) "vt100"

if {[string compare $method $telnetMethod] ==0} {

   spawn -noecho telnet $host $port

}

if {[string compare $method $sshMethod] ==0}  {

   spawn -noecho ssh -p $port $user@$host

}

expect {

    timeout { puts $fh "Error: no response"; exit}

    -nocase "yes/no)?" {send "yes\r"; set sentYes 1;}

    -nocase "connection refused" { puts $fh "Error: connection refused"; exit}

    -nocase "Username:" { send "$user\r"; set sentUser 1;}

    -nocase "login:" { send "$user\r"; set sentUser 1;}

    -nocase "Password:" { send "$passwd\r"; set sentPasswd 1;}

    -re {[#$>][ \t]*(\e.*)?$} { send "$finalCmd\r";}

}

if {$sentYes ==1} {

  expect {

    timeout { puts $fh "Error: no response"; exit}

    -nocase "connection refused" { puts $fh "Error: connection refused"; exit}

    -nocase "Username:" { send "$user\r"; set sentUser 1;}

    -nocase "Password:" { send "$passwd\r"; set sentPasswd 1;}

    -re {[#$>][ \t]*(\e.*)?$} { send "$finalCmd\r";}

  }

}

if {$sentUser == 1} {

  expect {

    timeout { puts $fh "Error: bad user name"; exit}

    -nocase "Password:" { send "$passwd\r"; set sentPasswd 1;}

  }

}

if {$sentPasswd == 1} {

  expect {

    timeout { puts $fh "Error: bad password"; exit}

    -re {[#$>][ \t]*(\e.*)?$} {send "$finalCmd\r";}

  }

}

set running 1

while {$running} {

    expect {

    timeout { puts $fh "Error: error in communication"; exit}

    "\n" { puts -nonewline $fh "$expect_out(buffer)";}

    eof {set running 0;}

    -re {[#$>][ \t]*(\e.*)?$} {set running 0; send "exit\r";}

    }

}

close $fh;

EOF

# Create local keys

# Add dr cert to known_hosts

rm -rf /opt/phoenix/bin/.ssh/

local_cmd=$(rm -rf /opt/phoenix/bin/.ssh && \

            mkdir -p /opt/phoenix/bin/.ssh && \

            chmod 700 /opt/phoenix/bin/.ssh && \

            ssh-keygen -b 2048 -t rsa -f /opt/phoenix/bin/.ssh/id_rsa -q -N "" && \

            ssh-keyscan -H $DR_IP > /opt/phoenix/bin/.ssh/known_hosts && \

            chown -R admin:admin /opt/phoenix/bin/.ssh )

if [ $? -eq 0 ] && [ -f /opt/phoenix/bin/.ssh/id_rsa.pub ]

then

        ho_pub_key=$(cat /opt/phoenix/bin/.ssh/id_rsa.pub)

        echo "local keys created"

fi

# add local public key to remote authorized_keys and add local id on remote known_hosts

$EXP ssh $DR_IP 22 root 'ProspectHills' "rm -rf /opt/phoenix/bin/.ssh && \

mkdir -p /opt/phoenix/bin/.ssh && \

chmod 700 /opt/phoenix/bin/.ssh && \

ssh-keyscan -H $(hostname -I) > /opt/phoenix/bin/.ssh/known_hosts && \

echo $ho_pub_key > /opt/phoenix/bin/.ssh/authorized_keys && \

chmod 600 /opt/phoenix/bin/.ssh/authorized_keys && \

chown -R admin:admin /opt/phoenix/bin/.ssh && \

echo OK" $CMD_RES

if grep -q "OK" $CMD_RES

then

    echo "Keys added successfully"

else

    echo "Couldn't add key" && exit

fi

# Create key pairs on the remote FSM

$EXP ssh $DR_IP 22 root 'ProspectHills' "ssh-keygen -b 2048 -t rsa -f /opt/phoenix/bin/.ssh/id_rsa -q -N \"\" && \

chown -R admin:admin /opt/phoenix/bin/.ssh && \

cat /opt/phoenix/bin/.ssh/id_rsa.pub" $CMD_RES

if grep -q "root@" $CMD_RES

then

    echo "Keys created successfully"

else

    echo "Couldn't create keys" && exit

fi

dr_pub_key=$(cat $CMD_RES | grep 'root@')

# Get DR UUID

$EXP ssh $DR_IP 22 root 'ProspectHills' "phgetUUID | grep UUID | cut -d':' -f2" $CMD_RES

dr_uuid=$(cat $CMD_RES|grep -Po '\w+-\w+\w+-\w+-\w+-\w+')

if grep -q "-" $CMD_RES

then

    echo -n "UUID retrieved successfully: "

    echo $dr_uuid

else

    echo "Couldn't retrieve UUID" && exit

fi

# Get HO UUID

ho_uuid=$(phgetUUID|grep -Po '\w+-\w+\w+-\w+-\w+-\w+')

echo $dr_pub_key >> /opt/phoenix/bin/.ssh/authorized_keys

chown -R admin:admin /opt/phoenix/bin/.ssh

chmod 600 /opt/phoenix/bin/.ssh/authorized_keys

ho_pub_key=${ho_pub_key//[$'\t\r\n']}

dr_pub_key=${dr_pub_key//[$'\t\r\n']}

#d_pub_key=$(echo $dr_pub_key | grep -Po '.*==\s+\w+@\w+')

echo "[" > $JSON

echo "{" >> $JSON

echo "\"role\": \"primary\"," >> $JSON

echo "\"frequency\": \"10\"," >> $JSON

echo "\"ip\": \"$HO_IP\"," >> $JSON

echo "\"uuid\": \"$ho_uuid\"," >> $JSON

echo "\"cmdbPath\": \"/cmdb\"," >> $JSON

echo "\"publicKey\": \"${ho_pub_key}\"," >> $JSON

echo "\"privateKey\": \"/opt/phoenix/bin/.ssh/id_rsa\"," >> $JSON

echo "\"doEventReplication\": true," >> $JSON

echo "\"host\": \"$HO_HOSTNAME\"" >> $JSON

echo "}," >> $JSON

echo "{" >> $JSON

echo "\"role\": \"secondary\"," >> $JSON

echo "\"frequency\": \"10\"," >> $JSON

echo "\"ip\": \"$DR_IP\"," >> $JSON

echo "\"uuid\": \"$dr_uuid\"," >> $JSON

echo "\"cmdbPath\": \"/cmdb\"," >> $JSON

echo "\"publicKey\": \"${dr_pub_key}\"," >> $JSON

echo "\"privateKey\": \"/opt/phoenix/bin/.ssh/id_rsa\"," >> $JSON

echo "\"doEventReplication\": true," >> $JSON

echo "\"host\": \"$DR_HOSTNAME\"" >> $JSON

echo "}" >> $JSON

echo "]" >> $JSON

cat $JSON

#clean up

rm -f $EXP

rm -f $CMD_RES