salt_gisworkstation/runme.sh

#!/bin/bash

# Salt module: GIS workstation environment for multiple computers
# Copyright (C) 2018  Pekka Helenius
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

###########################################################

# Salt master requirement: Ubuntu 18.04 LTS or variants
# Salt minion requirement: Microsoft Windows and/or Ubuntu 18.04 LTS or variants

###########################################################

# Test connection to minion computers and ask for confirmation to continue. 
# May be useful in some situations but consider disabling the check 
# in large automated cluster environments.
#
# If accepted minion computer can't be reached and the value is true, 
# user is asked for input
#
# Default value: true
#
TEST_MINION_PING="true"

# Automatically accept all minion computers?
# If not true, user is asked for input for each unaccepted minion ID
#
# Default value: true
#
AUTOACCEPT_MINIONS="true"

# Use more verbose debug output for Salt master?
#
# Default value: true
#
SALT_DEBUG="true"

# Run salt only locally (I.E. test localhost minion)?
#
# Another implementation would be using 'salt-call' command
#
# Default value: true
#
SALT_LOCALONLY="false"

###########################################################
# Check root
function checkRoot() {
    if [ $(id -u) -ne 0 ]; then
        echo "Run the script with root permissions (sudo or root)."
        exit 1
    fi
}

checkRoot

###########################################################
# Colors

bash_red='\e[91m'
bash_color_default='\e[0m'

###########################################################
# Check for command dependencies

function checkCommands() {
    if [[ $(which --help 2>/dev/null) ]] && [[ $(echo --help 2>/dev/null) ]]; then

        COMMANDS=(grep mkdir systemctl id wget cat tee awk sed tr chmod cp rm mv touch dpkg apt-cache apt-get)

        a=0
        for command in ${COMMANDS[@]}; do
            if [[ ! $(which $command 2>/dev/null) ]]; then
                COMMANDS_NOTFOUND[$a]=$command
                let a++
            fi
        done

        if [[ -v COMMANDS_NOTFOUND ]]; then
            echo -e "\n${bash_red}Error:${bash_color_default} The following commands could not be found: ${COMMANDS_NOTFOUND[*]}\nAborting\n"
            exit 1
        fi
    else
        exit 1
    fi
}

checkCommands

###########################################################
# Check Salt master Linux distribution

if [[ -f /etc/os-release ]]; then

    MASTER_DISTRO=$(grep ^VERSION_ID /etc/os-release | grep -oP '(?<=")[^"]*')

    if [[ $MASTER_DISTRO != "18.04" ]]; then
        echo -e "This script is supported only on Ubuntu 18.04 LTS. Aborting.\n"
        exit 1
    fi

else

    echo -e "Can't recognize your Linux distribution. Aborting.\n"
    exit 1

fi

###########################################################
# Welcome message

function welcomeMessage() {

    echo -e "This script will install GIS workstation environment for multiple Ubuntu 18.04 LTS & MS Windows computers.\n\nSoftware:\n\n -QGIS\n -CloudCompare\n -LASTools\n -QuickRoute\n -Merkaartor\n -etc.\n"

    read -r -p "Continue? [y/N] " answer

    if [[ ! $(echo $answer | sed 's/ //g') =~ ^([yY][eE][sS]|[yY])$ ]]; then
        echo -e "Aborting.\n"
        exit 1
    fi

    unset answer

}

welcomeMessage

###########################################################
# Check network connection

function checkInternet() {
    if [[ $(echo $(wget --delete-after -q -T 5 github.com -o -)$?) -ne 0 ]]; then
        echo -e "\nInternet connection test failed (GitHub). Please check your connection and try again.\n"
        exit 1
    fi
}

###########################################################

if [[ ! -d /srv/{salt,pillar} ]]; then
    mkdir -p /srv/{pillar,salt/win/repo-ng/installers}
fi

cp -R srv_salt/* /srv/salt/
cp -R srv_salt_winrepo/* /srv/salt/win/repo-ng/
cp -R srv_pillar/* /srv/pillar

###########################################################

if [[ ! -d compiled ]]; then
    mkdir compiled
fi

###########################################################
# Run scripts

checkInternet

if [[ $(systemctl is-active salt-master) != "active" ]]; then
    bash ./saltscripts/1-setup-salt-env.sh
fi

function minionAcceptPolicy() {

    echo -e "\nApplying minion ID key policy\n"

    if [[ $AUTOACCEPT_MINIONS == "true" ]]; then
        salt-key -y -A | sed '/^The key glob/d'
    else

        for accept_minion in $(salt-key | sed '/Unaccepted Keys/,/Rejected Keys/!d;//d'); do
            salt-key -a $accept_minion
        done

    fi

    echo -e "Current policy:\n$(salt-key)"

}

if [[ $SALT_LOCALONLY != "true" ]]; then
    minionAcceptPolicy
else
    salt-key -y -D &> /dev/null
    systemctl restart salt-minion.service
    if [[ $? -eq 0 ]]; then
        sleep 5
        salt-key -y -a defaultMinion

        if [[ $(salt-key | sed '/Accepted Keys/,/Unaccepted Keys/!d;//d' | grep defaultMinion | wc -w) -eq 0 ]]; then
            exit 1
        fi

    fi
fi

function checkWinMinions() {

    local IFS=$'\n'

    WINMINIONS=""

    if [[ $SALT_LOCALONLY != "true" ]]; then

        echo -e "Checking for accepted Windows minions. Please wait.\n"

        for win_minion in $(salt-key | sed '/Accepted Keys/,/Unaccepted Keys/!d;//d'); do

            if [[ $(salt $win_minion grains.item os 2> /dev/null | sed '$!d; s/^\s*//; /^No minions/d') == "Windows" ]]; then
                WINMINIONS="true"
            fi

        done
    fi

}

echo -e "\nSalt master - version: $(salt-call grains.item saltversion | sed -n '$p' | sed -e 's/\s//g')"
echo -e "Salt master - reported IP addresses: $(salt-call network.ip_addrs  | sed '1d; s/^.*\-\s//' | sed ':a;N;$!ba;s/\n/, /g')\n"

checkWinMinions

bash ./saltscripts/2-get-programs-on-master.sh ${SALT_LOCALONLY} ${WINMINIONS}

###########################################################
# Fix permissions

for dir in /srv/salt /srv/pillar; do
    find $dir -type d -exec chmod u=rwx,go=rx {} +
    find $dir -type f -exec chmod u=rw,go=r {} +
done

###########################################################
# Test minion connectivity

function testMinions() {

    local IFS=$'\n'

    MASTER_VERSION=$(dpkg -s salt-master | grep '^Version:' | awk '{print $2}')

    ACCEPTED_MINIONS=$(salt-key | sed '/Accepted Keys/,/Denied Keys/!d;//d')

    i=0
    m=1
    for check_minion in $ACCEPTED_MINIONS; do

        echo -e "Testing minion '$check_minion' ($m / $(echo $ACCEPTED_MINIONS | wc -w))"

        if [[ $(echo $(salt $check_minion test.ping) | awk '{print $NF}') != "True" ]]; then
            echo -e "Can't connect to Salt minion '\e[91m$check_minion\e[0m'. Make sure the minion computer is connected and\nSalt minion program version matches with the Salt master version ($MASTER_VERSION)\nIf they do match, delete old master public key from the minion computer, restart Salt minion service and try again."
            echo -e "\nThe master key is stored at the following locations:\n\n -Windows: C:\salt\\\\conf\pki\minion\minion_master.pub\n -Linux: /etc/salt/minion/minion_master.pub\n\nSystem administration rights are required to access it.\n"
            read -r -p "Continue? [y/N] " answer
            if [[ $(echo $answer | sed 's/ //g') =~ ^([nN][oO]|[nN])$ ]]; then
                echo -e "Aborting.\n"
                exit 1
            fi
        else
            echo -e "\tSalt minion '$check_minion' - version: $(salt $check_minion grains.item saltversion | sed -n '$p' | sed -e 's/\s//g')"
            OK_MINIONS[$i]=$check_minion
            let i++
        fi
        let m++
    done
}

if [[ $TEST_MINION_PING == "true" ]]; then
    echo -e "\nTesting connections to the accepted minion computers. This takes a while.\n"
    testMinions
fi

###########################################################
# Install & set up software

TIMEOUT="600" # 10 minutes

# Reason for increasing the timeout from 5 to 300:
# Installing CloudCompare & QGIS programs take very long time
# During the installation, Windows minions do not return anything
# to the master computer, thus the master computer
# thinks that the minions have timed out, although, in reality,
# they are likely installing the applications
#
# Extra long timeout especially required by Windows QGIS installation!!
#
# Proper fix for this would be needed
# but this behavior may be linked with options and output
# by Windows NSIS (Nullsoft) installers

# NOTE: Windows minions return "Failed" (retcode 2) status
# for installed programs, although they return
# alphabetical "install status: success" as well.
# This is likely a retcode bug in Saltstack.
# Salt is known to have these issues in the past.

# Failed minions (failed)
f=0

# Succeeded minions (succeeded)
s=0

# All minions (total)
t=0

for minion in ${OK_MINIONS[*]}; do

    minion_ips=$(salt $minion network.ip_addrs | sed '1d; s/^.*\-\s//' | sed ':a;N;$!ba;s/\n/, /g')

    echo -e "\n(Minion $(( $t + 1 )) / ${#OK_MINIONS[@]}) Refreshing Salt grains & pillars for minion computer '\e[95m$minion\e[0m' (IPs: $minion_ips).\n"
    salt $minion saltutil.refresh_grains
    salt $minion saltutil.refresh_pillar

    # Update Salt package databases, especially for Windows minions
    echo -e "\n(Minion $(( $t + 1 )) / ${#OK_MINIONS[@]}) Updating Salt minion package databases for minion computer '\e[95m$minion\e[0m' (IPs: $minion_ips).\n"
    salt $minion pkg.refresh_db

    if [[ $SALT_DEBUG == "true" ]]; then
        saltdbg='-l debug'
    else
        saltdbg=''
    fi

    echo -e "\n(Minion $(( $t + 1 )) / ${#OK_MINIONS[@]}) Applying Salt state updates to the minion computer '\e[95m$minion\e[0m' (IPs: $minion_ips).\n"
    salt $saltdbg -t $TIMEOUT $minion state.highstate --state-output terse

    if [[ $? -ne 0 ]]; then
        let f++
        FAILED_MINIONS[$f]=$(echo "$minion (IPs: $minion_ips)")
    else
        let s++
    fi

    let t++

done

echo -e "\nSucceeded minions: $s of $t\nFailed minions: $f of $t\n"

if [[ $f -ne 0 ]]; then
    echo -e "The following minions returned failed status:\n\n \e[91m$(echo ${FAILED_MINIONS[@]})\e[0m\n"
fi

unset OK_MINIONS