path: root/ssh-tunneler-tests

#!/bin/sh

BOOT_WAIT_DEFAULT=25
# set GO_PLAY_KIDS to no to have a longer sequence of tests
GO_PLAY_KIDS=no

YESCOUNT=0
NOCOUNT=0

function testcount()
{
    test $1 = $2 && echo SUCCESS || echo FAILED
    test $1 = $2 && YESCOUNT=$(($YESCOUNT + 1)) || NOCOUNT=$(($NOCOUNT + 1))
}

function dirtyssh()
{
    ssh -q \
        -o UserKnownHostsFile=/dev/null \
        -o StrictHostKeyChecking=no \
        -p $1 \
        root@localhost \
        "$2"
}

function herd_service_status()
{
    echo "herd status of $2 on ${VM[$1]}:"
    dirtyssh ${PORT[$1]} "herd status $2"
    echo
}

function herd_test_service_status()
{
    echo "Test if the $2 service is $3 on ${VM[$1]}."
    RESULT=$(herd_service_status $1 $2 | awk -- '/It is '$3'/ {print $3}')
    testcount ${RESULT} $3
}

function test_forward()
{
    echo "From ${VM[$1]}, take the [reverse] port forward opened by
${VM[$2]} to connect to ${VM[$3]} through
sshd of ${VM[$4]}."
    EXITHOST=reset
    EXITHOST=$(dirtyssh \
                   ${PORT[$1]} \
                   "ssh -q \\
                        -o UserKnownHostsFile=/dev/null \\
                        -o StrictHostKeyChecking=no \\
                        -p $5 \\
                        root@$6 \\
                        hostname")
    echo "Hostname at forward exit: ${EXITHOST}, expected: ${VM[$3]}"
    testcount ${EXITHOST} ${VM[$3]}
}

function test_forwards()
{
    echo "*** REVERSE PORT FORWARD TESTS"
    echo
    test_forward $SSHD_WILD01 \
                 $SSH_CLIENT01 \
                 $SSH_CLIENT01 \
                 $SSHD_SOCKS \
                 ${EXTRAPORT_HOST[$SSHD_SOCKS]} \
                 10.0.2.2
    echo
    test_forward $SSHD_WILD01 \
                 $SSH_CLIENT02 \
                 $SSHD_WILD02 \
                 $SSHD_REMOTE \
                 ${EXTRAPORT_HOST[$SSHD_REMOTE]} \
                 10.0.2.2
    echo
    test_forward $SSHD_WILD02 \
                 $SSH_CLIENT02 \
                 $SSHD_WILD01 \
                 $SSHD_REMOTE \
                 ${ULTIMAPORT_HOST[$SSHD_REMOTE]} \
                 10.0.2.2
    echo
    echo
    echo "*** PORT FORWARD TESTS"
    echo
    test_forward $SSH_CLIENT01 \
                 $SSH_CLIENT01 \
                 $SSHD_WILD02 \
                 $SSHD_SOCKS \
                 ${ULTIMAPORT_VM[$SSH_CLIENT01]} \
                 127.0.0.1
    echo
    test_forward $SSH_CLIENT02 \
                 $SSH_CLIENT02 \
                 $SSHD_WILD01 \
                 $SSHD_SOCKS \
                 ${EXTRAPORT_VM[$SSH_CLIENT02]} \
                 127.0.0.1
    echo
    test_forward $SSHD_WILD02 \
                 $SSH_CLIENT02 \
                 $SSHD_WILD01 \
                 $SSHD_SOCKS \
                 ${EXTRAPORT_HOST[$SSH_CLIENT02]} \
                 10.0.2.2
    echo
    echo
}

echo "*** DEFINING VM INSTANCIATION ARRAYS"
echo

I=0
SSHD_WILD01=$I

VM[$I]=sshd-wild01
echo "* VM: ${VM[$I]}"
EXTRA_SERVICES[$I]=""
PORT[$I]=$((10022+$I))
EXTRAPORT_HOST[$I]=$((${PORT[$I]}+1000))
EXTRAPORT_VM[$I]=$((${PORT[$I]}+2000))
ULTIMAPORT_HOST[$I]=$((${PORT[$I]}+3000))
ULTIMAPORT_VM[$I]=$((${PORT[$I]}+4000))
echo

I=$(($I+1))
SSHD_WILD02=$I

VM[$I]=sshd-wild02
echo "* VM: ${VM[$I]}"
ALLOW_TUNDEV[$I]="yes"
EXTRA_SERVICES[$I]=""
PORT[$I]=$((10022+$I))
EXTRAPORT_HOST[$I]=$((${PORT[$I]}+1000))
EXTRAPORT_VM[$I]=$((${PORT[$I]}+2000))
ULTIMAPORT_HOST[$I]=$((${PORT[$I]}+3000))
ULTIMAPORT_VM[$I]=$((${PORT[$I]}+4000))
echo

I=$(($I+1))
SSHD_REMOTE=$I

VM[$I]=sshd-remote
echo "* VM: ${VM[$I]}"
ALLOW_GATEWAY[$I]="yes"
EXTRA_SERVICES[$I]=""
PORT[$I]=$((10022+$I))
EXTRAPORT_HOST[$I]=$((${PORT[$I]}+1000))
EXTRAPORT_VM[$I]=$((${PORT[$I]}+2000))
ULTIMAPORT_HOST[$I]=$((${PORT[$I]}+3000))
ULTIMAPORT_VM[$I]=$((${PORT[$I]}+4000))
EXTRA_SERVICES[$I]=""
echo

I=$(($I+1))
SSHD_SOCKS=$I

VM[$I]=sshd-socks
echo "* VM: ${VM[$I]}"
ALLOW_GATEWAY[$I]="yes"
BOOT_WAIT[$I]=25
EXTRA_SERVICES[$I]=""
PORT[$I]=$((10022+$I))
EXTRAPORT_HOST[$I]=$((${PORT[$I]}+1000))
EXTRAPORT_VM[$I]=$((${PORT[$I]}+2000))
ULTIMAPORT_HOST[$I]=$((${PORT[$I]}+3000))
ULTIMAPORT_VM[$I]=$((${PORT[$I]}+4000))
echo

I=$(($I+1))
SSH_CLIENT01=$I

VM[$I]=ssh-client01
echo "* VM: ${VM[$I]}"
PORT[$I]=$((10022+$I))
EXTRAPORT_HOST[$I]=$((${PORT[$I]}+1000))
EXTRAPORT_VM[$I]=$((${PORT[$I]}+2000))
ULTIMAPORT_HOST[$I]=$((${PORT[$I]}+3000))
ULTIMAPORT_VM[$I]=$((${PORT[$I]}+4000))
EXTRA_SERVICES[$I]="
           (service elogind-service-type)
           (service persistent-ssh-service-type
            (ssh-connection-configuration
             (%auto-start? #t)
             (sshd-host \"10.0.2.2\")
             (sshd-port ${PORT[$SSHD_SOCKS]})
             (forwards
              (list (dynamic-forward-configuration)))))
           (service persistent-ssh-service-type
            (ssh-connection-configuration
             (%auto-start? #t)
             (sshd-host \"10.0.2.2\")
             (sshd-port ${PORT[$SSHD_SOCKS]})
             (forwards
              (list (reverse-port-forward-configuration
                     (entry-port ${EXTRAPORT_VM[$SSHD_SOCKS]}))))))
           (service persistent-ssh-service-type
            (ssh-connection-configuration
             (%auto-start? #t)
             (sshd-host \"10.0.2.2\")
             (sshd-port ${PORT[$SSHD_SOCKS]})
             (socks-proxy-config
              (socks-proxy-configuration
               (use-proxy? #t)
               (extend? #f)))
             (forwards
              (list (port-forward-configuration
                    (entry-port ${ULTIMAPORT_VM[$I]})
                    (forward-host \"10.0.2.2\")
                    (exit-port ${PORT[$SSHD_WILD02]}))))))
           (service persistent-ssh-service-type
            (ssh-connection-configuration
             (%auto-start? #t)
             (sshd-host \"10.0.2.2\")
             (sshd-port ${PORT[$SSHD_WILD02]})
             (forwards
              (list (tunnel-forward-configuration)))))"
echo

I=$(($I+1))
SSH_CLIENT02=$I

VM[$I]=ssh-client02
echo "* VM: ${VM[$I]}"
PORT[$I]=$((10022+$I))
EXTRAPORT_HOST[$I]=$((${PORT[$I]}+1000))
EXTRAPORT_VM[$I]=$((${PORT[$I]}+2000))
ULTIMAPORT_HOST[$I]=$((${PORT[$I]}+3000))
ULTIMAPORT_VM[$I]=$((${PORT[$I]}+4000))
EXTRA_SERVICES[$I]="
           (service persistent-ssh-service-type
            (ssh-connection-configuration
             (%auto-start? #t)
             (socks-proxy-config
              (socks-proxy-configuration
               (use-proxy? #t)
               (extend? #f)))
             (sshd-host \"10.0.2.2\")
             (sshd-port ${PORT[$SSHD_REMOTE]})
             (forwards
              (list (reverse-port-forward-configuration
                     (entry-port ${EXTRAPORT_VM[$SSHD_REMOTE]})
                     (forward-host \"10.0.2.2\")
                     (exit-port ${PORT[$SSHD_WILD02]}))))))
           (service persistent-ssh-service-type
            (ssh-connection-configuration
             (%auto-start? #t)
             (socks-proxy-config
              (socks-proxy-configuration
               (use-proxy? #t)
               (dynamic-forward
                (ssh-connection-configuration
                 (sshd-host \"10.0.2.2\")
                 (sshd-port ${PORT[$SSHD_SOCKS]})
                 (forwards
                  (list (dynamic-forward-configuration)))))))
             (sshd-host \"10.0.2.2\")
             (sshd-port ${PORT[$SSHD_REMOTE]})
             (forwards
              (list (reverse-port-forward-configuration
                     (entry-port ${ULTIMAPORT_VM[$SSHD_REMOTE]})
                     (forward-host \"10.0.2.2\")
                     (exit-port ${PORT[$SSHD_WILD01]}))))))
           (service persistent-ssh-service-type
            (ssh-connection-configuration
             (%auto-start? #t)
             (sshd-host \"10.0.2.2\")
             (sshd-port ${PORT[$SSHD_SOCKS]})
             (gateway-ports? #t)
             (forwards
              (list (port-forward-configuration
                     (entry-port ${EXTRAPORT_VM[$I]})
                     (forward-host \"10.0.2.2\")
                     (exit-port ${PORT[$SSHD_WILD01]}))))))"
echo

for I in ${!PORT[@]}
do
    EXTRACONTENT[$I]=""
    if [ -v ALLOW_TUNDEV[$I] ]
    then
        EXTRACONTENT[$I]="
                     (extra-content \"
PermitTunnel=point-to-point\")"
    fi
    GATEWAY[$I]=""
    if [ -v ALLOW_GATEWAY[$I] ]
    then
        GATEWAY[$I]="
                     (gateway-ports? #t)"
    fi
    WAIT[$I]=$BOOT_WAIT_DEFAULT
    if [ -v BOOT_WAIT[$I] ]
    then
        WAIT[$I]=${BOOT_WAIT[$I]}
    fi
done
echo

echo "*** CONCATENATING SYSTEM CONFIGURATIONS"
echo
for I in ${!PORT[@]}
do
    echo "* VM: ${VM[$I]}"
    echo "(use-modules (guix records)
                   (gnu)
                   (whispers services ssh-tunneler))
(use-service-modules networking ssh desktop)
(use-package-modules ssh)

(operating-system
  (host-name \"${VM[$I]}\")
  (timezone \"Asia/Shanghai\")
  (locale \"en_US.utf8\")

  (bootloader (bootloader-configuration
                (bootloader grub-bootloader)
                (targets '(\"/dev/sda\"))))

  (file-systems (cons (file-system
                        (device (file-system-label \"root\"))
                        (mount-point \"/\")
                        (type \"ext4\"))
                      %base-file-systems))

  (users %base-user-accounts)

  (packages %base-packages)

  (services
    (append
     (list (service dhcp-client-service-type)
           (service openssh-service-type
                    (openssh-configuration
                     (permit-root-login #t)
                     (allow-empty-passwords? #t)
                     (openssh openssh-sans-x)
                     (port-number 22)${GATEWAY[$I]}${EXTRACONTENT[$I]}))${EXTRA_SERVICES[$I]})
     %base-services)))" > /tmp/${VM[$I]}.scm
    echo
done
echo

echo "*** INSTANCIATING VMs"
echo
for I in ${!PORT[@]}
do
    echo "* VM: ${VM[$I]}"
    VMRUN[$I]=$(guix system vm /tmp/${VM[$I]}.scm)
    echo
done
echo

echo "*** SILENTLY BOOTING VMs..."
echo
for I in ${!PORT[@]}
do
    NICSTANCE=-"nic user,model=virtio-net-pci,hostfwd=tcp::${PORT[$I]}-:22,hostfwd=tcp::${EXTRAPORT_HOST[$I]}-:${EXTRAPORT_VM[$I]},hostfwd=tcp::${ULTIMAPORT_HOST[$I]}-:${ULTIMAPORT_VM[$I]}"
    if [ $I = $SSHD_REMOTE ]
    then
        for CURPORT in $VPNPORTS
        do
            NICSTANCE=${NICSTANCE},hostfwd=tcp::${CURPORT}-:${CURPORT}
        done
    fi
        echo "* VM: ${VM[$I]}"
        ${VMRUN[$I]} \
            $NICSTANCE \
            -display none &
        sleep ${WAIT[$I]}
    echo
done
echo

echo "*** REPORTING SHEPHERD STATUS"
echo
for I in ${!PORT[@]}
do
    echo "* VM: ${VM[$I]}"
    dirtyssh ${PORT[$I]} 'herd status'
    echo
done
echo

test_forwards

function full_tests()
{

echo "*** RESURRECTION TESTS ON STOPPED AND DISABLED NON-WHISPERS CONNECTIONS"
echo
echo "* Tests on the resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT01]}."
echo "NOTE: test the socks proxy first otherwise the connections which
require the socks proxy will be stopped not be restarted automatically
by the Shepherd."
echo
for CONNECTION in ssh-forwards_dynamic@8971 \
                  ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_SOCKS]}:127.0.0.1:22 \
                  ssh-forwards_port@${ULTIMAPORT_VM[$SSH_CLIENT01]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_tunnel@any:any
do
    echo "Stop and Disable the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd stop $CONNECTION"
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd disable $CONNECTION"
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "disabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION stopped
    echo
    echo "Resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION running
    echo
done
echo "* Tests on the resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT02]}."
echo "NOTE: test the socks proxy first otherwise the connections which
require the socks proxy will be stopped not be restarted automatically
by the Shepherd."
echo
for CONNECTION in ssh-forwards_dynamic@8971 \
                  ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_reverse-port@${ULTIMAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD01]}_proxy-port_8971 \
                  ssh-forwards_port@${EXTRAPORT_VM[$SSH_CLIENT02]}:10.0.2.2:${PORT[$SSHD_WILD01]}
do
    echo "Stop and Disable the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd stop $CONNECTION"
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd disable $CONNECTION"
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "disabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION stopped
    echo
    echo "Resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION running
    echo
done
echo

test_forwards

echo "*** RESURRECTION TESTS ON STOPPED AND ENABLED NON-WHISPERS CONNECTIONS"
echo
echo "* Tests on the resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT01]}."
echo "NOTE: test the socks proxy first otherwise the connections which
require the socks proxy will be stopped not be restarted automatically
by the Shepherd."
echo
for CONNECTION in ssh-forwards_dynamic@8971 \
                  ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_SOCKS]}:127.0.0.1:22 \
                  ssh-forwards_port@${ULTIMAPORT_VM[$SSH_CLIENT01]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_tunnel@any:any
do
    echo "Stop the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd stop $CONNECTION"
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION stopped
    echo
    echo "Resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION running
    echo
done
echo "* Tests on the resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT02]}."
echo "NOTE: test the socks proxy first otherwise the connections which
require the socks proxy will be stopped not be restarted automatically
by the Shepherd."
echo
for CONNECTION in ssh-forwards_dynamic@8971 \
                  ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_reverse-port@${ULTIMAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD01]}_proxy-port_8971 \
                  ssh-forwards_port@${EXTRAPORT_VM[$SSH_CLIENT02]}:10.0.2.2:${PORT[$SSHD_WILD01]}
do
    echo "Stop the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd stop $CONNECTION"
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION stopped
    echo
    echo "Resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION running
    echo
done
echo

test_forwards

echo "*** RESURRECTION TESTS ON STARTED AND ENABLED NON-WHISPERS CONNECTIONS"
echo
echo "* Tests on the resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT01]}."
echo
for CONNECTION in ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_SOCKS]}:127.0.0.1:22 \
                  ssh-forwards_port@${ULTIMAPORT_VM[$SSH_CLIENT01]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_tunnel@any:any \
                  ssh-forwards_dynamic@8971
do
    echo "Resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION running
    echo
done
echo "* Tests on the resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT02]}."
echo
for CONNECTION in ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_reverse-port@${ULTIMAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD01]}_proxy-port_8971 \
                  ssh-forwards_port@${EXTRAPORT_VM[$SSH_CLIENT02]}:10.0.2.2:${PORT[$SSHD_WILD01]} \
                  ssh-forwards_dynamic@8971
do
    echo "Resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION running
    echo
done
echo

test_forwards

echo "*** RESURRECTION TESTS ON NON-WHISPERS CONNECTIONS
*** WHOSE SOCKS PROXIES HAVE BEEN STOPPED AND DISABLED"
echo
echo "* Tests on the resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT01]}."
echo
for CONNECTION in ssh-forwards_port@${ULTIMAPORT_VM[$SSH_CLIENT01]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971
do
    echo "Stop and Disable the socks proxy of the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd stop ssh-forwards_dynamic@8971"
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd disable ssh-forwards_dynamic@8971"
    herd_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 "disabled."
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 stopped
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION stopped
    echo
    echo "Resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 running
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION running
    echo
done
echo "* Tests on the resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT02]}."
echo
for CONNECTION in ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_reverse-port@${ULTIMAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD01]}_proxy-port_8971
do
    echo "Stop and Disable the socks proxy of the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd stop ssh-forwards_dynamic@8971"
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd disable ssh-forwards_dynamic@8971"
    herd_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 "disabled."
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 stopped
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION stopped
    echo
    echo "Resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 running
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION running
    echo
done
echo

echo "NOTE: have to restart one service because it does not restart
automatically when its inferior socks proxy is restarted"
dirtyssh \
    ${PORT[$SSH_CLIENT02]} \
    "herd start ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971"
test_forwards

echo "*** RESURRECTION TESTS ON NON-WHISPERS CONNECTIONS
*** WHOSE SOCKS PROXIES ARE STOPPED AND ENABLED"
echo
echo "* Tests on the resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT01]}."
echo
for CONNECTION in ssh-forwards_port@${ULTIMAPORT_VM[$SSH_CLIENT01]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971
do
    echo "Stop the socks proxy of the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd stop ssh-forwards_dynamic@8971"
    herd_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 stopped
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION stopped
    echo
    echo "Resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 running
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION running
    echo
done
echo "* Tests on the resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT02]}."
echo
for CONNECTION in ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_reverse-port@${ULTIMAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD01]}_proxy-port_8971
do
    echo "Stop the socks proxy of the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd stop ssh-forwards_dynamic@8971"
    herd_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 stopped
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION stopped
    echo
    echo "Resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 running
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION running
    echo
done

echo "NOTE: have to restart one service because it does not restart
automatically when its inferior socks proxy is restarted"
dirtyssh \
    ${PORT[$SSH_CLIENT02]} \
    "herd start ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971"
test_forwards

echo "*** FORCE-RESURRECTION TESTS ON STOPPED AND DISABLED NON-WHISPERS CONNECTIONS"
echo
echo "* Tests on the force-resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT01]}."
echo "NOTE: test the socks proxy first otherwise the connections which
require the socks proxy will be stopped not be restarted automatically
by the Shepherd."
echo
for CONNECTION in ssh-forwards_dynamic@8971 \
                  ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_SOCKS]}:127.0.0.1:22 \
                  ssh-forwards_port@${ULTIMAPORT_VM[$SSH_CLIENT01]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_tunnel@any:any
do
    echo "Stop and Disable the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd stop $CONNECTION"
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd disable $CONNECTION"
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "disabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION stopped
    echo
    echo "Force-resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd force-resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION running
    echo
done
echo "* Tests on the force-resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT02]}."
echo "NOTE: test the socks proxy first otherwise the connections which
require the socks proxy will be stopped not be restarted automatically
by the Shepherd."
echo
for CONNECTION in ssh-forwards_dynamic@8971 \
                  ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_reverse-port@${ULTIMAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD01]}_proxy-port_8971 \
                  ssh-forwards_port@${EXTRAPORT_VM[$SSH_CLIENT02]}:10.0.2.2:${PORT[$SSHD_WILD01]}
do
    echo "Stop and Disable the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd stop $CONNECTION"
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd disable $CONNECTION"
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "disabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION stopped
    echo
    echo "Force-resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd force-resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION running
    echo
done
echo

echo "NOTE: have to restart one service because it does not restart
automatically when its inferior socks proxy is restarted"
dirtyssh \
    ${PORT[$SSH_CLIENT02]} \
test_forwards

echo "*** FORCE-RESURRECTION TESTS ON STOPPED AND ENABLED NON-WHISPERS CONNECTIONS"
echo
echo "* Tests on the force-resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT01]}."
echo "NOTE: test the socks proxy first otherwise the connections which
require the socks proxy will be stopped not be restarted automatically
by the Shepherd."
echo
for CONNECTION in ssh-forwards_dynamic@8971 \
                  ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_SOCKS]}:127.0.0.1:22 \
                  ssh-forwards_port@${ULTIMAPORT_VM[$SSH_CLIENT01]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_tunnel@any:any
do
    echo "Stop the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd stop $CONNECTION"
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION stopped
    echo
    echo "Force-resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd force-resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION running
    echo
done
echo "* Tests on the force-resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT02]}."
echo "NOTE: test the socks proxy first otherwise the connections which
require the socks proxy will be stopped not be restarted automatically
by the Shepherd."
echo
for CONNECTION in ssh-forwards_dynamic@8971 \
                  ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_reverse-port@${ULTIMAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD01]}_proxy-port_8971 \
                  ssh-forwards_port@${EXTRAPORT_VM[$SSH_CLIENT02]}:10.0.2.2:${PORT[$SSHD_WILD01]}
do
    echo "Stop the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd stop $CONNECTION"
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION stopped
    echo
    echo "Force-resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd force-resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION running
    echo
done
echo

echo "NOTE: have to restart one service because it does not restart
automatically when its inferior socks proxy is restarted"
dirtyssh \
    ${PORT[$SSH_CLIENT02]} \
    "herd start ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971"
test_forwards

echo "*** FORCE-RESURRECTION TESTS ON STARTED AND ENABLED NON-WHISPERS CONNECTIONS"
echo
echo "* Tests on the force-resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT01]}."
echo "NOTE: test the socks proxy first otherwise the connections which
require the socks proxy will be stopped not be restarted automatically
by the Shepherd."
echo
for CONNECTION in ssh-forwards_dynamic@8971 \
                  ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_SOCKS]}:127.0.0.1:22 \
                  ssh-forwards_port@${ULTIMAPORT_VM[$SSH_CLIENT01]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_tunnel@any:any
do
    echo "Force-resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd force-resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION running
    echo
done
echo "* Tests on the force-resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT02]}."
echo "NOTE: test the socks proxy first otherwise the connections which
require the socks proxy will be stopped not be restarted automatically
by the Shepherd."
echo
for CONNECTION in ssh-forwards_dynamic@8971 \
                  ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_reverse-port@${ULTIMAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD01]}_proxy-port_8971 \
                  ssh-forwards_port@${EXTRAPORT_VM[$SSH_CLIENT02]}:10.0.2.2:${PORT[$SSHD_WILD01]}
do
    echo "Force-resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd force-resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION running
    echo
done
echo

echo "NOTE: have to restart one service because it does not restart
automatically when its inferior socks proxy is restarted"
dirtyssh \
    ${PORT[$SSH_CLIENT02]} \
    "herd start ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971"
test_forwards
echo "*** FORCE-RESURRECTION TESTS ON NON-WHISPERS CONNECTIONS
*** WHOSE SOCKS PROXIES HAVE BEEN STOPPED AND DISABLED"
echo
echo "* Tests on the force-resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT01]}."
echo
for CONNECTION in ssh-forwards_port@${ULTIMAPORT_VM[$SSH_CLIENT01]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971
do
    echo "Stop and Disable the socks proxy of the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd stop ssh-forwards_dynamic@8971"
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd disable ssh-forwards_dynamic@8971"
    herd_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 "disabled."
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 stopped
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION stopped
    echo
    echo "Force-resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd force-resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 running
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION running
    echo
done
echo "* Tests on the force-resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT02]}."
echo
for CONNECTION in ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_reverse-port@${ULTIMAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD01]}_proxy-port_8971
do
    echo "Stop and Disable the socks proxy of the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd stop ssh-forwards_dynamic@8971"
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd disable ssh-forwards_dynamic@8971"
    herd_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 "disabled."
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 stopped
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION stopped
    echo
    echo "Force-resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd force-resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 running
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION running
    echo
done
echo

echo "NOTE: have to restart one service because it does not restart
automatically when its inferior socks proxy is restarted"
dirtyssh \
    ${PORT[$SSH_CLIENT02]} \
    "herd start ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971"
test_forwards

echo "*** FORCE-RESURRECTION TESTS ON NON-WHISPERS CONNECTIONS
*** WHOSE SOCKS PROXIES ARE STOPPED AND ENABLED"
echo
echo "* Tests on the force-resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT01]}."
echo
for CONNECTION in ssh-forwards_port@${ULTIMAPORT_VM[$SSH_CLIENT01]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971
do
    echo "Stop the socks proxy of the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd stop ssh-forwards_dynamic@8971"
    herd_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 stopped
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION stopped
    echo
    echo "Force-resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT01]} \
        "herd force-resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT01 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 ssh-forwards_dynamic@8971 running
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT01 $CONNECTION running
    echo
done
echo "* Tests on the force-resurrect actions of the persistent ssh connections of
${VM[$SSH_CLIENT02]}."
echo
for CONNECTION in ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971 \
                  ssh-forwards_reverse-port@${ULTIMAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD01]}_proxy-port_8971
do
    echo "Stop the socks proxy of the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd stop ssh-forwards_dynamic@8971"
    herd_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 stopped
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION stopped
    echo
    echo "Force-resurrect the $CONNECTION service."
    dirtyssh \
        ${PORT[$SSH_CLIENT02]} \
        "herd force-resurrect $CONNECTION"
    herd_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971
    echo
    herd_service_status $SSH_CLIENT02 $CONNECTION
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 ssh-forwards_dynamic@8971 running
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION "enabled."
    echo
    herd_test_service_status $SSH_CLIENT02 $CONNECTION running
    echo
done

echo "NOTE: have to restart one service because it does not restart
automatically when its inferior socks proxy is restarted"
dirtyssh \
    ${PORT[$SSH_CLIENT02]} \
    "herd start ssh-forwards_reverse-port@${EXTRAPORT_VM[$SSHD_REMOTE]}:10.0.2.2:${PORT[$SSHD_WILD02]}_proxy-port_8971{}"
test_forwards

}

if [[ foo$GO_PLAY_KIDS == foono ]]
then
    full_tests
fi

echo "*** TESTS SUMMARRY"
echo
echo Successes: $YESCOUNT
echo Failures: $NOCOUNT
echo
echo

echo "You can ssh into the VMs from another terminal of this host"
for I in ${!PORT[@]}
do
    echo "* VM: ${VM[$I]}, port: ${PORT[$I]}"
done
echo
read -n 1 -r -s -p "When done playing, press any key to halt the VMs..."
echo
echo

echo "*** HALTING VMs"
echo
for I in ${!PORT[@]}
do
    echo "* VM: ${VM[$I]}"
    dirtyssh ${PORT[$I]} halt
    echo
done