fast-reboot

#!/bin/bash -e

REBOOT_USER=$(logname)
REBOOT_TIME=$(date)
REBOOT_CAUSE_FILE="/host/reboot-cause/reboot-cause.txt"
WARM_DIR=/host/warmboot
REDIS_FILE=dump.rdb
CONFIG_DB_FILE=/etc/sonic/config_db.json
REBOOT_SCRIPT_NAME=$(basename $0)
REBOOT_TYPE="${REBOOT_SCRIPT_NAME}"
SHUTDOWN_ORDER_FILE="/etc/sonic/${REBOOT_TYPE}_order"
VERBOSE=no
FORCE=no
IGNORE_ASIC=no
IGNORE_DB_CHECK=no
STRICT=no
REBOOT_METHOD="/sbin/kexec -e"
ASSISTANT_IP_LIST=""
ASSISTANT_SCRIPT="/usr/local/bin/neighbor_advertiser"
WATCHDOG_UTIL="/usr/local/bin/watchdogutil"
DEVPATH="/usr/share/sonic/device"
PLATFORM=$(sonic-cfggen -H -v DEVICE_METADATA.localhost.platform)
PLATFORM_PLUGIN="${REBOOT_TYPE}_plugin"
LOG_SSD_HEALTH="/usr/local/bin/log_ssd_health"
SSD_FW_UPDATE="ssd-fw-upgrade"
TAG_LATEST=yes

# Require 100M available on the hard drive for warm reboot temp files,
# Size is in 1K blocks:
MIN_HD_SPACE_NEEDED=100000

EXIT_SUCCESS=0
EXIT_FAILURE=1
EXIT_NOT_SUPPORTED=2
EXIT_FILE_SYSTEM_FULL=3
EXIT_NEXT_IMAGE_NOT_EXISTS=4
EXIT_ORCHAGENT_SHUTDOWN=10
EXIT_SYNCD_SHUTDOWN=11
EXIT_FAST_REBOOT_DUMP_FAILURE=12
EXIT_FILTER_FDB_ENTRIES_FAILURE=13
EXIT_COUNTERPOLL_DELAY_FAILURE=14
EXIT_DB_INTEGRITY_FAILURE=15
EXIT_NO_CONTROL_PLANE_ASSISTANT=20
EXIT_SONIC_INSTALLER_VERIFY_REBOOT=21

function error()
{
    echo $@ >&2
}

function debug()
{
    if [[ x"${VERBOSE}" == x"yes" ]]; then
        echo $(date) $@
    fi
    logger "$@"
}

function showHelpAndExit()
{
    echo "Usage: ${REBOOT_SCRIPT_NAME} [options]"
    echo "    -h,-? : get this help"
    echo "    -v    : turn on verbose"
    echo "    -f    : force execution - ignore Orchagent RESTARTCHECK failure"
    echo "    -i    : force execution - ignore ASIC MD5-checksum-verification"
    echo "    -d    : force execution - ignore database integrity check"
    echo "    -r    : reboot with /sbin/reboot"
    echo "    -k    : reboot with /sbin/kexec -e [default]"
    echo "    -x    : execute script with -x flag"
    echo "    -c    : specify control plane assistant IP list"
    echo "    -s    : strict mode: do not proceed without:"
    echo "            - control plane assistant IP list."
    echo "    -t    : Don't tag the current kube images as latest"

    exit "${EXIT_SUCCESS}"
}

function parseOptions()
{
    while getopts "vfidh?rkxc:s" opt; do
        case ${opt} in
            h|\? )
                showHelpAndExit
                ;;
            v )
                VERBOSE=yes
                ;;
            f )
                FORCE=yes
                ;;
            i )
                IGNORE_ASIC=yes
                ;;
            d )
                IGNORE_DB_CHECK=yes
                ;;
            r )
                REBOOT_METHOD="/sbin/reboot"
                ;;
            k )
                REBOOT_METHOD="/sbin/kexec -e"
                ;;
            x )
                set -x
                ;;
            c )
                ASSISTANT_IP_LIST=${OPTARG}
                ;;
            s )
                STRICT=yes
                ;;
            t )
                TAG_LATEST=no
                ;;
        esac
    done
}

function common_clear()
{
    debug "${REBOOT_TYPE} failure ($?) cleanup ..."

    /sbin/kexec -u || /bin/true

    teardown_control_plane_assistant
}

function clear_fast_boot()
{
    common_clear

    sonic-db-cli STATE_DB DEL "FAST_REBOOT|system" &>/dev/null || /bin/true
}

function clear_warm_boot()
{
    common_clear

    result=$(timeout 10s config warm_restart disable; res=$?; if [[ $res == 124 ]]; then echo timeout; else echo "code ($res)"; fi) || /bin/true
    debug "Cancel warm-reboot: ${result}"

    TIMESTAMP=$(date +%Y%m%d-%H%M%S)
    if [[ -f ${WARM_DIR}/${REDIS_FILE} ]]; then
        mv -f ${WARM_DIR}/${REDIS_FILE} ${WARM_DIR}/${REDIS_FILE}.${TIMESTAMP} || /bin/true
    fi
}

function init_warm_reboot_states()
{
    # If the current running instance was booted up with warm reboot. Then
    # the current DB contents will likely mark warm reboot is done.
    # Clear these states so that the next boot up image won't get confused.
    if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
        sonic-db-cli STATE_DB eval "
            for _, key in ipairs(redis.call('keys', 'WARM_RESTART_TABLE|*')) do
                redis.call('hdel', key, 'state')
            end
        " 0 >/dev/null
    fi
}

function initialize_pre_shutdown()
{
    debug "Initialize pre-shutdown ..."
    TABLE="WARM_RESTART_TABLE|warm-shutdown"
    RESTORE_COUNT=$(sonic-db-cli STATE_DB hget "${TABLE}" restore_count)
    if [[ -z "$RESTORE_COUNT" ]]; then
        sonic-db-cli STATE_DB hset "${TABLE}" "restore_count" "0" > /dev/null
    fi
    sonic-db-cli STATE_DB hset "${TABLE}" "state" "requesting" > /dev/null
}

function request_pre_shutdown()
{
    debug "Requesting pre-shutdown ..."
    STATE=$(timeout 5s docker exec syncd /usr/bin/syncd_request_shutdown --pre &> /dev/null; if [[ $? == 124 ]]; then echo "timed out"; fi)
    if [[ x"${STATE}" == x"timed out" ]]; then
        error "Failed to request pre-shutdown"
    fi
}

function recover_issu_bank_file()
{
    debug "Recovering the (${ISSU_BANK_FILE}) file"
    docker exec -i syncd sx_api_dbg_generate_dump.py
    issu_bank_value=`docker exec -i syncd cat /tmp/sdkdump | grep 'ISSU Bank' | grep -o -E '[0-9]+'`
    printf $issu_bank_value > /host/warmboot/issu_bank.txt
}

function check_issu_bank_file()
{
    ISSU_BANK_FILE=/host/warmboot/issu_bank.txt

    if [[ ! -s "$ISSU_BANK_FILE" ]]; then
        error "(${ISSU_BANK_FILE}) does NOT exist or empty ..."
        recover_issu_bank_file
        return
    fi

    issu_file_chars_count=`stat -c %s ${ISSU_BANK_FILE}`;
    issu_file_content=`awk '{print $0}' ${ISSU_BANK_FILE}`

    if [[ $issu_file_chars_count != 1 ]] ||
        [[ "$issu_file_content" != "0" && "$issu_file_content" != "1" ]]; then
        error "(${ISSU_BANK_FILE}) is broken ..."
        recover_issu_bank_file
    fi
}

function wait_for_pre_shutdown_complete_or_fail()
{
    debug "Waiting for pre-shutdown ..."
    TABLE="WARM_RESTART_TABLE|warm-shutdown"
    STATE="requesting"
    declare -i waitcount
    declare -i retrycount
    retrycount=0
    start_time=$SECONDS
    elapsed_time=$(($SECONDS - $start_time))
    # Wait up to 60 seconds for pre-shutdown to complete
    while [[ ${elapsed_time} -lt 60 ]]; do
        # timeout doesn't work with -i option of "docker exec". Therefore we have
        # to invoke docker exec directly below.
        STATE=$(timeout 5s sonic-db-cli STATE_DB hget "${TABLE}" state; if [[ $? == 124 ]]; then echo "timed out"; fi)

        if [[ x"${STATE}" == x"timed out" ]]; then
            retrycount+=1
            debug "Timed out getting pre-shutdown state, retry count ${retrycount} ..."
            if [[ retrycount -gt 2 ]]; then
                break
            fi
        elif [[ x"${STATE}" != x"requesting" ]]; then
            break
        else
            sleep 0.1
        fi
        elapsed_time=$(($SECONDS - $start_time))
    done

    if [[ x"${STATE}" != x"pre-shutdown-succeeded" ]]; then
        debug "Syncd pre-shutdown failed, state: ${STATE} ..."
    else
        debug "Pre-shutdown succeeded, state: ${STATE} ..."
    fi
}

function backup_database()
{
    debug "Backing up database ..."
    # Dump redis content to a file 'dump.rdb' in warmboot directory
    mkdir -p $WARM_DIR
    # Delete keys in stateDB except FDB_TABLE|*, MIRROR_SESSION_TABLE|*, WARM_RESTART_ENABLE_TABLE|*, FG_ROUTE_TABLE|*
    sonic-db-cli STATE_DB eval "
        for _, k in ipairs(redis.call('keys', '*')) do
            if not string.match(k, 'FDB_TABLE|') and not string.match(k, 'WARM_RESTART_TABLE|') \
                                          and not string.match(k, 'MIRROR_SESSION_TABLE|') \
                                          and not string.match(k, 'FG_ROUTE_TABLE|') \
                                          and not string.match(k, 'WARM_RESTART_ENABLE_TABLE|') \
                                          and not string.match(k, 'VXLAN_TUNNEL_TABLE|') \
                                          and not string.match(k, 'BUFFER_MAX_PARAM_TABLE|') then
                redis.call('del', k)
            end
        end
    " 0 > /dev/null

    # move all db data into the instance where APPL_DB locates
    target_db_inst=`centralize_database APPL_DB`

    # Dump redis content to a file 'dump.rdb' in warmboot directory
    docker cp database:/var/lib/$target_db_inst/$REDIS_FILE $WARM_DIR
    STATE=$(timeout 5s docker exec database rm /var/lib/$target_db_inst/$REDIS_FILE; if [[ $? == 124 ]]; then echo "timed out"; fi)
    if [[ x"${STATE}" == x"timed out" ]]; then
        error "Timed out during attempting to remove Redis dump file from database container"
    fi
}

function setup_control_plane_assistant()
{
    if [[ -n "${ASSISTANT_IP_LIST}" && -x ${ASSISTANT_SCRIPT} ]]; then
        debug "Setting up control plane assistant: ${ASSISTANT_IP_LIST} ..."
        ${ASSISTANT_SCRIPT} -s ${ASSISTANT_IP_LIST} -m set
    elif [[ X"${STRICT}" == X"yes" ]]; then
        debug "Strict mode: fail due to lack of control plane assistant ..."
        exit ${EXIT_NO_CONTROL_PLANE_ASSISTANT}
    fi
}

function teardown_control_plane_assistant()
{
    if [[ -n "${ASSISTANT_IP_LIST}" && -x ${ASSISTANT_SCRIPT} ]]; then
        debug "Tearing down control plane assistant: ${ASSISTANT_IP_LIST} ..."
        ${ASSISTANT_SCRIPT} -s ${ASSISTANT_IP_LIST} -m reset
    fi
}

function is_secureboot() {
    grep -Eq 'secure_boot_enable=[1y]' /proc/cmdline
}

function setup_reboot_variables()
{
    # Kernel and initrd image
    NEXT_SONIC_IMAGE=$(sonic-installer list | grep "Next: " | cut -d ' ' -f 2)
    IMAGE_PATH="/host/image-${NEXT_SONIC_IMAGE#SONiC-OS-}"
    if grep -q aboot_platform= /host/machine.conf; then
        if is_secureboot; then
            KERNEL_IMAGE=""
            BOOT_OPTIONS="SONIC_BOOT_TYPE=${BOOT_TYPE_ARG} secure_boot_enable=1"
        else
            KERNEL_IMAGE="$(ls $IMAGE_PATH/boot/vmlinuz-*)"
            BOOT_OPTIONS="$(cat "$IMAGE_PATH/kernel-cmdline" | tr '\n' ' ') SONIC_BOOT_TYPE=${BOOT_TYPE_ARG}"
        fi
    elif grep -q onie_platform= /host/machine.conf; then
        KERNEL_OPTIONS=$(cat /host/grub/grub.cfg | sed "/$NEXT_SONIC_IMAGE'/,/}/"'!'"g" | grep linux)
        KERNEL_IMAGE="/host$(echo $KERNEL_OPTIONS | cut -d ' ' -f 2)"
        BOOT_OPTIONS="$(echo $KERNEL_OPTIONS | sed -e 's/\s*linux\s*/BOOT_IMAGE=/') SONIC_BOOT_TYPE=${BOOT_TYPE_ARG}"
    else
        error "Unknown bootloader. ${REBOOT_TYPE} is not supported."
        exit "${EXIT_NOT_SUPPORTED}"
    fi
    INITRD=$(echo $KERNEL_IMAGE | sed 's/vmlinuz/initrd.img/g')
}

function check_docker_exec()
{
    containers="radv bgp lldp swss database teamd syncd"
    for container in $containers; do
        STATE=$(timeout 1s docker exec $container echo "success"; if [[ $? == 124 ]]; then echo "timed out"; fi)
        if [[ x"${STATE}" == x"timed out" ]]; then
            error "Docker exec on $container timedout"
            exit "${EXIT_FAILURE}"
        fi
    done
}

function check_db_integrity()
{
    if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
        CHECK_DB_INTEGRITY=0
        /usr/local/bin/check_db_integrity.py || CHECK_DB_INTEGRITY=$?
        if [[ CHECK_DB_INTEGRITY -ne 0 ]]; then
            if [[ x"${IGNORE_DB_CHECK}" == x"yes" ]]; then
                debug "Ignoring Database integrity checks..."
            else
                error "Failed to validate DB's integrity. Exit code: ${CHECK_DB_INTEGRITY}. \
                    Use '-d' option to force ignore this check."
                exit ${EXIT_DB_INTEGRITY_FAILURE}
            fi
        fi
    fi
}

function reboot_pre_check()
{
    check_docker_exec
    # Make sure that the file system is normal: read-write able
    filename="/host/test-$(date +%Y%m%d-%H%M%S)"
    if [[ ! -f ${filename} ]]; then
        touch ${filename}
    fi
    rm ${filename}

    check_db_integrity

    # Make sure /host has enough space for warm reboot temp files
    avail=$(df -k /host | tail -1 | awk '{ print $4 }')
    if [[ ${avail} -lt ${MIN_HD_SPACE_NEEDED} ]]; then
        debug "/host has ${avail}K bytes available, not enough for warm reboot."
        exit ${EXIT_FILE_SYSTEM_FULL}
    fi

    # Verify the next image by sonic-installer
    INSTALLER_VERIFY_RC=0
    sonic-installer verify-next-image > /dev/null || INSTALLER_VERIFY_RC=$?
    if [[ INSTALLER_VERIFY_RC -ne 0 ]]; then
        error "Failed to verify next image. Exit code: $INSTALLER_VERIFY_RC"
        exit ${EXIT_SONIC_INSTALLER_VERIFY_REBOOT}
    fi
    
    # Make sure ASIC configuration has not changed between images
    ASIC_CONFIG_CHECK_SCRIPT="/usr/local/bin/asic_config_check"
    ASIC_CONFIG_CHECK_SUCCESS=0
    if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
        ASIC_CONFIG_CHECK_EXIT_CODE=0
        ${ASIC_CONFIG_CHECK_SCRIPT} || ASIC_CONFIG_CHECK_EXIT_CODE=$?

        if [[ "${ASIC_CONFIG_CHECK_EXIT_CODE}" != "${ASIC_CONFIG_CHECK_SUCCESS}" ]]; then
            if [[ x"${IGNORE_ASIC}" == x"yes" ]]; then
                debug "Ignoring ASIC config checksum failure..."
            else
                error "ASIC config may have changed: errno=${ASIC_CONFIG_CHECK_EXIT_CODE}"
                exit "${EXIT_FAILURE}"
            fi
        fi
    fi
}

function load_aboot_secureboot_kernel() {
    local next_image="$IMAGE_PATH/sonic.swi"
    echo "Loading next image from $next_image"
    unzip -qp "$next_image" boot0 | \
        swipath=$next_image kexec=true loadonly=true ENV_EXTRA_CMDLINE="$BOOT_OPTIONS" bash -
}

function load_kernel() {
    # Load kernel into the memory
    /sbin/kexec -l "$KERNEL_IMAGE" --initrd="$INITRD" --append="$BOOT_OPTIONS"
}

function unload_kernel()
{
    # Unload the previously loaded kernel if any loaded
    if [[ "$(cat /sys/kernel/kexec_loaded)" -eq 1 ]]; then
        /sbin/kexec -u
    fi
}

function save_counters_folder() {
    debug "Saving counters folder before warmboot..."

    counters_folder="/host/counters"
    if [[ ! -d $counters_folder ]]; then
        mkdir $counters_folder
    fi
    if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
        modules=("portstat-0" "dropstat" "pfcstat-0" "queuestat-0" "intfstat-0")
        for module in ${modules[@]}
        do
            statfile="/tmp/$module"
            if [[ -d $statfile ]]; then
                cp -rf $statfile $counters_folder
            fi
        done
    fi
}

function check_warm_restart_in_progress() {
    sonic-db-cli STATE_DB keys "WARM_RESTART_ENABLE_TABLE|*" | while read key ; do
        if [ -n "$key" ] && [[ x"$(sonic-db-cli STATE_DB hget $key enable)" == x"true" ]]; then
            if [[ x"${FORCE}" == x"yes" ]]; then
                debug "Ignoring warm restart flag for ${key#*|}"
            else
                echo "Warm restart flag for ${key#*|} is set. Please check if a warm restart for ${key#*|} is in progress."
                exit "${EXIT_FAILURE}"
            fi
        fi
    done
}

# main starts here
parseOptions $@

# Check root privileges
if [[ "$EUID" -ne 0 ]]
then
    echo "This command must be run as root" >&2
    exit "${EXIT_FAILURE}"
fi

sonic_asic_type=$(sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type)

# Check reboot type supported
BOOT_TYPE_ARG="cold"
case "$REBOOT_TYPE" in
    "fast-reboot")
        BOOT_TYPE_ARG=$REBOOT_TYPE
        trap clear_fast_boot EXIT HUP INT QUIT TERM KILL ABRT ALRM
        sonic-db-cli STATE_DB SET "FAST_REBOOT|system" "1" "EX" "180" &>/dev/null
        ;;
    "warm-reboot")
        check_warm_restart_in_progress
        if [[ "$sonic_asic_type" == "mellanox" ]]; then
            REBOOT_TYPE="fastfast-reboot"
            BOOT_TYPE_ARG="fastfast"
            # source mlnx-ffb.sh file with
            # functions to check ISSU upgrade possibility
            source mlnx-ffb.sh
        else
            BOOT_TYPE_ARG="warm"
        fi
        trap clear_warm_boot EXIT HUP INT QUIT TERM KILL ABRT ALRM
        config warm_restart enable system
        ;;
    *)
        error "Not supported reboot type: $REBOOT_TYPE"
        exit "${EXIT_NOT_SUPPORTED}"
        ;;
esac

save_counters_folder

unload_kernel

setup_reboot_variables

reboot_pre_check

if test -f /usr/local/bin/ctrmgr_tools.py
then
    if [[ x"${TAG_LATEST}" == x"yes" ]]; then
        /usr/local/bin/ctrmgr_tools.py tag-all
    fi
fi

# Install new FW for mellanox platforms before control plane goes down
# So on boot switch will not spend time to upgrade FW increasing the CP downtime
if [[ "$sonic_asic_type" == "mellanox" ]]; then
    MLNX_EXIT_SUCCESS=0
    MLNX_EXIT_FW_ERROR=100
    MLNX_EXIT_FFB_FAILURE=101

    MLNX_FW_UPGRADE_SCRIPT="/usr/bin/mlnx-fw-upgrade.sh"


    if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
        check_ffb || {
            error "Warm reboot is not supported"
            exit "${MLNX_EXIT_FFB_FAILURE}"
        }
    fi

    debug "Prepare MLNX ASIC to ${REBOOT_TYPE}: install new FW if required"

    ${MLNX_FW_UPGRADE_SCRIPT} --upgrade
    MLNX_EXIT_CODE="$?"
    if [[ "${MLNX_EXIT_CODE}" != "${MLNX_EXIT_SUCCESS}" ]]; then
        error "Failed to burn MLNX FW: errno=${MLNX_EXIT_CODE}"
        exit "${MLNX_EXIT_FW_ERROR}"
    fi
fi


if is_secureboot && grep -q aboot_machine= /host/machine.conf; then
    load_aboot_secureboot_kernel
else
    load_kernel
fi

if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
    # Dump the ARP and FDB tables to files also as default routes for both IPv4 and IPv6
    # into /host/fast-reboot
    DUMP_DIR=/host/fast-reboot
    mkdir -p $DUMP_DIR
    FAST_REBOOT_DUMP_RC=0
    /usr/local/bin/fast-reboot-dump.py -t $DUMP_DIR || FAST_REBOOT_DUMP_RC=$?
    if [[ FAST_REBOOT_DUMP_RC -ne 0 ]]; then
        error "Failed to run fast-reboot-dump.py. Exit code: $FAST_REBOOT_DUMP_RC"
        unload_kernel
        exit "${EXIT_FAST_REBOOT_DUMP_FAILURE}"
    fi

    FILTER_FDB_ENTRIES_RC=0
    # Filter FDB entries using MAC addresses from ARP table
    /usr/local/bin/filter_fdb_entries -f $DUMP_DIR/fdb.json -a $DUMP_DIR/arp.json -c $CONFIG_DB_FILE || FILTER_FDB_ENTRIES_RC=$?
    if [[ FILTER_FDB_ENTRIES_RC -ne 0 ]]; then
        error "Failed to filter FDb entries. Exit code: $FILTER_FDB_ENTRIES_RC"
        unload_kernel
        exit "${EXIT_FILTER_FDB_ENTRIES_FAILURE}"
    fi
fi

init_warm_reboot_states

setup_control_plane_assistant

if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
    # Freeze orchagent for warm restart
    # Ask orchagent_restart_check to try freeze 5 times with interval of 2 seconds,
    # it is possible that the orchagent is in transient state and no opportunity to freeze
    # Note: assume that 2*5 seconds is enough for orchagent to process the request and response to freeze or not
    debug "Pausing orchagent ..."
    docker exec -i swss /usr/bin/orchagent_restart_check -w 2000 -r 5 > /dev/null || RESTARTCHECK_RC=$?
    if [[ RESTARTCHECK_RC -ne 0 ]]; then
        error "RESTARTCHECK failed"
        if [[ x"${FORCE}" == x"yes" ]]; then
            debug "Ignoring orchagent pausing failure ..."
        else
            exit "${EXIT_ORCHAGENT_SHUTDOWN}"
        fi
    fi
fi

if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
    COUNTERPOLL_DELAY_RC=0
    # Delay counters in config_db.json
    /usr/local/bin/counterpoll config-db delay $CONFIG_DB_FILE || COUNTERPOLL_DELAY_RC=$?
    if [[ COUNTERPOLL_DELAY_RC -ne 0 ]]; then
        error "Failed to delay counterpoll. Exit code: $COUNTERPOLL_DELAY_RC"
        unload_kernel
        exit "${EXIT_COUNTERPOLL_DELAY_FAILURE}"
    fi
fi

# We are fully committed to reboot from this point on because critical
# service will go down and we cannot recover from it.
set +e

# disable trap-handlers which were set before
trap '' EXIT HUP INT QUIT TERM KILL ABRT ALRM

if [ -x ${LOG_SSD_HEALTH} ]; then
    debug "Collecting logs to check ssd health before ${REBOOT_TYPE}..."
    ${LOG_SSD_HEALTH}
fi


if [[ -f ${SHUTDOWN_ORDER_FILE} ]]; then
    SERVICES_TO_STOP="$(cat ${SHUTDOWN_ORDER_FILE})"
else
    # TODO: to be removed once sonic-buildimage change is in
    if [[ "${REBOOT_TYPE}" == "fast-reboot" ]]; then
        SERVICES_TO_STOP="nat radv bgp sflow lldp swss teamd syncd"
    elif [[ "${REBOOT_TYPE}" == "fastfast-reboot" || "${REBOOT_TYPE}" == "warm-reboot" ]]; then
        SERVICES_TO_STOP="nat radv bgp sflow lldp teamd swss syncd"
    else
        error "Unexpected reboot type ${REBOOT_TYPE}"
        exit $EXIT_FAILURE
    fi
fi

for service in ${SERVICES_TO_STOP}; do
    debug "Stopping ${service} ..."

    # TODO: These exceptions for nat, sflow, lldp
    # have to be coded in corresponding service scripts

    if [[ "${service}" = "nat" ]]; then
        /usr/local/bin/dump_nat_entries.py
    fi

    if [[ "${service}" = "nat" || "${service}" = "sflow" || "${service}" = "lldp" ]]; then
        container kill "${service}"  &> /dev/null || debug "Docker ${service} is not running ($?) ..."
    fi

    if [[ "${service}" = "syncd" ]]; then
        systemctl stop ${service} || debug "Ignore stopping ${service} service error $?"
    else
        systemctl stop ${service}
    fi

    debug "Stopped ${service}"

    if [[ "${service}" = "swss" ]]; then
        if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
            # Pre-shutdown syncd
            initialize_pre_shutdown

            if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
                check_issu_bank_file
            fi

            request_pre_shutdown

            wait_for_pre_shutdown_complete_or_fail

            if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
                check_issu_bank_file
            fi

            # Warm reboot: dump state to host disk
            if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
                sonic-db-cli ASIC_DB FLUSHDB > /dev/null
                sonic-db-cli COUNTERS_DB FLUSHDB > /dev/null
                sonic-db-cli FLEX_COUNTER_DB FLUSHDB > /dev/null
            fi

            # TODO: backup_database preserves FDB_TABLE
            # need to cleanup as well for fastfast boot case
            backup_database
        fi
    fi
done

# Kill other containers to make the reboot faster
# We call `docker kill ...` to ensure the container stops as quickly as possible,
# then immediately call `systemctl stop ...` to prevent the service from
# restarting the container automatically.
debug "Stopping all remaining containers ..."
if test -f /usr/local/bin/ctrmgr_tools.py
then
    /usr/local/bin/ctrmgr_tools.py kill-all
else
    for CONTAINER_NAME in $(docker ps --format '{{.Names}}'); do
        CONTAINER_STOP_RC=0
        docker kill $CONTAINER_NAME &> /dev/null || CONTAINER_STOP_RC=$?
        systemctl stop $CONTAINER_NAME || debug "Ignore stopping $CONTAINER_NAME error $?"
        if [[ CONTAINER_STOP_RC -ne 0 ]]; then
            debug "Failed killing container $CONTAINER_NAME RC $CONTAINER_STOP_RC ."
        fi
    done
fi
debug "Stopped all remaining containers ..."

# Stop the docker container engine. Otherwise we will have a broken docker storage
systemctl stop docker.service || debug "Ignore stopping docker service error $?"

# Stop kernel modules for Nephos platform
if [[ "$sonic_asic_type" = 'nephos' ]];
then
  systemctl stop nps-modules-`uname -r`.service || debug "Ignore stopping nps service error $?"
fi

# Stop opennsl modules for Broadcom platform
if [[ "$sonic_asic_type" = 'broadcom' ]];
then
  service_name=$(systemctl list-units --plain --no-pager --no-legend --type=service | grep opennsl | cut -f 1 -d' ')
  systemctl stop "$service_name"
fi

# Update the reboot cause file to reflect that user issued this script
# Upon next boot, the contents of this file will be used to determine the
# cause of the previous reboot
echo "User issued '${REBOOT_SCRIPT_NAME}' command [User: ${REBOOT_USER}, Time: ${REBOOT_TIME}]" > ${REBOOT_CAUSE_FILE}

# Wait until all buffers synced with disk
sync
sleep 1
sync

# sync the current system time to CMOS
if [ -x /sbin/hwclock ]; then
    /sbin/hwclock -w || /bin/true
fi

if [ -x ${DEVPATH}/${PLATFORM}/${SSD_FW_UPDATE} ]; then
    debug "updating ssd fw for${REBOOT_TYPE}"
    ${DEVPATH}/${PLATFORM}/${SSD_FW_UPDATE} ${REBOOT_TYPE}
fi

# Enable Watchdog Timer
if [ -x ${WATCHDOG_UTIL} ]; then
    debug "Enabling Watchdog before ${REBOOT_TYPE}"
    ${WATCHDOG_UTIL} arm
fi

# Run platform specific reboot plugin
if [ -x ${DEVPATH}/${PLATFORM}/${PLATFORM_PLUGIN} ]; then
    debug "Running ${PLATFORM} specific plugin..."
    ${DEVPATH}/${PLATFORM}/${PLATFORM_PLUGIN}
fi

# Reboot: explicitly call Linux native reboot under sbin
debug "Rebooting with ${REBOOT_METHOD} to ${NEXT_SONIC_IMAGE} ..."
exec ${REBOOT_METHOD}

# Should never reach here
error "${REBOOT_TYPE} failed!"
exit "${EXIT_FAILURE}"