"""XBee update tool

This tool runs on the gateway and updates all the radios in a network.

The script has two main functionalities, which are kept separate:
Creating a prioritized list of radios to update, and updating all the
radios in a list. See `run_update_step()` and `run_discover_step()` for
more details on those steps.

When updating XBee 3 radios with firmware older than 1009, the update is
most likely to succeed if other transmissions are kept to a minimum. So,
if additional functionality is added to this script that would cause
network traffic (neighbor discovery, setting remote parameters, etc.) it
should be added to the discovery step, not the update step.
"""
from distutils.log import debug
import json
from random import randrange
import sys
import struct

if sys.version_info > (3, 5):
    # The gateway doesn't have python 3, but including this helps for
    # development if your IDE does type checking.
    from typing import List, Optional, Dict, Tuple, BinaryIO, TextIO

from xml.etree import ElementTree
import binascii
import time
import logging
import itertools
import re
import xbee
import os
import shutil
import subprocess
from datetime import datetime
from contextlib import contextmanager

from socket import *
from rci import process_request
from _xbee import ddo_get_param, ddo_set_param

root_log = logging.getLogger()
result_log = root_log.getChild("result")

SCRIPT_VERSION = 3.7

UPDATE_CHECK_RATE = 60
MAX_ERROR_LOGS = 10
num_ota_logs = 0
good_xbee3_log_found = False
good_s2c_log_found = False

RECOVERY_CHANNEL = 0xD

###############################################################################
# Configurable values
###############################################################################

# If this is True, radios with newer firmware will be prioritized first.
# If it's false, radios with older firmware will be prioritized first.
NEWEST_FIRST = False

# If this is set to False, updates targeting XBee 3 radios will be skipped
UPDATE_XB3_RADIOS = True
# XBee 3 target version: Update these values to match the number and
# filename of the version to update XBee 3 radios to.
XBEE3_TARGET_VERSION = 0x1010
XBEE3_TARGET_FILE = "XB3-24Z_1010.otb"

# If this is set to False, updates targeting S2C radios will be skipped
UPDATE_S2C_RADIOS = True
# S2C target version: Update these values to match the version number
# and file names for the S2C radios to update to.
XBEE_S2C_TARGET_VERSION = 0x40A7
XBEE_S2C_PRO_SMT_TARGET = "xbp24c-s2c_40A7.ebl"
XBEE_S2C_PRO_TH_TARGET = "xbp24c-s2c-th_40A7.ebl"
XBEE_S2C_NONPRO_SMT_TARGET = "xb24c-s2c_40A7.ebl"
XBEE_S2C_NONPRO_TH_TARGET = "xb24c-s2c-th_40A7.ebl"

# File location of where to store error logs
ERR_LOG_DIR_NAME = "error_logs"
LOG_DIR_NAME = "copied_logs"
LOG_DIR_PATH = "/userfs/WEB/python/" + LOG_DIR_NAME

# File location to store S2C neighbor info
NEIGHBOR_LIST_PATH = "/userfs/WEB/python/neighbors.json"

###############################################################################
# End of configurable values
###############################################################################

# Versions that need to be updated to 1B04 then the latest version
XBEE3_UPDATE_INTERIM = [
    0x1003,
]
XBEE3_INTERIM_VERSION = 0x1B04
XBEE3_INTERIM_FILE = "XB3-24Z_1B04.otb"

# S2C TH/SMT HV values
S2C_FW_BY_HV = {
    0x21: XBEE_S2C_PRO_SMT_TARGET,  # S2C SMT Pro
    0x22: XBEE_S2C_NONPRO_SMT_TARGET,  # S2C SMT Non-Pro
    0x30: XBEE_S2C_PRO_SMT_TARGET,  # S2C SMT Pro new revision
    0x2D: XBEE_S2C_PRO_TH_TARGET,  # S2C TH  Pro
    0x2E: XBEE_S2C_NONPRO_TH_TARGET,  # S2C TH  Non-Pro
}


def is_xb3_version(version):
    # type: (int) -> bool
    """Determine whether the given version number corresponds to an XBee 3.

    Since we know all the XBees on the network are either XBee 3 Zigbee
    or S2C Zigbee, we can tell just by looking at the most significant nibble.
    """
    return version & 0xF000 == 0x1000


def is_up_to_date_xb3(version):
    # type: (int) -> bool
    """Determine whether this version number is one we want to update."""
    if version == XBEE3_INTERIM_VERSION:
        return False
    else:
        return version == XBEE3_TARGET_VERSION


def is_s2c_version(version):
    """Determine whether the given version number corresponds to an S2C."""
    # type: (int) -> bool
    return version & 0xF000 == 0x4000


def is_up_to_date_s2c(version):
    # type: (int) -> bool
    """Determine whether this version number is one we want to update."""
    return version == XBEE_S2C_TARGET_VERSION


def is_good_s2c_recovery_broadcast_version(version):
    # type: (int) -> bool
    """
        Determine whether this version number is one that can do a 'broadcast
        search' to find any nearby orphaned nodes.
    """
    # If it can't recover S2Cs, it obviously can't to broadcast recovery.
    if not is_good_s2c_update_version(version, recovery=True):
        return False
    # Otherwise, if it's an S2C, make sure it's new enough to have the broadcast feature.
    if is_s2c_version(version) and version < 0x40A8:
        return False

    # Anything else (including any XB3 version with S2C support) will work.
    return True

def is_good_s2c_update_version(version, recovery):
    # type: (int, bool) -> bool
    """Determine whether this version can be used for S2C updates.

    If recovery is True, only allow radios that can safely be used in recovery
    mode.
    """
    # S2C radios with version 4061 or older should not be used for
    # recovery--they might get stuck on the recovery channel.
    if recovery:
        s2c_ok = version >= 0x4062
    else:
        s2c_ok = True
    return (
        (is_s2c_version(version) and s2c_ok)
        or (is_xb3_version(version) and (version & 0xF0FF) >= 0x1011)
    )

dry_run = False
save_logs = False


class DeviceInfo:
    @classmethod
    def from_xml(cls, el):
        # type: (ElementTree.Element) -> DeviceInfo
        address = el.find("ext_addr").text
        status = el.find(
            "fw_status/status"
        ).text  # Error, Unknown, Scheduled, Complete, Updated, Cancelled, Updating
        status_text = el.find("fw_status/error_text").text
        err_code = int(el.find("fw_status/error_code").text, 16)
        fw_version = int(el.find("fw_status/fw_version").text, 16)

        hw_result = int(el.find("fw_status/hw_version").text, 16)
        hw_revision = "%s%s" % (chr((hw_result & 0xFF00) >> 8), chr(hw_result & 0xFF))
        retries = int(el.find("fw_status/tries").text)

        return DeviceInfo(
            address, fw_version, hw_revision, status, err_code, retries, status_text
        )

    def __repr__(self):
        return "DeviceInfo(address={address}, version={version:X}, revision=0x{revision}, status={status})".format(
            address=self.address,
            version=self.version,
            revision=binascii.hexlify(self.revision),
            status=self.status,
        )

    def __init__(
        self, address, version, revision, status, error, retries, status_text=""
    ):
        # type: (str, int, str, str, int, int, str) -> None
        self.address = address
        self.version = version
        self.revision = revision
        self.status = status
        self.error = error
        self.retries = retries
        self.status_text = status_text

    def is_updating(self):
        return self.status in ("Scheduled", "Updating")

    def compare(self, other, previous_radios=None):
        # type: (DeviceInfo, Optional[List[str]]) -> int
        """Compare this device to another to determine which should be updated first.

        return:
            < 0 if `self` should be first,
            0 if don't care
            > 0 if `other` should be first
        """
        # aliases for readability
        SELF = -1
        ANY = 0
        OTHER = 1

        ## 1) Any updates we're retrying after a previous failure should go last

        if previous_radios is not None:
            # If there is an update that we know failed in the past, put it
            # off until the end in the hope that the other updates will
            # leave it with a more stable route.
            try:
                self_i = previous_radios.index(self.address)
            except ValueError:
                self_i = None
            try:
                other_i = previous_radios.index(other.address)
            except ValueError:
                other_i = None

            if self_i is not None and other_i is not None:
                # if they both failed, do them in the opposite order from
                # last time in case one is in the route for another.
                return SELF if self_i > other_i else OTHER
            elif self_i is not None:
                return OTHER
            elif other_i is not None:
                return SELF

        # Further prioritization is based on version, so if they are the
        # same version we don't need to check more.
        if self.version == other.version:
            return ANY

        ## 2) Do XBEE3 updates before S2C updates

        if is_xb3_version(self.version) and is_s2c_version(other.version):
            return SELF
        elif is_xb3_version(other.version) and is_s2c_version(self.version):
            return OTHER

        ## 3) For XBee3, catch any 1B04 stragglers first

        # 1B04 is a stripped-down firmware, only meant to be used for a
        # short time while moving from 1003 to a new firmware. Since
        # this script does 1003->1B04->latest one after another, there
        # usually won't be any of these just sitting on the network. If
        # there are, do them first.
        if self.version == XBEE3_INTERIM_VERSION:
            return SELF
        elif other.version == XBEE3_INTERIM_VERSION:
            return OTHER

        ## 4) All else being equal, pick the newer version first.

        if NEWEST_FIRST:
            # Update newer versions first
            if self.version > other.version:
                return SELF
            else:
                return OTHER
        else:
            # Update older versions first
            if self.version < other.version:
                return SELF
            else:
                return OTHER

    def fetch_missing_info(self):
        """Fetch any info that the gateway may not have given us, but we need.

        This function is used during network discovery, and may trigger
        network traffic.
        """
        if self.version == 0:
            try:
                root_log.debug("Querying version on %s...", self.address)
                version_bytes = query_remote_parameter(self.address, "VR")
                version = int(binascii.hexlify(version_bytes), 16)
                root_log.debug("%s has version %X", self.address, version)
                self.version = version
            except RemoteQueryFailed:
                root_log.debug("Failed to read VR on %s", self.address)
                pass

        if self.revision == "\0\0":
            try:
                root_log.debug("Querying HV on %s...", self.address)
                hv_bytes = query_remote_parameter(self.address, "HV")
                root_log.debug("%s has HV %s", self.address, binascii.hexlify(hv_bytes))
                self.revision = hv_bytes
            except RemoteQueryFailed:
                root_log.debug("Failed to read HV on %s", self.address)
                pass


class UpdateFailed(Exception):
    """Raised when an update performed by the gateway fails."""

    def __init__(self, message, address, version=None):
        # type: (str, str, Optional[str]) -> None
        self.msg = "Failed to update [{}]".format(address)
        if version is not None:
            self.msg += " to {}".format(version)
        self.msg += ": {}".format(message)

class NoUpdatersFound(Exception):
    """Raised when no valid updaters are found for an S2C"""

class SkipUpdate(Exception):
    """Raised when an update is skipped"""

class RemoteQueryFailed(Exception):
    """Raised when querying a remote AT command fails."""

@contextmanager
def use_socket(protocol, endpoint, timeout):
    # type: (int, int, int) -> socket
    # Create the socket, datagram mode, proprietary transport:
    root_log.debug("Creating socket with endpoint {:x}".format(endpoint))
    s = socket(AF_XBEE, SOCK_DGRAM, protocol)
    s.bind(('', endpoint, 0, 0))
    s.settimeout(timeout)
    try:
        yield s
    finally:
        root_log.debug("Closing socket endpoint {:x}".format(endpoint))
        s.close()

def get_cached_radio_info(radio_addr=None):
    # type: (Optional[str]) -> List[DeviceInfo]
    """Get the DeviceInfo for radios from the gateway's internal table.

    if radio_addr is an address, the returned list will have only one
    element, the DeviceInfo for the radio with that address.

    This function only uses data stored in the gateway, nothing should
    be sent over the air as a result of calling this.
    """
    if radio_addr is None:
        # Parsing this list for a large network can take a while
        root_log.info("Gathering a list of all devices from the gateway...")
        request = '<rci_request version="1.1"><do_command target="zigbee"><discover option="current"/></do_command></rci_request>'
    else:
        request = '<rci_request version="1.1"><do_command target="zigbee"><discover option="current" addr="{}" /></do_command></rci_request>'.format(
            radio_addr
        )
    result = process_request(request)

    e = ElementTree.XML(result)
    devices = e.find("do_command/discover")
    assert devices is not None, "Invalid command response"
    devices = devices.findall("device")

    return [DeviceInfo.from_xml(el) for el in devices]

def attempt_get_cached_radio_info(radio_addr, attempts=3):
    # type: (str, Optional[int]) -> DeviceInfo
    """Get the DeviceInfo for a specific radio from the gateway's internal table.

    This helps to prevent indexing into an empty list since the gateway can sometimes
    clear the cached data. We retry after waiting a few seconds for the gateway
    to update the internal table.
    """
    for _ in range(attempts):
        # Get info from the internal table
        root_log.debug("Attempting to find cached radio info...")
        device_info_list = get_cached_radio_info(radio_addr)

        # If we found the entry, return the DeviceInfo object
        if len(device_info_list):
            return device_info_list[0]

        # Otherwise, wait a few seconds and try to get the entry again
        root_log.debug("Unable to find radio info...")
        time.sleep(5)

    # If we get here, we weren't able to get the radio information
    # Raise an update failed exeception to be able to move on to the next radio
    root_log.info("Unable to get cached radio info...")
    raise UpdateFailed("Was unable to get cached radio info", radio_addr)

def get_radios_to_update(previous_radios=None):
    # type: (Optional[List[str]]) -> List[DeviceInfo]
    """Go through the gateway's cached radios and see which ones need to be updated.

    This function may cause network traffic--if the gateway doesn't have
    HV or VR for a radio, this function will request them.
    """
    devices = get_cached_radio_info()

    updates = []  # type: List[DeviceInfo]

    # For each device, decide whether to update it.
    for device in devices:
        device.fetch_missing_info()

        hv_val = ord(device.revision[0:1])

        if (
            # HV indicates a thru-hole or SMT xbee 3
            hv_val in [0x41, 0x42]
            # Firmware version is any of the ones we're targeting
            and not is_up_to_date_xb3(device.version)
        ):
            updates.append(device)

        elif (
            # HV indicates a thru-hole or SMT xbee s2c
            hv_val in S2C_FW_BY_HV.keys()
            # Firmware version is any of the ones we're targeting
            and not is_up_to_date_s2c(device.version)
        ):
            updates.append(device)

        else:
            root_log.debug("%s does not need updating", device)

    # Order the updates as determined in DeviceInfo.compare()
    return sorted(
        updates, cmp=lambda a, b: a.compare(b, previous_radios=previous_radios)
    )

def rci_ota_update(address, version, recovery=False, updater="", normal_ota_recovery=False):
    # type: (str, str, bool, str, bool) -> None
    """Update a radio OTA using RCI commands."""
    log = root_log.getChild("update[{}]".format(address))
    log.info("Update to %s", version)
    if dry_run:
        log.info("Skipping update because of --dry-run")
        return

    # Start the update
    if recovery:
        log.info("Updating with recovery mode with updater:{}...".format(updater))
        cmd = '<rci_request version="1.1"><do_command target="zigbee"><fw_update file="{}" target="{}" recover="only" updater="{}"/></do_command></rci_request>'.format(
            version, address, updater
        )
    elif normal_ota_recovery:
        log.info("Updating with normal ota recovery mode with updater:{}...".format(updater))
        cmd = '<rci_request version="1.1"><do_command target="zigbee"><fw_update file="{}" target="{}" updater="{}" recover="never"/></do_command></rci_request>'.format(
            version, address, updater
        )
    else:
        log.info("Updating with ota mode...")
        cmd = '<rci_request version="1.1"><do_command target="zigbee"><fw_update file="{}" target="{}"/></do_command></rci_request>'.format(
            version, address
        )
    log.debug("Send cmd: %s", cmd)
    response = process_request(cmd)
    log.debug("Response: %s", response)
    time.sleep(UPDATE_CHECK_RATE)

    # Wait for the update to finish.
    wait_for_update_to_finish(address)

    device_info = attempt_get_cached_radio_info(address)

    if device_info.status not in ["Complete", "Updated"]:
        raise UpdateFailed(
            "Gateway update failed with status {}: {}".format(
                device_info.status, device_info.status_text
            ),
            address,
            version,
        )
    else:
        result_log.info("[%s] successfully updated to %s", address, version)

        # Attempt to communicate with the recently updated radio to show its still on the network
        time.sleep(5) # Wait for the firmware update to boot
        try:
            root_log.info("Attempting to communicate post update...")
            query_remote_parameter(address, "HV")
            root_log.info("Update was successful and communication post update succeeded")
        except RemoteQueryFailed:
            root_log.info("UPDATED RADIO FAILED TO RESPOND TO REMOTE QUERY")
        finally:
            time.sleep(10) # Wait for the gateway to log these commands

def rci_ota_update_logging_wrapper(address, version, recovery=False, updater="", normal_ota_recovery=False):
    # Wrapper function to save off the logging from the xbee.log file for a given node
    global save_logs
    global num_ota_logs
    update_successful = True
    try:
        if save_logs and num_ota_logs < MAX_ERROR_LOGS:
            log_tuple = start_logging(address, LOG_DIR_PATH)
        rci_ota_update(address, version, recovery, updater, normal_ota_recovery)
    except UpdateFailed as err:
        root_log.error("Error: {}".format(err.msg))
        update_successful = False
    finally:
        if save_logs and num_ota_logs < MAX_ERROR_LOGS:
            save_initial_successful_update_logs(update_successful, log_tuple[3], version)
            end_logging(log_tuple[0], log_tuple[1], log_tuple[2])
            remove_log(update_successful, log_tuple[3])
        if not update_successful:
            raise UpdateFailed("Gateway update failed to update router", address)

def start_logging(name, log_dir_path):
    # type: (str, str) -> Tuple[TextIO, BinaryIO, subprocess.Popen[bytes], str]
    # Starts a process that constantly copies the xbee.log to another file
    if not os.path.exists(log_dir_path):
        root_log.debug("Creating {} folder...".format(log_dir_path))
        os.mkdir(log_dir_path)
    else:
        root_log.debug("{} folder exists...".format(log_dir_path))

    root_log.debug("Attempting to start subprocess")
    output_file = "{}/{}-{}.log".format(log_dir_path, datetime.now(), name)
    root_log.debug("Creating log file at {}".format(output_file))
    log_file = open(output_file, 'w')
    quiet_errors = open(os.devnull, 'wb')
    process = subprocess.Popen(["tail", "/userfs/WEB/logging/xbee.log", "-F"], stdout=log_file, stderr=quiet_errors)
    return (log_file, quiet_errors, process, output_file)

def end_logging(log_file, quiet_errors, process):
    # Ends a logging process
    root_log.debug("Ending logging process...")
    process.terminate()
    log_file.close()
    quiet_errors.close()

def save_initial_successful_update_logs(update_successful, output_file, version):
    # Deletes logs created from successful ota updates
    # Saves the first successful S2C and XBee3 updates for comparison purposes
    global good_xbee3_log_found
    global good_s2c_log_found
    if update_successful:
        # Save the first successful S2C and XBee3 Logs of in another folder
        good_logs_dir = "{}/good_logs".format(LOG_DIR_PATH)
        if not os.path.exists(good_logs_dir):
            root_log.debug("Creating {} folder...".format(good_logs_dir))
            os.mkdir(good_logs_dir)
        else:
            root_log.debug("{} folder exists...".format(good_logs_dir))

        if version is None:
            return
        if not good_xbee3_log_found and 'XB3' in version:
            shutil.copy(output_file, good_logs_dir)
            good_xbee3_log_found = True
        if not good_s2c_log_found and 's2c' in version:
            shutil.copy(output_file, good_logs_dir)
            good_s2c_log_found = True

def remove_log(update_successful, output_file):
    # Removes a log file if the update was successful
    global num_ota_logs
    if update_successful:
        # Remove the successful log to make room for more log files
        try:
            root_log.debug("Removing logs from {} folder...".format(output_file))
            os.remove(output_file)
            num_ota_logs -= 1
            root_log.debug("Logs removed")
        except:
            root_log.debug("Error occurred removing logs...")
    else:
        root_log.debug("Error logs retained")
        num_ota_logs += 1

def wait_for_update_to_finish(address):
    # type: (str) -> None
    """Wait for the gateway to finish an update in progress."""
    log = root_log.getChild("update[{}]".format(address))
    log.info("Waiting for update to finish...")
    device_info = attempt_get_cached_radio_info(address)
    log.info("Update status: %s: %s", device_info.status, device_info.status_text)

    count = 0
    while device_info.is_updating():
        time.sleep(UPDATE_CHECK_RATE)
        try:
            device_info = attempt_get_cached_radio_info(address)
        except IndexError:
            root_log.info("Error: Radio info not found. Skipping update")
            break
        # Log progress less frequently, so the log doesn't get too bloated.
        count += 1
        if count >= 2:
            log.info(
                "Update status: %s: %s", device_info.status, device_info.status_text
            )
            count = 0


def discover_radios(iterations, clear=False):
    # type: (int, bool) -> None
    """Trigger the gateway to discover nodes on the network.

    This will cause the gateway to do a discovery. Nodes that it
    discovers are stored in the gateway, and can be retrieved with
    get_cached_radio_info().
    """
    # NT is the time across which nodes will stagger their responses to
    # a Node Discovery broadcast. A larger value means nodes are less
    # likely to talk over each other when responding, which helps to
    # enumerate the radios on the network. For our case, we'll
    # temporarily set it to its max value, 0xFF=25.5 seconds.
    nt_val = ddo_get_param(None, "NT")
    ddo_set_param(None, "NT", 0xFF)

    try:
        for i in range(iterations):
            root_log.info("Trigger node discovery...")
            if dry_run:
                root_log.info("Skipping discovery because of --dry-run")
                continue

            option = "clear" if clear and i == 0 else "discover"
            request = '<rci_request version="1.1"><do_command target="zigbee"><discover option="{}"/></do_command></rci_request>'.format(
                option
            )
            root_log.debug("Send cmd: %s", request)
            process_request(request)
            root_log.debug("Discovery complete.")
            # Wait for a bit between discoveries to let everything
            # settle out. We'll wait about another NT time to make sure all
            # the transmissions have finished.
            time.sleep(25)
    finally:
        nt_val = ddo_set_param(None, "NT", nt_val)


def query_remote_parameter(address, cmd, attempts=5):
    # type: (str, str, int) -> bytes
    """Send a remote AT command to a radio to query a value."""
    log = root_log.getChild("update[{}]".format(address))
    for i in range(attempts):
        try:
            response = ddo_get_param(address, cmd, 30)
            return response
        except Exception:
            log.exception("Couldn't read command")
            continue
    else:
        log.error("Failed to read %s after %d attempts.", cmd, i)
        raise RemoteQueryFailed("Couldn't read {} after {} attempts".format(cmd, i))


def update_radio_logging_wrapper(address):
    # type: (str) -> None
    # Wrapper function to save off the logging from the xbee.log file for a given node
    global save_logs
    global num_ota_logs
    update_successful = True
    version = None
    try:
        if save_logs:
            (log_file, quiet_errors, process, output_file) = start_logging(address, LOG_DIR_PATH)
        version = update_radio(address)
    except UpdateFailed as err:
        root_log.error("Error: {}".format(err.msg))
        update_successful = False
    finally:
        if save_logs:
            save_initial_successful_update_logs(update_successful, output_file, version)
            end_logging(log_file, quiet_errors, process)
            store_neighbor_list_from_log(address, output_file)
            remove_log(update_successful or num_ota_logs > MAX_ERROR_LOGS, output_file)
        if not update_successful:
            raise UpdateFailed("Gateway update failed to update router", address)

def update_radio(address):
    # type: (str) -> None
    """Update the radio at the given address.

    This function will query the current version of the radio, determine
    what update is needed, then update the radio.
    """

    log = root_log.getChild("update[{}]".format(address))
    log.info("Starting update on [%s].", address)

    # Query the current version on the device, just in case what's in
    # the gateway is out of date. This also serves as a sanity check,
    # making sure we can actually talk to the device.
    log.debug("Checking device version...")
    try:
        version_bytes = query_remote_parameter(address, "VR")
    except RemoteQueryFailed as err:
        raise UpdateFailed(err.args[0] + " Update not started.", address)

    version = int(binascii.hexlify(version_bytes), 16)
    log.info("target has version %X", version)

    if is_xb3_version(version) and not UPDATE_XB3_RADIOS:
        log.info("Skipping update on XBee 3 module")
        raise SkipUpdate()
    elif is_s2c_version(version) and not UPDATE_S2C_RADIOS:
        log.info("Skipping update on S2C module")
        raise SkipUpdate()

    if is_xb3_version(version) and not is_up_to_date_xb3(version):
        if version in XBEE3_UPDATE_INTERIM:
            log.info("Update to the interim firmware, then to the latest")
            rci_ota_update(address, XBEE3_INTERIM_FILE)
            rci_ota_update(address, XBEE3_TARGET_FILE)

        else:
            log.info("Update directly to the latest firmware")
            rci_ota_update(address, XBEE3_TARGET_FILE)
        return 'XB3'

    elif is_s2c_version(version) and not is_up_to_date_s2c(version):
        log.info("Update S2C directly to the latest firmware")
        log.debug("Checking hardware version...")
        try:
            hv_bytes = query_remote_parameter(address, "HV")
        except RemoteQueryFailed as err:
            raise UpdateFailed(err.args[0] + " Update not started.", address)
        hv_val = int(binascii.hexlify(hv_bytes[0:1]), 16)
        log.info("target has hardware version %X", hv_val)

        try:
            updater_list = gather_neighbor_information(address)
        except Exception as e:
            raise UpdateFailed("Module was not able to gather neighbor information due to: '{}'".format(e), address)

        # Attempt to try up to a certain number of updaters before stopping attempting to update
        num_updaters_to_try = min(5, len(updater_list))
        for updater_index in range(num_updaters_to_try):
            try:
                firmware = S2C_FW_BY_HV[hv_val]
                rci_ota_update(address, firmware, updater=updater_list[updater_index], normal_ota_recovery=True)
                return 's2c'
            except UpdateFailed:
                root_log.info("Regular update failed.")
                if updater_index >= num_updaters_to_try - 1:
                    # Update failed after exhausting all updaters, attempt to use recovery mode before moving on
                    root_log.info("Attempting recovery mode...")
                    network_devices = get_cached_radio_info()
                    results_dict = recover_via_recovery_update(network_devices, [address], False)
                    if results_dict[address] == "Recovery Successful":
                        return 's2c'

                    raise UpdateFailed(
                        "Failed to update S2C after using {} updaters.".format(updater_index + 1),
                        address
                    )
            except KeyError:
                raise UpdateFailed(
                    "Unrecognized HV value {02X} for S2C,update not started.".format(hv_val),
                    address
                )

    else:
        log.info("No update available for this version, skipping...")
        raise SkipUpdate()

def gather_neighbor_information(address):
    # type: (str) -> List[str]
    """
    Takes an address and attempts to find updaters for the given node. For all addresses found, sort them by LQI
    and filter out any devices we don't want to use as updaters. Then save the list to a json file and return the
    sorted list of updaters.
    """
    root_log.debug("Gathering neighbor device information...")
    sequence_number = 1
    neighbor_table_index = 0
    neighbor_table_size = 1  # Assume one entry before receiving the acutal value
    neighbor_info_list = []

    root_log.debug("Looking for a response frame...")
    while neighbor_table_index < neighbor_table_size:
        with use_socket(XBS_PROT_TRANSPORT, 0x0, 5) as send_0x11_socket:
            # Tuple format: (address_string, endpoint, profile_id, cluster_id)
            DESTINATION=(address, 0x0, 0x0, 0x31)
            root_log.debug("Destination parameters {}".format(DESTINATION))
            payload = struct.pack("B", sequence_number) + struct.pack("B", neighbor_table_index)
            send_0x11_socket.sendto(payload, 0, DESTINATION)

        # Block until a single frame is received, up to 255 bytes:
        root_log.debug("Looking for a response frame...")
        with use_socket(XBS_PROT_XAPI, 0x91, 5) as rec_0x91_socket:
            payload, src_addr = rec_0x91_socket.recvfrom(255)

        recv_frame = binascii.hexlify(payload)
        root_log.debug("Response: {}".format(recv_frame))
        rx_seq_num = int(recv_frame[42:44], 16)  # Sequence Number
        if rx_seq_num != sequence_number:
            root_log.debug("Didn't receive a frame with the expected sequence number. Try again.")
            continue
        sequence_number += 1
        # recv_frame[44:46]  # Status
        neighbor_table_size = int(recv_frame[46:48], 16)  # Neighbor table size
        root_log.debug("Neighbor table size {}".format(neighbor_table_size))
        # recv_frame[48:50]  # Start Index
        list_count = int(recv_frame[50:52], 16)  # List Count
        if list_count > 3:
            root_log.debug("List count can only be up to 3")
            break
        neighbor_table_index += list_count
        start_index = 52
        for _ in range(list_count):
            # recv_frame[start_index:68]  # Pan ID
            neighbor_addr = recv_frame[start_index+16:start_index+32]  # neighbor addr
            # recv_frame[start_index+32:start_index+36]  # eui 16
            # recv_frame[start_index+36:start_index+38]  # device property
            # recv_frame[start_index+38:start_index+40]  # permit joining
            # recv_frame[start_index+40:start_index+42]  # depth
            neighbor_lqi = recv_frame[start_index+42:start_index+44]  # LQI
            neighbor_addr = ':'.join(neighbor_addr[i:i+2] for i in range(len(neighbor_addr)-2, -2, -2)) + "!"
            root_log.debug("Neighbor Address found: {} LQI:{}".format(neighbor_addr, neighbor_lqi))
            start_index += 44
            neighbor_info_list.append((neighbor_addr, neighbor_lqi))

    if len(neighbor_info_list) == 0:
        raise NoUpdatersFound("Was unable to find any updaters for the given module")

    root_log.debug("Sorting neighbors by LQI...")
    neighbor_info_list.sort(key=lambda y: y[1], reverse=True)
    neighbor_list = []
    for addr, lqi in neighbor_info_list:
        neighbor_list.append(addr)

    network_devices = get_cached_radio_info()
    root_log.debug("Filtering out devices not considered a good updater...")
    for device in network_devices:
        if device.address in neighbor_list and not is_good_s2c_update_version(version=device.version, recovery=False):
            root_log.debug("Removing {}".format(device.address))
            neighbor_list.remove(device.address)

    if len(neighbor_list) == 0:
        raise NoUpdatersFound("Was unable to find any valid updaters after filtering devices out")

    root_log.info("---List of valid updaters sorted by LQI found---")
    for i, neighbor in enumerate(neighbor_list):
        root_log.info("#{} Addr:{}".format(i, neighbor))
    store_neighbor_list(address, neighbor_list)

    return neighbor_list


def set_logging_level(level):
    # type: (int) -> None
    root_log.info("Setting logging level to {}".format(level))
    cmd = '<rci_request><set_setting><log_level><xbee>{}</xbee></log_level></set_setting></rci_request>'.format(level)
    root_log.debug(cmd)
    process_request(cmd)

def run_update_step(addresses, excluded_addresses=[]):
    # type: (List[str], List[str]) -> None
    """Update radios on the network.

    This runs the portion of the script that does OTA updates. It takes
    a list of addresses, and goes through them one by one and updates them.

    When updating nodes that are older than version 1009, this will have
    the highest chance of success if this script is the only thing
    causing transmissions on the network while the updates are going.
    """
    process_request("<rci_request><set_setting><xbee><fw_stop_on_error>on</fw_stop_on_error></xbee></set_setting></rci_request>")

    sh_val = ddo_get_param(None, "SH").rjust(4, "\0")
    sl_val = ddo_get_param(None, "SL").rjust(4, "\0")
    internal_xbee_addr = ":".join(binascii.hexlify(b) for b in sh_val + sl_val) + "!"

    if len(excluded_addresses):
        root_log.info("Addresses to exclude {}".format(excluded_addresses))
    filtered_addresses = [addr for addr in addresses if addr not in excluded_addresses]
    for addr in filtered_addresses:
        root_log.info("")

        wait = True
        if addr == internal_xbee_addr:
            result_log.info(
                "[%s] is the xbee inside this gateway, and can't be updated OTA. Skipping.",
                addr,
            )
            continue

        try:
            update_radio_logging_wrapper(addr)
        except UpdateFailed as err:
            result_log.error(err.msg)
        except SkipUpdate:
            # Don't wait between skipped updates
            wait = False
        # If the update succeeded, the radio will rejoin the network,
        # which can cause some traffic and an announcement broadcast. If
        # the update failed, it may have been due to transient network
        # conidition. In either case, we've seen more success if we wait
        # for a bit after the update is done before starting the next
        # one.
        if wait:
            time.sleep(45)

def run_discover_step(iterations, clear=False, previous_radios=None):
    # type: (int, bool, Optional[List[str]]) -> List[str]
    """Discover radios to update on the network.

    This runs the portion of the script that discovers radios to update.
    It triggers `iterations` node discoveries, then looks through the
    radios the gateway has found to see what needs updates.

    if clear is True, clear the gateways cache before starting the
    discovery. Usually you want to keep all the radio info you can, but
    clearing the list can be helpful for testing how effective the
    discovery is.
    """
    root_log.info("Checking for radios to update...")
    discover_radios(iterations, clear)
    to_update = get_radios_to_update(previous_radios=previous_radios)

    root_log.info("Found %s radios to update: ", len(to_update))
    for device in to_update:
        root_log.info("  %s", device)

    update_addrs = [d.address for d in to_update]
    return update_addrs

def filter_by_device_on_network(orphan_list, devices):
    # type: (List[str], List[DeviceInfo]) -> List[str]
    """
    Filters a given list of EUI against the current EUIs of devices on the network.
    Returns a list of EUI strings of devices not on the network currently
    """
    excluded_orphan_list = []
    for orphan_eui in orphan_list:
        for device in devices:
            if device.address.lower() == orphan_eui.lower():
                root_log.info("Found {} on the network. Skipping...".format(orphan_eui))
                excluded_orphan_list.append(orphan_eui)
                break

    filtered_list = [eui for eui in orphan_list if eui not in excluded_orphan_list]
    return filtered_list

def discover_and_recover_orphans(network_devices, orphan_list, listed_only):
    # type: (List[DeviceInfo], List[str], bool) -> Dict[str, str]
    results_dict = {}
    if not orphan_list and listed_only is True:
        root_log.info("A list of orphans was given and they all appeared in the network scan, so we can skip searching the network.")

    root_log.info("Attempting to discover orphaned radios...")
    for device in network_devices:
        root_log.info("Attempting to search with Addr: {} VR: {:x} HV: {}".format(device.address, int(device.version), binascii.hexlify(device.revision)))

        # STEP 1: Only attempt searches on S2C's that have the broadcast feature
        sh_val = ddo_get_param(None, "SH").rjust(4, "\0")
        sl_val = ddo_get_param(None, "SL").rjust(4, "\0")
        internal_xbee_addr = ":".join(binascii.hexlify(b) for b in sh_val + sl_val) + "!"
        if device.address != internal_xbee_addr:
            if not is_good_s2c_recovery_broadcast_version(device.version):
                root_log.info("    Radio doesn't support broadcast orphan searching. Skipping...")
                continue
        else:
            root_log.info("    Radio is an S2C gateway. Attempting broadcast recovery regardless of version")

        # STEP 2: Attempt to use broadcasts to find orphaned node addresses still on the network channel
        try:
            root_log.info("    Setting DH/DL to the broadcast address")
            ddo_set_param(device.address, "DH", 0x0000)
            ddo_set_param(device.address, "DL", 0xFFFF)
        except:
            root_log.info("    Could not set DH/DL")
            continue

        try:
            root_log.info("    Attempting to discover orphaned nodes...")
            ddo_set_param(None, "AO", 1) # AO must be set to one for 0x91 frames
            found_orphan_addr = attempt_to_find_query_response(device)
            results_dict[found_orphan_addr] = "Recovery Failed"
            root_log.info("    Found an orphaned node! Addr:{}".format(found_orphan_addr))
            if listed_only and (found_orphan_addr not in orphan_list):
                root_log.info("A list of orphans was given and {} isn't on the filtered list, so we're skipping it.")
                continue
            root_log.info("    Attempting to recover node...")
            run_recovery_update(found_orphan_addr, device, normal_ota_recovery=True)
            root_log.info("    Recovery was successful!")
            results_dict[found_orphan_addr] = "Recovery Successful"
            continue
        except:
            root_log.info("    Couldn't find query response/update found orphan address")
            pass

        # STEP 3: Attempt to use broadcasts to find orphaned node addresses on the recovery channel
        try:
            ddo_set_param(None, "AO", 0) # Set to zero to get A0 frames instead of 91 frames
            found_orphan_addr = find_valid_updater(device, 0x0000FFFF)
            results_dict[found_orphan_addr] = "Recovery Failed"
            root_log.info("    Found an orphaned node! Addr:{}".format(found_orphan_addr))
            if listed_only and (found_orphan_addr not in orphan_list):
                root_log.info("A list of orphans was given and {} isn't on the filtered list, so we're skipping it.")
                continue
            root_log.info("    Attempting to recover node...")
            ddo_set_param(None, "AO", 1) # Set AO back to 1
            run_recovery_update(found_orphan_addr, device, recovery=True)
            root_log.info("    Recovery was successful!")
            results_dict[found_orphan_addr] = "Recovery Successful"
        except:
            root_log.info("    Couldn't find A0 frame response/update found orphan address")
            pass
        finally:
            ddo_set_param(None, "AO", 1)

    return results_dict

def run_orphan_recovery_step(iterations):
    # type: (int) -> None
    """Discover and attempt to recover orphaned S2C nodes

    This step reads in a list of eui's from a text file,
    loops around the network to find nodes stuck in the bootloader
    on the network channel as well as attempts to use recovery mode
    to recover orphaned S2Cs. If an updater is found, we attempt to do an
    OTA update in normal mode first, then recovery mode to get the orphaned
    node back on the network.
    """
    root_log.debug("Reading orphans.txt file")
    orphan_list = []
    try:
        p = re.compile(r"((?:[0-9a-fA-F]{2}:){7}[0-9a-fA-F]{2}!)")
        with open("orphans.txt") as f:
            for line in f.readlines():
                orphan_list.extend(p.findall(line))

        root_log.info("List of orphaned nodes to find listed in orphans.txt")
        root_log.info(orphan_list)
    except IOError:
        root_log.info("File not found. Proceeding to do discover and recover step anyway")

    overall_results_list = []

    # If there aren't any orphans specifically listed, then we want to look for
    # any orphans we can find. If there are any listed, only update those.
    listed_only = len(orphan_list) > 0

    for recovery_method, recovery_name in [
        (recover_via_normal_update, "Normal Update"),
        (recover_via_recovery_update, "Recovery Update"),
        (discover_and_recover_orphans, "Discover Orphans and Recover"),
    ]:

        # STEP 1: Discover all radios on the network
        discover_radios(iterations, clear=True)
        network_devices = get_cached_radio_info()
        for device in network_devices:
            device.fetch_missing_info()

        # STEP 2: Filter orphans we're looking for if they are already on the network
        root_log.info("Filtering orphans already on the network...")
        filtered_orphan_list = filter_by_device_on_network(orphan_list, network_devices)
        root_log.info("Filtered list of orphaned nodes to find")
        root_log.info(filtered_orphan_list)

        # STEP 3: Use the given recovery method to attempt orphan recovery
        results_dict = recovery_method(network_devices, filtered_orphan_list, listed_only)

        # STEP 4: Print out results of recovery process
        root_log.info("")
        root_log.info("{} Method- Number of results:{}".format(recovery_name, len(results_dict)))
        for eui, status in results_dict.items():
            result = "Orphan: {} | Method: {} | Recovery Status: {}".format(eui, recovery_name, status)
            root_log.info(result)
            overall_results_list.append(result)
        root_log.info("")

    # Summary of orphan euis and update status
    root_log.info("Overall OTA Recovery Results...")
    for result in overall_results_list:
        root_log.info(result)

def run_recovery_update(orphan_eui, device, retries=3, normal_ota_recovery=False, recovery=False):
    """
    Attempt to do a recovery ota update on a S2C with 3 retries.
    """
    for i in range(retries):
        try:
            root_log.info("OTA Recovery attempt #:{}".format(i))
            rci_ota_update_logging_wrapper(orphan_eui, XBEE_S2C_NONPRO_SMT_TARGET, recovery=recovery, updater=device.address, normal_ota_recovery=normal_ota_recovery)
            return
        except:
            pass
    raise Exception

def find_valid_updater(device, orphan_eui, retries=3):
    for i in range(retries):
        root_log.info("    %U attempt #:{}".format(i))
        try:
            root_log.info("    Issuing %U on S2C {}".format(device.address))
            ddo_set_param(device.address, "%U", orphan_eui)
            root_log.info("    Remote AT command response received")
        except:
            root_log.info("    %U command response not found")
            time.sleep(1)
            continue
        try:
            root_log.info("    Looking for 0xA0 frame...")
            with use_socket(XBS_PROT_XAPI, 0xA0, 10) as rec_0xA0_socket:
                # If we receive any A0 frame on the coordinator, it means we've found our updater node
                payload, src_addr = rec_0xA0_socket.recvfrom(255) # recvfrom() calls on `s` will give you any 0xA0 API frames that are seen
            root_log.info("    0xA0 frame found. Valid updater found")
            time.sleep(1)
        except:
            root_log.info("    0xA0 frame not found")
            continue

        recv_frame = binascii.hexlify(payload)
        root_log.info("    Response: {} Payload Byte: {}".format(recv_frame, recv_frame[30:32]))
        if recv_frame[30:32] == '52':
            root_log.info("    Query Response frame found!")
            target_addr = recv_frame[34:50]
            target_addr = ':'.join(target_addr[i:i+2] for i in range(0, len(target_addr), 2)) + "!"
            return target_addr
        else:
            root_log.info("    Query Response frame not found!")
    raise Exception

def recover_via_recovery_update(network_devices, orphan_list, listed_only):
    # type: (List[DeviceInfo], List[str], bool) -> Dict[str, str]
    ddo_set_param(None, "AO", 0) # Set to zero to get A0 frames instead of 91 frames
    results_dict = {}
    for orphan_eui in orphan_list:
        root_log.info("Searching for orphan addr {}".format(orphan_eui))
        results_dict[orphan_eui] = "Recovery Failed"

        # If we have any neighbors defined, try to use those for this step. We
        # can skip any nodes that aren't listed as neighbors--if this doesn't
        # work, we'll try them in the next step.
        neighbor_euis = retreive_neighbor_list(orphan_eui)
        if neighbor_euis:
            search_devices = sort_devices_by_neighbors(network_devices, neighbor_euis, False)
        else:
            search_devices = network_devices

        for device in search_devices:
            root_log.info("  Addr: {} VR: {:x} HV: {}".format(device.address, int(device.version), binascii.hexlify(device.revision)))
            if not is_good_s2c_update_version(device.version, recovery=False):
                root_log.info("    Radio can't safely recover S2Cs. Skipping...")
                continue

            try:
                find_valid_updater(device, int(orphan_eui.replace(':', '')[:-1], 16))
            except:
                root_log.info("    Node is not found as a valid updater")
                continue

            try:
                ddo_set_param(None, "AO", 1) # Set AO back to 1
                run_recovery_update(orphan_eui, device, recovery=True)
                root_log.info("    Recovery Successful")
                results_dict[orphan_eui] = "Recovery Successful"
                break
            except:
                root_log.info("    Recovery Failed")
                results_dict[orphan_eui] = "Recovery Failed"
            finally:
                ddo_set_param(None, "AO", 0) # Set to zero to get A0 frames instead of 91 frames

    return results_dict

def attempt_to_find_query_response(device, retries=3):
    for i in range(retries):
        root_log.info("    Bootloader Query attempt #:{}".format(i))
        with use_socket(XBS_PROT_TRANSPORT, 0xE8, 5) as send_0x11_socket:
            # Tuple format: (address_string, endpoint, profile_id, cluster_id)
            DESTINATION=(device.address, 0xE8, 0xC105, 0x71FF)
            root_log.debug("    Destination parameters {}".format(DESTINATION))
            payload = "0151".decode("hex")  # Bootloader QUERY
            send_0x11_socket.sendto(payload, 0, DESTINATION)

        try:
            # Block until a single frame is received, up to 255 bytes:
            root_log.info("    Looking for a bootloader response frame...")
            with use_socket(XBS_PROT_XAPI, 0x91, 5) as rec_0x91_socket:
                payload, src_addr = rec_0x91_socket.recvfrom(255)
        except:
            root_log.info("    No response to bootloader query frame.")
            continue

        recv_frame = binascii.hexlify(payload)
        root_log.info("    Response: {} Payload Byte: {}".format(recv_frame, recv_frame[42:44]))
        if recv_frame[42:44] == '52':
            root_log.info("    Query Response frame found!")
            target_addr = recv_frame[46:62]
            target_addr = ':'.join(target_addr[i:i+2] for i in range(0, len(target_addr), 2)) + "!"
            return target_addr
        elif recv_frame[42:44] == '40':
            root_log.info("    Potential updater cannot find the desired node...")
            break
        else:
            root_log.info("    Query Response frame not found!")

    raise Exception

def recover_via_normal_update(network_devices, orphan_list, listed_only):
    # type: (List[DeviceInfo], List[str], bool) -> Dict[str, str]
    ddo_set_param(None, "AO", 1) # Set to one to get 91 frames
    results_dict = {}
    for orphan_eui in orphan_list:
        root_log.info("Searching for orphan addr {}".format(orphan_eui))
        results_dict[orphan_eui] = "Recovery Failed"

        # If we have any neighbors defined, try to use those first. We should
        # still try all of them if we need to though, since the later steps will
        # be trying on a different channel.
        neighbor_euis = retreive_neighbor_list(orphan_eui)
        if neighbor_euis:
            search_devices = sort_devices_by_neighbors(network_devices, neighbor_euis, True)
        else:
            search_devices = network_devices

        for device in search_devices:
            root_log.info("    Searching with Addr: {} VR: {:x} HV: {}".format(device.address, int(device.version), binascii.hexlify(device.revision)))
            if not is_good_s2c_update_version(device.version, recovery=False):
                root_log.info("    Radio can't update S2Cs. Skipping...")
                continue

            try:
                root_log.info("    Setting DH/DL of the updater to the target")
                ddo_set_param(device.address, "DH", int(orphan_eui.replace(':', '')[:8], 16))
                ddo_set_param(device.address, "DL", int(orphan_eui.replace(':', '')[8:16], 16))
            except:
                root_log.info("    Could not set DH/DL")

            try:
                target_addr = attempt_to_find_query_response(device)
            except:
                root_log.info("    Did not find any query responses")
                time.sleep(.5)
                continue

            try:
                run_recovery_update(target_addr, device, normal_ota_recovery=True)
                results_dict[orphan_eui] = "Recovery Successful"
                root_log.info("    Recovery Successful")
                break
            except:
                root_log.info("    Recovery Failed")
                time.sleep(.5)
    return results_dict

def remove_error_logs(clear_logs=False):
    if clear_logs:
        try:
            root_log.info("Removing logs from {} folder...".format(LOG_DIR_NAME))
            shutil.rmtree(LOG_DIR_NAME)
            root_log.info("Logs removed")
        except:
            root_log.debug("Error occurred removing logs...")

def run_update_process(dry_run, parser, args):
    # Before doing anything, make sure there aren't already updates in
    # progress.
    if not dry_run:
        root_log.info("Checking for updates already in progress...")
        updating_radios = [d for d in get_cached_radio_info() if d.is_updating()]
        if updating_radios:
            root_log.warn("Updates were already in progress when the script started:")
            for radio in updating_radios:
                root_log.warn("  [%s]: %s", radio.address, radio.status)
            root_log.warn("We'll wait for those to finish before starting.")

            for updating_radio in updating_radios:
                wait_for_update_to_finish(updating_radio.address)

    if args.command == "discover":
        to_update = run_discover_step(args.discover_loop, clear=args.clear)
        for addr in to_update:
            print(addr)

    elif args.command == "update":
        to_update = [s.strip() for s in sys.stdin.readlines()]
        run_update_step(to_update)

    elif args.command == "rescue-orphans":
        try:
            run_orphan_recovery_step(args.discover_loop)
        finally:
            # Set changed config commands to what they were
            ddo_set_param(None, "AO", 1)

    elif args.command == "recover":
        try:
            (log_file, quiet_errors, process, output_file) = start_logging("Recover", LOG_DIR_PATH)
            run_recover_command()
        except Exception as err:
            root_log.error("Error: {}".format(err))
        finally:
            end_logging(log_file, quiet_errors, process)

    elif args.command == "run":
        # Get list of radios to exclude if needed
        to_exclude = []
        if args.exclude_radios:
            root_log.info("Reading in file of radios to exclude")
            to_exclude = [s.strip() for s in sys.stdin.readlines()]
        # Determine how many times to loop
        if args.dry_run:
            loop_iter = range(1)
        elif args.loop is None:
            # Loop forever with an iterator that will never finish
            loop_iter = itertools.count()
        else:
            loop_iter = range(int(args.loop))

        previous_radios = None
        for _ in loop_iter:
            to_update = run_discover_step(
                args.discover_loop, clear=args.clear, previous_radios=previous_radios
            )
            if to_update:
                run_update_step(to_update, to_exclude)

                # Keep track of the radios we just tried to update. If
                # any of them still need updating the next time through,
                # it's because the update failed, and we should put them
                # at the end of the list next time.
                previous_radios = to_update

            else:
                root_log.info(
                    "No devices found to update. Waiting a while before trying again..."
                )
                time.sleep(60 * 5)
    elif args.command == "send":
        send_frame(args.frame)
    else:
        parser.print_help()
        parser.exit()

def run_recover_command():
    # type: () -> None
    """
    Attempt to recover S2C's stuck on the recovery channel but not stuck in the bootloader.
    """
    log = root_log.getChild("recover")  # Setup logging
    log.info("Running S2C channel recovery command")
    log.info("NOTE: Your gateway needs to be running firmware capable of changing CH (0x40A1)")

    log.info("Reading original channel from coordinator...")
    original_channel = int(binascii.hexlify(ddo_get_param(None, "CH")), 16)
    log.info("Original channel is 0x{:x}...".format(original_channel))
    if original_channel == RECOVERY_CHANNEL:
        log.info("Network channel is already on the recovery channel... Quitting recover process.")
        return

    log.info("Changing coordinator over to the recover channel...")
    ddo_set_param(None, "CH", RECOVERY_CHANNEL)
    ddo_set_param(None, "WR")
    log.info("Verifying that the coordinator changed channels...")
    cur_ch = int(binascii.hexlify(ddo_get_param(None, "CH")), 16)
    if cur_ch != RECOVERY_CHANNEL:
        log.info("Coordinator did not change to the recovery channel. CH was 0x{:x}... Quitting recover process.".format(cur_ch))
        ddo_set_param(None, "CH", original_channel)
        return
    log.info("Coordinator CH is 0x{:x}".format(cur_ch))

    log.info("Discovering devices on the recovery channel...")
    discover_radios(args.discover_loop, clear=True)
    devices_to_recover = get_cached_radio_info()

    log.info("Verifying that the coordinator changed channels...")
    cur_ch = int(binascii.hexlify(ddo_get_param(None, "CH")), 16)
    if cur_ch != RECOVERY_CHANNEL:
        log.info("Coordinator did not change to the recovery channel. CH was 0x{:x}... Quitting recover process.".format(cur_ch))
        ddo_set_param(None, "CH", original_channel)
        return

    log.info("Closing the join window...")
    original_nj = int(binascii.hexlify(ddo_get_param(None, "NJ")), 16)
    ddo_set_param(None, "NJ", 0)

    sh_val = ddo_get_param(None, "SH").rjust(4, "\0")
    sl_val = ddo_get_param(None, "SL").rjust(4, "\0")
    internal_xbee_addr = ":".join(binascii.hexlify(b) for b in sh_val + sl_val) + "!"
    device_addresses = []
    for device in devices_to_recover:
        # If its not the gateway address and the device is an S2C
        if device.address != internal_xbee_addr:
            log.info("Device found:{}".format(device.address))
            device_addresses.append(device.address)

    for address in device_addresses:
        try:
            log.info("Checking to make sure the remote is on the recovery channel...")
            cur_ch = int(binascii.hexlify(ddo_get_param(address, "CH")), 16)
            log.info("{} is on channel 0x{:x}".format(address, cur_ch))
            if cur_ch != RECOVERY_CHANNEL:
                log.info("Remote node is not on the recovery channel. Quitting recover process...")
                cur_ch = int(binascii.hexlify(ddo_get_param(None, "CH")), 16)
                log.info("Coordinator CH is 0x{:x}".format(cur_ch))
                break

            log.info("Issuing remote commands on address {}...".format(address))
            ddo_set_param(address, "NJ", 0)
            ddo_set_param(address, "NR", 0)
        except Exception:
            log.info("ERROR: Was unable to issue remote commands")
            continue

    log.info("Switching coordinator back to the original channel...")
    ddo_set_param(None, "CH", original_channel)
    time.sleep(0.5)
    if len(device_addresses) == 0:
        ddo_set_param(None, "NJ", original_nj)
        log.info("Didn't find any devices to recover")
        return

    log.info("Issuing CB2 from the gateway and discovering radios on the original channel...")
    ddo_set_param(None, "CB", 2)
    discover_radios(args.discover_loop, clear=True)
    devices = get_cached_radio_info()
    ddo_set_param(None, "NJ", original_nj)

    log.info("***Recovery results***")
    for address_to_recover in device_addresses:
        found_device = False
        for device in devices:
            if device.address == address_to_recover:
                found_device = True
                break

        if found_device:
            try:
                ddo_get_param(address_to_recover, "NI")
                log.info("Success: Address {} recovered".format(address_to_recover))
            except Exception:
                log.info("Failure: Address {} cached on gateway but remote command failed".format(address_to_recover))
        else:
            log.info("Failure: Address {} not found by the gateway".format(address_to_recover))

def send_frame(frame):
    frame = frame.replace(" ", "")
    root_log.info("Frame to parse {}".format(frame))
    # frame_length = frame[2:6]
    # frame_type = frame[6:8]
    # frame_id = frame[8:10]
    addr64 = frame[10:26]
    # addr16 = frame[26:30]
    # source_endpoint = frame[30:32]
    dest_endpoint = frame[32:34]
    cluster_id = frame[34:38]
    profile_id = frame[38:42]
    # broadcast_radius = frame[42:44]
    # options = frame[44:46]
    payload = frame[46:-2].decode("hex")
    root_log.info("Addr 64:{}".format(addr64))
    root_log.info("Destination endpoint:{}".format(dest_endpoint))
    root_log.info("Cluster ID:{}".format(cluster_id))
    root_log.info("Profile ID:{}".format(profile_id))
    root_log.info("Payload ASCII:{}".format(payload))

    root_log.info("Sending 0x11 frame")
    dest_addr = [addr64[i:i + 2] for i in range(0, len(addr64), 2)]
    dest_addr = "[" + ':'.join(dest_addr) + "]!"
    root_log.info("Formatted destination addr: {}".format(dest_addr))
    # Tuple format: (address_string, endpoint, profile_id, cluster_id)
    DESTINATION=(dest_addr, int(dest_endpoint, 16), int(profile_id, 16), int(cluster_id, 16))

    root_log.info("Destination parameters {}".format(DESTINATION))
    with use_socket(XBS_PROT_TRANSPORT, 0xE8, 5) as send_0x11_socket:
        send_0x11_socket.sendto(payload, 0, DESTINATION)

    # Block until a single frame is received, up to 255 bytes:
    root_log.info("Receiving response")
    with use_socket(XBS_PROT_XAPI, 0x91, 5) as rec_0x91_socket:
        payload, src_addr = rec_0x91_socket.recvfrom(255)
    root_log.info("Response: {}".format(binascii.hexlify(payload)))

def store_neighbor_list_from_log(address, log_path):
    # type: (str, str) -> Optional[List[str]]
    """Look in the given file for a list of `address`'s neighbors, then store it."""
    prefix = r".* xbeed\[\d+\]:\s+\d+"
    addr = r"([0-9A-Fa-f]{2}:){7}[0-9A-Fa-f]{2}!"
    header = r"{}:otaGetNeighbors: trying updaters in this order:".format(prefix)
    neighbor = r"{}:otaGetNeighbors: *\d+: (?P<neighbor>{})".format(prefix, addr)
    footer = r"{}:otaSetup: target=(?P<target>{})".format(prefix, addr)

    re_neighbor = re.compile(neighbor)
    re_footer = re.compile(footer)
    re_report = re.compile(r"{}\n({}\n)+{}".format(header, neighbor, footer))

    with open(log_path) as f:
        lines = "".join(
            l for l in f.readlines() if "otaGetNeighbors" in l or "otaSetup" in l
        )

    neighbor_lists = []  # type: List[List[str]]
    for m in re_report.finditer(lines):
        report = lines[m.start() : m.end()]
        target = re_footer.search(report)
        if target is None or target.group("target") != address:
            continue
        neighbor_lists.append(
            [m.group("neighbor") for m in re_neighbor.finditer(report)]
        )

    if not neighbor_lists:
        return None
    neighbors = neighbor_lists[-1]

    store_neighbor_list(address, neighbors)

    return neighbors

def store_neighbor_list(address, neighbors):
    # type: (str, List[str]) -> None
    """Save a list of neighbors for the given address."""
    neighbor_lists = {}  # type: Dict[str, List[str]]
    if os.path.exists(NEIGHBOR_LIST_PATH):
        try:
            with open(NEIGHBOR_LIST_PATH) as f:
                neighbor_lists = json.load(f)
        except ValueError:
            # If the neighbor list is in an invalid format, we'll just have to
            # toss it.
            pass
    neighbor_lists[address] = neighbors
    with open(NEIGHBOR_LIST_PATH, "w") as f:
        json.dump(neighbor_lists, f, indent=2)

def retreive_neighbor_list(address):
    # type: (str) -> Optional[List[str]]
    """Retrieve a list of neighbors from the neighbors.json file."""
    if not os.path.exists(NEIGHBOR_LIST_PATH):
        return None
    try:
        with open(NEIGHBOR_LIST_PATH, 'r') as f:
            neighbor_dict = json.load(f)
    except ValueError:
        # If it's invalid json, we can't get any answers from it.
        return None
    return neighbor_dict.get(address)

def sort_devices_by_neighbors(devices, neighbors, keep_all):
    # type: (List[DeviceInfo], List[str], bool) -> List[DeviceInfo]
    """Return a sorted copy of the device list, sorted by best neighbor.

    If keep_all is true, all non-neighbor devices will be left in the list after
    the neighbors.
    """
    # Make a copy of the list, potentially filtering out non-neighbors
    result = [d for d in devices if keep_all or d.address in neighbors]

    # Sort the devices in the same order they are in the neighbors list
    def key(d):
        # type: (DeviceInfo) -> int
        try:
            return neighbors.index(d.address)
        except ValueError:
            return len(neighbors)
    result.sort(key=key)

    return result


if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()

    common_args = argparse.ArgumentParser(add_help=False)
    common_args.add_argument(
        "-v", action="store_true", help="Enable more verbose logging."
    )
    common_args.add_argument(
        "--dry-run",
        action="store_true",
        help="""Run the script without actually updating any radios or
             triggering any network-intensive operations (broadcasts, node
             discoveries, etc.)""",
    )
    common_args.add_argument(
       "--save-logs",
        default=True,
        type=bool,
        help="""Set to True to save logs over the course of OTA updates""",
    )
    common_args.add_argument(
       "--clear-logs",
        default=True,
        type=bool,
        help="""Set to True to clear out the error logs folder for this run.""",
    )
    common_args.add_argument(
       "--logging-level",
        default=7,
        type=int,
        help="""Set the logging level on the gateway (default 7).""",
    )
    common_args.add_argument(
       "--exclude-radios",
        default=False,
        type=bool,
        help="""Set to True to read in text file of EUIs to exclude from updating.""",
    )
    common_args.add_argument(
        "--discover-loop",
        default=3,
        nargs=1,
        help="""How many times to go through the discovery process when building
               the list of nodes to update (default 3).""",
    )
    common_args.add_argument(
        "--ar-value",
        default=None,
        type=int,
        help="""The value to set AR to for the duration of the script""",
    )
    common_args.add_argument(
        "--nh-value",
        default=None,
        type=int,
        help="""The value to set AR to for the duration of the script""",
    )

    discover_args = argparse.ArgumentParser(add_help=False)
    discover_args.add_argument(
        "--clear",
        action="store_true",
        help="""Clear the gateway's internal list of radios before starting the
                discovery. Useful for testing the discovery code, don't use on
                production networks.""",
    )

    update_args = argparse.ArgumentParser(add_help=False)
    update_args.add_argument(
        "--loop",
        nargs="?",
        default=2,
        metavar="N",
        help="""Loop through the whole discovery/update process N times.
                Specify --loop without a number to loop indefinitely.""",
    )

    send_args = argparse.ArgumentParser(add_help=False)
    send_args.add_argument(
        "--frame",
        type=str,
        help="""The frame to send.""",
    )

    commands = parser.add_subparsers(
        title="Commands (run each with -h for available options)", dest="command"
    )
    discover_cmd = commands.add_parser(
        "discover",
        parents=[common_args, discover_args],
        help="Discover XBee nodes on the network that need updating.",
    )
    update_cmd = commands.add_parser(
        "update",
        parents=[common_args, update_args],
        help="Update XBee nodes from a list",
    )
    run_cmd = commands.add_parser(
        "run",
        parents=[common_args, discover_args, update_args],
        help="Discover XBee nodes to update, then update them",
    )
    orphan_cmd = commands.add_parser(
        "rescue-orphans",
        parents=[common_args],
        help="Discover orphaned XBee nodes, then update them",
    )
    recover_cmd = commands.add_parser(
        "recover",
        parents=[common_args],
        help="Find S2C's stuck on the recovery channel and recover them.",
    )
    send_cmd = commands.add_parser(
        "send",
        parents=[common_args, send_args],
        help="Send a 0x11 frame and see the response.",
    )
    args = parser.parse_args()

    dry_run = args.dry_run
    save_logs = args.save_logs

    logging.basicConfig(
        stream=sys.stderr,
        level=logging.DEBUG if args.v else logging.INFO,
        format="%(asctime)s:%(levelname)5s:%(name)s: %(message)s",
    )

    root_log.info("XBee Gateway Update Tool Version:{}".format(SCRIPT_VERSION))

    set_logging_level(args.logging_level)
    remove_error_logs(args.clear_logs)
    ddo_set_param(None, "AO", 1)
    ddo_set_param(None, "CR", 0x3F)
    try:
        if args.ar_value is not None:
            root_log.info("Setting AR to {} for the duration of the script".format(args.ar_value))
            ar_val = ddo_get_param(None, "AR")
            ddo_set_param(None, "AR", args.ar_value)
        if args.nh_value is not None:
            root_log.info("Setting NH to {} for the duration of the script".format(args.nh_value))
            ar_val = ddo_get_param(None, "NH")
            ddo_set_param(None, "NH", args.nh_value)

        run_update_process(dry_run, parser, args)
    finally:
        if args.ar_value is not None:
            ddo_set_param(None, "AR", ar_val)
        if args.nh_value is not None:
            ddo_set_param(None, "NH", nh_val)


# Rev: 687d27a29214bcad11c4fd7268d9a6069c67fe2a
