import cubeds.pylogger
import cubeds.exceptions
import cubeds.config
import yaml
from bitarray import bitarray
import struct
import os
import numpy as np
import csv
import json
import datetime as dt


def read_yaml(file):
    """
    Reads in yaml (.yml) file
    :param file: yaml file location
    :return: dictionary of information
    """
    with open(file, mode='r') as stream:
        out = yaml.load(stream)

    return out


def extract_bits_from_bytes(data, start_bit, num_bits):
    """
    Extracts bits from a group of bytes
    :param data: byte data
    :param start_bit: starting bit to begin extraction
    :param num_bits: number of bits to extract
    :return: bitarray of requested bits
    """
    b = bitarray(endian='big')
    b.frombytes(data)
    out = b[start_bit:start_bit+num_bits]
    return out


def bitstring_to_int(bitstr):
    """
    Converts a bitstring to an integer
    :param bitstr: bitstring
    :return: integer representation of bitstring
    """
    b_list = bitstr.tolist()
    mystring = ''
    for s in b_list:
        if s:
            mystring += '1'
        else:
            mystring += '0'
    b = int(mystring, 2)
    return b


def get_unpack_format(in_dtype, dsize, endian='big', logger=cubeds.pylogger.get_logger(__name__)):
    """
    Given data type and size, generate a format for struct.unpack.
    ASSUMES BIG ENDIAN for the data that will be unpacked. can specifiy by using endian="little"
    :param in_dtype: Type of data. Valid types are dn (unsigned), sn (signed), char, float, or double
    :param dsize: size of data in BITS!!!!
    :param endian: endianess of the data
    :return:
    """
    # logger = cubeds.pylogger.get_logger(__name__)
    if dsize <= 8:
        # bits
        letter = 'b'
    elif dsize == 16:
        # short
        letter = 'h'
    elif dsize == 32:
        # int
        letter = 'i'
    elif dsize == 64:
        # long long
        letter = 'q'
    elif dsize == -1:
        # padding
        letter = str(dsize//8)+'x'  # want self.size bytes of padding
        in_dtype = 'int'
        dsize = 1
    else:
        logger.fatal("parsing error for datatype size")
        raise cubeds.exceptions.UnpackFormatError(msg="parsing error for datatype size")

    if in_dtype == 'dn':
        letter = letter.upper()
    elif in_dtype == 'sn':
        letter = letter.lower()
    elif in_dtype == 'char':
        letter = str(int(dsize/8)) + 's'
    elif in_dtype == 'float' or in_dtype == 'double':
        letter = 'd'
    else:
        logger.fatal("parsing error for datatype \"" + in_dtype + "\" with size \"" + str(dsize) + "\"")
        raise cubeds.exceptions.UnpackFormatError(msg="parsing error for datatype \"" + in_dtype + "\" with size \""
                                                      + str(dsize) + "\"")

    if endian == 'big':
        letter = '>' + letter
    elif endian == 'little':
        letter = '<' + letter
    else:
        logger.fatal("DID NOT SPECIFY ENDIANNESS CORRECTLY")
        raise cubeds.exceptions.UnpackFormatError("Endianness not specified correctly.")
    return letter


def extract_bits(data, bit, length=1):
    """
    Extracts bits given data, start bit, and length
    :param data: data to extract bits from
    :param bit: starting position for extraction
    :param length: number of bits to extract, default=1
    :return: resulting integer of extracted bits
    """
    bits = bitarray(data, endian='big')
    if length > 1:
        out = bits[bit:bit+length]
        try:
            out = struct.unpack('>B', out.tobytes())[0]
        except struct.error:
            out = 0
    else:
        try:
            out = bits[bit]
        except IndexError:
            out = 0
    return int(out)


def get_csv_encoding(config):
    """
    Grabs the CSV encoding type form the config file, csv section, encoding variable.
    :return: csv encoding type string
    """
    logger = cubeds.pylogger.get_logger(__name__)
    try:
        csv_encoding = config.config['telemetry'][config.yaml_key]['encoding']
    except KeyError:
        csv_encoding = 'utf-8'
        logger.debug('Using utf-8 encoding because couldn\'t find encoding type in config file')

    return csv_encoding


def get_csv_info(config):
    """
    Reads in CSV info from top level file
    :param csv_filename: csv file name
    :return: dictionary of csv lines
    """
    logger = cubeds.pylogger.get_logger(__name__)
    csv_filename = config.config['telemetry'][config.yaml_key]['packet_definitions']

    # make sure file exists

    file = os.path.join(os.path.dirname(os.path.realpath(__file__)), csv_filename)

    try:
        assert os.path.exists(file)
    except AssertionError as e:
        logger.fatal(e)
        logger.fatal("Cannot find packet definition file "+file)
        raise cubeds.exceptions.DecoderError

    csv_encoding = get_csv_encoding(config)

    with open(file, mode='r', encoding=csv_encoding) as csv_f:
        reader = csv.DictReader(csv_f)
        csv_dict_list = []
        for row in reader:
            csv_dict_list.append(row)
    return csv_dict_list


def get_tlm_points(points_file, config):
    """
    This function will get all the points from the points file input to the function
    :param points_file: csv file with all the points.
    :param config: cubeds.config.Config object.
    :return: List of dictionaries of points.
    """
    # make sure file exists
    points_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), points_file)
    assert os.path.exists(points_file)

    csv_encoding = get_csv_encoding(config)

    with open(points_file, mode='r', encoding=csv_encoding) as csv_f:
        # create a dictionary from the top level csv file
        reader = csv.DictReader(csv_f)

        # init list for saving the points dict
        points_dict_list = []
        for row in reader:
            # if the "state" column includes a "/", that means that there is a properly formatted state pair,
            # i.e. "0/ON 1/OFF"
            # ALL PAIRS SHOULD BE SEPERATED BY A SPACE AND FORMATTED AS FOLLOWS: VAL/STR VAL/STR
            if '/' in row['state']:
                row['unit'] = ''

                # Seperate pairs and strip whitespace
                state_array = [x.strip() for x in row['state'].split(' ')]

                # init empty dictionary
                state_dict = {}

                # loop through the pairs and stor them in the dictionary with the integer as the key and the string as
                # the value
                for keyStrPair in state_array:
                    # extract the "key/str" pair
                    state_val, state_str = keyStrPair.split('/')
                    state_dict[state_val] = state_str

                row['state'] = json.dumps(state_dict)
            else:
                row['state'] = ''

            # append the new dictionary to the list
            points_dict_list.append(dict(row))
    return points_dict_list


def add_utc_to_df(df, time_index, conversion_function, config):
    """
    Adds a utc column to dataframe
    :param df: dataframe that needs utc column
    :param time_index: column to convert to utc
    :param conversion_function: function to convert values in time_index function to utc
    :return: updated dataframe
    """
    df['UTC'] = df[time_index].map(lambda x: conversion_function(x, config))
    return df


def tai_to_utc(tai, config, time_format="%Y/%j-%H:%M:%S"):
    """
    Converts TAI time to UTC time
    :param tai: TAI time
    :param time_format: Format of output UTC time
    :return: UTC time string
    """
    epoch = dt.datetime.strptime(config.config['runtime']['epoch'], "%m/%d/%Y-%H:%M:%S")
    try:
        utc = epoch + dt.timedelta(seconds=int(tai))
    except OverflowError:
        utc = epoch
    return utc.strftime(time_format)


def get_apids(config):
    """
    Returns list of apids from file specified in cfg file
    :return: list of apids
    """
    csv_info = get_csv_info(config)
    apids = []
    for line in csv_info:
        if int(line['apid']) not in apids:
            apids.append(int(line['apid']))
    return apids

3298286bbbacd523845ab730a69bd09be4198582
<mask token>


def read_yaml(file):
    """
    Reads in yaml (.yml) file
    :param file: yaml file location
    :return: dictionary of information
    """
    with open(file, mode='r') as stream:
        out = yaml.load(stream)
    return out


def extract_bits_from_bytes(data, start_bit, num_bits):
    """
    Extracts bits from a group of bytes
    :param data: byte data
    :param start_bit: starting bit to begin extraction
    :param num_bits: number of bits to extract
    :return: bitarray of requested bits
    """
    b = bitarray(endian='big')
    b.frombytes(data)
    out = b[start_bit:start_bit + num_bits]
    return out


def bitstring_to_int(bitstr):
    """
    Converts a bitstring to an integer
    :param bitstr: bitstring
    :return: integer representation of bitstring
    """
    b_list = bitstr.tolist()
    mystring = ''
    for s in b_list:
        if s:
            mystring += '1'
        else:
            mystring += '0'
    b = int(mystring, 2)
    return b


def get_unpack_format(in_dtype, dsize, endian='big', logger=cubeds.pylogger
    .get_logger(__name__)):
    """
    Given data type and size, generate a format for struct.unpack.
    ASSUMES BIG ENDIAN for the data that will be unpacked. can specifiy by using endian="little"
    :param in_dtype: Type of data. Valid types are dn (unsigned), sn (signed), char, float, or double
    :param dsize: size of data in BITS!!!!
    :param endian: endianess of the data
    :return:
    """
    if dsize <= 8:
        letter = 'b'
    elif dsize == 16:
        letter = 'h'
    elif dsize == 32:
        letter = 'i'
    elif dsize == 64:
        letter = 'q'
    elif dsize == -1:
        letter = str(dsize // 8) + 'x'
        in_dtype = 'int'
        dsize = 1
    else:
        logger.fatal('parsing error for datatype size')
        raise cubeds.exceptions.UnpackFormatError(msg=
            'parsing error for datatype size')
    if in_dtype == 'dn':
        letter = letter.upper()
    elif in_dtype == 'sn':
        letter = letter.lower()
    elif in_dtype == 'char':
        letter = str(int(dsize / 8)) + 's'
    elif in_dtype == 'float' or in_dtype == 'double':
        letter = 'd'
    else:
        logger.fatal('parsing error for datatype "' + in_dtype +
            '" with size "' + str(dsize) + '"')
        raise cubeds.exceptions.UnpackFormatError(msg=
            'parsing error for datatype "' + in_dtype + '" with size "' +
            str(dsize) + '"')
    if endian == 'big':
        letter = '>' + letter
    elif endian == 'little':
        letter = '<' + letter
    else:
        logger.fatal('DID NOT SPECIFY ENDIANNESS CORRECTLY')
        raise cubeds.exceptions.UnpackFormatError(
            'Endianness not specified correctly.')
    return letter


<mask token>


def get_csv_info(config):
    """
    Reads in CSV info from top level file
    :param csv_filename: csv file name
    :return: dictionary of csv lines
    """
    logger = cubeds.pylogger.get_logger(__name__)
    csv_filename = config.config['telemetry'][config.yaml_key][
        'packet_definitions']
    file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
        csv_filename)
    try:
        assert os.path.exists(file)
    except AssertionError as e:
        logger.fatal(e)
        logger.fatal('Cannot find packet definition file ' + file)
        raise cubeds.exceptions.DecoderError
    csv_encoding = get_csv_encoding(config)
    with open(file, mode='r', encoding=csv_encoding) as csv_f:
        reader = csv.DictReader(csv_f)
        csv_dict_list = []
        for row in reader:
            csv_dict_list.append(row)
    return csv_dict_list


<mask token>


def tai_to_utc(tai, config, time_format='%Y/%j-%H:%M:%S'):
    """
    Converts TAI time to UTC time
    :param tai: TAI time
    :param time_format: Format of output UTC time
    :return: UTC time string
    """
    epoch = dt.datetime.strptime(config.config['runtime']['epoch'],
        '%m/%d/%Y-%H:%M:%S')
    try:
        utc = epoch + dt.timedelta(seconds=int(tai))
    except OverflowError:
        utc = epoch
    return utc.strftime(time_format)


def get_apids(config):
    """
    Returns list of apids from file specified in cfg file
    :return: list of apids
    """
    csv_info = get_csv_info(config)
    apids = []
    for line in csv_info:
        if int(line['apid']) not in apids:
            apids.append(int(line['apid']))
    return apids

<mask token>


def read_yaml(file):
    """
    Reads in yaml (.yml) file
    :param file: yaml file location
    :return: dictionary of information
    """
    with open(file, mode='r') as stream:
        out = yaml.load(stream)
    return out


def extract_bits_from_bytes(data, start_bit, num_bits):
    """
    Extracts bits from a group of bytes
    :param data: byte data
    :param start_bit: starting bit to begin extraction
    :param num_bits: number of bits to extract
    :return: bitarray of requested bits
    """
    b = bitarray(endian='big')
    b.frombytes(data)
    out = b[start_bit:start_bit + num_bits]
    return out


def bitstring_to_int(bitstr):
    """
    Converts a bitstring to an integer
    :param bitstr: bitstring
    :return: integer representation of bitstring
    """
    b_list = bitstr.tolist()
    mystring = ''
    for s in b_list:
        if s:
            mystring += '1'
        else:
            mystring += '0'
    b = int(mystring, 2)
    return b


def get_unpack_format(in_dtype, dsize, endian='big', logger=cubeds.pylogger
    .get_logger(__name__)):
    """
    Given data type and size, generate a format for struct.unpack.
    ASSUMES BIG ENDIAN for the data that will be unpacked. can specifiy by using endian="little"
    :param in_dtype: Type of data. Valid types are dn (unsigned), sn (signed), char, float, or double
    :param dsize: size of data in BITS!!!!
    :param endian: endianess of the data
    :return:
    """
    if dsize <= 8:
        letter = 'b'
    elif dsize == 16:
        letter = 'h'
    elif dsize == 32:
        letter = 'i'
    elif dsize == 64:
        letter = 'q'
    elif dsize == -1:
        letter = str(dsize // 8) + 'x'
        in_dtype = 'int'
        dsize = 1
    else:
        logger.fatal('parsing error for datatype size')
        raise cubeds.exceptions.UnpackFormatError(msg=
            'parsing error for datatype size')
    if in_dtype == 'dn':
        letter = letter.upper()
    elif in_dtype == 'sn':
        letter = letter.lower()
    elif in_dtype == 'char':
        letter = str(int(dsize / 8)) + 's'
    elif in_dtype == 'float' or in_dtype == 'double':
        letter = 'd'
    else:
        logger.fatal('parsing error for datatype "' + in_dtype +
            '" with size "' + str(dsize) + '"')
        raise cubeds.exceptions.UnpackFormatError(msg=
            'parsing error for datatype "' + in_dtype + '" with size "' +
            str(dsize) + '"')
    if endian == 'big':
        letter = '>' + letter
    elif endian == 'little':
        letter = '<' + letter
    else:
        logger.fatal('DID NOT SPECIFY ENDIANNESS CORRECTLY')
        raise cubeds.exceptions.UnpackFormatError(
            'Endianness not specified correctly.')
    return letter


def extract_bits(data, bit, length=1):
    """
    Extracts bits given data, start bit, and length
    :param data: data to extract bits from
    :param bit: starting position for extraction
    :param length: number of bits to extract, default=1
    :return: resulting integer of extracted bits
    """
    bits = bitarray(data, endian='big')
    if length > 1:
        out = bits[bit:bit + length]
        try:
            out = struct.unpack('>B', out.tobytes())[0]
        except struct.error:
            out = 0
    else:
        try:
            out = bits[bit]
        except IndexError:
            out = 0
    return int(out)


<mask token>


def get_csv_info(config):
    """
    Reads in CSV info from top level file
    :param csv_filename: csv file name
    :return: dictionary of csv lines
    """
    logger = cubeds.pylogger.get_logger(__name__)
    csv_filename = config.config['telemetry'][config.yaml_key][
        'packet_definitions']
    file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
        csv_filename)
    try:
        assert os.path.exists(file)
    except AssertionError as e:
        logger.fatal(e)
        logger.fatal('Cannot find packet definition file ' + file)
        raise cubeds.exceptions.DecoderError
    csv_encoding = get_csv_encoding(config)
    with open(file, mode='r', encoding=csv_encoding) as csv_f:
        reader = csv.DictReader(csv_f)
        csv_dict_list = []
        for row in reader:
            csv_dict_list.append(row)
    return csv_dict_list


<mask token>


def add_utc_to_df(df, time_index, conversion_function, config):
    """
    Adds a utc column to dataframe
    :param df: dataframe that needs utc column
    :param time_index: column to convert to utc
    :param conversion_function: function to convert values in time_index function to utc
    :return: updated dataframe
    """
    df['UTC'] = df[time_index].map(lambda x: conversion_function(x, config))
    return df


def tai_to_utc(tai, config, time_format='%Y/%j-%H:%M:%S'):
    """
    Converts TAI time to UTC time
    :param tai: TAI time
    :param time_format: Format of output UTC time
    :return: UTC time string
    """
    epoch = dt.datetime.strptime(config.config['runtime']['epoch'],
        '%m/%d/%Y-%H:%M:%S')
    try:
        utc = epoch + dt.timedelta(seconds=int(tai))
    except OverflowError:
        utc = epoch
    return utc.strftime(time_format)


def get_apids(config):
    """
    Returns list of apids from file specified in cfg file
    :return: list of apids
    """
    csv_info = get_csv_info(config)
    apids = []
    for line in csv_info:
        if int(line['apid']) not in apids:
            apids.append(int(line['apid']))
    return apids

<mask token>


def read_yaml(file):
    """
    Reads in yaml (.yml) file
    :param file: yaml file location
    :return: dictionary of information
    """
    with open(file, mode='r') as stream:
        out = yaml.load(stream)
    return out


def extract_bits_from_bytes(data, start_bit, num_bits):
    """
    Extracts bits from a group of bytes
    :param data: byte data
    :param start_bit: starting bit to begin extraction
    :param num_bits: number of bits to extract
    :return: bitarray of requested bits
    """
    b = bitarray(endian='big')
    b.frombytes(data)
    out = b[start_bit:start_bit + num_bits]
    return out


def bitstring_to_int(bitstr):
    """
    Converts a bitstring to an integer
    :param bitstr: bitstring
    :return: integer representation of bitstring
    """
    b_list = bitstr.tolist()
    mystring = ''
    for s in b_list:
        if s:
            mystring += '1'
        else:
            mystring += '0'
    b = int(mystring, 2)
    return b


def get_unpack_format(in_dtype, dsize, endian='big', logger=cubeds.pylogger
    .get_logger(__name__)):
    """
    Given data type and size, generate a format for struct.unpack.
    ASSUMES BIG ENDIAN for the data that will be unpacked. can specifiy by using endian="little"
    :param in_dtype: Type of data. Valid types are dn (unsigned), sn (signed), char, float, or double
    :param dsize: size of data in BITS!!!!
    :param endian: endianess of the data
    :return:
    """
    if dsize <= 8:
        letter = 'b'
    elif dsize == 16:
        letter = 'h'
    elif dsize == 32:
        letter = 'i'
    elif dsize == 64:
        letter = 'q'
    elif dsize == -1:
        letter = str(dsize // 8) + 'x'
        in_dtype = 'int'
        dsize = 1
    else:
        logger.fatal('parsing error for datatype size')
        raise cubeds.exceptions.UnpackFormatError(msg=
            'parsing error for datatype size')
    if in_dtype == 'dn':
        letter = letter.upper()
    elif in_dtype == 'sn':
        letter = letter.lower()
    elif in_dtype == 'char':
        letter = str(int(dsize / 8)) + 's'
    elif in_dtype == 'float' or in_dtype == 'double':
        letter = 'd'
    else:
        logger.fatal('parsing error for datatype "' + in_dtype +
            '" with size "' + str(dsize) + '"')
        raise cubeds.exceptions.UnpackFormatError(msg=
            'parsing error for datatype "' + in_dtype + '" with size "' +
            str(dsize) + '"')
    if endian == 'big':
        letter = '>' + letter
    elif endian == 'little':
        letter = '<' + letter
    else:
        logger.fatal('DID NOT SPECIFY ENDIANNESS CORRECTLY')
        raise cubeds.exceptions.UnpackFormatError(
            'Endianness not specified correctly.')
    return letter


def extract_bits(data, bit, length=1):
    """
    Extracts bits given data, start bit, and length
    :param data: data to extract bits from
    :param bit: starting position for extraction
    :param length: number of bits to extract, default=1
    :return: resulting integer of extracted bits
    """
    bits = bitarray(data, endian='big')
    if length > 1:
        out = bits[bit:bit + length]
        try:
            out = struct.unpack('>B', out.tobytes())[0]
        except struct.error:
            out = 0
    else:
        try:
            out = bits[bit]
        except IndexError:
            out = 0
    return int(out)


<mask token>


def get_csv_info(config):
    """
    Reads in CSV info from top level file
    :param csv_filename: csv file name
    :return: dictionary of csv lines
    """
    logger = cubeds.pylogger.get_logger(__name__)
    csv_filename = config.config['telemetry'][config.yaml_key][
        'packet_definitions']
    file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
        csv_filename)
    try:
        assert os.path.exists(file)
    except AssertionError as e:
        logger.fatal(e)
        logger.fatal('Cannot find packet definition file ' + file)
        raise cubeds.exceptions.DecoderError
    csv_encoding = get_csv_encoding(config)
    with open(file, mode='r', encoding=csv_encoding) as csv_f:
        reader = csv.DictReader(csv_f)
        csv_dict_list = []
        for row in reader:
            csv_dict_list.append(row)
    return csv_dict_list


def get_tlm_points(points_file, config):
    """
    This function will get all the points from the points file input to the function
    :param points_file: csv file with all the points.
    :param config: cubeds.config.Config object.
    :return: List of dictionaries of points.
    """
    points_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
        points_file)
    assert os.path.exists(points_file)
    csv_encoding = get_csv_encoding(config)
    with open(points_file, mode='r', encoding=csv_encoding) as csv_f:
        reader = csv.DictReader(csv_f)
        points_dict_list = []
        for row in reader:
            if '/' in row['state']:
                row['unit'] = ''
                state_array = [x.strip() for x in row['state'].split(' ')]
                state_dict = {}
                for keyStrPair in state_array:
                    state_val, state_str = keyStrPair.split('/')
                    state_dict[state_val] = state_str
                row['state'] = json.dumps(state_dict)
            else:
                row['state'] = ''
            points_dict_list.append(dict(row))
    return points_dict_list


def add_utc_to_df(df, time_index, conversion_function, config):
    """
    Adds a utc column to dataframe
    :param df: dataframe that needs utc column
    :param time_index: column to convert to utc
    :param conversion_function: function to convert values in time_index function to utc
    :return: updated dataframe
    """
    df['UTC'] = df[time_index].map(lambda x: conversion_function(x, config))
    return df


def tai_to_utc(tai, config, time_format='%Y/%j-%H:%M:%S'):
    """
    Converts TAI time to UTC time
    :param tai: TAI time
    :param time_format: Format of output UTC time
    :return: UTC time string
    """
    epoch = dt.datetime.strptime(config.config['runtime']['epoch'],
        '%m/%d/%Y-%H:%M:%S')
    try:
        utc = epoch + dt.timedelta(seconds=int(tai))
    except OverflowError:
        utc = epoch
    return utc.strftime(time_format)


def get_apids(config):
    """
    Returns list of apids from file specified in cfg file
    :return: list of apids
    """
    csv_info = get_csv_info(config)
    apids = []
    for line in csv_info:
        if int(line['apid']) not in apids:
            apids.append(int(line['apid']))
    return apids

<mask token>


def read_yaml(file):
    """
    Reads in yaml (.yml) file
    :param file: yaml file location
    :return: dictionary of information
    """
    with open(file, mode='r') as stream:
        out = yaml.load(stream)
    return out


def extract_bits_from_bytes(data, start_bit, num_bits):
    """
    Extracts bits from a group of bytes
    :param data: byte data
    :param start_bit: starting bit to begin extraction
    :param num_bits: number of bits to extract
    :return: bitarray of requested bits
    """
    b = bitarray(endian='big')
    b.frombytes(data)
    out = b[start_bit:start_bit + num_bits]
    return out


def bitstring_to_int(bitstr):
    """
    Converts a bitstring to an integer
    :param bitstr: bitstring
    :return: integer representation of bitstring
    """
    b_list = bitstr.tolist()
    mystring = ''
    for s in b_list:
        if s:
            mystring += '1'
        else:
            mystring += '0'
    b = int(mystring, 2)
    return b


def get_unpack_format(in_dtype, dsize, endian='big', logger=cubeds.pylogger
    .get_logger(__name__)):
    """
    Given data type and size, generate a format for struct.unpack.
    ASSUMES BIG ENDIAN for the data that will be unpacked. can specifiy by using endian="little"
    :param in_dtype: Type of data. Valid types are dn (unsigned), sn (signed), char, float, or double
    :param dsize: size of data in BITS!!!!
    :param endian: endianess of the data
    :return:
    """
    if dsize <= 8:
        letter = 'b'
    elif dsize == 16:
        letter = 'h'
    elif dsize == 32:
        letter = 'i'
    elif dsize == 64:
        letter = 'q'
    elif dsize == -1:
        letter = str(dsize // 8) + 'x'
        in_dtype = 'int'
        dsize = 1
    else:
        logger.fatal('parsing error for datatype size')
        raise cubeds.exceptions.UnpackFormatError(msg=
            'parsing error for datatype size')
    if in_dtype == 'dn':
        letter = letter.upper()
    elif in_dtype == 'sn':
        letter = letter.lower()
    elif in_dtype == 'char':
        letter = str(int(dsize / 8)) + 's'
    elif in_dtype == 'float' or in_dtype == 'double':
        letter = 'd'
    else:
        logger.fatal('parsing error for datatype "' + in_dtype +
            '" with size "' + str(dsize) + '"')
        raise cubeds.exceptions.UnpackFormatError(msg=
            'parsing error for datatype "' + in_dtype + '" with size "' +
            str(dsize) + '"')
    if endian == 'big':
        letter = '>' + letter
    elif endian == 'little':
        letter = '<' + letter
    else:
        logger.fatal('DID NOT SPECIFY ENDIANNESS CORRECTLY')
        raise cubeds.exceptions.UnpackFormatError(
            'Endianness not specified correctly.')
    return letter


def extract_bits(data, bit, length=1):
    """
    Extracts bits given data, start bit, and length
    :param data: data to extract bits from
    :param bit: starting position for extraction
    :param length: number of bits to extract, default=1
    :return: resulting integer of extracted bits
    """
    bits = bitarray(data, endian='big')
    if length > 1:
        out = bits[bit:bit + length]
        try:
            out = struct.unpack('>B', out.tobytes())[0]
        except struct.error:
            out = 0
    else:
        try:
            out = bits[bit]
        except IndexError:
            out = 0
    return int(out)


def get_csv_encoding(config):
    """
    Grabs the CSV encoding type form the config file, csv section, encoding variable.
    :return: csv encoding type string
    """
    logger = cubeds.pylogger.get_logger(__name__)
    try:
        csv_encoding = config.config['telemetry'][config.yaml_key]['encoding']
    except KeyError:
        csv_encoding = 'utf-8'
        logger.debug(
            "Using utf-8 encoding because couldn't find encoding type in config file"
            )
    return csv_encoding


def get_csv_info(config):
    """
    Reads in CSV info from top level file
    :param csv_filename: csv file name
    :return: dictionary of csv lines
    """
    logger = cubeds.pylogger.get_logger(__name__)
    csv_filename = config.config['telemetry'][config.yaml_key][
        'packet_definitions']
    file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
        csv_filename)
    try:
        assert os.path.exists(file)
    except AssertionError as e:
        logger.fatal(e)
        logger.fatal('Cannot find packet definition file ' + file)
        raise cubeds.exceptions.DecoderError
    csv_encoding = get_csv_encoding(config)
    with open(file, mode='r', encoding=csv_encoding) as csv_f:
        reader = csv.DictReader(csv_f)
        csv_dict_list = []
        for row in reader:
            csv_dict_list.append(row)
    return csv_dict_list


def get_tlm_points(points_file, config):
    """
    This function will get all the points from the points file input to the function
    :param points_file: csv file with all the points.
    :param config: cubeds.config.Config object.
    :return: List of dictionaries of points.
    """
    points_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
        points_file)
    assert os.path.exists(points_file)
    csv_encoding = get_csv_encoding(config)
    with open(points_file, mode='r', encoding=csv_encoding) as csv_f:
        reader = csv.DictReader(csv_f)
        points_dict_list = []
        for row in reader:
            if '/' in row['state']:
                row['unit'] = ''
                state_array = [x.strip() for x in row['state'].split(' ')]
                state_dict = {}
                for keyStrPair in state_array:
                    state_val, state_str = keyStrPair.split('/')
                    state_dict[state_val] = state_str
                row['state'] = json.dumps(state_dict)
            else:
                row['state'] = ''
            points_dict_list.append(dict(row))
    return points_dict_list


def add_utc_to_df(df, time_index, conversion_function, config):
    """
    Adds a utc column to dataframe
    :param df: dataframe that needs utc column
    :param time_index: column to convert to utc
    :param conversion_function: function to convert values in time_index function to utc
    :return: updated dataframe
    """
    df['UTC'] = df[time_index].map(lambda x: conversion_function(x, config))
    return df


def tai_to_utc(tai, config, time_format='%Y/%j-%H:%M:%S'):
    """
    Converts TAI time to UTC time
    :param tai: TAI time
    :param time_format: Format of output UTC time
    :return: UTC time string
    """
    epoch = dt.datetime.strptime(config.config['runtime']['epoch'],
        '%m/%d/%Y-%H:%M:%S')
    try:
        utc = epoch + dt.timedelta(seconds=int(tai))
    except OverflowError:
        utc = epoch
    return utc.strftime(time_format)


def get_apids(config):
    """
    Returns list of apids from file specified in cfg file
    :return: list of apids
    """
    csv_info = get_csv_info(config)
    apids = []
    for line in csv_info:
        if int(line['apid']) not in apids:
            apids.append(int(line['apid']))
    return apids

import cubeds.pylogger
import cubeds.exceptions
import cubeds.config
import yaml
from bitarray import bitarray
import struct
import os
import numpy as np
import csv
import json
import datetime as dt


def read_yaml(file):
    """
    Reads in yaml (.yml) file
    :param file: yaml file location
    :return: dictionary of information
    """
    with open(file, mode='r') as stream:
        out = yaml.load(stream)

    return out


def extract_bits_from_bytes(data, start_bit, num_bits):
    """
    Extracts bits from a group of bytes
    :param data: byte data
    :param start_bit: starting bit to begin extraction
    :param num_bits: number of bits to extract
    :return: bitarray of requested bits
    """
    b = bitarray(endian='big')
    b.frombytes(data)
    out = b[start_bit:start_bit+num_bits]
    return out


def bitstring_to_int(bitstr):
    """
    Converts a bitstring to an integer
    :param bitstr: bitstring
    :return: integer representation of bitstring
    """
    b_list = bitstr.tolist()
    mystring = ''
    for s in b_list:
        if s:
            mystring += '1'
        else:
            mystring += '0'
    b = int(mystring, 2)
    return b


def get_unpack_format(in_dtype, dsize, endian='big', logger=cubeds.pylogger.get_logger(__name__)):
    """
    Given data type and size, generate a format for struct.unpack.
    ASSUMES BIG ENDIAN for the data that will be unpacked. can specifiy by using endian="little"
    :param in_dtype: Type of data. Valid types are dn (unsigned), sn (signed), char, float, or double
    :param dsize: size of data in BITS!!!!
    :param endian: endianess of the data
    :return:
    """
    # logger = cubeds.pylogger.get_logger(__name__)
    if dsize <= 8:
        # bits
        letter = 'b'
    elif dsize == 16:
        # short
        letter = 'h'
    elif dsize == 32:
        # int
        letter = 'i'
    elif dsize == 64:
        # long long
        letter = 'q'
    elif dsize == -1:
        # padding
        letter = str(dsize//8)+'x'  # want self.size bytes of padding
        in_dtype = 'int'
        dsize = 1
    else:
        logger.fatal("parsing error for datatype size")
        raise cubeds.exceptions.UnpackFormatError(msg="parsing error for datatype size")

    if in_dtype == 'dn':
        letter = letter.upper()
    elif in_dtype == 'sn':
        letter = letter.lower()
    elif in_dtype == 'char':
        letter = str(int(dsize/8)) + 's'
    elif in_dtype == 'float' or in_dtype == 'double':
        letter = 'd'
    else:
        logger.fatal("parsing error for datatype \"" + in_dtype + "\" with size \"" + str(dsize) + "\"")
        raise cubeds.exceptions.UnpackFormatError(msg="parsing error for datatype \"" + in_dtype + "\" with size \""
                                                      + str(dsize) + "\"")

    if endian == 'big':
        letter = '>' + letter
    elif endian == 'little':
        letter = '<' + letter
    else:
        logger.fatal("DID NOT SPECIFY ENDIANNESS CORRECTLY")
        raise cubeds.exceptions.UnpackFormatError("Endianness not specified correctly.")
    return letter


def extract_bits(data, bit, length=1):
    """
    Extracts bits given data, start bit, and length
    :param data: data to extract bits from
    :param bit: starting position for extraction
    :param length: number of bits to extract, default=1
    :return: resulting integer of extracted bits
    """
    bits = bitarray(data, endian='big')
    if length > 1:
        out = bits[bit:bit+length]
        try:
            out = struct.unpack('>B', out.tobytes())[0]
        except struct.error:
            out = 0
    else:
        try:
            out = bits[bit]
        except IndexError:
            out = 0
    return int(out)


def get_csv_encoding(config):
    """
    Grabs the CSV encoding type form the config file, csv section, encoding variable.
    :return: csv encoding type string
    """
    logger = cubeds.pylogger.get_logger(__name__)
    try:
        csv_encoding = config.config['telemetry'][config.yaml_key]['encoding']
    except KeyError:
        csv_encoding = 'utf-8'
        logger.debug('Using utf-8 encoding because couldn\'t find encoding type in config file')

    return csv_encoding


def get_csv_info(config):
    """
    Reads in CSV info from top level file
    :param csv_filename: csv file name
    :return: dictionary of csv lines
    """
    logger = cubeds.pylogger.get_logger(__name__)
    csv_filename = config.config['telemetry'][config.yaml_key]['packet_definitions']

    # make sure file exists

    file = os.path.join(os.path.dirname(os.path.realpath(__file__)), csv_filename)

    try:
        assert os.path.exists(file)
    except AssertionError as e:
        logger.fatal(e)
        logger.fatal("Cannot find packet definition file "+file)
        raise cubeds.exceptions.DecoderError

    csv_encoding = get_csv_encoding(config)

    with open(file, mode='r', encoding=csv_encoding) as csv_f:
        reader = csv.DictReader(csv_f)
        csv_dict_list = []
        for row in reader:
            csv_dict_list.append(row)
    return csv_dict_list


def get_tlm_points(points_file, config):
    """
    This function will get all the points from the points file input to the function
    :param points_file: csv file with all the points.
    :param config: cubeds.config.Config object.
    :return: List of dictionaries of points.
    """
    # make sure file exists
    points_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), points_file)
    assert os.path.exists(points_file)

    csv_encoding = get_csv_encoding(config)

    with open(points_file, mode='r', encoding=csv_encoding) as csv_f:
        # create a dictionary from the top level csv file
        reader = csv.DictReader(csv_f)

        # init list for saving the points dict
        points_dict_list = []
        for row in reader:
            # if the "state" column includes a "/", that means that there is a properly formatted state pair,
            # i.e. "0/ON 1/OFF"
            # ALL PAIRS SHOULD BE SEPERATED BY A SPACE AND FORMATTED AS FOLLOWS: VAL/STR VAL/STR
            if '/' in row['state']:
                row['unit'] = ''

                # Seperate pairs and strip whitespace
                state_array = [x.strip() for x in row['state'].split(' ')]

                # init empty dictionary
                state_dict = {}

                # loop through the pairs and stor them in the dictionary with the integer as the key and the string as
                # the value
                for keyStrPair in state_array:
                    # extract the "key/str" pair
                    state_val, state_str = keyStrPair.split('/')
                    state_dict[state_val] = state_str

                row['state'] = json.dumps(state_dict)
            else:
                row['state'] = ''

            # append the new dictionary to the list
            points_dict_list.append(dict(row))
    return points_dict_list


def add_utc_to_df(df, time_index, conversion_function, config):
    """
    Adds a utc column to dataframe
    :param df: dataframe that needs utc column
    :param time_index: column to convert to utc
    :param conversion_function: function to convert values in time_index function to utc
    :return: updated dataframe
    """
    df['UTC'] = df[time_index].map(lambda x: conversion_function(x, config))
    return df


def tai_to_utc(tai, config, time_format="%Y/%j-%H:%M:%S"):
    """
    Converts TAI time to UTC time
    :param tai: TAI time
    :param time_format: Format of output UTC time
    :return: UTC time string
    """
    epoch = dt.datetime.strptime(config.config['runtime']['epoch'], "%m/%d/%Y-%H:%M:%S")
    try:
        utc = epoch + dt.timedelta(seconds=int(tai))
    except OverflowError:
        utc = epoch
    return utc.strftime(time_format)


def get_apids(config):
    """
    Returns list of apids from file specified in cfg file
    :return: list of apids
    """
    csv_info = get_csv_info(config)
    apids = []
    for line in csv_info:
        if int(line['apid']) not in apids:
            apids.append(int(line['apid']))
    return apids
