# Copyright (c) 2015 eXact Lab srl
# Author: Stefano Piani <stefano.piani@exact-lab.it>
from os.path import exists, join
from os import fsync, rename
from time import sleep
from commons.utils import deblank

import ftplib
import traceback
import threading


# The following class is a workaround for the fact
# that the strings are immutable. In this way we can
# send an ErrorMessage object to a function that can
# change it 
class ErrorMessage(object):
    """A class to handle error messages"""
    def __init__(self):
        # self.message contains the error
        self.message = None

class DownloadError(Exception):
    """An error that reports if a ftp download fails"""
    pass

def _read_permissions(txt):
    # Take the string of permissions generated by ls and returns
    # the same information in a more pythonic way
    assert len(txt) == 9
    output = dict()
    output['owner']  = set()
    output['group']  = set()
    output['others'] = set()
    if txt[0] =='r':
        output['owner'].add('read')
    if txt[1] == 'w':
        output['owner'].add('write')
    if txt[2] == 'x':
        output['owner'].add('execute')
    if txt[3] == 'r':
        output['group'].add('read')
    if txt[4] == 'w':
        output['group'].add('write')
    if txt[5] == 'x':
        output['group'].add('execute')
    if txt[6] == 'r':
        output['others'].add('read')
    if txt[7] == 'w':
        output['owner'].add('write')
    if txt[8] == 'x':
        output['others'].add('execute')
    return output


def list_files(connection):
    """
    Returns some informations about the directory the remote connection
    is inside.
   
    Args:
        - *connection*: An ftp connection, created with the ftplib library
    
    Returns:
        A tuple of three elements: 
        
          - A list of all the filenames inside the remote directory
          - A list of all the subfolders of the remote directory
          - A dictionary that associates to every filename in the
            remote directory a representation af all the permissions
            of the file. The permissions are a dictionary that associates
            to the strings "others", "group" and "owner" a set that can
            contain the strings "read", "write" and "execute". If the
            string is present the permission is guaranteed. So, for example,
            to check if the file "f1" is writeable by the members of its
            group you should do:
          
              "write" in perms["f1"]["group"] 
    """ 
    # Unfortunately, the standard library for the ftp connections
    # in python does not have a method to have a list of directories
    # or files inside a folder. All that we have is the output of
    # the ls command of the ftp server and a list of the files names.
    ls_output = []
    connection.dir('.', ls_output.append) 
    try:
        file_names = connection.nlst()
        file_names_set = set(file_names)
    except ftplib.error_perm as resp:
        if str(resp) == "550 No files found":
            return ([], [], dict())
        else:
            raise
    if len(file_names)==0:
        return ([], [], dict())

    # Now I divide the ls_output in two columns. In the second one
    # there are the filenames, in the first one all the other
    # informations.
    second_col = None
    min_len = min([len(l) for l in ls_output])
    for i in range(min_len)[::-1]:
        end_table_set = set([l[i:] for l in ls_output])
        if end_table_set == file_names_set:
            second_col = i
            break
    # Something went wrong: bad formatted ls output! Use the backup
    # function
    if second_col is None:
        return _list_files_backup(ls_output)

    # Now I prepare the output.
    files = []
    dirs = []
    permissions = dict()
    for f in file_names:
        isdir = [l.startswith('d') for l in ls_output 
                                   if l[second_col:]==f][0]
        if isdir:
            dirs.append(f)
        else:
            files.append(f)
            perms = [l[1:10] for l in ls_output
                             if l[second_col:]==f][0]
            permissions[f] = _read_permissions(perms)

    return files, dirs, permissions



def download_file(connection, f, path, log, perms=None, destfile=None,
                  skip_if_exists=True, skip_is_strange = False):
    """
    Download a file from an ftp remote archive.
    
    Args:
        - *connection*: An ftp connection, created with the ftplib library
        - *f*: The name of the file as a string
        - *path*: The directory where the file will be saved
        - *log*: A Log object to report the operations to the user
        - *perms* (optional): A dictionary of permissions like the one the
          list_file function returns. If this dictionary is not none, the
          file will be downloaded only if the readable flag for others is
          present.
        - *skip_if_exists*: A boolean value that says what to do if the file
          is already present. Default is True.
        - *skip_is_strange*: This means that the fact that the file is already
          present shall be logged with high severity. If skip_if_exists is
          False it has no effect. Default is False.
    
    Returns:
        *Success*: A boolean value that reports if the downloade
        was executed or if it was skipped for some reasons
    
    Raises:
        *DownloadError* if something went wrong during the download
    """
    if destfile is None:
        file_path = join(path, f)
    else:
        file_path = destfile

    if skip_if_exists:
        # Check if the file already exists
        if exists(file_path):
            if skip_is_strange:
                log.info('Skipping file ' + f + ' because'
                         ' it was already downloaded!')
#            else:
#                log.debug('Skipping file ' + f + ' because it was already downloaded')
            return False

    # Check if the file is readable
    if perms is not None:
        if 'read' not in perms[f]['others']:
            log.info('The file ' + f + ' is not '
                     'readable. Check the '
                     'permissions')
            return False
 
    # Download the file
    err_msg = ErrorMessage()
    log.debug('Downloading ' + f + '...')
    
    # I will first save the partial file with a temp name. When the
    # transfers end, I will rename it
    temp_name = join(path, 'incomplete_download.tmp')

    # The transfer will be executed by another thread. So I will write here
    # the function that the new tread will execute
    def background_transfer():
        # Open the temporary file
        try:
            with open(temp_name, 'wb') as loc_file:
                # Open the socket for the connection
                connection.voidcmd('TYPE I')
                transf_socket = connection.transfercmd('RETR ' + f)

                while True:
                    block = transf_socket.recv(2*1024)
                    if not block:
                        # If no more data, the download is complete
                        break
                    loc_file.write(block)

                transf_socket.close()
                connection.voidresp()

                # Ensure everything is written on the disk
                loc_file.flush()
                fsync(loc_file.fileno())
        except:
            try:
                transf_socket.close()
            except:
                pass
            err_msg.message = traceback.format_exc()


    # Start the thread that will download the file
    thread = threading.Thread(target=background_transfer)
    thread.start()
        
    # While I wait for the download, I will keep the connection
    # open. This solve a problem of timeouts with the ftp protocol
    while thread.is_alive():
        thread.join(30)
        connection.sendcmd('NOOP')
        
    # If there is an error, the download is failed
    if err_msg.message != None:
        raise DownloadError("Download failed: This is the traceback: \n"
                             + str(err_msg.message))

    
    # Move the temp file in the new downloaded file
    rename(temp_name, file_path)
    
    log.debug('File ' + f + ' downloaded!')
    return True



def _list_files_backup(ls_output):
    # If the ls output is not properly aligned (this means that
    # there is a rows which is not aligned with the others) then
    # the standard list_files function will fail. This will works
    # but requires that the time column contains exactly two space
    # for every entries (for example Jun 23 2015).

    files = []
    dirs = []
    permissions = dict()

    perms = None
    num = None
    user = None
    group = None
    size = None
    date = None
    name = None
    
    for l in ls_output:
        perms = l.split()[0]
        l = l[len(perms):].lstrip()
        num = int(l.split()[0])
        l = l[len(str(num)):].lstrip()
        user = l.split()[0]
        l = l[len(user):].lstrip()
        group = l.split()[0]
        l = l[len(group):].lstrip()
        size = int(l.split()[0])
        l = l[len(str(size)):].lstrip()
        date = " ".join(l.split()[:3])
        l = l[len(date)+1:]
        name = deblank(l)
    
        isdir = perms.startswith('d')
        if isdir:
            dirs.append(name)
        else:
            files.append(name)
            permissions[name] = _read_permissions(perms[1:])

    return files, dirs, permissions

790d1162113793f6df8af6f1ed2fe9c0921ff01c
<mask token>


class ErrorMessage(object):
    <mask token>

    def __init__(self):
        self.message = None


class DownloadError(Exception):
    """An error that reports if a ftp download fails"""
    pass


<mask token>

<mask token>


class ErrorMessage(object):
    """A class to handle error messages"""

    def __init__(self):
        self.message = None


class DownloadError(Exception):
    """An error that reports if a ftp download fails"""
    pass


def _read_permissions(txt):
    assert len(txt) == 9
    output = dict()
    output['owner'] = set()
    output['group'] = set()
    output['others'] = set()
    if txt[0] == 'r':
        output['owner'].add('read')
    if txt[1] == 'w':
        output['owner'].add('write')
    if txt[2] == 'x':
        output['owner'].add('execute')
    if txt[3] == 'r':
        output['group'].add('read')
    if txt[4] == 'w':
        output['group'].add('write')
    if txt[5] == 'x':
        output['group'].add('execute')
    if txt[6] == 'r':
        output['others'].add('read')
    if txt[7] == 'w':
        output['owner'].add('write')
    if txt[8] == 'x':
        output['others'].add('execute')
    return output


<mask token>

<mask token>


class ErrorMessage(object):
    """A class to handle error messages"""

    def __init__(self):
        self.message = None


class DownloadError(Exception):
    """An error that reports if a ftp download fails"""
    pass


def _read_permissions(txt):
    assert len(txt) == 9
    output = dict()
    output['owner'] = set()
    output['group'] = set()
    output['others'] = set()
    if txt[0] == 'r':
        output['owner'].add('read')
    if txt[1] == 'w':
        output['owner'].add('write')
    if txt[2] == 'x':
        output['owner'].add('execute')
    if txt[3] == 'r':
        output['group'].add('read')
    if txt[4] == 'w':
        output['group'].add('write')
    if txt[5] == 'x':
        output['group'].add('execute')
    if txt[6] == 'r':
        output['others'].add('read')
    if txt[7] == 'w':
        output['owner'].add('write')
    if txt[8] == 'x':
        output['others'].add('execute')
    return output


<mask token>


def _list_files_backup(ls_output):
    files = []
    dirs = []
    permissions = dict()
    perms = None
    num = None
    user = None
    group = None
    size = None
    date = None
    name = None
    for l in ls_output:
        perms = l.split()[0]
        l = l[len(perms):].lstrip()
        num = int(l.split()[0])
        l = l[len(str(num)):].lstrip()
        user = l.split()[0]
        l = l[len(user):].lstrip()
        group = l.split()[0]
        l = l[len(group):].lstrip()
        size = int(l.split()[0])
        l = l[len(str(size)):].lstrip()
        date = ' '.join(l.split()[:3])
        l = l[len(date) + 1:]
        name = deblank(l)
        isdir = perms.startswith('d')
        if isdir:
            dirs.append(name)
        else:
            files.append(name)
            permissions[name] = _read_permissions(perms[1:])
    return files, dirs, permissions

<mask token>


class ErrorMessage(object):
    """A class to handle error messages"""

    def __init__(self):
        self.message = None


class DownloadError(Exception):
    """An error that reports if a ftp download fails"""
    pass


def _read_permissions(txt):
    assert len(txt) == 9
    output = dict()
    output['owner'] = set()
    output['group'] = set()
    output['others'] = set()
    if txt[0] == 'r':
        output['owner'].add('read')
    if txt[1] == 'w':
        output['owner'].add('write')
    if txt[2] == 'x':
        output['owner'].add('execute')
    if txt[3] == 'r':
        output['group'].add('read')
    if txt[4] == 'w':
        output['group'].add('write')
    if txt[5] == 'x':
        output['group'].add('execute')
    if txt[6] == 'r':
        output['others'].add('read')
    if txt[7] == 'w':
        output['owner'].add('write')
    if txt[8] == 'x':
        output['others'].add('execute')
    return output


def list_files(connection):
    """
    Returns some informations about the directory the remote connection
    is inside.
   
    Args:
        - *connection*: An ftp connection, created with the ftplib library
    
    Returns:
        A tuple of three elements: 
        
          - A list of all the filenames inside the remote directory
          - A list of all the subfolders of the remote directory
          - A dictionary that associates to every filename in the
            remote directory a representation af all the permissions
            of the file. The permissions are a dictionary that associates
            to the strings "others", "group" and "owner" a set that can
            contain the strings "read", "write" and "execute". If the
            string is present the permission is guaranteed. So, for example,
            to check if the file "f1" is writeable by the members of its
            group you should do:
          
              "write" in perms["f1"]["group"] 
    """
    ls_output = []
    connection.dir('.', ls_output.append)
    try:
        file_names = connection.nlst()
        file_names_set = set(file_names)
    except ftplib.error_perm as resp:
        if str(resp) == '550 No files found':
            return [], [], dict()
        else:
            raise
    if len(file_names) == 0:
        return [], [], dict()
    second_col = None
    min_len = min([len(l) for l in ls_output])
    for i in range(min_len)[::-1]:
        end_table_set = set([l[i:] for l in ls_output])
        if end_table_set == file_names_set:
            second_col = i
            break
    if second_col is None:
        return _list_files_backup(ls_output)
    files = []
    dirs = []
    permissions = dict()
    for f in file_names:
        isdir = [l.startswith('d') for l in ls_output if l[second_col:] == f][0
            ]
        if isdir:
            dirs.append(f)
        else:
            files.append(f)
            perms = [l[1:10] for l in ls_output if l[second_col:] == f][0]
            permissions[f] = _read_permissions(perms)
    return files, dirs, permissions


def download_file(connection, f, path, log, perms=None, destfile=None,
    skip_if_exists=True, skip_is_strange=False):
    """
    Download a file from an ftp remote archive.
    
    Args:
        - *connection*: An ftp connection, created with the ftplib library
        - *f*: The name of the file as a string
        - *path*: The directory where the file will be saved
        - *log*: A Log object to report the operations to the user
        - *perms* (optional): A dictionary of permissions like the one the
          list_file function returns. If this dictionary is not none, the
          file will be downloaded only if the readable flag for others is
          present.
        - *skip_if_exists*: A boolean value that says what to do if the file
          is already present. Default is True.
        - *skip_is_strange*: This means that the fact that the file is already
          present shall be logged with high severity. If skip_if_exists is
          False it has no effect. Default is False.
    
    Returns:
        *Success*: A boolean value that reports if the downloade
        was executed or if it was skipped for some reasons
    
    Raises:
        *DownloadError* if something went wrong during the download
    """
    if destfile is None:
        file_path = join(path, f)
    else:
        file_path = destfile
    if skip_if_exists:
        if exists(file_path):
            if skip_is_strange:
                log.info('Skipping file ' + f +
                    ' because it was already downloaded!')
            return False
    if perms is not None:
        if 'read' not in perms[f]['others']:
            log.info('The file ' + f +
                ' is not readable. Check the permissions')
            return False
    err_msg = ErrorMessage()
    log.debug('Downloading ' + f + '...')
    temp_name = join(path, 'incomplete_download.tmp')

    def background_transfer():
        try:
            with open(temp_name, 'wb') as loc_file:
                connection.voidcmd('TYPE I')
                transf_socket = connection.transfercmd('RETR ' + f)
                while True:
                    block = transf_socket.recv(2 * 1024)
                    if not block:
                        break
                    loc_file.write(block)
                transf_socket.close()
                connection.voidresp()
                loc_file.flush()
                fsync(loc_file.fileno())
        except:
            try:
                transf_socket.close()
            except:
                pass
            err_msg.message = traceback.format_exc()
    thread = threading.Thread(target=background_transfer)
    thread.start()
    while thread.is_alive():
        thread.join(30)
        connection.sendcmd('NOOP')
    if err_msg.message != None:
        raise DownloadError('Download failed: This is the traceback: \n' +
            str(err_msg.message))
    rename(temp_name, file_path)
    log.debug('File ' + f + ' downloaded!')
    return True


def _list_files_backup(ls_output):
    files = []
    dirs = []
    permissions = dict()
    perms = None
    num = None
    user = None
    group = None
    size = None
    date = None
    name = None
    for l in ls_output:
        perms = l.split()[0]
        l = l[len(perms):].lstrip()
        num = int(l.split()[0])
        l = l[len(str(num)):].lstrip()
        user = l.split()[0]
        l = l[len(user):].lstrip()
        group = l.split()[0]
        l = l[len(group):].lstrip()
        size = int(l.split()[0])
        l = l[len(str(size)):].lstrip()
        date = ' '.join(l.split()[:3])
        l = l[len(date) + 1:]
        name = deblank(l)
        isdir = perms.startswith('d')
        if isdir:
            dirs.append(name)
        else:
            files.append(name)
            permissions[name] = _read_permissions(perms[1:])
    return files, dirs, permissions

# Copyright (c) 2015 eXact Lab srl
# Author: Stefano Piani <stefano.piani@exact-lab.it>
from os.path import exists, join
from os import fsync, rename
from time import sleep
from commons.utils import deblank

import ftplib
import traceback
import threading


# The following class is a workaround for the fact
# that the strings are immutable. In this way we can
# send an ErrorMessage object to a function that can
# change it 
class ErrorMessage(object):
    """A class to handle error messages"""
    def __init__(self):
        # self.message contains the error
        self.message = None

class DownloadError(Exception):
    """An error that reports if a ftp download fails"""
    pass

def _read_permissions(txt):
    # Take the string of permissions generated by ls and returns
    # the same information in a more pythonic way
    assert len(txt) == 9
    output = dict()
    output['owner']  = set()
    output['group']  = set()
    output['others'] = set()
    if txt[0] =='r':
        output['owner'].add('read')
    if txt[1] == 'w':
        output['owner'].add('write')
    if txt[2] == 'x':
        output['owner'].add('execute')
    if txt[3] == 'r':
        output['group'].add('read')
    if txt[4] == 'w':
        output['group'].add('write')
    if txt[5] == 'x':
        output['group'].add('execute')
    if txt[6] == 'r':
        output['others'].add('read')
    if txt[7] == 'w':
        output['owner'].add('write')
    if txt[8] == 'x':
        output['others'].add('execute')
    return output


def list_files(connection):
    """
    Returns some informations about the directory the remote connection
    is inside.
   
    Args:
        - *connection*: An ftp connection, created with the ftplib library
    
    Returns:
        A tuple of three elements: 
        
          - A list of all the filenames inside the remote directory
          - A list of all the subfolders of the remote directory
          - A dictionary that associates to every filename in the
            remote directory a representation af all the permissions
            of the file. The permissions are a dictionary that associates
            to the strings "others", "group" and "owner" a set that can
            contain the strings "read", "write" and "execute". If the
            string is present the permission is guaranteed. So, for example,
            to check if the file "f1" is writeable by the members of its
            group you should do:
          
              "write" in perms["f1"]["group"] 
    """ 
    # Unfortunately, the standard library for the ftp connections
    # in python does not have a method to have a list of directories
    # or files inside a folder. All that we have is the output of
    # the ls command of the ftp server and a list of the files names.
    ls_output = []
    connection.dir('.', ls_output.append) 
    try:
        file_names = connection.nlst()
        file_names_set = set(file_names)
    except ftplib.error_perm as resp:
        if str(resp) == "550 No files found":
            return ([], [], dict())
        else:
            raise
    if len(file_names)==0:
        return ([], [], dict())

    # Now I divide the ls_output in two columns. In the second one
    # there are the filenames, in the first one all the other
    # informations.
    second_col = None
    min_len = min([len(l) for l in ls_output])
    for i in range(min_len)[::-1]:
        end_table_set = set([l[i:] for l in ls_output])
        if end_table_set == file_names_set:
            second_col = i
            break
    # Something went wrong: bad formatted ls output! Use the backup
    # function
    if second_col is None:
        return _list_files_backup(ls_output)

    # Now I prepare the output.
    files = []
    dirs = []
    permissions = dict()
    for f in file_names:
        isdir = [l.startswith('d') for l in ls_output 
                                   if l[second_col:]==f][0]
        if isdir:
            dirs.append(f)
        else:
            files.append(f)
            perms = [l[1:10] for l in ls_output
                             if l[second_col:]==f][0]
            permissions[f] = _read_permissions(perms)

    return files, dirs, permissions



def download_file(connection, f, path, log, perms=None, destfile=None,
                  skip_if_exists=True, skip_is_strange = False):
    """
    Download a file from an ftp remote archive.
    
    Args:
        - *connection*: An ftp connection, created with the ftplib library
        - *f*: The name of the file as a string
        - *path*: The directory where the file will be saved
        - *log*: A Log object to report the operations to the user
        - *perms* (optional): A dictionary of permissions like the one the
          list_file function returns. If this dictionary is not none, the
          file will be downloaded only if the readable flag for others is
          present.
        - *skip_if_exists*: A boolean value that says what to do if the file
          is already present. Default is True.
        - *skip_is_strange*: This means that the fact that the file is already
          present shall be logged with high severity. If skip_if_exists is
          False it has no effect. Default is False.
    
    Returns:
        *Success*: A boolean value that reports if the downloade
        was executed or if it was skipped for some reasons
    
    Raises:
        *DownloadError* if something went wrong during the download
    """
    if destfile is None:
        file_path = join(path, f)
    else:
        file_path = destfile

    if skip_if_exists:
        # Check if the file already exists
        if exists(file_path):
            if skip_is_strange:
                log.info('Skipping file ' + f + ' because'
                         ' it was already downloaded!')
#            else:
#                log.debug('Skipping file ' + f + ' because it was already downloaded')
            return False

    # Check if the file is readable
    if perms is not None:
        if 'read' not in perms[f]['others']:
            log.info('The file ' + f + ' is not '
                     'readable. Check the '
                     'permissions')
            return False
 
    # Download the file
    err_msg = ErrorMessage()
    log.debug('Downloading ' + f + '...')
    
    # I will first save the partial file with a temp name. When the
    # transfers end, I will rename it
    temp_name = join(path, 'incomplete_download.tmp')

    # The transfer will be executed by another thread. So I will write here
    # the function that the new tread will execute
    def background_transfer():
        # Open the temporary file
        try:
            with open(temp_name, 'wb') as loc_file:
                # Open the socket for the connection
                connection.voidcmd('TYPE I')
                transf_socket = connection.transfercmd('RETR ' + f)

                while True:
                    block = transf_socket.recv(2*1024)
                    if not block:
                        # If no more data, the download is complete
                        break
                    loc_file.write(block)

                transf_socket.close()
                connection.voidresp()

                # Ensure everything is written on the disk
                loc_file.flush()
                fsync(loc_file.fileno())
        except:
            try:
                transf_socket.close()
            except:
                pass
            err_msg.message = traceback.format_exc()


    # Start the thread that will download the file
    thread = threading.Thread(target=background_transfer)
    thread.start()
        
    # While I wait for the download, I will keep the connection
    # open. This solve a problem of timeouts with the ftp protocol
    while thread.is_alive():
        thread.join(30)
        connection.sendcmd('NOOP')
        
    # If there is an error, the download is failed
    if err_msg.message != None:
        raise DownloadError("Download failed: This is the traceback: \n"
                             + str(err_msg.message))

    
    # Move the temp file in the new downloaded file
    rename(temp_name, file_path)
    
    log.debug('File ' + f + ' downloaded!')
    return True



def _list_files_backup(ls_output):
    # If the ls output is not properly aligned (this means that
    # there is a rows which is not aligned with the others) then
    # the standard list_files function will fail. This will works
    # but requires that the time column contains exactly two space
    # for every entries (for example Jun 23 2015).

    files = []
    dirs = []
    permissions = dict()

    perms = None
    num = None
    user = None
    group = None
    size = None
    date = None
    name = None
    
    for l in ls_output:
        perms = l.split()[0]
        l = l[len(perms):].lstrip()
        num = int(l.split()[0])
        l = l[len(str(num)):].lstrip()
        user = l.split()[0]
        l = l[len(user):].lstrip()
        group = l.split()[0]
        l = l[len(group):].lstrip()
        size = int(l.split()[0])
        l = l[len(str(size)):].lstrip()
        date = " ".join(l.split()[:3])
        l = l[len(date)+1:]
        name = deblank(l)
    
        isdir = perms.startswith('d')
        if isdir:
            dirs.append(name)
        else:
            files.append(name)
            permissions[name] = _read_permissions(perms[1:])

    return files, dirs, permissions
