#!/usr/bin/python
# -*-coding:utf-8-*-

import io
import os
import sys
import logging
import urllib2
import string
import chardet
from optparse import OptionParser, Option, IndentedHelpFormatter
import json
import requests
from ast import literal_eval

from wcs.commons.putpolicy import PutPolicy
from wcs.commons.error_deal import ParameterError
from wcs.commons.util import file_crc64
from wcs.__init__ import __version__


def output(message):
    sys.stdout.write(message + '\n')
    sys.stdout.flush()


def format_commands(progname, commands_list):
    # Define commands display format
    help = "Commands:" + "\n"
    for cmd in commands_list:
        help += "   %s\n    	%s %s %s \n" % (cmd["label"], progname, cmd["cmd"], cmd["param"])
    return help


def get_commands_list():
    # Return all commands list of wcscmd
    return [
        {"cmd": "list", "label": "List objects", "param": "wcs://BUCKET RESFILE", "func": cmd_list, "argc": 2},
        {"cmd": "listbucket", "label": "List buckets", "param": "", "func": cmd_listbucket, "argc": 0},
        {"cmd": "get", "label": "Download file", "param": "URL", "func": cmd_get, "argc": 2},
        {"cmd": "del", "label": "Delete a file", "param": "wcs://BUCKET/OBJECT", "func": cmd_del, "argc": 1},
        {"cmd": "mv", "label": "Move a file from src bucket to des bucket",  "param": "wcs://srcBUCKET/srcOBJECT wcs://dstBUCKET/desOBJECT", "func": cmd_mv, "argc": 2},
        {"cmd": "cp", "label": "Copy a file from src bucket to des bucket",  "param": "wcs://srcBUCKET/srcOBJECT wcs://dstBUCKET/desOBJECT", "func": cmd_cp, "argc": 2},
        {"cmd": "setdeadline", "label": "Set deadline of file", "param": "wcs://BUCKET/OBJECT deadline", "func": cmd_setdeadline, "argc": 2},
        {"cmd": "stat", "label": "Get file info", "param": "wcs://BUCKET/OBJECT", "func": cmd_stat, "argc": 1},
        {"cmd": "put", "label": "Upload a local file to WCS", "param": "wcs://BUCKET/OBJECT LOCALFILE", "func": cmd_put,"argc": 2},
        {"cmd": "multiput", "label": "Multipart upload a local file to WCS", "param": "wcs://BUCKET/OBJECT LOCALFILE ", "func": cmd_multiput, "argc": 2},
        {"cmd": "deletePrefix", "label": "Delete multiple files according to prefix", "param": "wcs://BUCKET PREFIX", "func": cmd_delprefix, "argc": 2},
        {"cmd": "fops", "label": "Fops audio/video processing ", "param": "wcs://BUCKET/OBJECT fopsparm", "func": cmd_fops, "argc": 2},
        {"cmd": "fopsStatus", "label": "Get fops task results ", "param": "persistentId", "func": cmd_fopsstatus, "argc": 1},
        {"cmd": "fmgrStatus", "label": "Get fmgrStatus task results ", "param": "persistentId", "func": cmd_fmgrstatus, "argc": 1},
        {"cmd": "crc64", "label": "Calculating 64-bit CRC value ", "param": "filePath", "func": cmd_crc64, "argc": 1}
    ]


def cmd_list(args):
    # Define implementation of List Objects API
    cfg = Config()
    cli = Client(cfg)
    bucket = args[0].split("wcs://")[1]
    resfile = os.path.join(os.getcwd(), args[1])
    try:
        code, response, reqid = cli.bucket_list(bucket, cfg.prefix, cfg.marker, int(cfg.limit), int(cfg.mode))
        if code == 200:
            try:
                with open(resfile, 'w') as f:
                    f.write(json.dumps(response))
            except IOError:
                debug("Save Fail")
                sys.exit()
            debug("Request result saved to %s" % (resfile))
            debug("Request ID is: %s" % (reqid['x-reqid']))
        else:
            debug("error-message:%s" % response)
            error("Request fail, more detail please pay attention to reqid: %s " % (reqid['x-reqid']))
    except Exception,E:
        error(E)


def cmd_get(args):
    # Define implementation of download file from URL
    cfg = Config()
    try:
        if chardet.detect(args[0]).get('encoding') == 'ISO-8859-1':
            url = args[0].decode('gbk').encode('utf8')
        else:
            url = args[0]
        url_encode = str(urllib2.quote(url, safe=string.printable))
        if len(args) > 1:
            key = os.path.join(os.getcwd(), args[1])
        else:
            url = urllib2.unquote(str(url)).decode('utf8')
            filename = url.split('?')
            if len(filename) >= 2:
                filename.pop(-1)
                filename = '/'.join('?'.join(filename).split('/')[3:])
                key = os.path.join(os.getcwd(), filename)
            else:
                filename = url.split('/')[-1]
                key = os.path.join(os.getcwd(), filename)
        headers = {}
        try:
            if int(cfg.traffic_limit):
                headers['x-wos-traffic-limit'] = '{0}'.format(cfg.traffic_limit)
        except Exception as e:
            raise ValueError('traffic_limit parameter configuration error：{0}'.format(cfg.traffic_limit))
        if cfg.isverify:
            res = requests.get(url_encode,headers=headers)
        else:
            res = requests.get(url_encode,verify=False,headers=headers)
        if res.status_code == 200:
            debug("Download suc.")
            debug("return headers:{0}".format(res.headers))
            try:
                with open(key, 'wb') as f:
                    f.write(res.content)
                debug('File saved successfully,File:[{0}]'.format(key))
            except IOError as e:
                error("Create local file fail for %s" % e)
                sys.exit()
        else:
            error_text = res.text
            debug("return headers:{0}".format(res.headers))
            error("Download Fail:%s" % error_text)
    except Exception as e:
        error("Download Fail:%s" % e)


def _simple_arg_parse(arg):
    # Parse commands with only one wcs://bucket/[prefix1]/[prefix2]/object
    try:
        tmp = arg.split("wcs://")[1]
        if '/' in tmp:
            bucket = tmp.split('/')[0]
            obj = tmp.split(bucket + '/')[1]
        else:
            bucket = tmp
            obj = ''
    except Exception:
        error("Invalid arguments")
        sys.exit()
    return bucket, obj


def cmd_del(args):
    # Define implementation of Object Delete API
    cli = Client(Config())
    bucket, obj = _simple_arg_parse(args[0])
    debug(cli.delete(bucket, obj))


def _couple_args_parse(args):
    # Parse commands with two wcs://srcbucket/[srcprefix1]/[srcprefix2]/srcobject wcs://dstbucket/[dstprefix1]/[dstprefix2]/dstobject
    try:
        src = args[0].split("wcs://")[1]
        dst = args[1].split("wcs://")[1]
        srcbucket = src.split('/')[0]
        srcobject = src.split('/')[1:]
        srcobject = '/'.join(srcobject)

        dstbucket = dst.split('/')[0]
        dstobject = dst.split('/')[1:]
        dstobject = '/'.join(dstobject)
    except Exception:
        error("Invalid arguments")
        sys.exit()
    return srcbucket, srcobject, dstbucket, dstobject


def cmd_mv(args):
    # Define implementation of Synchronous move API
    cli = Client(Config())
    srcbk, srcob, dstbk, dstob = _couple_args_parse(args)
    debug(cli.move(srcbk, srcob, dstbk, dstob))


def cmd_cp(args):
    # Define implementation of Synchronous copy API
    cli = Client(Config())
    srcbk, srcob, dstbk, dstob = _couple_args_parse(args)
    debug(cli.copy(srcbk, srcob, dstbk, dstob))


def cmd_setdeadline(args):
    # Define implementation of Set Deadline API
    cli = Client(Config())
    bucket, obj = _simple_arg_parse(args[0])
    deadline = literal_eval(args[1])
    debug(cli.setdeadline(bucket, obj, deadline))


def cmd_stat(args):
    # Define implementation of stat API
    cli = Client(Config())
    bucket, obj = _simple_arg_parse(args[0])
    debug(cli.stat(bucket, obj))


def cmd_put(args):
    # Define implementation of simple upload API
    cfg = Config()
    cli = Client(cfg)
    bucket, obj = _simple_arg_parse(args[0])
    local = args[1]
    if not obj:
        obj = os.path.basename(local)
    debug(cli.simple_upload(local, bucket, obj))


def cmd_multiput(args):
    # Define implementation of multipart upload
    cfg = Config()
    cli = Client(cfg)
    bucket, obj = _simple_arg_parse(args[0])
    local = args[1]
    if not obj:
        obj = os.path.basename(local)
    debug(cli.multipart_upload(local, bucket, obj, cfg.upload_id))


def cmd_listbucket(args):
    #
    cli = Client(Config())
    debug(cli.list_buckets())


def cmd_delprefix(args):
    bucket = args[0].split("wcs://")[1]
    prefix = args[1]
    cli = Client(Config())
    fops = 'bucket/%s/prefix/%s' % (urlsafe_base64_encode(bucket), urlsafe_base64_encode(prefix))
    debug(cli.prefix_delete(fops))


def cmd_fops(args):
    # Define implementation of Fops API
    cli = Client(Config())
    bucket,obj = _simple_arg_parse(args[0])
    fops = args[1]
    debug(cli.ops_execute(fops,bucket,obj))


def cmd_fopsstatus(args):
    # Define implementation of fops status API
    cli = Client(Config())
    persistentId = args[0]
    debug(cli.ops_status(persistentId))


def cmd_fmgrstatus(args):
    # Define implementation of fmgr status API
    cli = Client(Config())
    persistentId = args[0]
    debug(cli.fmgr_status(persistentId))

def cmd_crc64(args):
    #calculating 64-bit CRC value
    try:
        file_path = args[0]
        if os.path.isfile(file_path):
            crc64 = file_crc64(file_path)
            debug("calculating suc.")
            debug(str({'file':file_path,'crc64Value':crc64}))
        else:
            error('Calculating faile,Reasons for failure: file not found .')
    except Exception as e:
        error('Some error occurred. {0}'.format(e))


def run_configure(config_file):
    if os.path.exists(config_file):
        cfg = Config(config_file)
    else:
        cfg = Config()
    options = [
        ("access_key", "Access Key", "Access key and Secret key are your identifiers for WCS"),
        ("secret_key", "Secret Key"),
        ("put_url", "PUT_URL", "Upload domain for WCS"),
        ("mgr_url", "MGR_URL", "Manager domain for WCS"),
        ("block_size", "Block_Size", "Mulpart Upload Block Size"),
        ("bput_size", "Bput_Size", "Mulpart Upload Bput Size"),
        ("connection_retries", "Connection_Retires", "Request Connection Retires"),
        ("connection_timeout", "Connection_Timeout", "Request Connection Timeout"),
        ("mkblk_retries", "Mkblk_Retries", "Multipart Upload Mkblk retries"),
        ("bput_retries", "Bput_Retries", "Multipart Upload Bput retries"),
        ("mkfile_retries", "Mkfile_Retries", "Multipart Upload Mkfile retries"),
        ("tmp_record_folder", "Tmp_Record_Folder", "Multipart Upload record folder"),
        ("concurrency", "Concurrency", "Multipart Upload Concurrency"),
        ("ishttps", "IsHttps", "Use https or not"),
        ("isverify", "isverify", "Use verify or not"),
        ("mode", "mode", "List objects mode"),
        ("keepalive", "keepalive", "request to use keepalive mode"),
        ("deadline", "deadline", "token expiration time"),
        ("traffic_limit", "traffic_limit", "traffic limit")
    ]
    try:
        while True:
            output(u"Enter new values or accept defaults in brackets with Enter")
            output(u"Refer to user manual for detailed description of all options")
            for option in options:
                prompt = option[1]
                try:

                    val = getattr(cfg, option[0])
                    if type(val) is bool:
                        val = val and "Yes" or "No"
                    if val not in (None, ""):
                        prompt += " [%s]" % val
                except AttributeError:
                    pass

                if len(option) >= 3:
                    output(u"\n%s" % option[2])
                val = raw_input(prompt + ": ")
                if val != "":
                    if type(getattr(cfg, option[0])) is bool:
                        val = val.lower().startswith('y')
                    setattr(cfg, option[0], val)
            output(u"\nNew settings:")
            for option in options:
                output(u"  %s: %s" % (option[0], getattr(cfg, option[0])))
            val = raw_input("\nSave settings? [y/N] ")
            if val.lower().startswith("y"):
                break
            val = raw_input("Retry configuration? [Y/n] ")
            if val.lower().startswith("n"):
                raise EOFError()
        try:
            with io.open(config_file, 'w') as fp:
                cfg.dump_config(fp)
        except IOError as e:
            raise e
        output(u"Configuration saved to '%s'" % config_file)

    except (EOFError, KeyboardInterrupt):
        output(u"\nConfiguration aborted. Changes were NOT saved.")
        return
    except IOError, e:
        error(u"Writing config file failed: %s: %s" % (config_file, e.strerror))
        sys.exit()


class MyHelpFormatter(IndentedHelpFormatter):
    def format_epilog(self, epilog):
        if epilog:
            return "\n" + epilog + "\n"
        else:
            return ""


def main():
    global cfg
    cfg = Config()
    commands_list = get_commands_list()
    commands = {}

    for cmd in commands_list:
        if 'cmd' in cmd:
            commands[cmd['cmd']] = cmd

    usage = 'Usage: wcscmd [options] COMMAND [parameters]'
    optparser = OptionParser(usage=usage, formatter=MyHelpFormatter())

    config_file = None
    from os.path import expanduser
    config_file = os.path.join(expanduser("~"), ".wcscfg")
    optparser.set_defaults(config=config_file)

    optparser.add_option("--configure", dest="run_config", action="store_true",
                         help="Invoke interactive (re)configuration tool.")
    optparser.add_option("-c", "--config", dest="config", metavar="FILE",
                         help="Config file name. Defaults to $HOME/.wcscfg")
    optparser.add_option("--dump-config", dest="dump_config", action="store_true",
                         help="Dump current configuration after parsing config file and command line options and exit.")
    # optparser.add_option(   "--access_key", dest="access_key", help="WCS Access Key")
    # optparser.add_option(   "--secret_key", dest="secret_key", help="WCS Secret Key")
    optparser.add_option("-s", "--ssl", dest="ishttps", action="store_true",
                         help="Use https connection when communicating with WCS")
    optparser.add_option("--upload-id", dest="upload_id",
                         help="UploadId for Multipart Upload, in case you want continue an existing upload")
    optparser.add_option("--limit", dest="limit", help="Limit for list objects of bucket")
    optparser.add_option("--prefix", dest="prefix", help="Prefix for list objects of bucket")
    optparser.add_option("--mode", dest="mode", help="Mode for list objects of bucket")
    optparser.add_option("--marker", dest="marker", help="Start string for list")
    optparser.add_option("--relevance", dest="relevance", action="store_true",
                         help="Whether modify deadline relevant .ts when modify deadline of m3u8 file")
    optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG,
                         help="Enable debug output.")
    optparser.add_option("--output", dest="output", help="Save request result to specified key, like:<bucket>:<key>")
    optparser.add_option("--notify", dest="notifyURL", help="Notify url for request result")
    optparser.add_option("--separate", dest="separate", help="Whether separate notify, bool type")
    optparser.add_option("--force", dest="force", help="Whether to enforce data processing, bool type")
    optparser.add_option("--overwrite", dest="overwrite", help="Whether overwrite existed key, bool type")
    optparser.add_option("--returnurl", dest="returl", help="Url for return")
    optparser.add_option("--returnbody", dest="retbody", help="Body of return")
    optparser.add_option("--callbackurl", dest="cburl", help="Url for callback")
    optparser.add_option("--callbackbody", dest="cbbody", help="Body of callback")
    optparser.add_option("--persistentntyurl", dest="persistentntyurl", help="Url for persistent ops notify")
    optparser.add_option("--persistentops", dest="persistentops", help="Persistent ops")
    optparser.add_option("--contentdetect", dest="contentdetect", help="Content detect type")
    optparser.add_option("--detectntyurl", dest="detectntyurl", help="Url for detect notify")
    optparser.add_option("--detectntyrule", dest="detectntyrule", help="RUle for ectect notify")
    optparser.add_option("--hashalgorithm", dest="hashalgorithm", help="Hash algorithm, currently supports 'crc64ecma' for calculating 64-bit CRC value.")#add by caiyz 2020-07-13
    optparser.add_option("--deadline", dest="deadline",help="Token expiration time,Unit is second")  # add by caiyz 2021-08-16
    optparser.add_option("--traffic_limit", dest="traffic_limit",help="Upload/Download traffic limit")  # add by caiyz 2022-03-15
    optparser.set_description("Wcscmd is a tool for managing objects in WCS Object Storage." + " It allows for uploading, downloading and removing objects form buckets")

    optparser.epilog = format_commands(optparser.get_prog_name(), commands_list)
    optparser.epilog += "\n\nFor more information, visit WCS website: https://wcs.chinanetcenter.com/index\n"

    (options, args) = optparser.parse_args()

    # if options.verbosity:
    #    logging.basicConfig(level=options.verbosity, format='%(levelname)s: %(message)s', stream=sys.stdderr)
    # if run_config:
    try:
        debug(u"wcscmd version {0}".format(__version__))
    except ValueError as e:
        debug(u"Get Version Fail :%s" % e)
    if options.run_config:
        run_configure(options.config)
        sys.exit()
    try:
        cfg = Config(options.config)
    except ValueError as exc:
        raise ParameterError(unicode(exc))
    except IOError as e:
        if options.run_configure:
            cfg = Config()
        else:
            error(u"%s: %s" % (options.config, e.strerror))
            error(u"Configuration file not available.")
            error(u"Consider using --configure parameter to create one.")
            sys.exit()

    if options.output:
        cfg.output = options.output
    if options.upload_id:
        cfg.upload_id = options.upload_id
    if options.notifyURL:
        cfg.notifyurl = options.notifyURL
    if options.separate:
        cfg.separate = options.separate
    if options.force:
        cfg.force = options.force
    if options.limit:
        cfg.limit = options.limit
    if options.prefix:
        cfg.prefix = options.prefix
    if options.mode:
        cfg.mode = options.mode
    if options.marker:
        cfg.marker = options.marker
    if options.overwrite:
        cfg.overwrite = options.overwrite
    if options.returl:
        cfg.returnUrl = options.returl
    if options.retbody:
        cfg.returnBody = options.retbody
    if options.cburl:
        cfg.callbackUrl = options.cburl
    if options.cbbody:
        cfg.callbackBody = options.cbbody
    if options.persistentntyurl:
        cfg.persistentNotifyUrl = options.persistentntyurl
    if options.persistentops:
        cfg.persistentOps = options.persistentops
    if options.contentdetect:
        cfg.contentDetect = options.contentdetect
    if options.detectntyurl:
        cfg.detectNotifyURL = options.detectntyurl
    if options.detectntyrule:
        cfg.detectNotifyRule = options.detectntyrule
    if options.hashalgorithm:
        cfg.hashAlgorithm = options.hashalgorithm
    if options.deadline:
        try:
            cfg.deadline = int(options.deadline)
        except ValueError:
            raise ParameterError('deadline must be an integer.')
    if options.traffic_limit:
        try:
            cfg.traffic_limit = int(options.traffic_limit)
        except ValueError:
            raise ParameterError('"traffic_limit" must be an integer.')
    if options.dump_config:
        cfg.dump_config(sys.stdout)
        sys.exit()
    for option in cfg.option_list():
        try:
            value = getattr(options, option)
            if value != None:
                type(value) == type(b'')
            # debug(u"Updating Config.Config %s->%s" % (option, value))
            cfg.update_option(option, value)
        except AttributeError:
            pass
        try:
            # caiyz 20180306 add option check
            if option == 'block_size':
                if int(cfg.block_size) <= 0 or (int(cfg.block_size) % 4194304) != 0:
                    raise ParameterError("Block size %d B is not available" % (int(cfg.block_size)))
            if option == 'bput_size':
                if int(cfg.bput_size) <= 0 or (int(cfg.bput_size)) % 2 != 0:
                    raise ParameterError("Bput size %d B is not available" % (int(cfg.bput_size)))
        except ParameterError as e:
            print e

    if len(args) < 1:
        optparser.print_help()
        sys.exit()
    command = args.pop(0)
    try:
        debug(u"Command: %s" % commands[command]["cmd"])
        cmd_func = commands[command]["func"]
    except KeyError as e:
        error(u"Invalid command: %s" % command)
        sys.exit()
    if command == 'get':
        if len(args) > commands[command]["argc"]:
            error(u"Too much parameters for command '%s'" % command)
            sys.exit()
    else:
        if len(args) < commands[command]["argc"]:
            error(u"Not enough parameters for command '%s'" % command)
            sys.exit()
        if len(args) > commands[command]["argc"]:
            error(u"Too much parameters for command '%s'" % command)
            sys.exit()
    rc = cmd_func(args)
    if rc is None:
        rc = "Nothing"
    return rc


if __name__ == '__main__':
    from wcs.commons.config import Config
    from wcs.commons.logme import debug, error
    from wcs.services.client import Client
    from wcs.commons.util import urlsafe_base64_encode

    main()

32d3aa168eb107af4a957205212ac57223bedf18
#!/usr/bin/python
# -*-coding:utf-8-*-

import io
import os
import sys
import logging
import urllib2
import string
import chardet
from optparse import OptionParser, Option, IndentedHelpFormatter
import json
import requests
from ast import literal_eval

from wcs.commons.putpolicy import PutPolicy
from wcs.commons.error_deal import ParameterError
from wcs.commons.util import file_crc64
from wcs.__init__ import __version__


def output(message):
    sys.stdout.write(message + '\n')
    sys.stdout.flush()


def format_commands(progname, commands_list):
    # Define commands display format
    help = "Commands:" + "\n"
    for cmd in commands_list:
        help += "   %s\n    	%s %s %s \n" % (cmd["label"], progname, cmd["cmd"], cmd["param"])
    return help


def get_commands_list():
    # Return all commands list of wcscmd
    return [
        {"cmd": "list", "label": "List objects", "param": "wcs://BUCKET RESFILE", "func": cmd_list, "argc": 2},
        {"cmd": "listbucket", "label": "List buckets", "param": "", "func": cmd_listbucket, "argc": 0},
        {"cmd": "get", "label": "Download file", "param": "URL", "func": cmd_get, "argc": 2},
        {"cmd": "del", "label": "Delete a file", "param": "wcs://BUCKET/OBJECT", "func": cmd_del, "argc": 1},
        {"cmd": "mv", "label": "Move a file from src bucket to des bucket",  "param": "wcs://srcBUCKET/srcOBJECT wcs://dstBUCKET/desOBJECT", "func": cmd_mv, "argc": 2},
        {"cmd": "cp", "label": "Copy a file from src bucket to des bucket",  "param": "wcs://srcBUCKET/srcOBJECT wcs://dstBUCKET/desOBJECT", "func": cmd_cp, "argc": 2},
        {"cmd": "setdeadline", "label": "Set deadline of file", "param": "wcs://BUCKET/OBJECT deadline", "func": cmd_setdeadline, "argc": 2},
        {"cmd": "stat", "label": "Get file info", "param": "wcs://BUCKET/OBJECT", "func": cmd_stat, "argc": 1},
        {"cmd": "put", "label": "Upload a local file to WCS", "param": "wcs://BUCKET/OBJECT LOCALFILE", "func": cmd_put,"argc": 2},
        {"cmd": "multiput", "label": "Multipart upload a local file to WCS", "param": "wcs://BUCKET/OBJECT LOCALFILE ", "func": cmd_multiput, "argc": 2},
        {"cmd": "deletePrefix", "label": "Delete multiple files according to prefix", "param": "wcs://BUCKET PREFIX", "func": cmd_delprefix, "argc": 2},
        {"cmd": "fops", "label": "Fops audio/video processing ", "param": "wcs://BUCKET/OBJECT fopsparm", "func": cmd_fops, "argc": 2},
        {"cmd": "fopsStatus", "label": "Get fops task results ", "param": "persistentId", "func": cmd_fopsstatus, "argc": 1},
        {"cmd": "fmgrStatus", "label": "Get fmgrStatus task results ", "param": "persistentId", "func": cmd_fmgrstatus, "argc": 1},
        {"cmd": "crc64", "label": "Calculating 64-bit CRC value ", "param": "filePath", "func": cmd_crc64, "argc": 1}
    ]


def cmd_list(args):
    # Define implementation of List Objects API
    cfg = Config()
    cli = Client(cfg)
    bucket = args[0].split("wcs://")[1]
    resfile = os.path.join(os.getcwd(), args[1])
    try:
        code, response, reqid = cli.bucket_list(bucket, cfg.prefix, cfg.marker, int(cfg.limit), int(cfg.mode))
        if code == 200:
            try:
                with open(resfile, 'w') as f:
                    f.write(json.dumps(response))
            except IOError:
                debug("Save Fail")
                sys.exit()
            debug("Request result saved to %s" % (resfile))
            debug("Request ID is: %s" % (reqid['x-reqid']))
        else:
            debug("error-message:%s" % response)
            error("Request fail, more detail please pay attention to reqid: %s " % (reqid['x-reqid']))
    except Exception,E:
        error(E)


def cmd_get(args):
    # Define implementation of download file from URL
    cfg = Config()
    try:
        if chardet.detect(args[0]).get('encoding') == 'ISO-8859-1':
            url = args[0].decode('gbk').encode('utf8')
        else:
            url = args[0]
        url_encode = str(urllib2.quote(url, safe=string.printable))
        if len(args) > 1:
            key = os.path.join(os.getcwd(), args[1])
        else:
            url = urllib2.unquote(str(url)).decode('utf8')
            filename = url.split('?')
            if len(filename) >= 2:
                filename.pop(-1)
                filename = '/'.join('?'.join(filename).split('/')[3:])
                key = os.path.join(os.getcwd(), filename)
            else:
                filename = url.split('/')[-1]
                key = os.path.join(os.getcwd(), filename)
        headers = {}
        try:
            if int(cfg.traffic_limit):
                headers['x-wos-traffic-limit'] = '{0}'.format(cfg.traffic_limit)
        except Exception as e:
            raise ValueError('traffic_limit parameter configuration error：{0}'.format(cfg.traffic_limit))
        if cfg.isverify:
            res = requests.get(url_encode,headers=headers)
        else:
            res = requests.get(url_encode,verify=False,headers=headers)
        if res.status_code == 200:
            debug("Download suc.")
            debug("return headers:{0}".format(res.headers))
            try:
                with open(key, 'wb') as f:
                    f.write(res.content)
                debug('File saved successfully,File:[{0}]'.format(key))
            except IOError as e:
                error("Create local file fail for %s" % e)
                sys.exit()
        else:
            error_text = res.text
            debug("return headers:{0}".format(res.headers))
            error("Download Fail:%s" % error_text)
    except Exception as e:
        error("Download Fail:%s" % e)


def _simple_arg_parse(arg):
    # Parse commands with only one wcs://bucket/[prefix1]/[prefix2]/object
    try:
        tmp = arg.split("wcs://")[1]
        if '/' in tmp:
            bucket = tmp.split('/')[0]
            obj = tmp.split(bucket + '/')[1]
        else:
            bucket = tmp
            obj = ''
    except Exception:
        error("Invalid arguments")
        sys.exit()
    return bucket, obj


def cmd_del(args):
    # Define implementation of Object Delete API
    cli = Client(Config())
    bucket, obj = _simple_arg_parse(args[0])
    debug(cli.delete(bucket, obj))


def _couple_args_parse(args):
    # Parse commands with two wcs://srcbucket/[srcprefix1]/[srcprefix2]/srcobject wcs://dstbucket/[dstprefix1]/[dstprefix2]/dstobject
    try:
        src = args[0].split("wcs://")[1]
        dst = args[1].split("wcs://")[1]
        srcbucket = src.split('/')[0]
        srcobject = src.split('/')[1:]
        srcobject = '/'.join(srcobject)

        dstbucket = dst.split('/')[0]
        dstobject = dst.split('/')[1:]
        dstobject = '/'.join(dstobject)
    except Exception:
        error("Invalid arguments")
        sys.exit()
    return srcbucket, srcobject, dstbucket, dstobject


def cmd_mv(args):
    # Define implementation of Synchronous move API
    cli = Client(Config())
    srcbk, srcob, dstbk, dstob = _couple_args_parse(args)
    debug(cli.move(srcbk, srcob, dstbk, dstob))


def cmd_cp(args):
    # Define implementation of Synchronous copy API
    cli = Client(Config())
    srcbk, srcob, dstbk, dstob = _couple_args_parse(args)
    debug(cli.copy(srcbk, srcob, dstbk, dstob))


def cmd_setdeadline(args):
    # Define implementation of Set Deadline API
    cli = Client(Config())
    bucket, obj = _simple_arg_parse(args[0])
    deadline = literal_eval(args[1])
    debug(cli.setdeadline(bucket, obj, deadline))


def cmd_stat(args):
    # Define implementation of stat API
    cli = Client(Config())
    bucket, obj = _simple_arg_parse(args[0])
    debug(cli.stat(bucket, obj))


def cmd_put(args):
    # Define implementation of simple upload API
    cfg = Config()
    cli = Client(cfg)
    bucket, obj = _simple_arg_parse(args[0])
    local = args[1]
    if not obj:
        obj = os.path.basename(local)
    debug(cli.simple_upload(local, bucket, obj))


def cmd_multiput(args):
    # Define implementation of multipart upload
    cfg = Config()
    cli = Client(cfg)
    bucket, obj = _simple_arg_parse(args[0])
    local = args[1]
    if not obj:
        obj = os.path.basename(local)
    debug(cli.multipart_upload(local, bucket, obj, cfg.upload_id))


def cmd_listbucket(args):
    #
    cli = Client(Config())
    debug(cli.list_buckets())


def cmd_delprefix(args):
    bucket = args[0].split("wcs://")[1]
    prefix = args[1]
    cli = Client(Config())
    fops = 'bucket/%s/prefix/%s' % (urlsafe_base64_encode(bucket), urlsafe_base64_encode(prefix))
    debug(cli.prefix_delete(fops))


def cmd_fops(args):
    # Define implementation of Fops API
    cli = Client(Config())
    bucket,obj = _simple_arg_parse(args[0])
    fops = args[1]
    debug(cli.ops_execute(fops,bucket,obj))


def cmd_fopsstatus(args):
    # Define implementation of fops status API
    cli = Client(Config())
    persistentId = args[0]
    debug(cli.ops_status(persistentId))


def cmd_fmgrstatus(args):
    # Define implementation of fmgr status API
    cli = Client(Config())
    persistentId = args[0]
    debug(cli.fmgr_status(persistentId))

def cmd_crc64(args):
    #calculating 64-bit CRC value
    try:
        file_path = args[0]
        if os.path.isfile(file_path):
            crc64 = file_crc64(file_path)
            debug("calculating suc.")
            debug(str({'file':file_path,'crc64Value':crc64}))
        else:
            error('Calculating faile,Reasons for failure: file not found .')
    except Exception as e:
        error('Some error occurred. {0}'.format(e))


def run_configure(config_file):
    if os.path.exists(config_file):
        cfg = Config(config_file)
    else:
        cfg = Config()
    options = [
        ("access_key", "Access Key", "Access key and Secret key are your identifiers for WCS"),
        ("secret_key", "Secret Key"),
        ("put_url", "PUT_URL", "Upload domain for WCS"),
        ("mgr_url", "MGR_URL", "Manager domain for WCS"),
        ("block_size", "Block_Size", "Mulpart Upload Block Size"),
        ("bput_size", "Bput_Size", "Mulpart Upload Bput Size"),
        ("connection_retries", "Connection_Retires", "Request Connection Retires"),
        ("connection_timeout", "Connection_Timeout", "Request Connection Timeout"),
        ("mkblk_retries", "Mkblk_Retries", "Multipart Upload Mkblk retries"),
        ("bput_retries", "Bput_Retries", "Multipart Upload Bput retries"),
        ("mkfile_retries", "Mkfile_Retries", "Multipart Upload Mkfile retries"),
        ("tmp_record_folder", "Tmp_Record_Folder", "Multipart Upload record folder"),
        ("concurrency", "Concurrency", "Multipart Upload Concurrency"),
        ("ishttps", "IsHttps", "Use https or not"),
        ("isverify", "isverify", "Use verify or not"),
        ("mode", "mode", "List objects mode"),
        ("keepalive", "keepalive", "request to use keepalive mode"),
        ("deadline", "deadline", "token expiration time"),
        ("traffic_limit", "traffic_limit", "traffic limit")
    ]
    try:
        while True:
            output(u"Enter new values or accept defaults in brackets with Enter")
            output(u"Refer to user manual for detailed description of all options")
            for option in options:
                prompt = option[1]
                try:

                    val = getattr(cfg, option[0])
                    if type(val) is bool:
                        val = val and "Yes" or "No"
                    if val not in (None, ""):
                        prompt += " [%s]" % val
                except AttributeError:
                    pass

                if len(option) >= 3:
                    output(u"\n%s" % option[2])
                val = raw_input(prompt + ": ")
                if val != "":
                    if type(getattr(cfg, option[0])) is bool:
                        val = val.lower().startswith('y')
                    setattr(cfg, option[0], val)
            output(u"\nNew settings:")
            for option in options:
                output(u"  %s: %s" % (option[0], getattr(cfg, option[0])))
            val = raw_input("\nSave settings? [y/N] ")
            if val.lower().startswith("y"):
                break
            val = raw_input("Retry configuration? [Y/n] ")
            if val.lower().startswith("n"):
                raise EOFError()
        try:
            with io.open(config_file, 'w') as fp:
                cfg.dump_config(fp)
        except IOError as e:
            raise e
        output(u"Configuration saved to '%s'" % config_file)

    except (EOFError, KeyboardInterrupt):
        output(u"\nConfiguration aborted. Changes were NOT saved.")
        return
    except IOError, e:
        error(u"Writing config file failed: %s: %s" % (config_file, e.strerror))
        sys.exit()


class MyHelpFormatter(IndentedHelpFormatter):
    def format_epilog(self, epilog):
        if epilog:
            return "\n" + epilog + "\n"
        else:
            return ""


def main():
    global cfg
    cfg = Config()
    commands_list = get_commands_list()
    commands = {}

    for cmd in commands_list:
        if 'cmd' in cmd:
            commands[cmd['cmd']] = cmd

    usage = 'Usage: wcscmd [options] COMMAND [parameters]'
    optparser = OptionParser(usage=usage, formatter=MyHelpFormatter())

    config_file = None
    from os.path import expanduser
    config_file = os.path.join(expanduser("~"), ".wcscfg")
    optparser.set_defaults(config=config_file)

    optparser.add_option("--configure", dest="run_config", action="store_true",
                         help="Invoke interactive (re)configuration tool.")
    optparser.add_option("-c", "--config", dest="config", metavar="FILE",
                         help="Config file name. Defaults to $HOME/.wcscfg")
    optparser.add_option("--dump-config", dest="dump_config", action="store_true",
                         help="Dump current configuration after parsing config file and command line options and exit.")
    # optparser.add_option(   "--access_key", dest="access_key", help="WCS Access Key")
    # optparser.add_option(   "--secret_key", dest="secret_key", help="WCS Secret Key")
    optparser.add_option("-s", "--ssl", dest="ishttps", action="store_true",
                         help="Use https connection when communicating with WCS")
    optparser.add_option("--upload-id", dest="upload_id",
                         help="UploadId for Multipart Upload, in case you want continue an existing upload")
    optparser.add_option("--limit", dest="limit", help="Limit for list objects of bucket")
    optparser.add_option("--prefix", dest="prefix", help="Prefix for list objects of bucket")
    optparser.add_option("--mode", dest="mode", help="Mode for list objects of bucket")
    optparser.add_option("--marker", dest="marker", help="Start string for list")
    optparser.add_option("--relevance", dest="relevance", action="store_true",
                         help="Whether modify deadline relevant .ts when modify deadline of m3u8 file")
    optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG,
                         help="Enable debug output.")
    optparser.add_option("--output", dest="output", help="Save request result to specified key, like:<bucket>:<key>")
    optparser.add_option("--notify", dest="notifyURL", help="Notify url for request result")
    optparser.add_option("--separate", dest="separate", help="Whether separate notify, bool type")
    optparser.add_option("--force", dest="force", help="Whether to enforce data processing, bool type")
    optparser.add_option("--overwrite", dest="overwrite", help="Whether overwrite existed key, bool type")
    optparser.add_option("--returnurl", dest="returl", help="Url for return")
    optparser.add_option("--returnbody", dest="retbody", help="Body of return")
    optparser.add_option("--callbackurl", dest="cburl", help="Url for callback")
    optparser.add_option("--callbackbody", dest="cbbody", help="Body of callback")
    optparser.add_option("--persistentntyurl", dest="persistentntyurl", help="Url for persistent ops notify")
    optparser.add_option("--persistentops", dest="persistentops", help="Persistent ops")
    optparser.add_option("--contentdetect", dest="contentdetect", help="Content detect type")
    optparser.add_option("--detectntyurl", dest="detectntyurl", help="Url for detect notify")
    optparser.add_option("--detectntyrule", dest="detectntyrule", help="RUle for ectect notify")
    optparser.add_option("--hashalgorithm", dest="hashalgorithm", help="Hash algorithm, currently supports 'crc64ecma' for calculating 64-bit CRC value.")#add by caiyz 2020-07-13
    optparser.add_option("--deadline", dest="deadline",help="Token expiration time,Unit is second")  # add by caiyz 2021-08-16
    optparser.add_option("--traffic_limit", dest="traffic_limit",help="Upload/Download traffic limit")  # add by caiyz 2022-03-15
    optparser.set_description("Wcscmd is a tool for managing objects in WCS Object Storage." + " It allows for uploading, downloading and removing objects form buckets")

    optparser.epilog = format_commands(optparser.get_prog_name(), commands_list)
    optparser.epilog += "\n\nFor more information, visit WCS website: https://wcs.chinanetcenter.com/index\n"

    (options, args) = optparser.parse_args()

    # if options.verbosity:
    #    logging.basicConfig(level=options.verbosity, format='%(levelname)s: %(message)s', stream=sys.stdderr)
    # if run_config:
    try:
        debug(u"wcscmd version {0}".format(__version__))
    except ValueError as e:
        debug(u"Get Version Fail :%s" % e)
    if options.run_config:
        run_configure(options.config)
        sys.exit()
    try:
        cfg = Config(options.config)
    except ValueError as exc:
        raise ParameterError(unicode(exc))
    except IOError as e:
        if options.run_configure:
            cfg = Config()
        else:
            error(u"%s: %s" % (options.config, e.strerror))
            error(u"Configuration file not available.")
            error(u"Consider using --configure parameter to create one.")
            sys.exit()

    if options.output:
        cfg.output = options.output
    if options.upload_id:
        cfg.upload_id = options.upload_id
    if options.notifyURL:
        cfg.notifyurl = options.notifyURL
    if options.separate:
        cfg.separate = options.separate
    if options.force:
        cfg.force = options.force
    if options.limit:
        cfg.limit = options.limit
    if options.prefix:
        cfg.prefix = options.prefix
    if options.mode:
        cfg.mode = options.mode
    if options.marker:
        cfg.marker = options.marker
    if options.overwrite:
        cfg.overwrite = options.overwrite
    if options.returl:
        cfg.returnUrl = options.returl
    if options.retbody:
        cfg.returnBody = options.retbody
    if options.cburl:
        cfg.callbackUrl = options.cburl
    if options.cbbody:
        cfg.callbackBody = options.cbbody
    if options.persistentntyurl:
        cfg.persistentNotifyUrl = options.persistentntyurl
    if options.persistentops:
        cfg.persistentOps = options.persistentops
    if options.contentdetect:
        cfg.contentDetect = options.contentdetect
    if options.detectntyurl:
        cfg.detectNotifyURL = options.detectntyurl
    if options.detectntyrule:
        cfg.detectNotifyRule = options.detectntyrule
    if options.hashalgorithm:
        cfg.hashAlgorithm = options.hashalgorithm
    if options.deadline:
        try:
            cfg.deadline = int(options.deadline)
        except ValueError:
            raise ParameterError('deadline must be an integer.')
    if options.traffic_limit:
        try:
            cfg.traffic_limit = int(options.traffic_limit)
        except ValueError:
            raise ParameterError('"traffic_limit" must be an integer.')
    if options.dump_config:
        cfg.dump_config(sys.stdout)
        sys.exit()
    for option in cfg.option_list():
        try:
            value = getattr(options, option)
            if value != None:
                type(value) == type(b'')
            # debug(u"Updating Config.Config %s->%s" % (option, value))
            cfg.update_option(option, value)
        except AttributeError:
            pass
        try:
            # caiyz 20180306 add option check
            if option == 'block_size':
                if int(cfg.block_size) <= 0 or (int(cfg.block_size) % 4194304) != 0:
                    raise ParameterError("Block size %d B is not available" % (int(cfg.block_size)))
            if option == 'bput_size':
                if int(cfg.bput_size) <= 0 or (int(cfg.bput_size)) % 2 != 0:
                    raise ParameterError("Bput size %d B is not available" % (int(cfg.bput_size)))
        except ParameterError as e:
            print e

    if len(args) < 1:
        optparser.print_help()
        sys.exit()
    command = args.pop(0)
    try:
        debug(u"Command: %s" % commands[command]["cmd"])
        cmd_func = commands[command]["func"]
    except KeyError as e:
        error(u"Invalid command: %s" % command)
        sys.exit()
    if command == 'get':
        if len(args) > commands[command]["argc"]:
            error(u"Too much parameters for command '%s'" % command)
            sys.exit()
    else:
        if len(args) < commands[command]["argc"]:
            error(u"Not enough parameters for command '%s'" % command)
            sys.exit()
        if len(args) > commands[command]["argc"]:
            error(u"Too much parameters for command '%s'" % command)
            sys.exit()
    rc = cmd_func(args)
    if rc is None:
        rc = "Nothing"
    return rc


if __name__ == '__main__':
    from wcs.commons.config import Config
    from wcs.commons.logme import debug, error
    from wcs.services.client import Client
    from wcs.commons.util import urlsafe_base64_encode

    main()
