#!/usr/bin/env python3
"""
Gather all the case metadata and send it to the experiments databases
via a web post and SVN check-in

Author: CSEG <cseg@cgd.ucar.edu>
"""
import argparse
import datetime
import filecmp
import getpass
import glob
import gzip
import json
import io
import os
from os.path import expanduser
import re
import shutil
import ssl
import subprocess
import sys
from string import Template
import configparser
import urllib

from standard_script_setup import *
from CIME.case import Case
from CIME.utils import is_last_process_complete

# define global constants
logger = logging.getLogger(__name__)
_svn_expdb_url = "https://svn-cesm2-expdb.cgd.ucar.edu"
_exp_types = ["CMIP6", "production", "tuning", "lens", "C1", "C2", "C3", "C4", "C5"]
_xml_vars = [
    "CASE",
    "COMPILER",
    "COMPSET",
    "CONTINUE_RUN",
    "DOUT_S",
    "DOUT_S_ROOT",
    "GRID",
    "MACH",
    "MPILIB",
    "MODEL",
    "MODEL_VERSION",
    "REST_N",
    "REST_OPTION",
    "RUNDIR",
    "RUN_REFCASE",
    "RUN_REFDATE",
    "RUN_STARTDATE",
    "RUN_TYPE",
    "STOP_N",
    "STOP_OPTION",
    "USER",
]
_run_vars = ["JOB_QUEUE", "JOB_WALLCLOCK_TIME", "PROJECT"]
_archive_list = [
    "Buildconf",
    "CaseDocs",
    "CaseStatus",
    "LockedFiles",
    "Macros.make",
    "README.case",
    "SourceMods",
    "software_environment.txt",
]
_call_template = Template(
    'in "$function" - Ignoring SVN repo update\n'
    'SVN error executing command "$cmd". \n'
    "$error: $strerror"
)
_copy_template = Template(
    'in "$function" - Unable to copy "$source" to "$dest"' "$error: $strerror"
)
_svn_error_template = Template(
    'in "$function" - SVN client unavailable\n'
    'SVN error executing command "$cmd". \n'
    "$error: $strerror"
)
_ignore_patterns = ["*.pyc", "^.git", "tmp", ".svn", "*~"]
_pp_xml_vars = {
    "atm": "ATMDIAG_test_path_climo",
    "glc": "",
    "lnd": "LNDDIAG_PTMPDIR_1",
    "ice": "ICEDIAG_PATH_CLIMO_CONT",
    "ocn": "OCNDIAG_TAVGDIR",
    "rof": "",
    "timeseries": "TIMESERIES_OUTPUT_ROOTDIR",
    "xconform": "CONFORM_OUTPUT_DIR",
}
_pp_diag_vars = {
    "atm": ["ATMDIAG_test_first_yr", "ATMDIAG_test_nyrs"],
    "ice": ["ICEDIAG_BEGYR_CONT", "ICEDIAG_ENDYR_CONT", "ICEDIAG_YRS_TO_AVG"],
    "lnd": [
        "LNDDIAG_clim_first_yr_1",
        "LNDDIAG_clim_num_yrs_1",
        "LNDDIAG_trends_first_yr_1",
        "LNDDIAG_trends_num_yrs_1",
    ],
    "ocn": [
        "OCNDIAG_YEAR0",
        "OCNDIAG_YEAR1",
        "OCNDIAG_TSERIES_YEAR0",
        "OCNDIAG_TSERIES_YEAR1",
    ],
}
_pp_tseries_comps = ["atm", "glc", "ice", "lnd", "ocn", "rof"]

# setting the ssl context to avoid issues with CGD certificates
_context = ssl._create_unverified_context()  # pylint:disable=protected-access

# -------------------------------------------------------------------------------
class PasswordPromptAction(argparse.Action):
    # -------------------------------------------------------------------------------
    """SVN developer's password class handler"""
    # pylint: disable=redefined-builtin
    def __init__(
        self,
        option_strings=None,
        dest=None,
        default=None,
        required=False,
        nargs=0,
        help=None,
    ):
        super(PasswordPromptAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            default=default,
            required=required,
            nargs=nargs,
            help=help,
        )

    def __call__(self, parser, args, values, option_string=None):
        # check if ~/.subversion/cmip6.conf exists
        home = expanduser("~")
        conf_path = os.path.join(home, ".subversion/cmip6.conf")
        if os.path.exists(conf_path):
            # read the .cmip6.conf file
            config = configparser.SafeConfigParser()
            config.read(conf_path)
            password = config.get("svn", "password")
        else:
            password = getpass.getpass()
        setattr(args, self.dest, password)


# ---------------------------------------------------------------------
def basic_authorization(user, password):
    # ---------------------------------------------------------------------
    """Basic authentication encoding"""
    sauth = user + ":" + password
    return "Basic " + sauth.encode("base64").rstrip()


# ---------------------------------------------------------------------
class SVNException(Exception):
    # ---------------------------------------------------------------------
    """SVN command exception handler"""

    def __init__(self, value):
        super(SVNException, self).__init__(value)
        self.value = value

    def __str__(self):
        return repr(self.value)


# -------------------------------------------------------------------------------
def commandline_options(args):
    # -------------------------------------------------------------------------------
    """Process the command line arguments."""
    parser = argparse.ArgumentParser(
        description="Query and parse the caseroot files to gather metadata information"
        " that can be posted to the CESM experiments database."
        " "
        " CMIP6 experiment case names must be reserved already in the"
        " experiment database. Please see:"
        " https://csesgweb.cgd.ucar.edu/expdb2.0 for details."
    )

    CIME.utils.setup_standard_logging_options(parser)

    parser.add_argument(
        "--user",
        dest="user",
        type=str,
        default=None,
        required=True,
        help="User name for SVN CESM developer access (required)",
    )

    parser.add_argument(
        "--password",
        dest="password",
        action=PasswordPromptAction,
        default="",
        required=True,
        help="Password for SVN CESM developer access (required)",
    )

    parser.add_argument(
        "--caseroot",
        nargs=1,
        required=False,
        help="Fully quailfied path to case root directory (optional). "
        "Defaults to current working directory.",
    )

    parser.add_argument(
        "--workdir",
        nargs=1,
        required=False,
        help="Fully quailfied path to directory for storing intermediate "
        "case files. A sub-directory called "
        "archive_temp_dir is created, populated "
        "with case files, and posted to the CESM experiments database and "
        'SVN repository at URL "{0}". '
        "This argument can be used to archive a caseroot when the user "
        "does not have write permission in the caseroot (optional). "
        "Defaults to current working directory.".format(_svn_expdb_url),
    )

    parser.add_argument(
        "--expType",
        dest="expType",
        nargs=1,
        required=True,
        choices=_exp_types,
        help="Experiment type. For CMIP6 experiments, the case must already "
        "exist in the experiments database at URL "
        ' "http://csegweb.cgd.ucar.edu/expdb2.0" (required). '
        'Must be one of "{0}"'.format(_exp_types),
    )

    parser.add_argument(
        "--title",
        nargs=1,
        required=False,
        default=None,
        help="Title of experiment (optional).",
    )

    parser.add_argument(
        "--ignore-logs",
        dest="ignore_logs",
        action="store_true",
        help="Ignore updating the SVN repository with the caseroot/logs files. "
        "The experiments database will be updated (optional).",
    )

    parser.add_argument(
        "--ignore-timing",
        dest="ignore_timing",
        action="store_true",
        help="Ignore updating the the SVN repository with caseroot/timing files."
        "The experiments database will be updated (optional).",
    )

    parser.add_argument(
        "--ignore-repo-update",
        dest="ignore_repo_update",
        action="store_true",
        help="Ignore updating the SVN repository with all the caseroot files. "
        "The experiments database will be updated (optional).",
    )

    parser.add_argument(
        "--add-files",
        dest="user_add_files",
        required=False,
        help="Comma-separated list with no spaces of files or directories to be "
        "added to the SVN repository. These are in addition to the default added "
        "caseroot files and directories: "
        '"{0}, *.xml, user_nl_*" (optional).'.format(_archive_list),
    )

    parser.add_argument(
        "--dryrun",
        action="store_true",
        help="Parse settings and print what actions will be taken but "
        "do not execute the action (optional).",
    )

    parser.add_argument(
        "--query_cmip6",
        nargs=2,
        required=False,
        help="Query the experiments database global attributes "
        "for specified CMIP6 casename as argument 1. "
        "Writes a json formatted output file, specified by argument 2, "
        "to subdir archive_files (optional).",
    )

    parser.add_argument(
        "--test-post",
        dest="test_post",
        action="store_true",
        help="Post metadata to the test expdb2.0 web application server "
        'at URL "http://csegwebdev.cgd.ucar.edu/expdb2.0". '
        "No --test-post argument defaults to posting metadata to the "
        "production expdb2.0 web application server "
        'at URL "http://csegweb.cgd.ucar.edu/expdb2.0" (optional).',
    )

    opts = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)

    return opts


# ---------------------------------------------------------------------
def get_case_vars(case_dict, case):
    # ---------------------------------------------------------------------
    """get_case_vars
    loop through the global list of XML vars and get the values
    from the case object into a case dictionary

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        case (object) - case object
    """
    logger.debug("get_case_vars")

    for xml_id in _xml_vars:
        case_dict[xml_id] = case.get_value(xml_id, resolved=True, subgroup=None)

    for xml_id in _run_vars:
        case_dict[xml_id] = case.get_value(xml_id, resolved=True, subgroup="case.run")

    return case_dict


# ---------------------------------------------------------------------
def get_disk_usage(path):
    # ---------------------------------------------------------------------
    """get_disk_usage
    return the total disk usage in bytes for a given path.

    Arguments:
        path - path to start
    """
    logger.debug("get_disk_usage")
    total_size = 0
    cwd = os.getcwd()
    if os.path.exists(path):
        os.chdir(path)
        cmd = ["du", "--summarize", "--block-size=1"]
        try:
            total_size = subprocess.check_output(cmd)
            total_size = total_size.replace("\t.\n", "")
        except subprocess.CalledProcessError:
            msg = "Error executing command = '{0}'".format(cmd)
            logger.warning(msg)
    os.chdir(cwd)
    return int(total_size)


# ---------------------------------------------------------------------
def get_ocn_disk_usage(path):
    # ---------------------------------------------------------------------
    """get_ocn_disk_usage
    return the total disk usage in bytes for a given path.

    Arguments:
        path - path to start
    """
    logger.debug("get_ocn_disk_usage")
    total_size = 0
    paths = glob.glob(path)
    for path in paths:
        total_size += get_disk_usage(path)
    return int(total_size)


# ---------------------------------------------------------------------
def get_pp_path(pp_dir, process):
    # ---------------------------------------------------------------------
    """get_pp_path
    return the XML path for process

    Arguments:
        pp_dir  - path to postprocess directory
        process - process name
    """
    logger.debug("get_pp_path")

    cwd = os.getcwd()
    os.chdir(pp_dir)

    pp_path_var = ""
    if process == "timeseries":
        pp_path_var = _pp_xml_vars["timeseries"]
    elif process == "xconform":
        pp_path_var = _pp_xml_vars["xconform"]

    cmd = ["./pp_config", "--get", pp_path_var, "--value"]
    try:
        pp_path = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        msg = "Error executing command = '{0}'".format(cmd)
        logger.warning(msg)

    if len(pp_path) > 2:
        pp_path = pp_path.rstrip()
    else:
        pp_path = ""

    os.chdir(cwd)
    return pp_path


# ---------------------------------------------------------------------
def get_diag_dates(comp, pp_dir):
    # ---------------------------------------------------------------------
    """get_diag_dates

    Query the postprocessing env_diags_[comp].xml file to get the model diag
    dates for the given component.
    """
    logger.debug("get_diag_dates")

    cwd = os.getcwd()
    os.chdir(pp_dir)

    model_dates = ""
    pp_vars = _pp_diag_vars.get(comp)
    for pp_var in pp_vars:
        cmd = ["./pp_config", "--get", pp_var, "--value"]
        try:
            pp_value = subprocess.check_output(cmd)
        except subprocess.CalledProcessError:
            msg = "Error executing command = '{0}'".format(cmd)
            logger.warning(msg)
        tmp_dates = "{0} = {1}".format(pp_var, pp_value)
        model_dates = model_dates + tmp_dates

    os.chdir(cwd)
    return model_dates


# ---------------------------------------------------------------------
def get_pp_status(case_dict):
    # ---------------------------------------------------------------------
    """get_pp_status
    Parse the postprocessing log files
    looking for status information

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug("get_pp_status")

    # initialize status variables
    msg_avg = dict()
    msg_diags = dict()
    diag_comps = ["atm", "ice", "lnd", "ocn"]

    pp_dir = os.path.join(case_dict["CASEROOT"], "postprocess")
    pp_log_dir = os.path.join(case_dict["CASEROOT"], "postprocess", "logs")

    msg_avg["atm"] = "COMPLETED SUCCESSFULLY"
    msg_diags["atm"] = "Successfully completed generating atmosphere diagnostics"
    case_dict["atm_avg_dates"] = case_dict["atm_diag_dates"] = get_diag_dates(
        "atm", pp_dir
    )

    msg_avg["ice"] = "Successfully completed generating ice climatology averages"
    msg_diags["ice"] = "Successfully completed generating ice diagnostics"
    case_dict["ice_avg_dates"] = case_dict["ice_diag_dates"] = get_diag_dates(
        "ice", pp_dir
    )

    msg_avg["lnd"] = "COMPLETED SUCCESSFULLY"
    msg_diags["lnd"] = "Successfully completed generating land diagnostics"
    case_dict["lnd_avg_dates"] = case_dict["lnd_diag_dates"] = get_diag_dates(
        "lnd", pp_dir
    )

    msg_avg["ocn"] = "Successfully completed generating ocean climatology averages"
    msg_diags["ocn"] = "Successfully completed generating ocean diagnostics"
    case_dict["ocn_avg_dates"] = case_dict["ocn_diag_dates"] = get_diag_dates(
        "ocn", pp_dir
    )

    for comp in diag_comps:
        case_dict[comp + "_avg_status"] = "Unknown"
        case_dict[comp + "_diag_status"] = "Unknown"

        if comp != "ocn":
            case_dict[comp + "_avg_path"] = os.path.join(
                case_dict["DOUT_S_ROOT"], comp, "proc/climo"
            )
            case_dict[comp + "_avg_size"] = get_disk_usage(
                case_dict[comp + "_avg_path"]
            )
            case_dict[comp + "_diag_path"] = os.path.join(
                case_dict["DOUT_S_ROOT"], comp, "proc/diag"
            )
            case_dict[comp + "_diag_size"] = get_disk_usage(
                case_dict[comp + "_diag_path"]
            )
        else:
            case_dict[comp + "_avg_path"] = os.path.join(
                case_dict["DOUT_S_ROOT"], comp, "proc/climo*"
            )
            case_dict[comp + "_avg_size"] = get_ocn_disk_usage(
                case_dict[comp + "_avg_path"]
            )
            case_dict[comp + "_diag_path"] = os.path.join(
                case_dict["DOUT_S_ROOT"], comp, "proc/diag*"
            )
            case_dict[comp + "_diag_size"] = get_ocn_disk_usage(
                case_dict[comp + "_diag_path"]
            )

        avg_logs = list()
        avg_file_pattern = "{0}/{1}_averages.log.*".format(pp_log_dir, comp)
        avg_logs = glob.glob(avg_file_pattern)

        if avg_logs:
            log_file = max(avg_logs, key=os.path.getctime)
            if is_last_process_complete(
                log_file, msg_avg[comp], "Average list complies with standards."
            ):
                case_dict[comp + "_avg_status"] = "Succeeded"
            else:
                case_dict[comp + "_avg_status"] = "Started"

        diag_logs = list()
        diag_file_pattern = "{0}/{1}_diagnostics.log.*".format(pp_log_dir, comp)
        diag_logs = glob.glob(diag_file_pattern)

        if diag_logs:
            log_file = max(diag_logs, key=os.path.getctime)
            if is_last_process_complete(log_file, msg_diags[comp], "ncks version"):
                case_dict[comp + "_diag_status"] = "Succeeded"
            else:
                case_dict[comp + "_diag_status"] = "Started"

    # get overall timeseries status
    case_dict["timeseries_status"] = "Unknown"
    case_dict["timeseries_path"] = get_pp_path(pp_dir, "timeseries")
    case_dict["timeseries_size"] = 0
    case_dict["timeseries_dates"] = "{0}-{1}".format(
        case_dict["RUN_STARTDATE"].replace("-", ""),
        case_dict["RUN_STARTDATE"].replace("-", ""),
    )
    case_dict["timeseries_total_time"] = 0
    tseries_logs = list()
    tseries_file_pattern = "{0}/timeseries.log.*".format(pp_log_dir)
    tseries_logs = glob.glob(tseries_file_pattern)
    if tseries_logs:
        log_file = max(tseries_logs, key=os.path.getctime)
        if is_last_process_complete(
            filepath=log_file, expect_text="Successfully completed", fail_text="opening"
        ):
            case_dict["timeseries_status"] = "Succeeded"
            with open(log_file, "r") as fname:
                log_content = fname.readlines()
            total_time = [line for line in log_content if "Total Time:" in line]
            case_dict["timeseries_total_time"] = " ".join(total_time[0].split())
        else:
            case_dict["timeseries_status"] = "Started"
        sta_dates = case_dict["sta_last_date"].split("-")
        case_dict["timeseries_dates"] = "{0}-{1}".format(
            case_dict["RUN_STARTDATE"].replace("-", ""), "".join(sta_dates[:-1])
        )
        for comp in _pp_tseries_comps:
            tseries_path = "{0}/{1}/proc/tseries".format(
                case_dict["timeseries_path"], comp
            )
            case_dict["timeseries_size"] += get_disk_usage(tseries_path)

    # get iconform status = this initializes files in the POSTPROCESS_PATH
    case_dict["iconform_status"] = "Unknown"
    case_dict["iconform_path"] = ""
    case_dict["iconform_size"] = 0
    case_dict["iconform_dates"] = case_dict["timeseries_dates"]

    iconform_logs = list()
    iconform_file_pattern = "{0}/iconform.log.*".format(pp_log_dir)
    iconform_logs = glob.glob(iconform_file_pattern)
    if iconform_logs:
        log_file = max(iconform_logs, key=os.path.getctime)
        if is_last_process_complete(
            log_file,
            "Successfully created the conform tool",
            "Running createOutputSpecs",
        ):
            case_dict["iconform_status"] = "Succeeded"
        else:
            case_dict["iconform_status"] = "Started"

    # get xconform status
    case_dict["xconform_path"] = ""
    case_dict["xconform_path"] = get_pp_path(pp_dir, "xconform")
    case_dict["xconform_status"] = "Unknown"
    case_dict["xconform_size"] = get_disk_usage(case_dict["xconform_path"])
    case_dict["xconform_dates"] = case_dict["timeseries_dates"]
    case_dict["xconform_total_time"] = 0

    xconform_logs = list()
    xconform_file_pattern = "{0}/xconform.log.*".format(pp_log_dir)
    xconform_logs = glob.glob(xconform_file_pattern)
    if xconform_logs:
        log_file = max(xconform_logs, key=os.path.getctime)
        if is_last_process_complete(
            log_file,
            "Successfully completed converting all files",
            "cesm_conform_generator INFO",
        ):
            case_dict["xconform_status"] = "Succeeded"
            case_dict["xconform_size"] = get_disk_usage(case_dict["xconform_path"])
            with open(log_file, "r") as fname:
                log_content = fname.readlines()
            total_time = [line for line in log_content if "Total Time:" in line]
            if total_time:
                case_dict["xconform_total_time"] = " ".join(total_time[0].split())
        else:
            case_dict["xconform_status"] = "Started"

    return case_dict


# ---------------------------------------------------------------------
def get_run_last_date(casename, run_path):
    # ---------------------------------------------------------------------
    """get_run_last_date
    parse the last cpl.r file in the run_path to retrieve that last date.

    Arguments:
        casename
        run_path - path to run directory
    """
    logger.debug("get_run_last_date")

    pattern = "{0}.cpl.r.*.nc".format(casename)
    cpl_files = sorted(glob.glob(os.path.join(run_path, pattern)))

    if cpl_files:
        _, cpl_file = os.path.split(cpl_files[-1])
        fparts = cpl_file.split(".")
        return fparts[-2]

    return "0000-00-00"


# ---------------------------------------------------------------------
def get_sta_last_date(sta_path):
    # ---------------------------------------------------------------------
    """get_sta_last_date
    parse the last rest directory in the sta_path to retrieve that last date.

    Arguments:
        sta_path - path to run directory
    """
    logger.debug("get_sta_last_date")

    rest_dirs = sorted(glob.glob(os.path.join(sta_path, "rest/*")))

    if rest_dirs:
        _, rest_dir = os.path.split(rest_dirs[-1])
        return rest_dir

    return "0000-00-00"


# ---------------------------------------------------------------------
def get_case_status(case_dict):
    # ---------------------------------------------------------------------
    """get_case_status
    Parse the CaseStatus and postprocessing log files
    looking for status information

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug("get_case_status")

    # initialize status variables
    case_dict["run_status"] = "Unknown"
    case_dict["run_path"] = case_dict["RUNDIR"]
    case_dict["run_size"] = 0
    case_dict["run_last_date"] = case_dict["RUN_STARTDATE"]

    case_dict["sta_status"] = "Unknown"
    case_dict["sta_path"] = case_dict["DOUT_S_ROOT"]
    case_dict["sta_size"] = 0
    case_dict["sta_last_date"] = case_dict["RUN_STARTDATE"]

    cstatus = case_dict["CASEROOT"] + "/CaseStatus"
    if os.path.exists(cstatus):
        # get the run status
        run_status_1 = is_last_process_complete(
            cstatus, "case.run success", "case.run starting"
        )
        run_status_2 = is_last_process_complete(
            cstatus, "model execution success", "model execution starting"
        )
        if run_status_1 is True or run_status_2 is True:
            case_dict["run_status"] = "Succeeded"
        case_dict["run_size"] = get_disk_usage(case_dict["run_path"])
        case_dict["run_last_date"] = get_run_last_date(
            case_dict["CASE"], case_dict["run_path"]
        )

        # get the STA status
        if case_dict["DOUT_S"]:
            # get only the history, rest and logs dir - ignoring the proc subdirs
            sta_status = is_last_process_complete(
                cstatus, "st_archive success", "st_archive starting"
            )
            case_dict["sta_last_date"] = get_sta_last_date(case_dict["DOUT_S_ROOT"])
            if sta_status is True:
                case_dict["sta_status"] = "Succeeded"
            # exclude the proc directories in the sta size estimates
            for subdir in [
                "atm/hist",
                "cpl/hist",
                "esp/hist",
                "ice/hist",
                "glc/hist",
                "lnd/hist",
                "logs",
                "ocn/hist",
                "rest",
                "rof/hist",
                "wav/hist",
                "iac/hist",
            ]:
                path = os.path.join(case_dict["sta_path"], subdir)
                if os.path.isdir(path):
                    case_dict["sta_size"] += get_disk_usage(path)

    # check if the postprocess dir exists in the caseroot
    case_dict["postprocess"] = False
    if os.path.exists(case_dict["CASEROOT"] + "/postprocess"):
        case_dict["postprocess"] = True
        case_dict = get_pp_status(case_dict)

    return case_dict


# ---------------------------------------------------------------------
def check_expdb_case(case_dict, username, password):
    # ---------------------------------------------------------------------
    """check_exp_case
    Cross check the casename with the database for a CMIP6 experiment

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        username (string) - SVN developer's username
        password (string) - SVN developer's password

    Return case_id value; 0 if does not exist or > 0 for exists.

    """
    logger.debug("check_expdb_case")
    data_dict = {
        "casename": case_dict["CASE"],
        "queryType": "checkCaseExists",
        "expType": case_dict["expType"],
    }
    data = json.dumps(data_dict)
    params = urllib.parse.urlencode(
        dict(username=username, password=password, data=data)
    )
    try:
        response = urllib.request.urlopen(
            case_dict["query_expdb_url"], params, context=_context
        )
        output = json.loads(response.read().decode())
    except urllib.error.HTTPError as http_e:
        logger.info('ERROR archive_metadata HTTP post failed "%s"', http_e.code)
        sys.exit(1)
    except urllib.error.URLError as url_e:
        logger.info('ERROR archive_metadata URL failed "%s"', url_e.reason)
        sys.exit(1)

    return int(output["case_id"])


# ---------------------------------------------------------------------
def query_expdb_cmip6(case_dict, username, password):
    # ---------------------------------------------------------------------
    """query_exp_case
    Query the expdb for CMIP6 casename = case_dict['q_casename'] metadata.
    Write out a json file to case_dict['q_outfile'].

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        username (string) - SVN developer's username
        password (string) - SVN developer's password

    """
    logger.debug("query_expdb_cmip6")
    exists = False
    data_dict = {
        "casename": case_dict["q_casename"],
        "queryType": "CMIP6GlobalAtts",
        "expType": "CMIP6",
    }
    data = json.dumps(data_dict)
    params = urllib.parse.urlencode(
        dict(username=username, password=password, data=data)
    )
    try:
        response = urllib.request.urlopen(
            case_dict["query_expdb_url"], params, context=_context
        )
        output = json.load(response)
    except urllib.error.HTTPError as http_e:
        logger.info('ERROR archive_metadata HTTP post failed "%s"', http_e.code)
    except urllib.error.URLError as url_e:
        logger.info('ERROR archive_metadata URL failed "%s"', url_e.reason)

    if output:
        if not os.path.exists("{0}/archive_files".format(case_dict["workdir"])):
            os.makedirs("{0}/archive_files".format(case_dict["workdir"]))

        filename = "{0}/archive_files/{1}".format(
            case_dict["workdir"], case_dict["q_outfile"]
        )
        with io.open(filename, "w+", encoding="utf-8") as fname:
            fname.write(json.dumps(output, ensure_ascii=False))
            fname.close()
        exists = True

    return exists


# ---------------------------------------------------------------------
def create_json(case_dict):
    # ---------------------------------------------------------------------
    """create_json
    Create a JSON file in the caseroot/archive_files dir.

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug("create_json")

    if not os.path.exists("{0}/archive_files".format(case_dict["workdir"])):
        os.makedirs("{0}/archive_files".format(case_dict["workdir"]))

    filename = "{0}/archive_files/json.{1}".format(
        case_dict["workdir"], datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    )
    with io.open(filename, "wb") as fname:
        jstr = str(json.dumps(case_dict, indent=4, sort_keys=True, ensure_ascii=False))
        if isinstance(jstr, str):
            jstr = jstr.decode("utf-8")
        fname.write(jstr)
        fname.close()


# ---------------------------------------------------------------------
def post_json(case_dict, username, password):
    # ---------------------------------------------------------------------
    """post_json
    Post a JSON file in the caseroot/archive_files to the
    remote expdb URL.

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        username (string) - SVN developers username
        password (string) - SVN developers password
    """
    logger.debug("post_json")

    case_dict["COMPSET"] = urllib.parse.quote(case_dict["COMPSET"])
    case_dict["GRID"] = urllib.parse.quote(case_dict["GRID"])
    data = json.dumps(case_dict)
    params = urllib.parse.urlencode(
        dict(username=username, password=password, data=data)
    )
    try:
        urllib.request.urlopen(case_dict["json_expdb_url"], params, context=_context)
    except urllib.error.HTTPError as http_e:
        logger.info('ERROR archive_metadata HTTP post failed "%s"', http_e.code)
    except urllib.error.URLError as url_e:
        logger.info('ERROR archive_metadata URL failed "%s"', url_e.reason)


# ---------------------------------------------------------------------
def check_svn():
    # ---------------------------------------------------------------------
    """check_svn

    make sure svn client is installed and accessible
    """
    logger.debug("check_svn")

    cmd = ["svn", "--version"]
    svn_exists = True
    result = ""
    try:
        result = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as error:
        msg = _svn_error_template.substitute(
            function="check_svn", cmd=cmd, error=error.returncode, strerror=error.output
        )
        svn_exists = False
        logger.info(msg)
        raise SVNException(msg)

    if "version" not in result:
        msg = "SVN is not available. Ignoring SVN update"
        svn_exists = False
        raise SVNException(msg)

    return svn_exists


# ---------------------------------------------------------------------
def create_temp_archive(case_dict):
    # ---------------------------------------------------------------------
    """create_temp_archive

    Create a temporary SVN sandbox directory in the current caseroot
    """
    archive_temp_dir = "{0}/archive_temp_dir".format(case_dict["workdir"])
    logger.debug("create_temp_archive %s", archive_temp_dir)

    if not os.path.exists(archive_temp_dir):
        os.makedirs(archive_temp_dir)
    else:
        logger.info(
            "ERROR archive_metadata archive_temp_dir already exists. exiting..."
        )
        sys.exit(1)

    return archive_temp_dir


# ---------------------------------------------------------------------
def check_svn_repo(case_dict, username, password):
    # ---------------------------------------------------------------------
    """check_svn_repo

    check if a SVN repo exists for this case
    """
    logger.debug("check_svn_repo")

    repo_exists = False
    svn_repo = "{0}/trunk".format(case_dict["svn_repo_url"])
    cmd = ["svn", "list", svn_repo, "--username", username, "--password", password]
    result = ""
    try:
        result = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        msg = "SVN repo does not exist for this case. A new one will be created."
        logger.warning(msg)

    if re.search("README.archive", result):
        repo_exists = True

    return repo_exists


# ---------------------------------------------------------------------
def get_trunk_tag(case_dict, username, password):
    # ---------------------------------------------------------------------
    """get_trunk_tag

    return the most recent trunk tag as an integer
    """
    logger.debug("get_trunk_tag")

    tag = 0
    svn_repo = "{0}/trunk_tags".format(case_dict["svn_repo_url"])
    cmd = ["svn", "list", svn_repo, "--username", username, "--password", password]
    result = ""
    try:
        result = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = [
            "svn",
            "list",
            svn_repo,
            "--username",
            username,
            "--password",
            "******",
        ]
        msg = _call_template.substitute(
            function="get_trunk_tag",
            cmd=cmd_nopasswd,
            error=error.returncode,
            strerror=error.output,
        )
        logger.warning(msg)
        raise SVNException(msg)

    if result:
        last_tag = [i for i in result.split("\n") if i][-1]
        last_tag = last_tag[:-1].split("_")[-1]
        tag = int(last_tag.lstrip("0"))

    return tag


# ---------------------------------------------------------------------
def checkout_repo(case_dict, username, password):
    # ---------------------------------------------------------------------
    """checkout_repo

    checkout the repo into the archive_temp_dir
    """
    logger.debug("checkout_repo")

    os.chdir(case_dict["archive_temp_dir"])
    svn_repo = "{0}/trunk".format(case_dict["svn_repo_url"])
    cmd = ["svn", "co", "--username", username, "--password", password, svn_repo, "."]
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = [
            "svn",
            "co",
            "--username",
            username,
            "--password",
            "******",
            svn_repo,
            ".",
        ]
        msg = _call_template.substitute(
            function="checkout_repo",
            cmd=cmd_nopasswd,
            error=error.returncode,
            strerror=error.output,
        )
        logger.warning(msg)
        raise SVNException(msg)

    os.chdir(case_dict["CASEROOT"])


# ---------------------------------------------------------------------
def create_readme(case_dict):
    # ---------------------------------------------------------------------
    """create_readme

    Create a generic README.archive file
    """
    logger.debug("create_readme")
    os.chdir(case_dict["archive_temp_dir"])

    fname = open("README.archive", "w")
    fname.write("Archived metadata is available for this case at URL:\n")
    fname.write(case_dict["base_expdb_url"])
    fname.close()


# ---------------------------------------------------------------------
def update_repo_add_file(filename, dir1, dir2):
    # ---------------------------------------------------------------------
    """update_repo_add_file

    Add a file to the SVN repository
    """
    src = os.path.join(dir1, filename)
    dest = os.path.join(dir2, filename)
    logger.debug("left_only: " + src + " -> " + dest)
    if not os.path.exists(dest):
        shutil.copy2(src, dest)
        cmd = ["svn", "add", "--parents", dest]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            msg = _call_template.substitute(
                function="update_lcoal_repo",
                cmd=cmd,
                error=error.returncode,
                strerror=error.output,
            )
            logger.warning(msg)
            raise SVNException(msg)


# ---------------------------------------------------------------------
def update_repo_rm_file(filename, dir1, dir2):
    # ---------------------------------------------------------------------
    """update_repo_rm_file

    Remove a file from the SVN repository
    """
    src = os.path.join(dir2, filename)
    dest = os.path.join(dir1, filename)
    logger.debug("right_only: " + src + " -> " + dest)
    if os.path.exists(dest):
        cmd = ["svn", "rm", dest]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            msg = _call_template.substitute(
                function="update_lcoal_repo",
                cmd=cmd,
                error=error.returncode,
                strerror=error.output,
            )
            logger.warning(msg)
            raise SVNException(msg)


# ---------------------------------------------------------------------
def update_repo_copy_file(filename, dir1, dir2):
    # ---------------------------------------------------------------------
    """update_repo_copy_file

    Copy a file into the SVN local repo
    """
    src = os.path.join(dir1, filename)
    dest = os.path.join(dir2, filename)
    shutil.copy2(src, dest)


# ---------------------------------------------------------------------
def compare_dir_trees(dir1, dir2, archive_list):
    # ---------------------------------------------------------------------
    """compare_dir_trees

    Compare two directories recursively. Files in each directory are
    assumed to be equal if their names and contents are equal.
    """
    xml_files = glob.glob(os.path.join(dir1, "*.xml"))
    user_nl_files = glob.glob(os.path.join(dir1, "user_nl_*"))
    dirs_cmp = filecmp.dircmp(dir1, dir2, _ignore_patterns)

    left_only = [
        fn
        for fn in dirs_cmp.left_only
        if not os.path.islink(fn)
        and (fn in xml_files or fn in user_nl_files or fn in archive_list)
    ]
    right_only = [
        fn
        for fn in dirs_cmp.right_only
        if not os.path.islink(fn)
        and (fn in xml_files or fn in user_nl_files or fn in archive_list)
    ]
    funny_files = [
        fn
        for fn in dirs_cmp.funny_files
        if not os.path.islink(fn)
        and (fn in xml_files or fn in user_nl_files or fn in archive_list)
    ]

    # files and directories need to be added to svn repo from the caseroot
    if left_only:
        for filename in left_only:
            if os.path.isfile(os.path.join(dir1, filename)) and filename[-1] != "~":
                update_repo_add_file(filename, dir1, dir2)
            else:
                new_dir1 = os.path.join(dir1, filename)
                new_dir2 = os.path.join(dir2, filename)
                os.makedirs(new_dir2)
                cmd = ["svn", "add", new_dir2]
                try:
                    subprocess.check_call(cmd)
                except subprocess.CalledProcessError as error:
                    msg = _call_template.substitute(
                        function="update_lcoal_repo",
                        cmd=cmd,
                        error=error.returncode,
                        strerror=error.output,
                    )
                    logger.warning(msg)
                    raise SVNException(msg)

                # recurse through this new subdir
                new_archive_list = [filename]
                compare_dir_trees(new_dir1, new_dir2, new_archive_list)

    # files need to be removed from svn repo that are no longer in the caseroot
    if right_only:
        for filename in right_only:
            if os.path.isfile(os.path.join(dir1, filename)) and filename[-1] != "~":
                update_repo_rm_file(filename, dir1, dir2)

    # files are the same but could not be compared so copy the caseroot version
    if funny_files:
        for filename in funny_files:
            if os.path.isfile(os.path.join(dir1, filename)) and filename[-1] != "~":
                update_repo_copy_file(filename, dir1, dir2)

    # common files have changed in the caseroot and need to be copied to the svn repo
    (_, mismatch, errors) = filecmp.cmpfiles(
        dir1, dir2, dirs_cmp.common_files, shallow=False
    )
    if mismatch:
        for filename in mismatch:
            if os.path.isfile(os.path.join(dir1, filename)) and filename[-1] != "~":
                update_repo_copy_file(filename, dir1, dir2)

    # error in file comparison so copy the caseroot file to the svn repo
    if errors:
        for filename in errors:
            if os.path.isfile(os.path.join(dir1, filename)) and filename[-1] != "~":
                update_repo_copy_file(filename, dir1, dir2)

    # recurse through the subdirs
    common_dirs = dirs_cmp.common_dirs
    if common_dirs:
        for common_dir in common_dirs:
            if common_dir in archive_list:
                new_dir1 = os.path.join(dir1, common_dir)
                new_dir2 = os.path.join(dir2, common_dir)
                compare_dir_trees(new_dir1, new_dir2, archive_list)
    else:
        return


# ---------------------------------------------------------------------
def update_local_repo(case_dict, ignore_logs, ignore_timing):
    # ---------------------------------------------------------------------
    """update_local_repo

    Compare and update local SVN sandbox
    """
    logger.debug("update_local_repo")
    from_dir = case_dict["CASEROOT"]
    to_dir = case_dict["archive_temp_dir"]

    compare_dir_trees(from_dir, to_dir, case_dict["archive_list"])

    # check if ignore_logs is specified
    if ignore_logs:
        os.chdir(to_dir)
        if os.path.isdir("./logs"):
            try:
                shutil.rmtree("./logs")
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "logs" in archive dir.'
                )

            cmd = ["svn", "delete", "./logs"]
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(
                    function="update_lcoal_repo",
                    cmd=cmd,
                    error=error.returncode,
                    strerror=error.output,
                )
                logger.warning(msg)
                raise SVNException(msg)

        if os.path.isdir("./postprocess/logs"):
            os.chdir("./postprocess")
            try:
                shutil.rmtree("./logs")
            except OSError:
                logger.warning(
                    'in "update_local_repo" - '
                    'Unable to remove "postprocess/logs" in archive dir.'
                )

            cmd = ["svn", "delete", "./logs"]
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(
                    function="update_lcoal_repo",
                    cmd=cmd,
                    error=error.returncode,
                    strerror=error.output,
                )
                logger.warning(msg)
                raise SVNException(msg)
    else:
        # add log files
        if os.path.exists("{0}/logs".format(from_dir)):
            if not os.path.exists("{0}/logs".format(to_dir)):
                os.makedirs("{0}/logs".format(to_dir))
            os.chdir(os.path.join(from_dir, "logs"))
            for filename in glob.glob("*.*"):
                update_repo_add_file(
                    filename,
                    os.path.join(from_dir, "logs"),
                    os.path.join(to_dir, "logs"),
                )

        if os.path.exists("{0}/postprocess/logs".format(from_dir)):
            if not os.path.exists("{0}/postprocess/logs".format(to_dir)):
                os.makedirs("{0}/postprocess/logs".format(to_dir))
            os.chdir(os.path.join(from_dir, "postprocess/logs"))
            for filename in glob.glob("*.*"):
                update_repo_add_file(
                    filename,
                    os.path.join(from_dir, "postprocess", "logs"),
                    os.path.join(to_dir, "postprocess", "logs"),
                )

    # check if ignore_timing is specified
    if ignore_timing:
        os.chdir(case_dict["archive_temp_dir"])
        if os.path.isdir("./timing"):
            try:
                shutil.rmtree("./timing")
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "timing" in archive dir.'
                )

            cmd = ["svn", "delete", "./timing"]
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(
                    function="update_lcoal_repo",
                    cmd=cmd,
                    error=error.returncode,
                    strerror=error.output,
                )
                logger.warning(msg)
                raise SVNException(msg)
    else:
        # add timing files
        if os.path.exists("{0}/timing".format(from_dir)):
            if not os.path.exists("{0}/timing".format(to_dir)):
                os.makedirs("{0}/timing".format(to_dir))
            os.chdir(os.path.join(from_dir, "timing"))
            for filename in glob.glob("*.*"):
                update_repo_add_file(
                    filename,
                    os.path.join(from_dir, "timing"),
                    os.path.join(to_dir, "timing"),
                )


# ---------------------------------------------------------------------
def populate_local_repo(case_dict, ignore_logs, ignore_timing):
    # ---------------------------------------------------------------------
    """populate_local_repo

    Populate local SVN sandbox
    """
    logger.debug("populate_local_repo")
    os.chdir(case_dict["CASEROOT"])

    # loop through the archive_list and copy to the temp archive dir
    for archive in case_dict["archive_list"]:
        if os.path.exists(archive):
            if os.path.isdir(archive):
                try:
                    target = case_dict["archive_temp_dir"] + "/" + archive
                    shutil.copytree(
                        archive,
                        target,
                        symlinks=False,
                        ignore=shutil.ignore_patterns(*_ignore_patterns),
                    )
                except OSError as error:
                    msg = _copy_template.substitute(
                        function="populate_local_repo",
                        source=archive,
                        dest=case_dict["archive_temp_dir"],
                        error=error.errno,
                        strerror=error.strerror,
                    )
                    logger.warning(msg)
            else:
                try:
                    shutil.copy2(archive, case_dict["archive_temp_dir"])
                except OSError as error:
                    msg = _copy_template.substitute(
                        function="populate_local_repo",
                        source=archive,
                        dest=case_dict["archive_temp_dir"],
                        error=error.errno,
                        strerror=error.strerror,
                    )
                    logger.warning(msg)

    # add files with .xml as the suffix
    xml_files = glob.glob("*.xml")
    for xml_file in xml_files:
        if os.path.isfile(xml_file):
            try:
                shutil.copy2(xml_file, case_dict["archive_temp_dir"])
            except OSError as error:
                msg = _copy_template.substitute(
                    function="populate_local_repo",
                    source=xml_file,
                    dest=case_dict["archive_temp_dir"],
                    error=error.errno,
                    strerror=error.strerror,
                )
                logger.warning(msg)

    # add files with .xml as the suffix from the postprocess directory
    if os.path.isdir("./postprocess"):
        pp_path = "{0}/{1}".format(case_dict["archive_temp_dir"], "postprocess")
        if not os.path.exists(pp_path):
            os.mkdir(pp_path)
        xml_files = glob.glob("./postprocess/*.xml")
        for xml_file in xml_files:
            if os.path.isfile(xml_file):
                try:
                    shutil.copy2(xml_file, pp_path)
                except OSError as error:
                    msg = _copy_template.substitute(
                        function="populate_local_repo",
                        source=xml_file,
                        dest=case_dict["archive_temp_dir"],
                        error=error.errno,
                        strerror=error.strerror,
                    )
                    logger.warning(msg)

    # add files with user_nl_ as the prefix
    user_files = glob.glob("user_nl_*")
    for user_file in user_files:
        if os.path.isfile(user_file):
            try:
                shutil.copy2(user_file, case_dict["archive_temp_dir"])
            except OSError as error:
                msg = _copy_template.substitute(
                    function="populate_local_repo",
                    source=user_file,
                    dest=case_dict["archive_temp_dir"],
                    error=error.errno,
                    strerror=error.strerror,
                )
                logger.warning(msg)

    # add files with Depends as the prefix
    conf_files = glob.glob("Depends.*")
    for conf_file in conf_files:
        if os.path.isfile(conf_file):
            try:
                shutil.copy2(conf_file, case_dict["archive_temp_dir"])
            except OSError as error:
                msg = _copy_template.substitute(
                    function="populate_local_repo",
                    source=conf_file,
                    dest=case_dict["archive_temp_dir"],
                    error=error.errno,
                    strerror=error.strerror,
                )
                logger.warning(msg)

    # check if ignore_logs is specified
    if ignore_logs:
        os.chdir(case_dict["archive_temp_dir"])
        if os.path.isdir("./logs"):
            try:
                shutil.rmtree("./logs")
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "logs" in archive_temp_dir.'
                )
        if os.path.isdir("./postprocess/logs"):
            os.chdir("./postprocess")
            try:
                shutil.rmtree("./logs")
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - '
                    'Unable to remove "postprocess/logs" in archive_temp_dir.'
                )
        os.chdir(case_dict["CASEROOT"])

    # check if ignore_timing is specified
    if ignore_timing:
        os.chdir(case_dict["archive_temp_dir"])
        if os.path.isdir("./timing"):
            try:
                shutil.rmtree("./timing")
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "timing" in archive_temp_dir.'
                )
        os.chdir(case_dict["CASEROOT"])


# ---------------------------------------------------------------------
def checkin_trunk(case_dict, svn_cmd, message, username, password):
    # ---------------------------------------------------------------------
    """checkin_trunk

    Check in the local SVN sandbox to the remote trunk
    """
    logger.debug("checkin_trunk")

    os.chdir(case_dict["archive_temp_dir"])
    svn_repo = "{0}/trunk".format(case_dict["svn_repo_url"])
    msg = '"{0}"'.format(message)
    cmd = [
        "svn",
        svn_cmd,
        "--username",
        username,
        "--password",
        password,
        ".",
        "--message",
        msg,
    ]

    if svn_cmd in ["import"]:
        # create the trunk dir
        msg = '"create trunk"'
        cmd = [
            "svn",
            "mkdir",
            "--parents",
            svn_repo,
            "--username",
            username,
            "--password",
            password,
            "--message",
            msg,
        ]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            cmd_nopasswd = [
                "svn",
                "mkdir",
                "--parents",
                svn_repo,
                "--username",
                username,
                "--password",
                "******",
                "--message",
                msg,
            ]
            msg = _call_template.substitute(
                function="checkin_trunk",
                cmd=cmd_nopasswd,
                error=error.returncode,
                strerror=error.output,
            )
            logger.warning(msg)
            raise SVNException(msg)

        # create the trunk_tags dir
        tags = "{0}/trunk_tags".format(case_dict["svn_repo_url"])
        msg = '"create trunk_tags"'
        cmd = [
            "svn",
            "mkdir",
            tags,
            "--username",
            username,
            "--password",
            password,
            "--message",
            msg,
        ]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            cmd_nopasswd = [
                "svn",
                "mkdir",
                tags,
                "--username",
                username,
                "--password",
                "******",
                "--message",
                msg,
            ]
            msg = _call_template.substitute(
                function="checkin_trunk",
                cmd=cmd_nopasswd,
                error=error.returncode,
                strerror=error.output,
            )
            logger.warning(msg)
            raise SVNException(msg)

        msg = '"{0}"'.format(message)
        cmd = [
            "svn",
            svn_cmd,
            "--username",
            username,
            "--password",
            password,
            ".",
            svn_repo,
            "--message",
            msg,
        ]

    # check-in the trunk to svn
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = [
            "svn",
            svn_cmd,
            "--username",
            username,
            "--password",
            "******",
            ".",
            "--message",
            msg,
        ]
        msg = _call_template.substitute(
            function="checkin_trunk",
            cmd=cmd_nopasswd,
            error=error.returncode,
            strerror=error.output,
        )
        logger.warning(msg)
        raise SVNException(msg)


# ---------------------------------------------------------------------
def create_tag(case_dict, new_tag, username, password):
    # ---------------------------------------------------------------------
    """create_tag

    create a new trunk tag in the remote repo
    """
    logger.debug("create_tag")

    # create a new trunk tag
    os.chdir(case_dict["archive_temp_dir"])
    svn_repo = "{0}/trunk".format(case_dict["svn_repo_url"])
    svn_repo_tag = "{0}/trunk_tags/{1}".format(case_dict["svn_repo_url"], new_tag)
    msg = '"create new trunk tag"'
    cmd = [
        "svn",
        "copy",
        "--username",
        username,
        "--password",
        password,
        svn_repo,
        svn_repo_tag,
        "--message",
        msg,
    ]
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = [
            "svn",
            "copy",
            "--username",
            username,
            "--password",
            "******",
            svn_repo,
            svn_repo_tag,
            "--message",
            msg,
        ]
        msg = _call_template.substitute(
            function="checkin_trunk",
            cmd=cmd_nopasswd,
            error=error.returncode,
            strerror=error.output,
        )
        logger.warning(msg)
        raise SVNException(msg)


# -------------------------------------------------------------------------
def update_repo(ignore_logs, ignore_timing, case_dict, username, password):
    # -------------------------------------------------------------------------
    """update_repo

    Update SVN repo
    """
    logger.debug("update_repo")

    try:
        # check if svn client is installed
        svn_exists = check_svn()

        if svn_exists:
            # check if the case repo exists
            case_dict["svn_repo_url"] = "{0}/{1}".format(
                _svn_expdb_url, case_dict["CASE"]
            )
            repo_exists = check_svn_repo(case_dict, username, password)
            case_dict["archive_temp_dir"] = create_temp_archive(case_dict)
            case_dict["archive_list"] = _archive_list + case_dict["user_add_files"]

            if repo_exists:
                # update trunk and make a new tag
                last_tag = get_trunk_tag(case_dict, username, password)
                new_tag = "{0}_{1}".format(
                    case_dict["CASE"], str(last_tag + 1).zfill(4)
                )
                checkout_repo(case_dict, username, password)
                update_local_repo(case_dict, ignore_logs, ignore_timing)
                msg = "update case metadata for {0} by {1}".format(
                    case_dict["CASE"], username
                )
                checkin_trunk(case_dict, "ci", msg, username, password)
                create_tag(case_dict, new_tag, username, password)
                logger.info(
                    'SVN repository trunk updated at URL "%s"',
                    case_dict["svn_repo_url"],
                )
                logger.info(' and a new trunk tag created "%s"', new_tag)
            else:
                # create a new case repo
                new_tag = "{0}_0001".format(case_dict["CASE"])
                create_readme(case_dict)
                populate_local_repo(case_dict, ignore_logs, ignore_timing)
                msg = "initial import of case metadata for {0} by {1}".format(
                    case_dict["CASE"], username
                )
                checkin_trunk(case_dict, "import", msg, username, password)
                create_tag(case_dict, new_tag, username, password)
                logger.info(
                    'SVN repository imported to trunk URL "%s"',
                    case_dict["svn_repo_url"],
                )
                logger.info('   and a new trunk tag created for  "%s"', new_tag)

    except SVNException:
        pass

    return case_dict


# ---------------------------------------------------------------------
def get_timing_data(case_dict):
    # ---------------------------------------------------------------------
    """get_timing_data
    parse the timing data file and add information to the case_dict

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug("get_timing_data")

    # initialize the timing values in the dictionary
    case_dict["model_cost"] = "undefined"
    case_dict["model_throughput"] = "undefined"

    timing_dir = case_dict["CASEROOT"] + "/timing"
    last_time = ""
    if os.path.exists(timing_dir):
        # check if timing files exists
        timing_file_pattern = "cesm_timing." + case_dict["CASE"]
        last_time = max(
            glob.glob(timing_dir + "/" + timing_file_pattern + ".*"),
            key=os.path.getctime,
        )
        if last_time:
            if "gz" in last_time:
                # gunzip file first
                with gzip.open(last_time, "rb") as fname:
                    file_content = fname.readlines()
            else:
                with open(last_time, "r") as fname:
                    file_content = fname.readlines()

            # search the file content for matching lines
            model_cost = [line for line in file_content if "Model Cost:" in line]
            model_throughput = [
                line for line in file_content if "Model Throughput:" in line
            ]

            case_dict["model_cost"] = " ".join(model_cost[0].split())
            case_dict["model_throughput"] = " ".join(model_throughput[0].split())

    return case_dict


# ---------------------------------------------------------------------
def initialize_main(options):
    # ---------------------------------------------------------------------
    """initialize_main

    Initialize the case dictionary data structure with command line options
    """
    logger.debug("intialize_main")

    case_dict = dict()

    case_dict["CASEROOT"] = os.getcwd()
    if options.caseroot:
        case_dict["CASEROOT"] = options.caseroot[0]

    case_dict["workdir"] = case_dict["CASEROOT"]
    if options.workdir:
        case_dict["workdir"] = options.workdir[0]

    username = None
    if options.user:
        username = options.user
        case_dict["svnlogin"] = username

    password = None
    if options.password:
        password = options.password

    if options.expType:
        case_dict["expType"] = options.expType[0]

    case_dict["title"] = None
    if options.title:
        case_dict["title"] = options.title[0]

    case_dict["dryrun"] = False
    if options.dryrun:
        case_dict["dryrun"] = True

    case_dict["archive_temp_dir"] = ""

    case_dict["user_add_files"] = list()
    if options.user_add_files:
        case_dict["user_add_files"] = options.user_add_files.split(",")

    case_dict["q_casename"] = ""
    case_dict["q_outfile"] = ""
    if options.query_cmip6:
        case_dict["q_casename"] = options.query_cmip6[0]
        case_dict["q_outfile"] = options.query_cmip6[1]

    case_dict["base_expdb_url"] = "https://csegweb.cgd.ucar.edu/expdb2.0"
    if options.test_post:
        case_dict["base_expdb_url"] = "https://csegwebdev.cgd.ucar.edu/expdb2.0"
    case_dict["json_expdb_url"] = (
        case_dict["base_expdb_url"] + "/cgi-bin/processJSON.cgi"
    )
    case_dict["query_expdb_url"] = case_dict["base_expdb_url"] + "/cgi-bin/query.cgi"

    return case_dict, username, password


# ---------------------------------------------------------------------
def main_func(options):
    # ---------------------------------------------------------------------
    """main function

    Arguments:
        options (list) - input options from command line
    """
    logger.debug("main_func")

    (case_dict, username, password) = initialize_main(options)

    # check if query_cmip6 argument is specified
    if options.query_cmip6:
        if case_dict["dryrun"]:
            logger.info("Dryrun - calling query_expdb_cmip6 for case metadata")
        else:
            if query_expdb_cmip6(case_dict, username, password):
                logger.info(
                    'Casename "%s" CMIP6 global attribute '
                    'metadata written to "%s/archive_files/%s" '
                    'from "%s"',
                    case_dict["workdir"],
                    case_dict["q_casename"],
                    case_dict["q_outfile"],
                    case_dict["query_expdb_url"],
                )
                logger.info("Successful completion of archive_metadata")
                sys.exit(0)
            else:
                logger.info(
                    'ERROR archive_metadata failed to find "%s" '
                    'in experiments database at "%s".',
                    case_dict["q_casename"],
                    case_dict["query_expdb_url"],
                )
                sys.exit(1)

    # loop through the _xml_vars gathering values
    with Case(case_dict["CASEROOT"], read_only=True) as case:
        if case_dict["dryrun"]:
            logger.info("Dryrun - calling get_case_vars")
        else:
            case_dict = get_case_vars(case_dict, case)

    # check reserved casename expdb for CMIP6 experiments
    if case_dict["expType"].lower() == "cmip6":
        if case_dict["dryrun"]:
            logger.info(
                "Dryrun - calling check_expdb_case for CMIP6 experiment reservation"
            )
        else:
            case_dict["case_id"] = check_expdb_case(case_dict, username, password)
            if case_dict["case_id"] < 1:
                logger.info(
                    "Unable to archive CMIP6 metadata. "
                    '"%s" casename does not exist in database. '
                    "All CMIP6 experiments casenames must be "
                    "reserved in the experiments database at URL: "
                    "https://csegweb.cgd.ucar.edu/expdb2.0 "
                    "prior to running archive_metadata.",
                    case_dict["CASE"],
                )
                sys.exit(1)

    # get the case status into the case_dict
    if case_dict["dryrun"]:
        logger.info("Dryrun - calling get_case_status")
    else:
        case_dict = get_case_status(case_dict)

    # create / update the cesm expdb repo with the caseroot files
    if not options.ignore_repo_update:
        if case_dict["dryrun"]:
            logger.info("Dryrun - calling update_repo")
        else:
            case_dict = update_repo(
                options.ignore_logs,
                options.ignore_timing,
                case_dict,
                username,
                password,
            )

    # parse the timing data into the case_dict
    if not options.ignore_timing:
        if case_dict["dryrun"]:
            logger.info("Dryrun - calling get_timing_data")
        else:
            case_dict = get_timing_data(case_dict)

    # Create a JSON file containing the case_dict with the date appended to the filename
    if case_dict["dryrun"]:
        logger.info("Dryrun - calling create_json")
    else:
        create_json(case_dict)

    # post the JSON to the remote DB
    if case_dict["dryrun"]:
        logger.info("Dryrun - calling post_json")
    else:
        post_json(case_dict, username, password)

    # clean-up the temporary archive files dir
    if case_dict["dryrun"]:
        logger.info('Dryrun - deleting "./archive_temp_dir"')
    else:
        if not options.ignore_repo_update and os.path.exists(
            case_dict["archive_temp_dir"]
        ):
            shutil.rmtree(case_dict["archive_temp_dir"])

    logger.info("Successful completion of archive_metadata")

    return 0


# ===================================
if __name__ == "__main__":

    try:
        __status__ = main_func(commandline_options(sys.argv))
        sys.exit(__status__)
    except Exception as error:
        print("{}".format(str(error)))
        sys.exit(1)

2ef957f1ff3d2b84821a8168c7028e1858a647d8
<mask token>


class PasswordPromptAction(argparse.Action):
    """SVN developer's password class handler"""

    def __init__(self, option_strings=None, dest=None, default=None,
        required=False, nargs=0, help=None):
        super(PasswordPromptAction, self).__init__(option_strings=
            option_strings, dest=dest, default=default, required=required,
            nargs=nargs, help=help)

    def __call__(self, parser, args, values, option_string=None):
        home = expanduser('~')
        conf_path = os.path.join(home, '.subversion/cmip6.conf')
        if os.path.exists(conf_path):
            config = configparser.SafeConfigParser()
            config.read(conf_path)
            password = config.get('svn', 'password')
        else:
            password = getpass.getpass()
        setattr(args, self.dest, password)


def basic_authorization(user, password):
    """Basic authentication encoding"""
    sauth = user + ':' + password
    return 'Basic ' + sauth.encode('base64').rstrip()


class SVNException(Exception):
    """SVN command exception handler"""

    def __init__(self, value):
        super(SVNException, self).__init__(value)
        self.value = value

    def __str__(self):
        return repr(self.value)


def commandline_options(args):
    """Process the command line arguments."""
    parser = argparse.ArgumentParser(description=
        'Query and parse the caseroot files to gather metadata information that can be posted to the CESM experiments database.  CMIP6 experiment case names must be reserved already in the experiment database. Please see: https://csesgweb.cgd.ucar.edu/expdb2.0 for details.'
        )
    CIME.utils.setup_standard_logging_options(parser)
    parser.add_argument('--user', dest='user', type=str, default=None,
        required=True, help=
        'User name for SVN CESM developer access (required)')
    parser.add_argument('--password', dest='password', action=
        PasswordPromptAction, default='', required=True, help=
        'Password for SVN CESM developer access (required)')
    parser.add_argument('--caseroot', nargs=1, required=False, help=
        'Fully quailfied path to case root directory (optional). Defaults to current working directory.'
        )
    parser.add_argument('--workdir', nargs=1, required=False, help=
        'Fully quailfied path to directory for storing intermediate case files. A sub-directory called archive_temp_dir is created, populated with case files, and posted to the CESM experiments database and SVN repository at URL "{0}". This argument can be used to archive a caseroot when the user does not have write permission in the caseroot (optional). Defaults to current working directory.'
        .format(_svn_expdb_url))
    parser.add_argument('--expType', dest='expType', nargs=1, required=True,
        choices=_exp_types, help=
        'Experiment type. For CMIP6 experiments, the case must already exist in the experiments database at URL  "http://csegweb.cgd.ucar.edu/expdb2.0" (required). Must be one of "{0}"'
        .format(_exp_types))
    parser.add_argument('--title', nargs=1, required=False, default=None,
        help='Title of experiment (optional).')
    parser.add_argument('--ignore-logs', dest='ignore_logs', action=
        'store_true', help=
        'Ignore updating the SVN repository with the caseroot/logs files. The experiments database will be updated (optional).'
        )
    parser.add_argument('--ignore-timing', dest='ignore_timing', action=
        'store_true', help=
        'Ignore updating the the SVN repository with caseroot/timing files.The experiments database will be updated (optional).'
        )
    parser.add_argument('--ignore-repo-update', dest='ignore_repo_update',
        action='store_true', help=
        'Ignore updating the SVN repository with all the caseroot files. The experiments database will be updated (optional).'
        )
    parser.add_argument('--add-files', dest='user_add_files', required=
        False, help=
        'Comma-separated list with no spaces of files or directories to be added to the SVN repository. These are in addition to the default added caseroot files and directories: "{0}, *.xml, user_nl_*" (optional).'
        .format(_archive_list))
    parser.add_argument('--dryrun', action='store_true', help=
        'Parse settings and print what actions will be taken but do not execute the action (optional).'
        )
    parser.add_argument('--query_cmip6', nargs=2, required=False, help=
        'Query the experiments database global attributes for specified CMIP6 casename as argument 1. Writes a json formatted output file, specified by argument 2, to subdir archive_files (optional).'
        )
    parser.add_argument('--test-post', dest='test_post', action=
        'store_true', help=
        'Post metadata to the test expdb2.0 web application server at URL "http://csegwebdev.cgd.ucar.edu/expdb2.0". No --test-post argument defaults to posting metadata to the production expdb2.0 web application server at URL "http://csegweb.cgd.ucar.edu/expdb2.0" (optional).'
        )
    opts = CIME.utils.parse_args_and_handle_standard_logging_options(args,
        parser)
    return opts


def get_case_vars(case_dict, case):
    """get_case_vars
    loop through the global list of XML vars and get the values
    from the case object into a case dictionary

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        case (object) - case object
    """
    logger.debug('get_case_vars')
    for xml_id in _xml_vars:
        case_dict[xml_id] = case.get_value(xml_id, resolved=True, subgroup=None
            )
    for xml_id in _run_vars:
        case_dict[xml_id] = case.get_value(xml_id, resolved=True, subgroup=
            'case.run')
    return case_dict


<mask token>


def get_ocn_disk_usage(path):
    """get_ocn_disk_usage
    return the total disk usage in bytes for a given path.

    Arguments:
        path - path to start
    """
    logger.debug('get_ocn_disk_usage')
    total_size = 0
    paths = glob.glob(path)
    for path in paths:
        total_size += get_disk_usage(path)
    return int(total_size)


def get_pp_path(pp_dir, process):
    """get_pp_path
    return the XML path for process

    Arguments:
        pp_dir  - path to postprocess directory
        process - process name
    """
    logger.debug('get_pp_path')
    cwd = os.getcwd()
    os.chdir(pp_dir)
    pp_path_var = ''
    if process == 'timeseries':
        pp_path_var = _pp_xml_vars['timeseries']
    elif process == 'xconform':
        pp_path_var = _pp_xml_vars['xconform']
    cmd = ['./pp_config', '--get', pp_path_var, '--value']
    try:
        pp_path = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        msg = "Error executing command = '{0}'".format(cmd)
        logger.warning(msg)
    if len(pp_path) > 2:
        pp_path = pp_path.rstrip()
    else:
        pp_path = ''
    os.chdir(cwd)
    return pp_path


def get_diag_dates(comp, pp_dir):
    """get_diag_dates

    Query the postprocessing env_diags_[comp].xml file to get the model diag
    dates for the given component.
    """
    logger.debug('get_diag_dates')
    cwd = os.getcwd()
    os.chdir(pp_dir)
    model_dates = ''
    pp_vars = _pp_diag_vars.get(comp)
    for pp_var in pp_vars:
        cmd = ['./pp_config', '--get', pp_var, '--value']
        try:
            pp_value = subprocess.check_output(cmd)
        except subprocess.CalledProcessError:
            msg = "Error executing command = '{0}'".format(cmd)
            logger.warning(msg)
        tmp_dates = '{0} = {1}'.format(pp_var, pp_value)
        model_dates = model_dates + tmp_dates
    os.chdir(cwd)
    return model_dates


def get_pp_status(case_dict):
    """get_pp_status
    Parse the postprocessing log files
    looking for status information

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug('get_pp_status')
    msg_avg = dict()
    msg_diags = dict()
    diag_comps = ['atm', 'ice', 'lnd', 'ocn']
    pp_dir = os.path.join(case_dict['CASEROOT'], 'postprocess')
    pp_log_dir = os.path.join(case_dict['CASEROOT'], 'postprocess', 'logs')
    msg_avg['atm'] = 'COMPLETED SUCCESSFULLY'
    msg_diags['atm'
        ] = 'Successfully completed generating atmosphere diagnostics'
    case_dict['atm_avg_dates'] = case_dict['atm_diag_dates'] = get_diag_dates(
        'atm', pp_dir)
    msg_avg['ice'
        ] = 'Successfully completed generating ice climatology averages'
    msg_diags['ice'] = 'Successfully completed generating ice diagnostics'
    case_dict['ice_avg_dates'] = case_dict['ice_diag_dates'] = get_diag_dates(
        'ice', pp_dir)
    msg_avg['lnd'] = 'COMPLETED SUCCESSFULLY'
    msg_diags['lnd'] = 'Successfully completed generating land diagnostics'
    case_dict['lnd_avg_dates'] = case_dict['lnd_diag_dates'] = get_diag_dates(
        'lnd', pp_dir)
    msg_avg['ocn'
        ] = 'Successfully completed generating ocean climatology averages'
    msg_diags['ocn'] = 'Successfully completed generating ocean diagnostics'
    case_dict['ocn_avg_dates'] = case_dict['ocn_diag_dates'] = get_diag_dates(
        'ocn', pp_dir)
    for comp in diag_comps:
        case_dict[comp + '_avg_status'] = 'Unknown'
        case_dict[comp + '_diag_status'] = 'Unknown'
        if comp != 'ocn':
            case_dict[comp + '_avg_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/climo')
            case_dict[comp + '_avg_size'] = get_disk_usage(case_dict[comp +
                '_avg_path'])
            case_dict[comp + '_diag_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/diag')
            case_dict[comp + '_diag_size'] = get_disk_usage(case_dict[comp +
                '_diag_path'])
        else:
            case_dict[comp + '_avg_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/climo*')
            case_dict[comp + '_avg_size'] = get_ocn_disk_usage(case_dict[
                comp + '_avg_path'])
            case_dict[comp + '_diag_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/diag*')
            case_dict[comp + '_diag_size'] = get_ocn_disk_usage(case_dict[
                comp + '_diag_path'])
        avg_logs = list()
        avg_file_pattern = '{0}/{1}_averages.log.*'.format(pp_log_dir, comp)
        avg_logs = glob.glob(avg_file_pattern)
        if avg_logs:
            log_file = max(avg_logs, key=os.path.getctime)
            if is_last_process_complete(log_file, msg_avg[comp],
                'Average list complies with standards.'):
                case_dict[comp + '_avg_status'] = 'Succeeded'
            else:
                case_dict[comp + '_avg_status'] = 'Started'
        diag_logs = list()
        diag_file_pattern = '{0}/{1}_diagnostics.log.*'.format(pp_log_dir, comp
            )
        diag_logs = glob.glob(diag_file_pattern)
        if diag_logs:
            log_file = max(diag_logs, key=os.path.getctime)
            if is_last_process_complete(log_file, msg_diags[comp],
                'ncks version'):
                case_dict[comp + '_diag_status'] = 'Succeeded'
            else:
                case_dict[comp + '_diag_status'] = 'Started'
    case_dict['timeseries_status'] = 'Unknown'
    case_dict['timeseries_path'] = get_pp_path(pp_dir, 'timeseries')
    case_dict['timeseries_size'] = 0
    case_dict['timeseries_dates'] = '{0}-{1}'.format(case_dict[
        'RUN_STARTDATE'].replace('-', ''), case_dict['RUN_STARTDATE'].
        replace('-', ''))
    case_dict['timeseries_total_time'] = 0
    tseries_logs = list()
    tseries_file_pattern = '{0}/timeseries.log.*'.format(pp_log_dir)
    tseries_logs = glob.glob(tseries_file_pattern)
    if tseries_logs:
        log_file = max(tseries_logs, key=os.path.getctime)
        if is_last_process_complete(filepath=log_file, expect_text=
            'Successfully completed', fail_text='opening'):
            case_dict['timeseries_status'] = 'Succeeded'
            with open(log_file, 'r') as fname:
                log_content = fname.readlines()
            total_time = [line for line in log_content if 'Total Time:' in line
                ]
            case_dict['timeseries_total_time'] = ' '.join(total_time[0].split()
                )
        else:
            case_dict['timeseries_status'] = 'Started'
        sta_dates = case_dict['sta_last_date'].split('-')
        case_dict['timeseries_dates'] = '{0}-{1}'.format(case_dict[
            'RUN_STARTDATE'].replace('-', ''), ''.join(sta_dates[:-1]))
        for comp in _pp_tseries_comps:
            tseries_path = '{0}/{1}/proc/tseries'.format(case_dict[
                'timeseries_path'], comp)
            case_dict['timeseries_size'] += get_disk_usage(tseries_path)
    case_dict['iconform_status'] = 'Unknown'
    case_dict['iconform_path'] = ''
    case_dict['iconform_size'] = 0
    case_dict['iconform_dates'] = case_dict['timeseries_dates']
    iconform_logs = list()
    iconform_file_pattern = '{0}/iconform.log.*'.format(pp_log_dir)
    iconform_logs = glob.glob(iconform_file_pattern)
    if iconform_logs:
        log_file = max(iconform_logs, key=os.path.getctime)
        if is_last_process_complete(log_file,
            'Successfully created the conform tool',
            'Running createOutputSpecs'):
            case_dict['iconform_status'] = 'Succeeded'
        else:
            case_dict['iconform_status'] = 'Started'
    case_dict['xconform_path'] = ''
    case_dict['xconform_path'] = get_pp_path(pp_dir, 'xconform')
    case_dict['xconform_status'] = 'Unknown'
    case_dict['xconform_size'] = get_disk_usage(case_dict['xconform_path'])
    case_dict['xconform_dates'] = case_dict['timeseries_dates']
    case_dict['xconform_total_time'] = 0
    xconform_logs = list()
    xconform_file_pattern = '{0}/xconform.log.*'.format(pp_log_dir)
    xconform_logs = glob.glob(xconform_file_pattern)
    if xconform_logs:
        log_file = max(xconform_logs, key=os.path.getctime)
        if is_last_process_complete(log_file,
            'Successfully completed converting all files',
            'cesm_conform_generator INFO'):
            case_dict['xconform_status'] = 'Succeeded'
            case_dict['xconform_size'] = get_disk_usage(case_dict[
                'xconform_path'])
            with open(log_file, 'r') as fname:
                log_content = fname.readlines()
            total_time = [line for line in log_content if 'Total Time:' in line
                ]
            if total_time:
                case_dict['xconform_total_time'] = ' '.join(total_time[0].
                    split())
        else:
            case_dict['xconform_status'] = 'Started'
    return case_dict


<mask token>


def check_svn_repo(case_dict, username, password):
    """check_svn_repo

    check if a SVN repo exists for this case
    """
    logger.debug('check_svn_repo')
    repo_exists = False
    svn_repo = '{0}/trunk'.format(case_dict['svn_repo_url'])
    cmd = ['svn', 'list', svn_repo, '--username', username, '--password',
        password]
    result = ''
    try:
        result = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        msg = (
            'SVN repo does not exist for this case. A new one will be created.'
            )
        logger.warning(msg)
    if re.search('README.archive', result):
        repo_exists = True
    return repo_exists


<mask token>


def update_repo_add_file(filename, dir1, dir2):
    """update_repo_add_file

    Add a file to the SVN repository
    """
    src = os.path.join(dir1, filename)
    dest = os.path.join(dir2, filename)
    logger.debug('left_only: ' + src + ' -> ' + dest)
    if not os.path.exists(dest):
        shutil.copy2(src, dest)
        cmd = ['svn', 'add', '--parents', dest]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            msg = _call_template.substitute(function='update_lcoal_repo',
                cmd=cmd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)


def update_repo_rm_file(filename, dir1, dir2):
    """update_repo_rm_file

    Remove a file from the SVN repository
    """
    src = os.path.join(dir2, filename)
    dest = os.path.join(dir1, filename)
    logger.debug('right_only: ' + src + ' -> ' + dest)
    if os.path.exists(dest):
        cmd = ['svn', 'rm', dest]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            msg = _call_template.substitute(function='update_lcoal_repo',
                cmd=cmd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)


<mask token>


def update_local_repo(case_dict, ignore_logs, ignore_timing):
    """update_local_repo

    Compare and update local SVN sandbox
    """
    logger.debug('update_local_repo')
    from_dir = case_dict['CASEROOT']
    to_dir = case_dict['archive_temp_dir']
    compare_dir_trees(from_dir, to_dir, case_dict['archive_list'])
    if ignore_logs:
        os.chdir(to_dir)
        if os.path.isdir('./logs'):
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "logs" in archive dir.'
                    )
            cmd = ['svn', 'delete', './logs']
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(function=
                    'update_lcoal_repo', cmd=cmd, error=error.returncode,
                    strerror=error.output)
                logger.warning(msg)
                raise SVNException(msg)
        if os.path.isdir('./postprocess/logs'):
            os.chdir('./postprocess')
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "postprocess/logs" in archive dir.'
                    )
            cmd = ['svn', 'delete', './logs']
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(function=
                    'update_lcoal_repo', cmd=cmd, error=error.returncode,
                    strerror=error.output)
                logger.warning(msg)
                raise SVNException(msg)
    else:
        if os.path.exists('{0}/logs'.format(from_dir)):
            if not os.path.exists('{0}/logs'.format(to_dir)):
                os.makedirs('{0}/logs'.format(to_dir))
            os.chdir(os.path.join(from_dir, 'logs'))
            for filename in glob.glob('*.*'):
                update_repo_add_file(filename, os.path.join(from_dir,
                    'logs'), os.path.join(to_dir, 'logs'))
        if os.path.exists('{0}/postprocess/logs'.format(from_dir)):
            if not os.path.exists('{0}/postprocess/logs'.format(to_dir)):
                os.makedirs('{0}/postprocess/logs'.format(to_dir))
            os.chdir(os.path.join(from_dir, 'postprocess/logs'))
            for filename in glob.glob('*.*'):
                update_repo_add_file(filename, os.path.join(from_dir,
                    'postprocess', 'logs'), os.path.join(to_dir,
                    'postprocess', 'logs'))
    if ignore_timing:
        os.chdir(case_dict['archive_temp_dir'])
        if os.path.isdir('./timing'):
            try:
                shutil.rmtree('./timing')
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "timing" in archive dir.'
                    )
            cmd = ['svn', 'delete', './timing']
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(function=
                    'update_lcoal_repo', cmd=cmd, error=error.returncode,
                    strerror=error.output)
                logger.warning(msg)
                raise SVNException(msg)
    elif os.path.exists('{0}/timing'.format(from_dir)):
        if not os.path.exists('{0}/timing'.format(to_dir)):
            os.makedirs('{0}/timing'.format(to_dir))
        os.chdir(os.path.join(from_dir, 'timing'))
        for filename in glob.glob('*.*'):
            update_repo_add_file(filename, os.path.join(from_dir, 'timing'),
                os.path.join(to_dir, 'timing'))


def populate_local_repo(case_dict, ignore_logs, ignore_timing):
    """populate_local_repo

    Populate local SVN sandbox
    """
    logger.debug('populate_local_repo')
    os.chdir(case_dict['CASEROOT'])
    for archive in case_dict['archive_list']:
        if os.path.exists(archive):
            if os.path.isdir(archive):
                try:
                    target = case_dict['archive_temp_dir'] + '/' + archive
                    shutil.copytree(archive, target, symlinks=False, ignore
                        =shutil.ignore_patterns(*_ignore_patterns))
                except OSError as error:
                    msg = _copy_template.substitute(function=
                        'populate_local_repo', source=archive, dest=
                        case_dict['archive_temp_dir'], error=error.errno,
                        strerror=error.strerror)
                    logger.warning(msg)
            else:
                try:
                    shutil.copy2(archive, case_dict['archive_temp_dir'])
                except OSError as error:
                    msg = _copy_template.substitute(function=
                        'populate_local_repo', source=archive, dest=
                        case_dict['archive_temp_dir'], error=error.errno,
                        strerror=error.strerror)
                    logger.warning(msg)
    xml_files = glob.glob('*.xml')
    for xml_file in xml_files:
        if os.path.isfile(xml_file):
            try:
                shutil.copy2(xml_file, case_dict['archive_temp_dir'])
            except OSError as error:
                msg = _copy_template.substitute(function=
                    'populate_local_repo', source=xml_file, dest=case_dict[
                    'archive_temp_dir'], error=error.errno, strerror=error.
                    strerror)
                logger.warning(msg)
    if os.path.isdir('./postprocess'):
        pp_path = '{0}/{1}'.format(case_dict['archive_temp_dir'], 'postprocess'
            )
        if not os.path.exists(pp_path):
            os.mkdir(pp_path)
        xml_files = glob.glob('./postprocess/*.xml')
        for xml_file in xml_files:
            if os.path.isfile(xml_file):
                try:
                    shutil.copy2(xml_file, pp_path)
                except OSError as error:
                    msg = _copy_template.substitute(function=
                        'populate_local_repo', source=xml_file, dest=
                        case_dict['archive_temp_dir'], error=error.errno,
                        strerror=error.strerror)
                    logger.warning(msg)
    user_files = glob.glob('user_nl_*')
    for user_file in user_files:
        if os.path.isfile(user_file):
            try:
                shutil.copy2(user_file, case_dict['archive_temp_dir'])
            except OSError as error:
                msg = _copy_template.substitute(function=
                    'populate_local_repo', source=user_file, dest=case_dict
                    ['archive_temp_dir'], error=error.errno, strerror=error
                    .strerror)
                logger.warning(msg)
    conf_files = glob.glob('Depends.*')
    for conf_file in conf_files:
        if os.path.isfile(conf_file):
            try:
                shutil.copy2(conf_file, case_dict['archive_temp_dir'])
            except OSError as error:
                msg = _copy_template.substitute(function=
                    'populate_local_repo', source=conf_file, dest=case_dict
                    ['archive_temp_dir'], error=error.errno, strerror=error
                    .strerror)
                logger.warning(msg)
    if ignore_logs:
        os.chdir(case_dict['archive_temp_dir'])
        if os.path.isdir('./logs'):
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "logs" in archive_temp_dir.'
                    )
        if os.path.isdir('./postprocess/logs'):
            os.chdir('./postprocess')
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "postprocess/logs" in archive_temp_dir.'
                    )
        os.chdir(case_dict['CASEROOT'])
    if ignore_timing:
        os.chdir(case_dict['archive_temp_dir'])
        if os.path.isdir('./timing'):
            try:
                shutil.rmtree('./timing')
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "timing" in archive_temp_dir.'
                    )
        os.chdir(case_dict['CASEROOT'])


def checkin_trunk(case_dict, svn_cmd, message, username, password):
    """checkin_trunk

    Check in the local SVN sandbox to the remote trunk
    """
    logger.debug('checkin_trunk')
    os.chdir(case_dict['archive_temp_dir'])
    svn_repo = '{0}/trunk'.format(case_dict['svn_repo_url'])
    msg = '"{0}"'.format(message)
    cmd = ['svn', svn_cmd, '--username', username, '--password', password,
        '.', '--message', msg]
    if svn_cmd in ['import']:
        msg = '"create trunk"'
        cmd = ['svn', 'mkdir', '--parents', svn_repo, '--username',
            username, '--password', password, '--message', msg]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            cmd_nopasswd = ['svn', 'mkdir', '--parents', svn_repo,
                '--username', username, '--password', '******', '--message',
                msg]
            msg = _call_template.substitute(function='checkin_trunk', cmd=
                cmd_nopasswd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)
        tags = '{0}/trunk_tags'.format(case_dict['svn_repo_url'])
        msg = '"create trunk_tags"'
        cmd = ['svn', 'mkdir', tags, '--username', username, '--password',
            password, '--message', msg]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            cmd_nopasswd = ['svn', 'mkdir', tags, '--username', username,
                '--password', '******', '--message', msg]
            msg = _call_template.substitute(function='checkin_trunk', cmd=
                cmd_nopasswd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)
        msg = '"{0}"'.format(message)
        cmd = ['svn', svn_cmd, '--username', username, '--password',
            password, '.', svn_repo, '--message', msg]
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = ['svn', svn_cmd, '--username', username,
            '--password', '******', '.', '--message', msg]
        msg = _call_template.substitute(function='checkin_trunk', cmd=
            cmd_nopasswd, error=error.returncode, strerror=error.output)
        logger.warning(msg)
        raise SVNException(msg)


<mask token>


def update_repo(ignore_logs, ignore_timing, case_dict, username, password):
    """update_repo

    Update SVN repo
    """
    logger.debug('update_repo')
    try:
        svn_exists = check_svn()
        if svn_exists:
            case_dict['svn_repo_url'] = '{0}/{1}'.format(_svn_expdb_url,
                case_dict['CASE'])
            repo_exists = check_svn_repo(case_dict, username, password)
            case_dict['archive_temp_dir'] = create_temp_archive(case_dict)
            case_dict['archive_list'] = _archive_list + case_dict[
                'user_add_files']
            if repo_exists:
                last_tag = get_trunk_tag(case_dict, username, password)
                new_tag = '{0}_{1}'.format(case_dict['CASE'], str(last_tag +
                    1).zfill(4))
                checkout_repo(case_dict, username, password)
                update_local_repo(case_dict, ignore_logs, ignore_timing)
                msg = 'update case metadata for {0} by {1}'.format(case_dict
                    ['CASE'], username)
                checkin_trunk(case_dict, 'ci', msg, username, password)
                create_tag(case_dict, new_tag, username, password)
                logger.info('SVN repository trunk updated at URL "%s"',
                    case_dict['svn_repo_url'])
                logger.info(' and a new trunk tag created "%s"', new_tag)
            else:
                new_tag = '{0}_0001'.format(case_dict['CASE'])
                create_readme(case_dict)
                populate_local_repo(case_dict, ignore_logs, ignore_timing)
                msg = 'initial import of case metadata for {0} by {1}'.format(
                    case_dict['CASE'], username)
                checkin_trunk(case_dict, 'import', msg, username, password)
                create_tag(case_dict, new_tag, username, password)
                logger.info('SVN repository imported to trunk URL "%s"',
                    case_dict['svn_repo_url'])
                logger.info('   and a new trunk tag created for  "%s"', new_tag
                    )
    except SVNException:
        pass
    return case_dict


<mask token>


def initialize_main(options):
    """initialize_main

    Initialize the case dictionary data structure with command line options
    """
    logger.debug('intialize_main')
    case_dict = dict()
    case_dict['CASEROOT'] = os.getcwd()
    if options.caseroot:
        case_dict['CASEROOT'] = options.caseroot[0]
    case_dict['workdir'] = case_dict['CASEROOT']
    if options.workdir:
        case_dict['workdir'] = options.workdir[0]
    username = None
    if options.user:
        username = options.user
        case_dict['svnlogin'] = username
    password = None
    if options.password:
        password = options.password
    if options.expType:
        case_dict['expType'] = options.expType[0]
    case_dict['title'] = None
    if options.title:
        case_dict['title'] = options.title[0]
    case_dict['dryrun'] = False
    if options.dryrun:
        case_dict['dryrun'] = True
    case_dict['archive_temp_dir'] = ''
    case_dict['user_add_files'] = list()
    if options.user_add_files:
        case_dict['user_add_files'] = options.user_add_files.split(',')
    case_dict['q_casename'] = ''
    case_dict['q_outfile'] = ''
    if options.query_cmip6:
        case_dict['q_casename'] = options.query_cmip6[0]
        case_dict['q_outfile'] = options.query_cmip6[1]
    case_dict['base_expdb_url'] = 'https://csegweb.cgd.ucar.edu/expdb2.0'
    if options.test_post:
        case_dict['base_expdb_url'
            ] = 'https://csegwebdev.cgd.ucar.edu/expdb2.0'
    case_dict['json_expdb_url'] = case_dict['base_expdb_url'
        ] + '/cgi-bin/processJSON.cgi'
    case_dict['query_expdb_url'] = case_dict['base_expdb_url'
        ] + '/cgi-bin/query.cgi'
    return case_dict, username, password


<mask token>

<mask token>


class PasswordPromptAction(argparse.Action):
    """SVN developer's password class handler"""

    def __init__(self, option_strings=None, dest=None, default=None,
        required=False, nargs=0, help=None):
        super(PasswordPromptAction, self).__init__(option_strings=
            option_strings, dest=dest, default=default, required=required,
            nargs=nargs, help=help)

    def __call__(self, parser, args, values, option_string=None):
        home = expanduser('~')
        conf_path = os.path.join(home, '.subversion/cmip6.conf')
        if os.path.exists(conf_path):
            config = configparser.SafeConfigParser()
            config.read(conf_path)
            password = config.get('svn', 'password')
        else:
            password = getpass.getpass()
        setattr(args, self.dest, password)


def basic_authorization(user, password):
    """Basic authentication encoding"""
    sauth = user + ':' + password
    return 'Basic ' + sauth.encode('base64').rstrip()


class SVNException(Exception):
    """SVN command exception handler"""

    def __init__(self, value):
        super(SVNException, self).__init__(value)
        self.value = value

    def __str__(self):
        return repr(self.value)


def commandline_options(args):
    """Process the command line arguments."""
    parser = argparse.ArgumentParser(description=
        'Query and parse the caseroot files to gather metadata information that can be posted to the CESM experiments database.  CMIP6 experiment case names must be reserved already in the experiment database. Please see: https://csesgweb.cgd.ucar.edu/expdb2.0 for details.'
        )
    CIME.utils.setup_standard_logging_options(parser)
    parser.add_argument('--user', dest='user', type=str, default=None,
        required=True, help=
        'User name for SVN CESM developer access (required)')
    parser.add_argument('--password', dest='password', action=
        PasswordPromptAction, default='', required=True, help=
        'Password for SVN CESM developer access (required)')
    parser.add_argument('--caseroot', nargs=1, required=False, help=
        'Fully quailfied path to case root directory (optional). Defaults to current working directory.'
        )
    parser.add_argument('--workdir', nargs=1, required=False, help=
        'Fully quailfied path to directory for storing intermediate case files. A sub-directory called archive_temp_dir is created, populated with case files, and posted to the CESM experiments database and SVN repository at URL "{0}". This argument can be used to archive a caseroot when the user does not have write permission in the caseroot (optional). Defaults to current working directory.'
        .format(_svn_expdb_url))
    parser.add_argument('--expType', dest='expType', nargs=1, required=True,
        choices=_exp_types, help=
        'Experiment type. For CMIP6 experiments, the case must already exist in the experiments database at URL  "http://csegweb.cgd.ucar.edu/expdb2.0" (required). Must be one of "{0}"'
        .format(_exp_types))
    parser.add_argument('--title', nargs=1, required=False, default=None,
        help='Title of experiment (optional).')
    parser.add_argument('--ignore-logs', dest='ignore_logs', action=
        'store_true', help=
        'Ignore updating the SVN repository with the caseroot/logs files. The experiments database will be updated (optional).'
        )
    parser.add_argument('--ignore-timing', dest='ignore_timing', action=
        'store_true', help=
        'Ignore updating the the SVN repository with caseroot/timing files.The experiments database will be updated (optional).'
        )
    parser.add_argument('--ignore-repo-update', dest='ignore_repo_update',
        action='store_true', help=
        'Ignore updating the SVN repository with all the caseroot files. The experiments database will be updated (optional).'
        )
    parser.add_argument('--add-files', dest='user_add_files', required=
        False, help=
        'Comma-separated list with no spaces of files or directories to be added to the SVN repository. These are in addition to the default added caseroot files and directories: "{0}, *.xml, user_nl_*" (optional).'
        .format(_archive_list))
    parser.add_argument('--dryrun', action='store_true', help=
        'Parse settings and print what actions will be taken but do not execute the action (optional).'
        )
    parser.add_argument('--query_cmip6', nargs=2, required=False, help=
        'Query the experiments database global attributes for specified CMIP6 casename as argument 1. Writes a json formatted output file, specified by argument 2, to subdir archive_files (optional).'
        )
    parser.add_argument('--test-post', dest='test_post', action=
        'store_true', help=
        'Post metadata to the test expdb2.0 web application server at URL "http://csegwebdev.cgd.ucar.edu/expdb2.0". No --test-post argument defaults to posting metadata to the production expdb2.0 web application server at URL "http://csegweb.cgd.ucar.edu/expdb2.0" (optional).'
        )
    opts = CIME.utils.parse_args_and_handle_standard_logging_options(args,
        parser)
    return opts


def get_case_vars(case_dict, case):
    """get_case_vars
    loop through the global list of XML vars and get the values
    from the case object into a case dictionary

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        case (object) - case object
    """
    logger.debug('get_case_vars')
    for xml_id in _xml_vars:
        case_dict[xml_id] = case.get_value(xml_id, resolved=True, subgroup=None
            )
    for xml_id in _run_vars:
        case_dict[xml_id] = case.get_value(xml_id, resolved=True, subgroup=
            'case.run')
    return case_dict


<mask token>


def get_ocn_disk_usage(path):
    """get_ocn_disk_usage
    return the total disk usage in bytes for a given path.

    Arguments:
        path - path to start
    """
    logger.debug('get_ocn_disk_usage')
    total_size = 0
    paths = glob.glob(path)
    for path in paths:
        total_size += get_disk_usage(path)
    return int(total_size)


def get_pp_path(pp_dir, process):
    """get_pp_path
    return the XML path for process

    Arguments:
        pp_dir  - path to postprocess directory
        process - process name
    """
    logger.debug('get_pp_path')
    cwd = os.getcwd()
    os.chdir(pp_dir)
    pp_path_var = ''
    if process == 'timeseries':
        pp_path_var = _pp_xml_vars['timeseries']
    elif process == 'xconform':
        pp_path_var = _pp_xml_vars['xconform']
    cmd = ['./pp_config', '--get', pp_path_var, '--value']
    try:
        pp_path = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        msg = "Error executing command = '{0}'".format(cmd)
        logger.warning(msg)
    if len(pp_path) > 2:
        pp_path = pp_path.rstrip()
    else:
        pp_path = ''
    os.chdir(cwd)
    return pp_path


def get_diag_dates(comp, pp_dir):
    """get_diag_dates

    Query the postprocessing env_diags_[comp].xml file to get the model diag
    dates for the given component.
    """
    logger.debug('get_diag_dates')
    cwd = os.getcwd()
    os.chdir(pp_dir)
    model_dates = ''
    pp_vars = _pp_diag_vars.get(comp)
    for pp_var in pp_vars:
        cmd = ['./pp_config', '--get', pp_var, '--value']
        try:
            pp_value = subprocess.check_output(cmd)
        except subprocess.CalledProcessError:
            msg = "Error executing command = '{0}'".format(cmd)
            logger.warning(msg)
        tmp_dates = '{0} = {1}'.format(pp_var, pp_value)
        model_dates = model_dates + tmp_dates
    os.chdir(cwd)
    return model_dates


def get_pp_status(case_dict):
    """get_pp_status
    Parse the postprocessing log files
    looking for status information

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug('get_pp_status')
    msg_avg = dict()
    msg_diags = dict()
    diag_comps = ['atm', 'ice', 'lnd', 'ocn']
    pp_dir = os.path.join(case_dict['CASEROOT'], 'postprocess')
    pp_log_dir = os.path.join(case_dict['CASEROOT'], 'postprocess', 'logs')
    msg_avg['atm'] = 'COMPLETED SUCCESSFULLY'
    msg_diags['atm'
        ] = 'Successfully completed generating atmosphere diagnostics'
    case_dict['atm_avg_dates'] = case_dict['atm_diag_dates'] = get_diag_dates(
        'atm', pp_dir)
    msg_avg['ice'
        ] = 'Successfully completed generating ice climatology averages'
    msg_diags['ice'] = 'Successfully completed generating ice diagnostics'
    case_dict['ice_avg_dates'] = case_dict['ice_diag_dates'] = get_diag_dates(
        'ice', pp_dir)
    msg_avg['lnd'] = 'COMPLETED SUCCESSFULLY'
    msg_diags['lnd'] = 'Successfully completed generating land diagnostics'
    case_dict['lnd_avg_dates'] = case_dict['lnd_diag_dates'] = get_diag_dates(
        'lnd', pp_dir)
    msg_avg['ocn'
        ] = 'Successfully completed generating ocean climatology averages'
    msg_diags['ocn'] = 'Successfully completed generating ocean diagnostics'
    case_dict['ocn_avg_dates'] = case_dict['ocn_diag_dates'] = get_diag_dates(
        'ocn', pp_dir)
    for comp in diag_comps:
        case_dict[comp + '_avg_status'] = 'Unknown'
        case_dict[comp + '_diag_status'] = 'Unknown'
        if comp != 'ocn':
            case_dict[comp + '_avg_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/climo')
            case_dict[comp + '_avg_size'] = get_disk_usage(case_dict[comp +
                '_avg_path'])
            case_dict[comp + '_diag_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/diag')
            case_dict[comp + '_diag_size'] = get_disk_usage(case_dict[comp +
                '_diag_path'])
        else:
            case_dict[comp + '_avg_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/climo*')
            case_dict[comp + '_avg_size'] = get_ocn_disk_usage(case_dict[
                comp + '_avg_path'])
            case_dict[comp + '_diag_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/diag*')
            case_dict[comp + '_diag_size'] = get_ocn_disk_usage(case_dict[
                comp + '_diag_path'])
        avg_logs = list()
        avg_file_pattern = '{0}/{1}_averages.log.*'.format(pp_log_dir, comp)
        avg_logs = glob.glob(avg_file_pattern)
        if avg_logs:
            log_file = max(avg_logs, key=os.path.getctime)
            if is_last_process_complete(log_file, msg_avg[comp],
                'Average list complies with standards.'):
                case_dict[comp + '_avg_status'] = 'Succeeded'
            else:
                case_dict[comp + '_avg_status'] = 'Started'
        diag_logs = list()
        diag_file_pattern = '{0}/{1}_diagnostics.log.*'.format(pp_log_dir, comp
            )
        diag_logs = glob.glob(diag_file_pattern)
        if diag_logs:
            log_file = max(diag_logs, key=os.path.getctime)
            if is_last_process_complete(log_file, msg_diags[comp],
                'ncks version'):
                case_dict[comp + '_diag_status'] = 'Succeeded'
            else:
                case_dict[comp + '_diag_status'] = 'Started'
    case_dict['timeseries_status'] = 'Unknown'
    case_dict['timeseries_path'] = get_pp_path(pp_dir, 'timeseries')
    case_dict['timeseries_size'] = 0
    case_dict['timeseries_dates'] = '{0}-{1}'.format(case_dict[
        'RUN_STARTDATE'].replace('-', ''), case_dict['RUN_STARTDATE'].
        replace('-', ''))
    case_dict['timeseries_total_time'] = 0
    tseries_logs = list()
    tseries_file_pattern = '{0}/timeseries.log.*'.format(pp_log_dir)
    tseries_logs = glob.glob(tseries_file_pattern)
    if tseries_logs:
        log_file = max(tseries_logs, key=os.path.getctime)
        if is_last_process_complete(filepath=log_file, expect_text=
            'Successfully completed', fail_text='opening'):
            case_dict['timeseries_status'] = 'Succeeded'
            with open(log_file, 'r') as fname:
                log_content = fname.readlines()
            total_time = [line for line in log_content if 'Total Time:' in line
                ]
            case_dict['timeseries_total_time'] = ' '.join(total_time[0].split()
                )
        else:
            case_dict['timeseries_status'] = 'Started'
        sta_dates = case_dict['sta_last_date'].split('-')
        case_dict['timeseries_dates'] = '{0}-{1}'.format(case_dict[
            'RUN_STARTDATE'].replace('-', ''), ''.join(sta_dates[:-1]))
        for comp in _pp_tseries_comps:
            tseries_path = '{0}/{1}/proc/tseries'.format(case_dict[
                'timeseries_path'], comp)
            case_dict['timeseries_size'] += get_disk_usage(tseries_path)
    case_dict['iconform_status'] = 'Unknown'
    case_dict['iconform_path'] = ''
    case_dict['iconform_size'] = 0
    case_dict['iconform_dates'] = case_dict['timeseries_dates']
    iconform_logs = list()
    iconform_file_pattern = '{0}/iconform.log.*'.format(pp_log_dir)
    iconform_logs = glob.glob(iconform_file_pattern)
    if iconform_logs:
        log_file = max(iconform_logs, key=os.path.getctime)
        if is_last_process_complete(log_file,
            'Successfully created the conform tool',
            'Running createOutputSpecs'):
            case_dict['iconform_status'] = 'Succeeded'
        else:
            case_dict['iconform_status'] = 'Started'
    case_dict['xconform_path'] = ''
    case_dict['xconform_path'] = get_pp_path(pp_dir, 'xconform')
    case_dict['xconform_status'] = 'Unknown'
    case_dict['xconform_size'] = get_disk_usage(case_dict['xconform_path'])
    case_dict['xconform_dates'] = case_dict['timeseries_dates']
    case_dict['xconform_total_time'] = 0
    xconform_logs = list()
    xconform_file_pattern = '{0}/xconform.log.*'.format(pp_log_dir)
    xconform_logs = glob.glob(xconform_file_pattern)
    if xconform_logs:
        log_file = max(xconform_logs, key=os.path.getctime)
        if is_last_process_complete(log_file,
            'Successfully completed converting all files',
            'cesm_conform_generator INFO'):
            case_dict['xconform_status'] = 'Succeeded'
            case_dict['xconform_size'] = get_disk_usage(case_dict[
                'xconform_path'])
            with open(log_file, 'r') as fname:
                log_content = fname.readlines()
            total_time = [line for line in log_content if 'Total Time:' in line
                ]
            if total_time:
                case_dict['xconform_total_time'] = ' '.join(total_time[0].
                    split())
        else:
            case_dict['xconform_status'] = 'Started'
    return case_dict


def get_run_last_date(casename, run_path):
    """get_run_last_date
    parse the last cpl.r file in the run_path to retrieve that last date.

    Arguments:
        casename
        run_path - path to run directory
    """
    logger.debug('get_run_last_date')
    pattern = '{0}.cpl.r.*.nc'.format(casename)
    cpl_files = sorted(glob.glob(os.path.join(run_path, pattern)))
    if cpl_files:
        _, cpl_file = os.path.split(cpl_files[-1])
        fparts = cpl_file.split('.')
        return fparts[-2]
    return '0000-00-00'


<mask token>


def query_expdb_cmip6(case_dict, username, password):
    """query_exp_case
    Query the expdb for CMIP6 casename = case_dict['q_casename'] metadata.
    Write out a json file to case_dict['q_outfile'].

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        username (string) - SVN developer's username
        password (string) - SVN developer's password

    """
    logger.debug('query_expdb_cmip6')
    exists = False
    data_dict = {'casename': case_dict['q_casename'], 'queryType':
        'CMIP6GlobalAtts', 'expType': 'CMIP6'}
    data = json.dumps(data_dict)
    params = urllib.parse.urlencode(dict(username=username, password=
        password, data=data))
    try:
        response = urllib.request.urlopen(case_dict['query_expdb_url'],
            params, context=_context)
        output = json.load(response)
    except urllib.error.HTTPError as http_e:
        logger.info('ERROR archive_metadata HTTP post failed "%s"', http_e.code
            )
    except urllib.error.URLError as url_e:
        logger.info('ERROR archive_metadata URL failed "%s"', url_e.reason)
    if output:
        if not os.path.exists('{0}/archive_files'.format(case_dict['workdir'])
            ):
            os.makedirs('{0}/archive_files'.format(case_dict['workdir']))
        filename = '{0}/archive_files/{1}'.format(case_dict['workdir'],
            case_dict['q_outfile'])
        with io.open(filename, 'w+', encoding='utf-8') as fname:
            fname.write(json.dumps(output, ensure_ascii=False))
            fname.close()
        exists = True
    return exists


<mask token>


def check_svn_repo(case_dict, username, password):
    """check_svn_repo

    check if a SVN repo exists for this case
    """
    logger.debug('check_svn_repo')
    repo_exists = False
    svn_repo = '{0}/trunk'.format(case_dict['svn_repo_url'])
    cmd = ['svn', 'list', svn_repo, '--username', username, '--password',
        password]
    result = ''
    try:
        result = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        msg = (
            'SVN repo does not exist for this case. A new one will be created.'
            )
        logger.warning(msg)
    if re.search('README.archive', result):
        repo_exists = True
    return repo_exists


<mask token>


def update_repo_add_file(filename, dir1, dir2):
    """update_repo_add_file

    Add a file to the SVN repository
    """
    src = os.path.join(dir1, filename)
    dest = os.path.join(dir2, filename)
    logger.debug('left_only: ' + src + ' -> ' + dest)
    if not os.path.exists(dest):
        shutil.copy2(src, dest)
        cmd = ['svn', 'add', '--parents', dest]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            msg = _call_template.substitute(function='update_lcoal_repo',
                cmd=cmd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)


def update_repo_rm_file(filename, dir1, dir2):
    """update_repo_rm_file

    Remove a file from the SVN repository
    """
    src = os.path.join(dir2, filename)
    dest = os.path.join(dir1, filename)
    logger.debug('right_only: ' + src + ' -> ' + dest)
    if os.path.exists(dest):
        cmd = ['svn', 'rm', dest]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            msg = _call_template.substitute(function='update_lcoal_repo',
                cmd=cmd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)


<mask token>


def update_local_repo(case_dict, ignore_logs, ignore_timing):
    """update_local_repo

    Compare and update local SVN sandbox
    """
    logger.debug('update_local_repo')
    from_dir = case_dict['CASEROOT']
    to_dir = case_dict['archive_temp_dir']
    compare_dir_trees(from_dir, to_dir, case_dict['archive_list'])
    if ignore_logs:
        os.chdir(to_dir)
        if os.path.isdir('./logs'):
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "logs" in archive dir.'
                    )
            cmd = ['svn', 'delete', './logs']
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(function=
                    'update_lcoal_repo', cmd=cmd, error=error.returncode,
                    strerror=error.output)
                logger.warning(msg)
                raise SVNException(msg)
        if os.path.isdir('./postprocess/logs'):
            os.chdir('./postprocess')
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "postprocess/logs" in archive dir.'
                    )
            cmd = ['svn', 'delete', './logs']
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(function=
                    'update_lcoal_repo', cmd=cmd, error=error.returncode,
                    strerror=error.output)
                logger.warning(msg)
                raise SVNException(msg)
    else:
        if os.path.exists('{0}/logs'.format(from_dir)):
            if not os.path.exists('{0}/logs'.format(to_dir)):
                os.makedirs('{0}/logs'.format(to_dir))
            os.chdir(os.path.join(from_dir, 'logs'))
            for filename in glob.glob('*.*'):
                update_repo_add_file(filename, os.path.join(from_dir,
                    'logs'), os.path.join(to_dir, 'logs'))
        if os.path.exists('{0}/postprocess/logs'.format(from_dir)):
            if not os.path.exists('{0}/postprocess/logs'.format(to_dir)):
                os.makedirs('{0}/postprocess/logs'.format(to_dir))
            os.chdir(os.path.join(from_dir, 'postprocess/logs'))
            for filename in glob.glob('*.*'):
                update_repo_add_file(filename, os.path.join(from_dir,
                    'postprocess', 'logs'), os.path.join(to_dir,
                    'postprocess', 'logs'))
    if ignore_timing:
        os.chdir(case_dict['archive_temp_dir'])
        if os.path.isdir('./timing'):
            try:
                shutil.rmtree('./timing')
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "timing" in archive dir.'
                    )
            cmd = ['svn', 'delete', './timing']
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(function=
                    'update_lcoal_repo', cmd=cmd, error=error.returncode,
                    strerror=error.output)
                logger.warning(msg)
                raise SVNException(msg)
    elif os.path.exists('{0}/timing'.format(from_dir)):
        if not os.path.exists('{0}/timing'.format(to_dir)):
            os.makedirs('{0}/timing'.format(to_dir))
        os.chdir(os.path.join(from_dir, 'timing'))
        for filename in glob.glob('*.*'):
            update_repo_add_file(filename, os.path.join(from_dir, 'timing'),
                os.path.join(to_dir, 'timing'))


def populate_local_repo(case_dict, ignore_logs, ignore_timing):
    """populate_local_repo

    Populate local SVN sandbox
    """
    logger.debug('populate_local_repo')
    os.chdir(case_dict['CASEROOT'])
    for archive in case_dict['archive_list']:
        if os.path.exists(archive):
            if os.path.isdir(archive):
                try:
                    target = case_dict['archive_temp_dir'] + '/' + archive
                    shutil.copytree(archive, target, symlinks=False, ignore
                        =shutil.ignore_patterns(*_ignore_patterns))
                except OSError as error:
                    msg = _copy_template.substitute(function=
                        'populate_local_repo', source=archive, dest=
                        case_dict['archive_temp_dir'], error=error.errno,
                        strerror=error.strerror)
                    logger.warning(msg)
            else:
                try:
                    shutil.copy2(archive, case_dict['archive_temp_dir'])
                except OSError as error:
                    msg = _copy_template.substitute(function=
                        'populate_local_repo', source=archive, dest=
                        case_dict['archive_temp_dir'], error=error.errno,
                        strerror=error.strerror)
                    logger.warning(msg)
    xml_files = glob.glob('*.xml')
    for xml_file in xml_files:
        if os.path.isfile(xml_file):
            try:
                shutil.copy2(xml_file, case_dict['archive_temp_dir'])
            except OSError as error:
                msg = _copy_template.substitute(function=
                    'populate_local_repo', source=xml_file, dest=case_dict[
                    'archive_temp_dir'], error=error.errno, strerror=error.
                    strerror)
                logger.warning(msg)
    if os.path.isdir('./postprocess'):
        pp_path = '{0}/{1}'.format(case_dict['archive_temp_dir'], 'postprocess'
            )
        if not os.path.exists(pp_path):
            os.mkdir(pp_path)
        xml_files = glob.glob('./postprocess/*.xml')
        for xml_file in xml_files:
            if os.path.isfile(xml_file):
                try:
                    shutil.copy2(xml_file, pp_path)
                except OSError as error:
                    msg = _copy_template.substitute(function=
                        'populate_local_repo', source=xml_file, dest=
                        case_dict['archive_temp_dir'], error=error.errno,
                        strerror=error.strerror)
                    logger.warning(msg)
    user_files = glob.glob('user_nl_*')
    for user_file in user_files:
        if os.path.isfile(user_file):
            try:
                shutil.copy2(user_file, case_dict['archive_temp_dir'])
            except OSError as error:
                msg = _copy_template.substitute(function=
                    'populate_local_repo', source=user_file, dest=case_dict
                    ['archive_temp_dir'], error=error.errno, strerror=error
                    .strerror)
                logger.warning(msg)
    conf_files = glob.glob('Depends.*')
    for conf_file in conf_files:
        if os.path.isfile(conf_file):
            try:
                shutil.copy2(conf_file, case_dict['archive_temp_dir'])
            except OSError as error:
                msg = _copy_template.substitute(function=
                    'populate_local_repo', source=conf_file, dest=case_dict
                    ['archive_temp_dir'], error=error.errno, strerror=error
                    .strerror)
                logger.warning(msg)
    if ignore_logs:
        os.chdir(case_dict['archive_temp_dir'])
        if os.path.isdir('./logs'):
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "logs" in archive_temp_dir.'
                    )
        if os.path.isdir('./postprocess/logs'):
            os.chdir('./postprocess')
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "postprocess/logs" in archive_temp_dir.'
                    )
        os.chdir(case_dict['CASEROOT'])
    if ignore_timing:
        os.chdir(case_dict['archive_temp_dir'])
        if os.path.isdir('./timing'):
            try:
                shutil.rmtree('./timing')
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "timing" in archive_temp_dir.'
                    )
        os.chdir(case_dict['CASEROOT'])


def checkin_trunk(case_dict, svn_cmd, message, username, password):
    """checkin_trunk

    Check in the local SVN sandbox to the remote trunk
    """
    logger.debug('checkin_trunk')
    os.chdir(case_dict['archive_temp_dir'])
    svn_repo = '{0}/trunk'.format(case_dict['svn_repo_url'])
    msg = '"{0}"'.format(message)
    cmd = ['svn', svn_cmd, '--username', username, '--password', password,
        '.', '--message', msg]
    if svn_cmd in ['import']:
        msg = '"create trunk"'
        cmd = ['svn', 'mkdir', '--parents', svn_repo, '--username',
            username, '--password', password, '--message', msg]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            cmd_nopasswd = ['svn', 'mkdir', '--parents', svn_repo,
                '--username', username, '--password', '******', '--message',
                msg]
            msg = _call_template.substitute(function='checkin_trunk', cmd=
                cmd_nopasswd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)
        tags = '{0}/trunk_tags'.format(case_dict['svn_repo_url'])
        msg = '"create trunk_tags"'
        cmd = ['svn', 'mkdir', tags, '--username', username, '--password',
            password, '--message', msg]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            cmd_nopasswd = ['svn', 'mkdir', tags, '--username', username,
                '--password', '******', '--message', msg]
            msg = _call_template.substitute(function='checkin_trunk', cmd=
                cmd_nopasswd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)
        msg = '"{0}"'.format(message)
        cmd = ['svn', svn_cmd, '--username', username, '--password',
            password, '.', svn_repo, '--message', msg]
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = ['svn', svn_cmd, '--username', username,
            '--password', '******', '.', '--message', msg]
        msg = _call_template.substitute(function='checkin_trunk', cmd=
            cmd_nopasswd, error=error.returncode, strerror=error.output)
        logger.warning(msg)
        raise SVNException(msg)


<mask token>


def update_repo(ignore_logs, ignore_timing, case_dict, username, password):
    """update_repo

    Update SVN repo
    """
    logger.debug('update_repo')
    try:
        svn_exists = check_svn()
        if svn_exists:
            case_dict['svn_repo_url'] = '{0}/{1}'.format(_svn_expdb_url,
                case_dict['CASE'])
            repo_exists = check_svn_repo(case_dict, username, password)
            case_dict['archive_temp_dir'] = create_temp_archive(case_dict)
            case_dict['archive_list'] = _archive_list + case_dict[
                'user_add_files']
            if repo_exists:
                last_tag = get_trunk_tag(case_dict, username, password)
                new_tag = '{0}_{1}'.format(case_dict['CASE'], str(last_tag +
                    1).zfill(4))
                checkout_repo(case_dict, username, password)
                update_local_repo(case_dict, ignore_logs, ignore_timing)
                msg = 'update case metadata for {0} by {1}'.format(case_dict
                    ['CASE'], username)
                checkin_trunk(case_dict, 'ci', msg, username, password)
                create_tag(case_dict, new_tag, username, password)
                logger.info('SVN repository trunk updated at URL "%s"',
                    case_dict['svn_repo_url'])
                logger.info(' and a new trunk tag created "%s"', new_tag)
            else:
                new_tag = '{0}_0001'.format(case_dict['CASE'])
                create_readme(case_dict)
                populate_local_repo(case_dict, ignore_logs, ignore_timing)
                msg = 'initial import of case metadata for {0} by {1}'.format(
                    case_dict['CASE'], username)
                checkin_trunk(case_dict, 'import', msg, username, password)
                create_tag(case_dict, new_tag, username, password)
                logger.info('SVN repository imported to trunk URL "%s"',
                    case_dict['svn_repo_url'])
                logger.info('   and a new trunk tag created for  "%s"', new_tag
                    )
    except SVNException:
        pass
    return case_dict


<mask token>


def initialize_main(options):
    """initialize_main

    Initialize the case dictionary data structure with command line options
    """
    logger.debug('intialize_main')
    case_dict = dict()
    case_dict['CASEROOT'] = os.getcwd()
    if options.caseroot:
        case_dict['CASEROOT'] = options.caseroot[0]
    case_dict['workdir'] = case_dict['CASEROOT']
    if options.workdir:
        case_dict['workdir'] = options.workdir[0]
    username = None
    if options.user:
        username = options.user
        case_dict['svnlogin'] = username
    password = None
    if options.password:
        password = options.password
    if options.expType:
        case_dict['expType'] = options.expType[0]
    case_dict['title'] = None
    if options.title:
        case_dict['title'] = options.title[0]
    case_dict['dryrun'] = False
    if options.dryrun:
        case_dict['dryrun'] = True
    case_dict['archive_temp_dir'] = ''
    case_dict['user_add_files'] = list()
    if options.user_add_files:
        case_dict['user_add_files'] = options.user_add_files.split(',')
    case_dict['q_casename'] = ''
    case_dict['q_outfile'] = ''
    if options.query_cmip6:
        case_dict['q_casename'] = options.query_cmip6[0]
        case_dict['q_outfile'] = options.query_cmip6[1]
    case_dict['base_expdb_url'] = 'https://csegweb.cgd.ucar.edu/expdb2.0'
    if options.test_post:
        case_dict['base_expdb_url'
            ] = 'https://csegwebdev.cgd.ucar.edu/expdb2.0'
    case_dict['json_expdb_url'] = case_dict['base_expdb_url'
        ] + '/cgi-bin/processJSON.cgi'
    case_dict['query_expdb_url'] = case_dict['base_expdb_url'
        ] + '/cgi-bin/query.cgi'
    return case_dict, username, password


<mask token>

<mask token>


class PasswordPromptAction(argparse.Action):
    """SVN developer's password class handler"""

    def __init__(self, option_strings=None, dest=None, default=None,
        required=False, nargs=0, help=None):
        super(PasswordPromptAction, self).__init__(option_strings=
            option_strings, dest=dest, default=default, required=required,
            nargs=nargs, help=help)

    def __call__(self, parser, args, values, option_string=None):
        home = expanduser('~')
        conf_path = os.path.join(home, '.subversion/cmip6.conf')
        if os.path.exists(conf_path):
            config = configparser.SafeConfigParser()
            config.read(conf_path)
            password = config.get('svn', 'password')
        else:
            password = getpass.getpass()
        setattr(args, self.dest, password)


def basic_authorization(user, password):
    """Basic authentication encoding"""
    sauth = user + ':' + password
    return 'Basic ' + sauth.encode('base64').rstrip()


class SVNException(Exception):
    """SVN command exception handler"""

    def __init__(self, value):
        super(SVNException, self).__init__(value)
        self.value = value

    def __str__(self):
        return repr(self.value)


def commandline_options(args):
    """Process the command line arguments."""
    parser = argparse.ArgumentParser(description=
        'Query and parse the caseroot files to gather metadata information that can be posted to the CESM experiments database.  CMIP6 experiment case names must be reserved already in the experiment database. Please see: https://csesgweb.cgd.ucar.edu/expdb2.0 for details.'
        )
    CIME.utils.setup_standard_logging_options(parser)
    parser.add_argument('--user', dest='user', type=str, default=None,
        required=True, help=
        'User name for SVN CESM developer access (required)')
    parser.add_argument('--password', dest='password', action=
        PasswordPromptAction, default='', required=True, help=
        'Password for SVN CESM developer access (required)')
    parser.add_argument('--caseroot', nargs=1, required=False, help=
        'Fully quailfied path to case root directory (optional). Defaults to current working directory.'
        )
    parser.add_argument('--workdir', nargs=1, required=False, help=
        'Fully quailfied path to directory for storing intermediate case files. A sub-directory called archive_temp_dir is created, populated with case files, and posted to the CESM experiments database and SVN repository at URL "{0}". This argument can be used to archive a caseroot when the user does not have write permission in the caseroot (optional). Defaults to current working directory.'
        .format(_svn_expdb_url))
    parser.add_argument('--expType', dest='expType', nargs=1, required=True,
        choices=_exp_types, help=
        'Experiment type. For CMIP6 experiments, the case must already exist in the experiments database at URL  "http://csegweb.cgd.ucar.edu/expdb2.0" (required). Must be one of "{0}"'
        .format(_exp_types))
    parser.add_argument('--title', nargs=1, required=False, default=None,
        help='Title of experiment (optional).')
    parser.add_argument('--ignore-logs', dest='ignore_logs', action=
        'store_true', help=
        'Ignore updating the SVN repository with the caseroot/logs files. The experiments database will be updated (optional).'
        )
    parser.add_argument('--ignore-timing', dest='ignore_timing', action=
        'store_true', help=
        'Ignore updating the the SVN repository with caseroot/timing files.The experiments database will be updated (optional).'
        )
    parser.add_argument('--ignore-repo-update', dest='ignore_repo_update',
        action='store_true', help=
        'Ignore updating the SVN repository with all the caseroot files. The experiments database will be updated (optional).'
        )
    parser.add_argument('--add-files', dest='user_add_files', required=
        False, help=
        'Comma-separated list with no spaces of files or directories to be added to the SVN repository. These are in addition to the default added caseroot files and directories: "{0}, *.xml, user_nl_*" (optional).'
        .format(_archive_list))
    parser.add_argument('--dryrun', action='store_true', help=
        'Parse settings and print what actions will be taken but do not execute the action (optional).'
        )
    parser.add_argument('--query_cmip6', nargs=2, required=False, help=
        'Query the experiments database global attributes for specified CMIP6 casename as argument 1. Writes a json formatted output file, specified by argument 2, to subdir archive_files (optional).'
        )
    parser.add_argument('--test-post', dest='test_post', action=
        'store_true', help=
        'Post metadata to the test expdb2.0 web application server at URL "http://csegwebdev.cgd.ucar.edu/expdb2.0". No --test-post argument defaults to posting metadata to the production expdb2.0 web application server at URL "http://csegweb.cgd.ucar.edu/expdb2.0" (optional).'
        )
    opts = CIME.utils.parse_args_and_handle_standard_logging_options(args,
        parser)
    return opts


def get_case_vars(case_dict, case):
    """get_case_vars
    loop through the global list of XML vars and get the values
    from the case object into a case dictionary

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        case (object) - case object
    """
    logger.debug('get_case_vars')
    for xml_id in _xml_vars:
        case_dict[xml_id] = case.get_value(xml_id, resolved=True, subgroup=None
            )
    for xml_id in _run_vars:
        case_dict[xml_id] = case.get_value(xml_id, resolved=True, subgroup=
            'case.run')
    return case_dict


<mask token>


def get_ocn_disk_usage(path):
    """get_ocn_disk_usage
    return the total disk usage in bytes for a given path.

    Arguments:
        path - path to start
    """
    logger.debug('get_ocn_disk_usage')
    total_size = 0
    paths = glob.glob(path)
    for path in paths:
        total_size += get_disk_usage(path)
    return int(total_size)


def get_pp_path(pp_dir, process):
    """get_pp_path
    return the XML path for process

    Arguments:
        pp_dir  - path to postprocess directory
        process - process name
    """
    logger.debug('get_pp_path')
    cwd = os.getcwd()
    os.chdir(pp_dir)
    pp_path_var = ''
    if process == 'timeseries':
        pp_path_var = _pp_xml_vars['timeseries']
    elif process == 'xconform':
        pp_path_var = _pp_xml_vars['xconform']
    cmd = ['./pp_config', '--get', pp_path_var, '--value']
    try:
        pp_path = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        msg = "Error executing command = '{0}'".format(cmd)
        logger.warning(msg)
    if len(pp_path) > 2:
        pp_path = pp_path.rstrip()
    else:
        pp_path = ''
    os.chdir(cwd)
    return pp_path


def get_diag_dates(comp, pp_dir):
    """get_diag_dates

    Query the postprocessing env_diags_[comp].xml file to get the model diag
    dates for the given component.
    """
    logger.debug('get_diag_dates')
    cwd = os.getcwd()
    os.chdir(pp_dir)
    model_dates = ''
    pp_vars = _pp_diag_vars.get(comp)
    for pp_var in pp_vars:
        cmd = ['./pp_config', '--get', pp_var, '--value']
        try:
            pp_value = subprocess.check_output(cmd)
        except subprocess.CalledProcessError:
            msg = "Error executing command = '{0}'".format(cmd)
            logger.warning(msg)
        tmp_dates = '{0} = {1}'.format(pp_var, pp_value)
        model_dates = model_dates + tmp_dates
    os.chdir(cwd)
    return model_dates


def get_pp_status(case_dict):
    """get_pp_status
    Parse the postprocessing log files
    looking for status information

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug('get_pp_status')
    msg_avg = dict()
    msg_diags = dict()
    diag_comps = ['atm', 'ice', 'lnd', 'ocn']
    pp_dir = os.path.join(case_dict['CASEROOT'], 'postprocess')
    pp_log_dir = os.path.join(case_dict['CASEROOT'], 'postprocess', 'logs')
    msg_avg['atm'] = 'COMPLETED SUCCESSFULLY'
    msg_diags['atm'
        ] = 'Successfully completed generating atmosphere diagnostics'
    case_dict['atm_avg_dates'] = case_dict['atm_diag_dates'] = get_diag_dates(
        'atm', pp_dir)
    msg_avg['ice'
        ] = 'Successfully completed generating ice climatology averages'
    msg_diags['ice'] = 'Successfully completed generating ice diagnostics'
    case_dict['ice_avg_dates'] = case_dict['ice_diag_dates'] = get_diag_dates(
        'ice', pp_dir)
    msg_avg['lnd'] = 'COMPLETED SUCCESSFULLY'
    msg_diags['lnd'] = 'Successfully completed generating land diagnostics'
    case_dict['lnd_avg_dates'] = case_dict['lnd_diag_dates'] = get_diag_dates(
        'lnd', pp_dir)
    msg_avg['ocn'
        ] = 'Successfully completed generating ocean climatology averages'
    msg_diags['ocn'] = 'Successfully completed generating ocean diagnostics'
    case_dict['ocn_avg_dates'] = case_dict['ocn_diag_dates'] = get_diag_dates(
        'ocn', pp_dir)
    for comp in diag_comps:
        case_dict[comp + '_avg_status'] = 'Unknown'
        case_dict[comp + '_diag_status'] = 'Unknown'
        if comp != 'ocn':
            case_dict[comp + '_avg_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/climo')
            case_dict[comp + '_avg_size'] = get_disk_usage(case_dict[comp +
                '_avg_path'])
            case_dict[comp + '_diag_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/diag')
            case_dict[comp + '_diag_size'] = get_disk_usage(case_dict[comp +
                '_diag_path'])
        else:
            case_dict[comp + '_avg_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/climo*')
            case_dict[comp + '_avg_size'] = get_ocn_disk_usage(case_dict[
                comp + '_avg_path'])
            case_dict[comp + '_diag_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/diag*')
            case_dict[comp + '_diag_size'] = get_ocn_disk_usage(case_dict[
                comp + '_diag_path'])
        avg_logs = list()
        avg_file_pattern = '{0}/{1}_averages.log.*'.format(pp_log_dir, comp)
        avg_logs = glob.glob(avg_file_pattern)
        if avg_logs:
            log_file = max(avg_logs, key=os.path.getctime)
            if is_last_process_complete(log_file, msg_avg[comp],
                'Average list complies with standards.'):
                case_dict[comp + '_avg_status'] = 'Succeeded'
            else:
                case_dict[comp + '_avg_status'] = 'Started'
        diag_logs = list()
        diag_file_pattern = '{0}/{1}_diagnostics.log.*'.format(pp_log_dir, comp
            )
        diag_logs = glob.glob(diag_file_pattern)
        if diag_logs:
            log_file = max(diag_logs, key=os.path.getctime)
            if is_last_process_complete(log_file, msg_diags[comp],
                'ncks version'):
                case_dict[comp + '_diag_status'] = 'Succeeded'
            else:
                case_dict[comp + '_diag_status'] = 'Started'
    case_dict['timeseries_status'] = 'Unknown'
    case_dict['timeseries_path'] = get_pp_path(pp_dir, 'timeseries')
    case_dict['timeseries_size'] = 0
    case_dict['timeseries_dates'] = '{0}-{1}'.format(case_dict[
        'RUN_STARTDATE'].replace('-', ''), case_dict['RUN_STARTDATE'].
        replace('-', ''))
    case_dict['timeseries_total_time'] = 0
    tseries_logs = list()
    tseries_file_pattern = '{0}/timeseries.log.*'.format(pp_log_dir)
    tseries_logs = glob.glob(tseries_file_pattern)
    if tseries_logs:
        log_file = max(tseries_logs, key=os.path.getctime)
        if is_last_process_complete(filepath=log_file, expect_text=
            'Successfully completed', fail_text='opening'):
            case_dict['timeseries_status'] = 'Succeeded'
            with open(log_file, 'r') as fname:
                log_content = fname.readlines()
            total_time = [line for line in log_content if 'Total Time:' in line
                ]
            case_dict['timeseries_total_time'] = ' '.join(total_time[0].split()
                )
        else:
            case_dict['timeseries_status'] = 'Started'
        sta_dates = case_dict['sta_last_date'].split('-')
        case_dict['timeseries_dates'] = '{0}-{1}'.format(case_dict[
            'RUN_STARTDATE'].replace('-', ''), ''.join(sta_dates[:-1]))
        for comp in _pp_tseries_comps:
            tseries_path = '{0}/{1}/proc/tseries'.format(case_dict[
                'timeseries_path'], comp)
            case_dict['timeseries_size'] += get_disk_usage(tseries_path)
    case_dict['iconform_status'] = 'Unknown'
    case_dict['iconform_path'] = ''
    case_dict['iconform_size'] = 0
    case_dict['iconform_dates'] = case_dict['timeseries_dates']
    iconform_logs = list()
    iconform_file_pattern = '{0}/iconform.log.*'.format(pp_log_dir)
    iconform_logs = glob.glob(iconform_file_pattern)
    if iconform_logs:
        log_file = max(iconform_logs, key=os.path.getctime)
        if is_last_process_complete(log_file,
            'Successfully created the conform tool',
            'Running createOutputSpecs'):
            case_dict['iconform_status'] = 'Succeeded'
        else:
            case_dict['iconform_status'] = 'Started'
    case_dict['xconform_path'] = ''
    case_dict['xconform_path'] = get_pp_path(pp_dir, 'xconform')
    case_dict['xconform_status'] = 'Unknown'
    case_dict['xconform_size'] = get_disk_usage(case_dict['xconform_path'])
    case_dict['xconform_dates'] = case_dict['timeseries_dates']
    case_dict['xconform_total_time'] = 0
    xconform_logs = list()
    xconform_file_pattern = '{0}/xconform.log.*'.format(pp_log_dir)
    xconform_logs = glob.glob(xconform_file_pattern)
    if xconform_logs:
        log_file = max(xconform_logs, key=os.path.getctime)
        if is_last_process_complete(log_file,
            'Successfully completed converting all files',
            'cesm_conform_generator INFO'):
            case_dict['xconform_status'] = 'Succeeded'
            case_dict['xconform_size'] = get_disk_usage(case_dict[
                'xconform_path'])
            with open(log_file, 'r') as fname:
                log_content = fname.readlines()
            total_time = [line for line in log_content if 'Total Time:' in line
                ]
            if total_time:
                case_dict['xconform_total_time'] = ' '.join(total_time[0].
                    split())
        else:
            case_dict['xconform_status'] = 'Started'
    return case_dict


def get_run_last_date(casename, run_path):
    """get_run_last_date
    parse the last cpl.r file in the run_path to retrieve that last date.

    Arguments:
        casename
        run_path - path to run directory
    """
    logger.debug('get_run_last_date')
    pattern = '{0}.cpl.r.*.nc'.format(casename)
    cpl_files = sorted(glob.glob(os.path.join(run_path, pattern)))
    if cpl_files:
        _, cpl_file = os.path.split(cpl_files[-1])
        fparts = cpl_file.split('.')
        return fparts[-2]
    return '0000-00-00'


<mask token>


def query_expdb_cmip6(case_dict, username, password):
    """query_exp_case
    Query the expdb for CMIP6 casename = case_dict['q_casename'] metadata.
    Write out a json file to case_dict['q_outfile'].

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        username (string) - SVN developer's username
        password (string) - SVN developer's password

    """
    logger.debug('query_expdb_cmip6')
    exists = False
    data_dict = {'casename': case_dict['q_casename'], 'queryType':
        'CMIP6GlobalAtts', 'expType': 'CMIP6'}
    data = json.dumps(data_dict)
    params = urllib.parse.urlencode(dict(username=username, password=
        password, data=data))
    try:
        response = urllib.request.urlopen(case_dict['query_expdb_url'],
            params, context=_context)
        output = json.load(response)
    except urllib.error.HTTPError as http_e:
        logger.info('ERROR archive_metadata HTTP post failed "%s"', http_e.code
            )
    except urllib.error.URLError as url_e:
        logger.info('ERROR archive_metadata URL failed "%s"', url_e.reason)
    if output:
        if not os.path.exists('{0}/archive_files'.format(case_dict['workdir'])
            ):
            os.makedirs('{0}/archive_files'.format(case_dict['workdir']))
        filename = '{0}/archive_files/{1}'.format(case_dict['workdir'],
            case_dict['q_outfile'])
        with io.open(filename, 'w+', encoding='utf-8') as fname:
            fname.write(json.dumps(output, ensure_ascii=False))
            fname.close()
        exists = True
    return exists


<mask token>


def post_json(case_dict, username, password):
    """post_json
    Post a JSON file in the caseroot/archive_files to the
    remote expdb URL.

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        username (string) - SVN developers username
        password (string) - SVN developers password
    """
    logger.debug('post_json')
    case_dict['COMPSET'] = urllib.parse.quote(case_dict['COMPSET'])
    case_dict['GRID'] = urllib.parse.quote(case_dict['GRID'])
    data = json.dumps(case_dict)
    params = urllib.parse.urlencode(dict(username=username, password=
        password, data=data))
    try:
        urllib.request.urlopen(case_dict['json_expdb_url'], params, context
            =_context)
    except urllib.error.HTTPError as http_e:
        logger.info('ERROR archive_metadata HTTP post failed "%s"', http_e.code
            )
    except urllib.error.URLError as url_e:
        logger.info('ERROR archive_metadata URL failed "%s"', url_e.reason)


<mask token>


def check_svn_repo(case_dict, username, password):
    """check_svn_repo

    check if a SVN repo exists for this case
    """
    logger.debug('check_svn_repo')
    repo_exists = False
    svn_repo = '{0}/trunk'.format(case_dict['svn_repo_url'])
    cmd = ['svn', 'list', svn_repo, '--username', username, '--password',
        password]
    result = ''
    try:
        result = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        msg = (
            'SVN repo does not exist for this case. A new one will be created.'
            )
        logger.warning(msg)
    if re.search('README.archive', result):
        repo_exists = True
    return repo_exists


def get_trunk_tag(case_dict, username, password):
    """get_trunk_tag

    return the most recent trunk tag as an integer
    """
    logger.debug('get_trunk_tag')
    tag = 0
    svn_repo = '{0}/trunk_tags'.format(case_dict['svn_repo_url'])
    cmd = ['svn', 'list', svn_repo, '--username', username, '--password',
        password]
    result = ''
    try:
        result = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = ['svn', 'list', svn_repo, '--username', username,
            '--password', '******']
        msg = _call_template.substitute(function='get_trunk_tag', cmd=
            cmd_nopasswd, error=error.returncode, strerror=error.output)
        logger.warning(msg)
        raise SVNException(msg)
    if result:
        last_tag = [i for i in result.split('\n') if i][-1]
        last_tag = last_tag[:-1].split('_')[-1]
        tag = int(last_tag.lstrip('0'))
    return tag


<mask token>


def update_repo_add_file(filename, dir1, dir2):
    """update_repo_add_file

    Add a file to the SVN repository
    """
    src = os.path.join(dir1, filename)
    dest = os.path.join(dir2, filename)
    logger.debug('left_only: ' + src + ' -> ' + dest)
    if not os.path.exists(dest):
        shutil.copy2(src, dest)
        cmd = ['svn', 'add', '--parents', dest]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            msg = _call_template.substitute(function='update_lcoal_repo',
                cmd=cmd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)


def update_repo_rm_file(filename, dir1, dir2):
    """update_repo_rm_file

    Remove a file from the SVN repository
    """
    src = os.path.join(dir2, filename)
    dest = os.path.join(dir1, filename)
    logger.debug('right_only: ' + src + ' -> ' + dest)
    if os.path.exists(dest):
        cmd = ['svn', 'rm', dest]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            msg = _call_template.substitute(function='update_lcoal_repo',
                cmd=cmd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)


def update_repo_copy_file(filename, dir1, dir2):
    """update_repo_copy_file

    Copy a file into the SVN local repo
    """
    src = os.path.join(dir1, filename)
    dest = os.path.join(dir2, filename)
    shutil.copy2(src, dest)


<mask token>


def update_local_repo(case_dict, ignore_logs, ignore_timing):
    """update_local_repo

    Compare and update local SVN sandbox
    """
    logger.debug('update_local_repo')
    from_dir = case_dict['CASEROOT']
    to_dir = case_dict['archive_temp_dir']
    compare_dir_trees(from_dir, to_dir, case_dict['archive_list'])
    if ignore_logs:
        os.chdir(to_dir)
        if os.path.isdir('./logs'):
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "logs" in archive dir.'
                    )
            cmd = ['svn', 'delete', './logs']
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(function=
                    'update_lcoal_repo', cmd=cmd, error=error.returncode,
                    strerror=error.output)
                logger.warning(msg)
                raise SVNException(msg)
        if os.path.isdir('./postprocess/logs'):
            os.chdir('./postprocess')
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "postprocess/logs" in archive dir.'
                    )
            cmd = ['svn', 'delete', './logs']
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(function=
                    'update_lcoal_repo', cmd=cmd, error=error.returncode,
                    strerror=error.output)
                logger.warning(msg)
                raise SVNException(msg)
    else:
        if os.path.exists('{0}/logs'.format(from_dir)):
            if not os.path.exists('{0}/logs'.format(to_dir)):
                os.makedirs('{0}/logs'.format(to_dir))
            os.chdir(os.path.join(from_dir, 'logs'))
            for filename in glob.glob('*.*'):
                update_repo_add_file(filename, os.path.join(from_dir,
                    'logs'), os.path.join(to_dir, 'logs'))
        if os.path.exists('{0}/postprocess/logs'.format(from_dir)):
            if not os.path.exists('{0}/postprocess/logs'.format(to_dir)):
                os.makedirs('{0}/postprocess/logs'.format(to_dir))
            os.chdir(os.path.join(from_dir, 'postprocess/logs'))
            for filename in glob.glob('*.*'):
                update_repo_add_file(filename, os.path.join(from_dir,
                    'postprocess', 'logs'), os.path.join(to_dir,
                    'postprocess', 'logs'))
    if ignore_timing:
        os.chdir(case_dict['archive_temp_dir'])
        if os.path.isdir('./timing'):
            try:
                shutil.rmtree('./timing')
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "timing" in archive dir.'
                    )
            cmd = ['svn', 'delete', './timing']
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(function=
                    'update_lcoal_repo', cmd=cmd, error=error.returncode,
                    strerror=error.output)
                logger.warning(msg)
                raise SVNException(msg)
    elif os.path.exists('{0}/timing'.format(from_dir)):
        if not os.path.exists('{0}/timing'.format(to_dir)):
            os.makedirs('{0}/timing'.format(to_dir))
        os.chdir(os.path.join(from_dir, 'timing'))
        for filename in glob.glob('*.*'):
            update_repo_add_file(filename, os.path.join(from_dir, 'timing'),
                os.path.join(to_dir, 'timing'))


def populate_local_repo(case_dict, ignore_logs, ignore_timing):
    """populate_local_repo

    Populate local SVN sandbox
    """
    logger.debug('populate_local_repo')
    os.chdir(case_dict['CASEROOT'])
    for archive in case_dict['archive_list']:
        if os.path.exists(archive):
            if os.path.isdir(archive):
                try:
                    target = case_dict['archive_temp_dir'] + '/' + archive
                    shutil.copytree(archive, target, symlinks=False, ignore
                        =shutil.ignore_patterns(*_ignore_patterns))
                except OSError as error:
                    msg = _copy_template.substitute(function=
                        'populate_local_repo', source=archive, dest=
                        case_dict['archive_temp_dir'], error=error.errno,
                        strerror=error.strerror)
                    logger.warning(msg)
            else:
                try:
                    shutil.copy2(archive, case_dict['archive_temp_dir'])
                except OSError as error:
                    msg = _copy_template.substitute(function=
                        'populate_local_repo', source=archive, dest=
                        case_dict['archive_temp_dir'], error=error.errno,
                        strerror=error.strerror)
                    logger.warning(msg)
    xml_files = glob.glob('*.xml')
    for xml_file in xml_files:
        if os.path.isfile(xml_file):
            try:
                shutil.copy2(xml_file, case_dict['archive_temp_dir'])
            except OSError as error:
                msg = _copy_template.substitute(function=
                    'populate_local_repo', source=xml_file, dest=case_dict[
                    'archive_temp_dir'], error=error.errno, strerror=error.
                    strerror)
                logger.warning(msg)
    if os.path.isdir('./postprocess'):
        pp_path = '{0}/{1}'.format(case_dict['archive_temp_dir'], 'postprocess'
            )
        if not os.path.exists(pp_path):
            os.mkdir(pp_path)
        xml_files = glob.glob('./postprocess/*.xml')
        for xml_file in xml_files:
            if os.path.isfile(xml_file):
                try:
                    shutil.copy2(xml_file, pp_path)
                except OSError as error:
                    msg = _copy_template.substitute(function=
                        'populate_local_repo', source=xml_file, dest=
                        case_dict['archive_temp_dir'], error=error.errno,
                        strerror=error.strerror)
                    logger.warning(msg)
    user_files = glob.glob('user_nl_*')
    for user_file in user_files:
        if os.path.isfile(user_file):
            try:
                shutil.copy2(user_file, case_dict['archive_temp_dir'])
            except OSError as error:
                msg = _copy_template.substitute(function=
                    'populate_local_repo', source=user_file, dest=case_dict
                    ['archive_temp_dir'], error=error.errno, strerror=error
                    .strerror)
                logger.warning(msg)
    conf_files = glob.glob('Depends.*')
    for conf_file in conf_files:
        if os.path.isfile(conf_file):
            try:
                shutil.copy2(conf_file, case_dict['archive_temp_dir'])
            except OSError as error:
                msg = _copy_template.substitute(function=
                    'populate_local_repo', source=conf_file, dest=case_dict
                    ['archive_temp_dir'], error=error.errno, strerror=error
                    .strerror)
                logger.warning(msg)
    if ignore_logs:
        os.chdir(case_dict['archive_temp_dir'])
        if os.path.isdir('./logs'):
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "logs" in archive_temp_dir.'
                    )
        if os.path.isdir('./postprocess/logs'):
            os.chdir('./postprocess')
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "postprocess/logs" in archive_temp_dir.'
                    )
        os.chdir(case_dict['CASEROOT'])
    if ignore_timing:
        os.chdir(case_dict['archive_temp_dir'])
        if os.path.isdir('./timing'):
            try:
                shutil.rmtree('./timing')
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "timing" in archive_temp_dir.'
                    )
        os.chdir(case_dict['CASEROOT'])


def checkin_trunk(case_dict, svn_cmd, message, username, password):
    """checkin_trunk

    Check in the local SVN sandbox to the remote trunk
    """
    logger.debug('checkin_trunk')
    os.chdir(case_dict['archive_temp_dir'])
    svn_repo = '{0}/trunk'.format(case_dict['svn_repo_url'])
    msg = '"{0}"'.format(message)
    cmd = ['svn', svn_cmd, '--username', username, '--password', password,
        '.', '--message', msg]
    if svn_cmd in ['import']:
        msg = '"create trunk"'
        cmd = ['svn', 'mkdir', '--parents', svn_repo, '--username',
            username, '--password', password, '--message', msg]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            cmd_nopasswd = ['svn', 'mkdir', '--parents', svn_repo,
                '--username', username, '--password', '******', '--message',
                msg]
            msg = _call_template.substitute(function='checkin_trunk', cmd=
                cmd_nopasswd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)
        tags = '{0}/trunk_tags'.format(case_dict['svn_repo_url'])
        msg = '"create trunk_tags"'
        cmd = ['svn', 'mkdir', tags, '--username', username, '--password',
            password, '--message', msg]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            cmd_nopasswd = ['svn', 'mkdir', tags, '--username', username,
                '--password', '******', '--message', msg]
            msg = _call_template.substitute(function='checkin_trunk', cmd=
                cmd_nopasswd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)
        msg = '"{0}"'.format(message)
        cmd = ['svn', svn_cmd, '--username', username, '--password',
            password, '.', svn_repo, '--message', msg]
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = ['svn', svn_cmd, '--username', username,
            '--password', '******', '.', '--message', msg]
        msg = _call_template.substitute(function='checkin_trunk', cmd=
            cmd_nopasswd, error=error.returncode, strerror=error.output)
        logger.warning(msg)
        raise SVNException(msg)


<mask token>


def update_repo(ignore_logs, ignore_timing, case_dict, username, password):
    """update_repo

    Update SVN repo
    """
    logger.debug('update_repo')
    try:
        svn_exists = check_svn()
        if svn_exists:
            case_dict['svn_repo_url'] = '{0}/{1}'.format(_svn_expdb_url,
                case_dict['CASE'])
            repo_exists = check_svn_repo(case_dict, username, password)
            case_dict['archive_temp_dir'] = create_temp_archive(case_dict)
            case_dict['archive_list'] = _archive_list + case_dict[
                'user_add_files']
            if repo_exists:
                last_tag = get_trunk_tag(case_dict, username, password)
                new_tag = '{0}_{1}'.format(case_dict['CASE'], str(last_tag +
                    1).zfill(4))
                checkout_repo(case_dict, username, password)
                update_local_repo(case_dict, ignore_logs, ignore_timing)
                msg = 'update case metadata for {0} by {1}'.format(case_dict
                    ['CASE'], username)
                checkin_trunk(case_dict, 'ci', msg, username, password)
                create_tag(case_dict, new_tag, username, password)
                logger.info('SVN repository trunk updated at URL "%s"',
                    case_dict['svn_repo_url'])
                logger.info(' and a new trunk tag created "%s"', new_tag)
            else:
                new_tag = '{0}_0001'.format(case_dict['CASE'])
                create_readme(case_dict)
                populate_local_repo(case_dict, ignore_logs, ignore_timing)
                msg = 'initial import of case metadata for {0} by {1}'.format(
                    case_dict['CASE'], username)
                checkin_trunk(case_dict, 'import', msg, username, password)
                create_tag(case_dict, new_tag, username, password)
                logger.info('SVN repository imported to trunk URL "%s"',
                    case_dict['svn_repo_url'])
                logger.info('   and a new trunk tag created for  "%s"', new_tag
                    )
    except SVNException:
        pass
    return case_dict


<mask token>


def initialize_main(options):
    """initialize_main

    Initialize the case dictionary data structure with command line options
    """
    logger.debug('intialize_main')
    case_dict = dict()
    case_dict['CASEROOT'] = os.getcwd()
    if options.caseroot:
        case_dict['CASEROOT'] = options.caseroot[0]
    case_dict['workdir'] = case_dict['CASEROOT']
    if options.workdir:
        case_dict['workdir'] = options.workdir[0]
    username = None
    if options.user:
        username = options.user
        case_dict['svnlogin'] = username
    password = None
    if options.password:
        password = options.password
    if options.expType:
        case_dict['expType'] = options.expType[0]
    case_dict['title'] = None
    if options.title:
        case_dict['title'] = options.title[0]
    case_dict['dryrun'] = False
    if options.dryrun:
        case_dict['dryrun'] = True
    case_dict['archive_temp_dir'] = ''
    case_dict['user_add_files'] = list()
    if options.user_add_files:
        case_dict['user_add_files'] = options.user_add_files.split(',')
    case_dict['q_casename'] = ''
    case_dict['q_outfile'] = ''
    if options.query_cmip6:
        case_dict['q_casename'] = options.query_cmip6[0]
        case_dict['q_outfile'] = options.query_cmip6[1]
    case_dict['base_expdb_url'] = 'https://csegweb.cgd.ucar.edu/expdb2.0'
    if options.test_post:
        case_dict['base_expdb_url'
            ] = 'https://csegwebdev.cgd.ucar.edu/expdb2.0'
    case_dict['json_expdb_url'] = case_dict['base_expdb_url'
        ] + '/cgi-bin/processJSON.cgi'
    case_dict['query_expdb_url'] = case_dict['base_expdb_url'
        ] + '/cgi-bin/query.cgi'
    return case_dict, username, password


<mask token>

<mask token>


class PasswordPromptAction(argparse.Action):
    """SVN developer's password class handler"""

    def __init__(self, option_strings=None, dest=None, default=None,
        required=False, nargs=0, help=None):
        super(PasswordPromptAction, self).__init__(option_strings=
            option_strings, dest=dest, default=default, required=required,
            nargs=nargs, help=help)

    def __call__(self, parser, args, values, option_string=None):
        home = expanduser('~')
        conf_path = os.path.join(home, '.subversion/cmip6.conf')
        if os.path.exists(conf_path):
            config = configparser.SafeConfigParser()
            config.read(conf_path)
            password = config.get('svn', 'password')
        else:
            password = getpass.getpass()
        setattr(args, self.dest, password)


def basic_authorization(user, password):
    """Basic authentication encoding"""
    sauth = user + ':' + password
    return 'Basic ' + sauth.encode('base64').rstrip()


class SVNException(Exception):
    """SVN command exception handler"""

    def __init__(self, value):
        super(SVNException, self).__init__(value)
        self.value = value

    def __str__(self):
        return repr(self.value)


def commandline_options(args):
    """Process the command line arguments."""
    parser = argparse.ArgumentParser(description=
        'Query and parse the caseroot files to gather metadata information that can be posted to the CESM experiments database.  CMIP6 experiment case names must be reserved already in the experiment database. Please see: https://csesgweb.cgd.ucar.edu/expdb2.0 for details.'
        )
    CIME.utils.setup_standard_logging_options(parser)
    parser.add_argument('--user', dest='user', type=str, default=None,
        required=True, help=
        'User name for SVN CESM developer access (required)')
    parser.add_argument('--password', dest='password', action=
        PasswordPromptAction, default='', required=True, help=
        'Password for SVN CESM developer access (required)')
    parser.add_argument('--caseroot', nargs=1, required=False, help=
        'Fully quailfied path to case root directory (optional). Defaults to current working directory.'
        )
    parser.add_argument('--workdir', nargs=1, required=False, help=
        'Fully quailfied path to directory for storing intermediate case files. A sub-directory called archive_temp_dir is created, populated with case files, and posted to the CESM experiments database and SVN repository at URL "{0}". This argument can be used to archive a caseroot when the user does not have write permission in the caseroot (optional). Defaults to current working directory.'
        .format(_svn_expdb_url))
    parser.add_argument('--expType', dest='expType', nargs=1, required=True,
        choices=_exp_types, help=
        'Experiment type. For CMIP6 experiments, the case must already exist in the experiments database at URL  "http://csegweb.cgd.ucar.edu/expdb2.0" (required). Must be one of "{0}"'
        .format(_exp_types))
    parser.add_argument('--title', nargs=1, required=False, default=None,
        help='Title of experiment (optional).')
    parser.add_argument('--ignore-logs', dest='ignore_logs', action=
        'store_true', help=
        'Ignore updating the SVN repository with the caseroot/logs files. The experiments database will be updated (optional).'
        )
    parser.add_argument('--ignore-timing', dest='ignore_timing', action=
        'store_true', help=
        'Ignore updating the the SVN repository with caseroot/timing files.The experiments database will be updated (optional).'
        )
    parser.add_argument('--ignore-repo-update', dest='ignore_repo_update',
        action='store_true', help=
        'Ignore updating the SVN repository with all the caseroot files. The experiments database will be updated (optional).'
        )
    parser.add_argument('--add-files', dest='user_add_files', required=
        False, help=
        'Comma-separated list with no spaces of files or directories to be added to the SVN repository. These are in addition to the default added caseroot files and directories: "{0}, *.xml, user_nl_*" (optional).'
        .format(_archive_list))
    parser.add_argument('--dryrun', action='store_true', help=
        'Parse settings and print what actions will be taken but do not execute the action (optional).'
        )
    parser.add_argument('--query_cmip6', nargs=2, required=False, help=
        'Query the experiments database global attributes for specified CMIP6 casename as argument 1. Writes a json formatted output file, specified by argument 2, to subdir archive_files (optional).'
        )
    parser.add_argument('--test-post', dest='test_post', action=
        'store_true', help=
        'Post metadata to the test expdb2.0 web application server at URL "http://csegwebdev.cgd.ucar.edu/expdb2.0". No --test-post argument defaults to posting metadata to the production expdb2.0 web application server at URL "http://csegweb.cgd.ucar.edu/expdb2.0" (optional).'
        )
    opts = CIME.utils.parse_args_and_handle_standard_logging_options(args,
        parser)
    return opts


def get_case_vars(case_dict, case):
    """get_case_vars
    loop through the global list of XML vars and get the values
    from the case object into a case dictionary

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        case (object) - case object
    """
    logger.debug('get_case_vars')
    for xml_id in _xml_vars:
        case_dict[xml_id] = case.get_value(xml_id, resolved=True, subgroup=None
            )
    for xml_id in _run_vars:
        case_dict[xml_id] = case.get_value(xml_id, resolved=True, subgroup=
            'case.run')
    return case_dict


def get_disk_usage(path):
    """get_disk_usage
    return the total disk usage in bytes for a given path.

    Arguments:
        path - path to start
    """
    logger.debug('get_disk_usage')
    total_size = 0
    cwd = os.getcwd()
    if os.path.exists(path):
        os.chdir(path)
        cmd = ['du', '--summarize', '--block-size=1']
        try:
            total_size = subprocess.check_output(cmd)
            total_size = total_size.replace('\t.\n', '')
        except subprocess.CalledProcessError:
            msg = "Error executing command = '{0}'".format(cmd)
            logger.warning(msg)
    os.chdir(cwd)
    return int(total_size)


def get_ocn_disk_usage(path):
    """get_ocn_disk_usage
    return the total disk usage in bytes for a given path.

    Arguments:
        path - path to start
    """
    logger.debug('get_ocn_disk_usage')
    total_size = 0
    paths = glob.glob(path)
    for path in paths:
        total_size += get_disk_usage(path)
    return int(total_size)


def get_pp_path(pp_dir, process):
    """get_pp_path
    return the XML path for process

    Arguments:
        pp_dir  - path to postprocess directory
        process - process name
    """
    logger.debug('get_pp_path')
    cwd = os.getcwd()
    os.chdir(pp_dir)
    pp_path_var = ''
    if process == 'timeseries':
        pp_path_var = _pp_xml_vars['timeseries']
    elif process == 'xconform':
        pp_path_var = _pp_xml_vars['xconform']
    cmd = ['./pp_config', '--get', pp_path_var, '--value']
    try:
        pp_path = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        msg = "Error executing command = '{0}'".format(cmd)
        logger.warning(msg)
    if len(pp_path) > 2:
        pp_path = pp_path.rstrip()
    else:
        pp_path = ''
    os.chdir(cwd)
    return pp_path


def get_diag_dates(comp, pp_dir):
    """get_diag_dates

    Query the postprocessing env_diags_[comp].xml file to get the model diag
    dates for the given component.
    """
    logger.debug('get_diag_dates')
    cwd = os.getcwd()
    os.chdir(pp_dir)
    model_dates = ''
    pp_vars = _pp_diag_vars.get(comp)
    for pp_var in pp_vars:
        cmd = ['./pp_config', '--get', pp_var, '--value']
        try:
            pp_value = subprocess.check_output(cmd)
        except subprocess.CalledProcessError:
            msg = "Error executing command = '{0}'".format(cmd)
            logger.warning(msg)
        tmp_dates = '{0} = {1}'.format(pp_var, pp_value)
        model_dates = model_dates + tmp_dates
    os.chdir(cwd)
    return model_dates


def get_pp_status(case_dict):
    """get_pp_status
    Parse the postprocessing log files
    looking for status information

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug('get_pp_status')
    msg_avg = dict()
    msg_diags = dict()
    diag_comps = ['atm', 'ice', 'lnd', 'ocn']
    pp_dir = os.path.join(case_dict['CASEROOT'], 'postprocess')
    pp_log_dir = os.path.join(case_dict['CASEROOT'], 'postprocess', 'logs')
    msg_avg['atm'] = 'COMPLETED SUCCESSFULLY'
    msg_diags['atm'
        ] = 'Successfully completed generating atmosphere diagnostics'
    case_dict['atm_avg_dates'] = case_dict['atm_diag_dates'] = get_diag_dates(
        'atm', pp_dir)
    msg_avg['ice'
        ] = 'Successfully completed generating ice climatology averages'
    msg_diags['ice'] = 'Successfully completed generating ice diagnostics'
    case_dict['ice_avg_dates'] = case_dict['ice_diag_dates'] = get_diag_dates(
        'ice', pp_dir)
    msg_avg['lnd'] = 'COMPLETED SUCCESSFULLY'
    msg_diags['lnd'] = 'Successfully completed generating land diagnostics'
    case_dict['lnd_avg_dates'] = case_dict['lnd_diag_dates'] = get_diag_dates(
        'lnd', pp_dir)
    msg_avg['ocn'
        ] = 'Successfully completed generating ocean climatology averages'
    msg_diags['ocn'] = 'Successfully completed generating ocean diagnostics'
    case_dict['ocn_avg_dates'] = case_dict['ocn_diag_dates'] = get_diag_dates(
        'ocn', pp_dir)
    for comp in diag_comps:
        case_dict[comp + '_avg_status'] = 'Unknown'
        case_dict[comp + '_diag_status'] = 'Unknown'
        if comp != 'ocn':
            case_dict[comp + '_avg_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/climo')
            case_dict[comp + '_avg_size'] = get_disk_usage(case_dict[comp +
                '_avg_path'])
            case_dict[comp + '_diag_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/diag')
            case_dict[comp + '_diag_size'] = get_disk_usage(case_dict[comp +
                '_diag_path'])
        else:
            case_dict[comp + '_avg_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/climo*')
            case_dict[comp + '_avg_size'] = get_ocn_disk_usage(case_dict[
                comp + '_avg_path'])
            case_dict[comp + '_diag_path'] = os.path.join(case_dict[
                'DOUT_S_ROOT'], comp, 'proc/diag*')
            case_dict[comp + '_diag_size'] = get_ocn_disk_usage(case_dict[
                comp + '_diag_path'])
        avg_logs = list()
        avg_file_pattern = '{0}/{1}_averages.log.*'.format(pp_log_dir, comp)
        avg_logs = glob.glob(avg_file_pattern)
        if avg_logs:
            log_file = max(avg_logs, key=os.path.getctime)
            if is_last_process_complete(log_file, msg_avg[comp],
                'Average list complies with standards.'):
                case_dict[comp + '_avg_status'] = 'Succeeded'
            else:
                case_dict[comp + '_avg_status'] = 'Started'
        diag_logs = list()
        diag_file_pattern = '{0}/{1}_diagnostics.log.*'.format(pp_log_dir, comp
            )
        diag_logs = glob.glob(diag_file_pattern)
        if diag_logs:
            log_file = max(diag_logs, key=os.path.getctime)
            if is_last_process_complete(log_file, msg_diags[comp],
                'ncks version'):
                case_dict[comp + '_diag_status'] = 'Succeeded'
            else:
                case_dict[comp + '_diag_status'] = 'Started'
    case_dict['timeseries_status'] = 'Unknown'
    case_dict['timeseries_path'] = get_pp_path(pp_dir, 'timeseries')
    case_dict['timeseries_size'] = 0
    case_dict['timeseries_dates'] = '{0}-{1}'.format(case_dict[
        'RUN_STARTDATE'].replace('-', ''), case_dict['RUN_STARTDATE'].
        replace('-', ''))
    case_dict['timeseries_total_time'] = 0
    tseries_logs = list()
    tseries_file_pattern = '{0}/timeseries.log.*'.format(pp_log_dir)
    tseries_logs = glob.glob(tseries_file_pattern)
    if tseries_logs:
        log_file = max(tseries_logs, key=os.path.getctime)
        if is_last_process_complete(filepath=log_file, expect_text=
            'Successfully completed', fail_text='opening'):
            case_dict['timeseries_status'] = 'Succeeded'
            with open(log_file, 'r') as fname:
                log_content = fname.readlines()
            total_time = [line for line in log_content if 'Total Time:' in line
                ]
            case_dict['timeseries_total_time'] = ' '.join(total_time[0].split()
                )
        else:
            case_dict['timeseries_status'] = 'Started'
        sta_dates = case_dict['sta_last_date'].split('-')
        case_dict['timeseries_dates'] = '{0}-{1}'.format(case_dict[
            'RUN_STARTDATE'].replace('-', ''), ''.join(sta_dates[:-1]))
        for comp in _pp_tseries_comps:
            tseries_path = '{0}/{1}/proc/tseries'.format(case_dict[
                'timeseries_path'], comp)
            case_dict['timeseries_size'] += get_disk_usage(tseries_path)
    case_dict['iconform_status'] = 'Unknown'
    case_dict['iconform_path'] = ''
    case_dict['iconform_size'] = 0
    case_dict['iconform_dates'] = case_dict['timeseries_dates']
    iconform_logs = list()
    iconform_file_pattern = '{0}/iconform.log.*'.format(pp_log_dir)
    iconform_logs = glob.glob(iconform_file_pattern)
    if iconform_logs:
        log_file = max(iconform_logs, key=os.path.getctime)
        if is_last_process_complete(log_file,
            'Successfully created the conform tool',
            'Running createOutputSpecs'):
            case_dict['iconform_status'] = 'Succeeded'
        else:
            case_dict['iconform_status'] = 'Started'
    case_dict['xconform_path'] = ''
    case_dict['xconform_path'] = get_pp_path(pp_dir, 'xconform')
    case_dict['xconform_status'] = 'Unknown'
    case_dict['xconform_size'] = get_disk_usage(case_dict['xconform_path'])
    case_dict['xconform_dates'] = case_dict['timeseries_dates']
    case_dict['xconform_total_time'] = 0
    xconform_logs = list()
    xconform_file_pattern = '{0}/xconform.log.*'.format(pp_log_dir)
    xconform_logs = glob.glob(xconform_file_pattern)
    if xconform_logs:
        log_file = max(xconform_logs, key=os.path.getctime)
        if is_last_process_complete(log_file,
            'Successfully completed converting all files',
            'cesm_conform_generator INFO'):
            case_dict['xconform_status'] = 'Succeeded'
            case_dict['xconform_size'] = get_disk_usage(case_dict[
                'xconform_path'])
            with open(log_file, 'r') as fname:
                log_content = fname.readlines()
            total_time = [line for line in log_content if 'Total Time:' in line
                ]
            if total_time:
                case_dict['xconform_total_time'] = ' '.join(total_time[0].
                    split())
        else:
            case_dict['xconform_status'] = 'Started'
    return case_dict


def get_run_last_date(casename, run_path):
    """get_run_last_date
    parse the last cpl.r file in the run_path to retrieve that last date.

    Arguments:
        casename
        run_path - path to run directory
    """
    logger.debug('get_run_last_date')
    pattern = '{0}.cpl.r.*.nc'.format(casename)
    cpl_files = sorted(glob.glob(os.path.join(run_path, pattern)))
    if cpl_files:
        _, cpl_file = os.path.split(cpl_files[-1])
        fparts = cpl_file.split('.')
        return fparts[-2]
    return '0000-00-00'


def get_sta_last_date(sta_path):
    """get_sta_last_date
    parse the last rest directory in the sta_path to retrieve that last date.

    Arguments:
        sta_path - path to run directory
    """
    logger.debug('get_sta_last_date')
    rest_dirs = sorted(glob.glob(os.path.join(sta_path, 'rest/*')))
    if rest_dirs:
        _, rest_dir = os.path.split(rest_dirs[-1])
        return rest_dir
    return '0000-00-00'


def get_case_status(case_dict):
    """get_case_status
    Parse the CaseStatus and postprocessing log files
    looking for status information

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug('get_case_status')
    case_dict['run_status'] = 'Unknown'
    case_dict['run_path'] = case_dict['RUNDIR']
    case_dict['run_size'] = 0
    case_dict['run_last_date'] = case_dict['RUN_STARTDATE']
    case_dict['sta_status'] = 'Unknown'
    case_dict['sta_path'] = case_dict['DOUT_S_ROOT']
    case_dict['sta_size'] = 0
    case_dict['sta_last_date'] = case_dict['RUN_STARTDATE']
    cstatus = case_dict['CASEROOT'] + '/CaseStatus'
    if os.path.exists(cstatus):
        run_status_1 = is_last_process_complete(cstatus, 'case.run success',
            'case.run starting')
        run_status_2 = is_last_process_complete(cstatus,
            'model execution success', 'model execution starting')
        if run_status_1 is True or run_status_2 is True:
            case_dict['run_status'] = 'Succeeded'
        case_dict['run_size'] = get_disk_usage(case_dict['run_path'])
        case_dict['run_last_date'] = get_run_last_date(case_dict['CASE'],
            case_dict['run_path'])
        if case_dict['DOUT_S']:
            sta_status = is_last_process_complete(cstatus,
                'st_archive success', 'st_archive starting')
            case_dict['sta_last_date'] = get_sta_last_date(case_dict[
                'DOUT_S_ROOT'])
            if sta_status is True:
                case_dict['sta_status'] = 'Succeeded'
            for subdir in ['atm/hist', 'cpl/hist', 'esp/hist', 'ice/hist',
                'glc/hist', 'lnd/hist', 'logs', 'ocn/hist', 'rest',
                'rof/hist', 'wav/hist', 'iac/hist']:
                path = os.path.join(case_dict['sta_path'], subdir)
                if os.path.isdir(path):
                    case_dict['sta_size'] += get_disk_usage(path)
    case_dict['postprocess'] = False
    if os.path.exists(case_dict['CASEROOT'] + '/postprocess'):
        case_dict['postprocess'] = True
        case_dict = get_pp_status(case_dict)
    return case_dict


def check_expdb_case(case_dict, username, password):
    """check_exp_case
    Cross check the casename with the database for a CMIP6 experiment

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        username (string) - SVN developer's username
        password (string) - SVN developer's password

    Return case_id value; 0 if does not exist or > 0 for exists.

    """
    logger.debug('check_expdb_case')
    data_dict = {'casename': case_dict['CASE'], 'queryType':
        'checkCaseExists', 'expType': case_dict['expType']}
    data = json.dumps(data_dict)
    params = urllib.parse.urlencode(dict(username=username, password=
        password, data=data))
    try:
        response = urllib.request.urlopen(case_dict['query_expdb_url'],
            params, context=_context)
        output = json.loads(response.read().decode())
    except urllib.error.HTTPError as http_e:
        logger.info('ERROR archive_metadata HTTP post failed "%s"', http_e.code
            )
        sys.exit(1)
    except urllib.error.URLError as url_e:
        logger.info('ERROR archive_metadata URL failed "%s"', url_e.reason)
        sys.exit(1)
    return int(output['case_id'])


def query_expdb_cmip6(case_dict, username, password):
    """query_exp_case
    Query the expdb for CMIP6 casename = case_dict['q_casename'] metadata.
    Write out a json file to case_dict['q_outfile'].

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        username (string) - SVN developer's username
        password (string) - SVN developer's password

    """
    logger.debug('query_expdb_cmip6')
    exists = False
    data_dict = {'casename': case_dict['q_casename'], 'queryType':
        'CMIP6GlobalAtts', 'expType': 'CMIP6'}
    data = json.dumps(data_dict)
    params = urllib.parse.urlencode(dict(username=username, password=
        password, data=data))
    try:
        response = urllib.request.urlopen(case_dict['query_expdb_url'],
            params, context=_context)
        output = json.load(response)
    except urllib.error.HTTPError as http_e:
        logger.info('ERROR archive_metadata HTTP post failed "%s"', http_e.code
            )
    except urllib.error.URLError as url_e:
        logger.info('ERROR archive_metadata URL failed "%s"', url_e.reason)
    if output:
        if not os.path.exists('{0}/archive_files'.format(case_dict['workdir'])
            ):
            os.makedirs('{0}/archive_files'.format(case_dict['workdir']))
        filename = '{0}/archive_files/{1}'.format(case_dict['workdir'],
            case_dict['q_outfile'])
        with io.open(filename, 'w+', encoding='utf-8') as fname:
            fname.write(json.dumps(output, ensure_ascii=False))
            fname.close()
        exists = True
    return exists


def create_json(case_dict):
    """create_json
    Create a JSON file in the caseroot/archive_files dir.

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug('create_json')
    if not os.path.exists('{0}/archive_files'.format(case_dict['workdir'])):
        os.makedirs('{0}/archive_files'.format(case_dict['workdir']))
    filename = '{0}/archive_files/json.{1}'.format(case_dict['workdir'],
        datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
    with io.open(filename, 'wb') as fname:
        jstr = str(json.dumps(case_dict, indent=4, sort_keys=True,
            ensure_ascii=False))
        if isinstance(jstr, str):
            jstr = jstr.decode('utf-8')
        fname.write(jstr)
        fname.close()


def post_json(case_dict, username, password):
    """post_json
    Post a JSON file in the caseroot/archive_files to the
    remote expdb URL.

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        username (string) - SVN developers username
        password (string) - SVN developers password
    """
    logger.debug('post_json')
    case_dict['COMPSET'] = urllib.parse.quote(case_dict['COMPSET'])
    case_dict['GRID'] = urllib.parse.quote(case_dict['GRID'])
    data = json.dumps(case_dict)
    params = urllib.parse.urlencode(dict(username=username, password=
        password, data=data))
    try:
        urllib.request.urlopen(case_dict['json_expdb_url'], params, context
            =_context)
    except urllib.error.HTTPError as http_e:
        logger.info('ERROR archive_metadata HTTP post failed "%s"', http_e.code
            )
    except urllib.error.URLError as url_e:
        logger.info('ERROR archive_metadata URL failed "%s"', url_e.reason)


<mask token>


def check_svn_repo(case_dict, username, password):
    """check_svn_repo

    check if a SVN repo exists for this case
    """
    logger.debug('check_svn_repo')
    repo_exists = False
    svn_repo = '{0}/trunk'.format(case_dict['svn_repo_url'])
    cmd = ['svn', 'list', svn_repo, '--username', username, '--password',
        password]
    result = ''
    try:
        result = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        msg = (
            'SVN repo does not exist for this case. A new one will be created.'
            )
        logger.warning(msg)
    if re.search('README.archive', result):
        repo_exists = True
    return repo_exists


def get_trunk_tag(case_dict, username, password):
    """get_trunk_tag

    return the most recent trunk tag as an integer
    """
    logger.debug('get_trunk_tag')
    tag = 0
    svn_repo = '{0}/trunk_tags'.format(case_dict['svn_repo_url'])
    cmd = ['svn', 'list', svn_repo, '--username', username, '--password',
        password]
    result = ''
    try:
        result = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = ['svn', 'list', svn_repo, '--username', username,
            '--password', '******']
        msg = _call_template.substitute(function='get_trunk_tag', cmd=
            cmd_nopasswd, error=error.returncode, strerror=error.output)
        logger.warning(msg)
        raise SVNException(msg)
    if result:
        last_tag = [i for i in result.split('\n') if i][-1]
        last_tag = last_tag[:-1].split('_')[-1]
        tag = int(last_tag.lstrip('0'))
    return tag


def checkout_repo(case_dict, username, password):
    """checkout_repo

    checkout the repo into the archive_temp_dir
    """
    logger.debug('checkout_repo')
    os.chdir(case_dict['archive_temp_dir'])
    svn_repo = '{0}/trunk'.format(case_dict['svn_repo_url'])
    cmd = ['svn', 'co', '--username', username, '--password', password,
        svn_repo, '.']
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = ['svn', 'co', '--username', username, '--password',
            '******', svn_repo, '.']
        msg = _call_template.substitute(function='checkout_repo', cmd=
            cmd_nopasswd, error=error.returncode, strerror=error.output)
        logger.warning(msg)
        raise SVNException(msg)
    os.chdir(case_dict['CASEROOT'])


<mask token>


def update_repo_add_file(filename, dir1, dir2):
    """update_repo_add_file

    Add a file to the SVN repository
    """
    src = os.path.join(dir1, filename)
    dest = os.path.join(dir2, filename)
    logger.debug('left_only: ' + src + ' -> ' + dest)
    if not os.path.exists(dest):
        shutil.copy2(src, dest)
        cmd = ['svn', 'add', '--parents', dest]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            msg = _call_template.substitute(function='update_lcoal_repo',
                cmd=cmd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)


def update_repo_rm_file(filename, dir1, dir2):
    """update_repo_rm_file

    Remove a file from the SVN repository
    """
    src = os.path.join(dir2, filename)
    dest = os.path.join(dir1, filename)
    logger.debug('right_only: ' + src + ' -> ' + dest)
    if os.path.exists(dest):
        cmd = ['svn', 'rm', dest]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            msg = _call_template.substitute(function='update_lcoal_repo',
                cmd=cmd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)


def update_repo_copy_file(filename, dir1, dir2):
    """update_repo_copy_file

    Copy a file into the SVN local repo
    """
    src = os.path.join(dir1, filename)
    dest = os.path.join(dir2, filename)
    shutil.copy2(src, dest)


<mask token>


def update_local_repo(case_dict, ignore_logs, ignore_timing):
    """update_local_repo

    Compare and update local SVN sandbox
    """
    logger.debug('update_local_repo')
    from_dir = case_dict['CASEROOT']
    to_dir = case_dict['archive_temp_dir']
    compare_dir_trees(from_dir, to_dir, case_dict['archive_list'])
    if ignore_logs:
        os.chdir(to_dir)
        if os.path.isdir('./logs'):
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "logs" in archive dir.'
                    )
            cmd = ['svn', 'delete', './logs']
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(function=
                    'update_lcoal_repo', cmd=cmd, error=error.returncode,
                    strerror=error.output)
                logger.warning(msg)
                raise SVNException(msg)
        if os.path.isdir('./postprocess/logs'):
            os.chdir('./postprocess')
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "postprocess/logs" in archive dir.'
                    )
            cmd = ['svn', 'delete', './logs']
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(function=
                    'update_lcoal_repo', cmd=cmd, error=error.returncode,
                    strerror=error.output)
                logger.warning(msg)
                raise SVNException(msg)
    else:
        if os.path.exists('{0}/logs'.format(from_dir)):
            if not os.path.exists('{0}/logs'.format(to_dir)):
                os.makedirs('{0}/logs'.format(to_dir))
            os.chdir(os.path.join(from_dir, 'logs'))
            for filename in glob.glob('*.*'):
                update_repo_add_file(filename, os.path.join(from_dir,
                    'logs'), os.path.join(to_dir, 'logs'))
        if os.path.exists('{0}/postprocess/logs'.format(from_dir)):
            if not os.path.exists('{0}/postprocess/logs'.format(to_dir)):
                os.makedirs('{0}/postprocess/logs'.format(to_dir))
            os.chdir(os.path.join(from_dir, 'postprocess/logs'))
            for filename in glob.glob('*.*'):
                update_repo_add_file(filename, os.path.join(from_dir,
                    'postprocess', 'logs'), os.path.join(to_dir,
                    'postprocess', 'logs'))
    if ignore_timing:
        os.chdir(case_dict['archive_temp_dir'])
        if os.path.isdir('./timing'):
            try:
                shutil.rmtree('./timing')
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "timing" in archive dir.'
                    )
            cmd = ['svn', 'delete', './timing']
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(function=
                    'update_lcoal_repo', cmd=cmd, error=error.returncode,
                    strerror=error.output)
                logger.warning(msg)
                raise SVNException(msg)
    elif os.path.exists('{0}/timing'.format(from_dir)):
        if not os.path.exists('{0}/timing'.format(to_dir)):
            os.makedirs('{0}/timing'.format(to_dir))
        os.chdir(os.path.join(from_dir, 'timing'))
        for filename in glob.glob('*.*'):
            update_repo_add_file(filename, os.path.join(from_dir, 'timing'),
                os.path.join(to_dir, 'timing'))


def populate_local_repo(case_dict, ignore_logs, ignore_timing):
    """populate_local_repo

    Populate local SVN sandbox
    """
    logger.debug('populate_local_repo')
    os.chdir(case_dict['CASEROOT'])
    for archive in case_dict['archive_list']:
        if os.path.exists(archive):
            if os.path.isdir(archive):
                try:
                    target = case_dict['archive_temp_dir'] + '/' + archive
                    shutil.copytree(archive, target, symlinks=False, ignore
                        =shutil.ignore_patterns(*_ignore_patterns))
                except OSError as error:
                    msg = _copy_template.substitute(function=
                        'populate_local_repo', source=archive, dest=
                        case_dict['archive_temp_dir'], error=error.errno,
                        strerror=error.strerror)
                    logger.warning(msg)
            else:
                try:
                    shutil.copy2(archive, case_dict['archive_temp_dir'])
                except OSError as error:
                    msg = _copy_template.substitute(function=
                        'populate_local_repo', source=archive, dest=
                        case_dict['archive_temp_dir'], error=error.errno,
                        strerror=error.strerror)
                    logger.warning(msg)
    xml_files = glob.glob('*.xml')
    for xml_file in xml_files:
        if os.path.isfile(xml_file):
            try:
                shutil.copy2(xml_file, case_dict['archive_temp_dir'])
            except OSError as error:
                msg = _copy_template.substitute(function=
                    'populate_local_repo', source=xml_file, dest=case_dict[
                    'archive_temp_dir'], error=error.errno, strerror=error.
                    strerror)
                logger.warning(msg)
    if os.path.isdir('./postprocess'):
        pp_path = '{0}/{1}'.format(case_dict['archive_temp_dir'], 'postprocess'
            )
        if not os.path.exists(pp_path):
            os.mkdir(pp_path)
        xml_files = glob.glob('./postprocess/*.xml')
        for xml_file in xml_files:
            if os.path.isfile(xml_file):
                try:
                    shutil.copy2(xml_file, pp_path)
                except OSError as error:
                    msg = _copy_template.substitute(function=
                        'populate_local_repo', source=xml_file, dest=
                        case_dict['archive_temp_dir'], error=error.errno,
                        strerror=error.strerror)
                    logger.warning(msg)
    user_files = glob.glob('user_nl_*')
    for user_file in user_files:
        if os.path.isfile(user_file):
            try:
                shutil.copy2(user_file, case_dict['archive_temp_dir'])
            except OSError as error:
                msg = _copy_template.substitute(function=
                    'populate_local_repo', source=user_file, dest=case_dict
                    ['archive_temp_dir'], error=error.errno, strerror=error
                    .strerror)
                logger.warning(msg)
    conf_files = glob.glob('Depends.*')
    for conf_file in conf_files:
        if os.path.isfile(conf_file):
            try:
                shutil.copy2(conf_file, case_dict['archive_temp_dir'])
            except OSError as error:
                msg = _copy_template.substitute(function=
                    'populate_local_repo', source=conf_file, dest=case_dict
                    ['archive_temp_dir'], error=error.errno, strerror=error
                    .strerror)
                logger.warning(msg)
    if ignore_logs:
        os.chdir(case_dict['archive_temp_dir'])
        if os.path.isdir('./logs'):
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "logs" in archive_temp_dir.'
                    )
        if os.path.isdir('./postprocess/logs'):
            os.chdir('./postprocess')
            try:
                shutil.rmtree('./logs')
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "postprocess/logs" in archive_temp_dir.'
                    )
        os.chdir(case_dict['CASEROOT'])
    if ignore_timing:
        os.chdir(case_dict['archive_temp_dir'])
        if os.path.isdir('./timing'):
            try:
                shutil.rmtree('./timing')
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "timing" in archive_temp_dir.'
                    )
        os.chdir(case_dict['CASEROOT'])


def checkin_trunk(case_dict, svn_cmd, message, username, password):
    """checkin_trunk

    Check in the local SVN sandbox to the remote trunk
    """
    logger.debug('checkin_trunk')
    os.chdir(case_dict['archive_temp_dir'])
    svn_repo = '{0}/trunk'.format(case_dict['svn_repo_url'])
    msg = '"{0}"'.format(message)
    cmd = ['svn', svn_cmd, '--username', username, '--password', password,
        '.', '--message', msg]
    if svn_cmd in ['import']:
        msg = '"create trunk"'
        cmd = ['svn', 'mkdir', '--parents', svn_repo, '--username',
            username, '--password', password, '--message', msg]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            cmd_nopasswd = ['svn', 'mkdir', '--parents', svn_repo,
                '--username', username, '--password', '******', '--message',
                msg]
            msg = _call_template.substitute(function='checkin_trunk', cmd=
                cmd_nopasswd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)
        tags = '{0}/trunk_tags'.format(case_dict['svn_repo_url'])
        msg = '"create trunk_tags"'
        cmd = ['svn', 'mkdir', tags, '--username', username, '--password',
            password, '--message', msg]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            cmd_nopasswd = ['svn', 'mkdir', tags, '--username', username,
                '--password', '******', '--message', msg]
            msg = _call_template.substitute(function='checkin_trunk', cmd=
                cmd_nopasswd, error=error.returncode, strerror=error.output)
            logger.warning(msg)
            raise SVNException(msg)
        msg = '"{0}"'.format(message)
        cmd = ['svn', svn_cmd, '--username', username, '--password',
            password, '.', svn_repo, '--message', msg]
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = ['svn', svn_cmd, '--username', username,
            '--password', '******', '.', '--message', msg]
        msg = _call_template.substitute(function='checkin_trunk', cmd=
            cmd_nopasswd, error=error.returncode, strerror=error.output)
        logger.warning(msg)
        raise SVNException(msg)


def create_tag(case_dict, new_tag, username, password):
    """create_tag

    create a new trunk tag in the remote repo
    """
    logger.debug('create_tag')
    os.chdir(case_dict['archive_temp_dir'])
    svn_repo = '{0}/trunk'.format(case_dict['svn_repo_url'])
    svn_repo_tag = '{0}/trunk_tags/{1}'.format(case_dict['svn_repo_url'],
        new_tag)
    msg = '"create new trunk tag"'
    cmd = ['svn', 'copy', '--username', username, '--password', password,
        svn_repo, svn_repo_tag, '--message', msg]
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = ['svn', 'copy', '--username', username, '--password',
            '******', svn_repo, svn_repo_tag, '--message', msg]
        msg = _call_template.substitute(function='checkin_trunk', cmd=
            cmd_nopasswd, error=error.returncode, strerror=error.output)
        logger.warning(msg)
        raise SVNException(msg)


def update_repo(ignore_logs, ignore_timing, case_dict, username, password):
    """update_repo

    Update SVN repo
    """
    logger.debug('update_repo')
    try:
        svn_exists = check_svn()
        if svn_exists:
            case_dict['svn_repo_url'] = '{0}/{1}'.format(_svn_expdb_url,
                case_dict['CASE'])
            repo_exists = check_svn_repo(case_dict, username, password)
            case_dict['archive_temp_dir'] = create_temp_archive(case_dict)
            case_dict['archive_list'] = _archive_list + case_dict[
                'user_add_files']
            if repo_exists:
                last_tag = get_trunk_tag(case_dict, username, password)
                new_tag = '{0}_{1}'.format(case_dict['CASE'], str(last_tag +
                    1).zfill(4))
                checkout_repo(case_dict, username, password)
                update_local_repo(case_dict, ignore_logs, ignore_timing)
                msg = 'update case metadata for {0} by {1}'.format(case_dict
                    ['CASE'], username)
                checkin_trunk(case_dict, 'ci', msg, username, password)
                create_tag(case_dict, new_tag, username, password)
                logger.info('SVN repository trunk updated at URL "%s"',
                    case_dict['svn_repo_url'])
                logger.info(' and a new trunk tag created "%s"', new_tag)
            else:
                new_tag = '{0}_0001'.format(case_dict['CASE'])
                create_readme(case_dict)
                populate_local_repo(case_dict, ignore_logs, ignore_timing)
                msg = 'initial import of case metadata for {0} by {1}'.format(
                    case_dict['CASE'], username)
                checkin_trunk(case_dict, 'import', msg, username, password)
                create_tag(case_dict, new_tag, username, password)
                logger.info('SVN repository imported to trunk URL "%s"',
                    case_dict['svn_repo_url'])
                logger.info('   and a new trunk tag created for  "%s"', new_tag
                    )
    except SVNException:
        pass
    return case_dict


<mask token>


def initialize_main(options):
    """initialize_main

    Initialize the case dictionary data structure with command line options
    """
    logger.debug('intialize_main')
    case_dict = dict()
    case_dict['CASEROOT'] = os.getcwd()
    if options.caseroot:
        case_dict['CASEROOT'] = options.caseroot[0]
    case_dict['workdir'] = case_dict['CASEROOT']
    if options.workdir:
        case_dict['workdir'] = options.workdir[0]
    username = None
    if options.user:
        username = options.user
        case_dict['svnlogin'] = username
    password = None
    if options.password:
        password = options.password
    if options.expType:
        case_dict['expType'] = options.expType[0]
    case_dict['title'] = None
    if options.title:
        case_dict['title'] = options.title[0]
    case_dict['dryrun'] = False
    if options.dryrun:
        case_dict['dryrun'] = True
    case_dict['archive_temp_dir'] = ''
    case_dict['user_add_files'] = list()
    if options.user_add_files:
        case_dict['user_add_files'] = options.user_add_files.split(',')
    case_dict['q_casename'] = ''
    case_dict['q_outfile'] = ''
    if options.query_cmip6:
        case_dict['q_casename'] = options.query_cmip6[0]
        case_dict['q_outfile'] = options.query_cmip6[1]
    case_dict['base_expdb_url'] = 'https://csegweb.cgd.ucar.edu/expdb2.0'
    if options.test_post:
        case_dict['base_expdb_url'
            ] = 'https://csegwebdev.cgd.ucar.edu/expdb2.0'
    case_dict['json_expdb_url'] = case_dict['base_expdb_url'
        ] + '/cgi-bin/processJSON.cgi'
    case_dict['query_expdb_url'] = case_dict['base_expdb_url'
        ] + '/cgi-bin/query.cgi'
    return case_dict, username, password


<mask token>

#!/usr/bin/env python3
"""
Gather all the case metadata and send it to the experiments databases
via a web post and SVN check-in

Author: CSEG <cseg@cgd.ucar.edu>
"""
import argparse
import datetime
import filecmp
import getpass
import glob
import gzip
import json
import io
import os
from os.path import expanduser
import re
import shutil
import ssl
import subprocess
import sys
from string import Template
import configparser
import urllib

from standard_script_setup import *
from CIME.case import Case
from CIME.utils import is_last_process_complete

# define global constants
logger = logging.getLogger(__name__)
_svn_expdb_url = "https://svn-cesm2-expdb.cgd.ucar.edu"
_exp_types = ["CMIP6", "production", "tuning", "lens", "C1", "C2", "C3", "C4", "C5"]
_xml_vars = [
    "CASE",
    "COMPILER",
    "COMPSET",
    "CONTINUE_RUN",
    "DOUT_S",
    "DOUT_S_ROOT",
    "GRID",
    "MACH",
    "MPILIB",
    "MODEL",
    "MODEL_VERSION",
    "REST_N",
    "REST_OPTION",
    "RUNDIR",
    "RUN_REFCASE",
    "RUN_REFDATE",
    "RUN_STARTDATE",
    "RUN_TYPE",
    "STOP_N",
    "STOP_OPTION",
    "USER",
]
_run_vars = ["JOB_QUEUE", "JOB_WALLCLOCK_TIME", "PROJECT"]
_archive_list = [
    "Buildconf",
    "CaseDocs",
    "CaseStatus",
    "LockedFiles",
    "Macros.make",
    "README.case",
    "SourceMods",
    "software_environment.txt",
]
_call_template = Template(
    'in "$function" - Ignoring SVN repo update\n'
    'SVN error executing command "$cmd". \n'
    "$error: $strerror"
)
_copy_template = Template(
    'in "$function" - Unable to copy "$source" to "$dest"' "$error: $strerror"
)
_svn_error_template = Template(
    'in "$function" - SVN client unavailable\n'
    'SVN error executing command "$cmd". \n'
    "$error: $strerror"
)
_ignore_patterns = ["*.pyc", "^.git", "tmp", ".svn", "*~"]
_pp_xml_vars = {
    "atm": "ATMDIAG_test_path_climo",
    "glc": "",
    "lnd": "LNDDIAG_PTMPDIR_1",
    "ice": "ICEDIAG_PATH_CLIMO_CONT",
    "ocn": "OCNDIAG_TAVGDIR",
    "rof": "",
    "timeseries": "TIMESERIES_OUTPUT_ROOTDIR",
    "xconform": "CONFORM_OUTPUT_DIR",
}
_pp_diag_vars = {
    "atm": ["ATMDIAG_test_first_yr", "ATMDIAG_test_nyrs"],
    "ice": ["ICEDIAG_BEGYR_CONT", "ICEDIAG_ENDYR_CONT", "ICEDIAG_YRS_TO_AVG"],
    "lnd": [
        "LNDDIAG_clim_first_yr_1",
        "LNDDIAG_clim_num_yrs_1",
        "LNDDIAG_trends_first_yr_1",
        "LNDDIAG_trends_num_yrs_1",
    ],
    "ocn": [
        "OCNDIAG_YEAR0",
        "OCNDIAG_YEAR1",
        "OCNDIAG_TSERIES_YEAR0",
        "OCNDIAG_TSERIES_YEAR1",
    ],
}
_pp_tseries_comps = ["atm", "glc", "ice", "lnd", "ocn", "rof"]

# setting the ssl context to avoid issues with CGD certificates
_context = ssl._create_unverified_context()  # pylint:disable=protected-access

# -------------------------------------------------------------------------------
class PasswordPromptAction(argparse.Action):
    # -------------------------------------------------------------------------------
    """SVN developer's password class handler"""
    # pylint: disable=redefined-builtin
    def __init__(
        self,
        option_strings=None,
        dest=None,
        default=None,
        required=False,
        nargs=0,
        help=None,
    ):
        super(PasswordPromptAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            default=default,
            required=required,
            nargs=nargs,
            help=help,
        )

    def __call__(self, parser, args, values, option_string=None):
        # check if ~/.subversion/cmip6.conf exists
        home = expanduser("~")
        conf_path = os.path.join(home, ".subversion/cmip6.conf")
        if os.path.exists(conf_path):
            # read the .cmip6.conf file
            config = configparser.SafeConfigParser()
            config.read(conf_path)
            password = config.get("svn", "password")
        else:
            password = getpass.getpass()
        setattr(args, self.dest, password)


# ---------------------------------------------------------------------
def basic_authorization(user, password):
    # ---------------------------------------------------------------------
    """Basic authentication encoding"""
    sauth = user + ":" + password
    return "Basic " + sauth.encode("base64").rstrip()


# ---------------------------------------------------------------------
class SVNException(Exception):
    # ---------------------------------------------------------------------
    """SVN command exception handler"""

    def __init__(self, value):
        super(SVNException, self).__init__(value)
        self.value = value

    def __str__(self):
        return repr(self.value)


# -------------------------------------------------------------------------------
def commandline_options(args):
    # -------------------------------------------------------------------------------
    """Process the command line arguments."""
    parser = argparse.ArgumentParser(
        description="Query and parse the caseroot files to gather metadata information"
        " that can be posted to the CESM experiments database."
        " "
        " CMIP6 experiment case names must be reserved already in the"
        " experiment database. Please see:"
        " https://csesgweb.cgd.ucar.edu/expdb2.0 for details."
    )

    CIME.utils.setup_standard_logging_options(parser)

    parser.add_argument(
        "--user",
        dest="user",
        type=str,
        default=None,
        required=True,
        help="User name for SVN CESM developer access (required)",
    )

    parser.add_argument(
        "--password",
        dest="password",
        action=PasswordPromptAction,
        default="",
        required=True,
        help="Password for SVN CESM developer access (required)",
    )

    parser.add_argument(
        "--caseroot",
        nargs=1,
        required=False,
        help="Fully quailfied path to case root directory (optional). "
        "Defaults to current working directory.",
    )

    parser.add_argument(
        "--workdir",
        nargs=1,
        required=False,
        help="Fully quailfied path to directory for storing intermediate "
        "case files. A sub-directory called "
        "archive_temp_dir is created, populated "
        "with case files, and posted to the CESM experiments database and "
        'SVN repository at URL "{0}". '
        "This argument can be used to archive a caseroot when the user "
        "does not have write permission in the caseroot (optional). "
        "Defaults to current working directory.".format(_svn_expdb_url),
    )

    parser.add_argument(
        "--expType",
        dest="expType",
        nargs=1,
        required=True,
        choices=_exp_types,
        help="Experiment type. For CMIP6 experiments, the case must already "
        "exist in the experiments database at URL "
        ' "http://csegweb.cgd.ucar.edu/expdb2.0" (required). '
        'Must be one of "{0}"'.format(_exp_types),
    )

    parser.add_argument(
        "--title",
        nargs=1,
        required=False,
        default=None,
        help="Title of experiment (optional).",
    )

    parser.add_argument(
        "--ignore-logs",
        dest="ignore_logs",
        action="store_true",
        help="Ignore updating the SVN repository with the caseroot/logs files. "
        "The experiments database will be updated (optional).",
    )

    parser.add_argument(
        "--ignore-timing",
        dest="ignore_timing",
        action="store_true",
        help="Ignore updating the the SVN repository with caseroot/timing files."
        "The experiments database will be updated (optional).",
    )

    parser.add_argument(
        "--ignore-repo-update",
        dest="ignore_repo_update",
        action="store_true",
        help="Ignore updating the SVN repository with all the caseroot files. "
        "The experiments database will be updated (optional).",
    )

    parser.add_argument(
        "--add-files",
        dest="user_add_files",
        required=False,
        help="Comma-separated list with no spaces of files or directories to be "
        "added to the SVN repository. These are in addition to the default added "
        "caseroot files and directories: "
        '"{0}, *.xml, user_nl_*" (optional).'.format(_archive_list),
    )

    parser.add_argument(
        "--dryrun",
        action="store_true",
        help="Parse settings and print what actions will be taken but "
        "do not execute the action (optional).",
    )

    parser.add_argument(
        "--query_cmip6",
        nargs=2,
        required=False,
        help="Query the experiments database global attributes "
        "for specified CMIP6 casename as argument 1. "
        "Writes a json formatted output file, specified by argument 2, "
        "to subdir archive_files (optional).",
    )

    parser.add_argument(
        "--test-post",
        dest="test_post",
        action="store_true",
        help="Post metadata to the test expdb2.0 web application server "
        'at URL "http://csegwebdev.cgd.ucar.edu/expdb2.0". '
        "No --test-post argument defaults to posting metadata to the "
        "production expdb2.0 web application server "
        'at URL "http://csegweb.cgd.ucar.edu/expdb2.0" (optional).',
    )

    opts = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)

    return opts


# ---------------------------------------------------------------------
def get_case_vars(case_dict, case):
    # ---------------------------------------------------------------------
    """get_case_vars
    loop through the global list of XML vars and get the values
    from the case object into a case dictionary

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        case (object) - case object
    """
    logger.debug("get_case_vars")

    for xml_id in _xml_vars:
        case_dict[xml_id] = case.get_value(xml_id, resolved=True, subgroup=None)

    for xml_id in _run_vars:
        case_dict[xml_id] = case.get_value(xml_id, resolved=True, subgroup="case.run")

    return case_dict


# ---------------------------------------------------------------------
def get_disk_usage(path):
    # ---------------------------------------------------------------------
    """get_disk_usage
    return the total disk usage in bytes for a given path.

    Arguments:
        path - path to start
    """
    logger.debug("get_disk_usage")
    total_size = 0
    cwd = os.getcwd()
    if os.path.exists(path):
        os.chdir(path)
        cmd = ["du", "--summarize", "--block-size=1"]
        try:
            total_size = subprocess.check_output(cmd)
            total_size = total_size.replace("\t.\n", "")
        except subprocess.CalledProcessError:
            msg = "Error executing command = '{0}'".format(cmd)
            logger.warning(msg)
    os.chdir(cwd)
    return int(total_size)


# ---------------------------------------------------------------------
def get_ocn_disk_usage(path):
    # ---------------------------------------------------------------------
    """get_ocn_disk_usage
    return the total disk usage in bytes for a given path.

    Arguments:
        path - path to start
    """
    logger.debug("get_ocn_disk_usage")
    total_size = 0
    paths = glob.glob(path)
    for path in paths:
        total_size += get_disk_usage(path)
    return int(total_size)


# ---------------------------------------------------------------------
def get_pp_path(pp_dir, process):
    # ---------------------------------------------------------------------
    """get_pp_path
    return the XML path for process

    Arguments:
        pp_dir  - path to postprocess directory
        process - process name
    """
    logger.debug("get_pp_path")

    cwd = os.getcwd()
    os.chdir(pp_dir)

    pp_path_var = ""
    if process == "timeseries":
        pp_path_var = _pp_xml_vars["timeseries"]
    elif process == "xconform":
        pp_path_var = _pp_xml_vars["xconform"]

    cmd = ["./pp_config", "--get", pp_path_var, "--value"]
    try:
        pp_path = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        msg = "Error executing command = '{0}'".format(cmd)
        logger.warning(msg)

    if len(pp_path) > 2:
        pp_path = pp_path.rstrip()
    else:
        pp_path = ""

    os.chdir(cwd)
    return pp_path


# ---------------------------------------------------------------------
def get_diag_dates(comp, pp_dir):
    # ---------------------------------------------------------------------
    """get_diag_dates

    Query the postprocessing env_diags_[comp].xml file to get the model diag
    dates for the given component.
    """
    logger.debug("get_diag_dates")

    cwd = os.getcwd()
    os.chdir(pp_dir)

    model_dates = ""
    pp_vars = _pp_diag_vars.get(comp)
    for pp_var in pp_vars:
        cmd = ["./pp_config", "--get", pp_var, "--value"]
        try:
            pp_value = subprocess.check_output(cmd)
        except subprocess.CalledProcessError:
            msg = "Error executing command = '{0}'".format(cmd)
            logger.warning(msg)
        tmp_dates = "{0} = {1}".format(pp_var, pp_value)
        model_dates = model_dates + tmp_dates

    os.chdir(cwd)
    return model_dates


# ---------------------------------------------------------------------
def get_pp_status(case_dict):
    # ---------------------------------------------------------------------
    """get_pp_status
    Parse the postprocessing log files
    looking for status information

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug("get_pp_status")

    # initialize status variables
    msg_avg = dict()
    msg_diags = dict()
    diag_comps = ["atm", "ice", "lnd", "ocn"]

    pp_dir = os.path.join(case_dict["CASEROOT"], "postprocess")
    pp_log_dir = os.path.join(case_dict["CASEROOT"], "postprocess", "logs")

    msg_avg["atm"] = "COMPLETED SUCCESSFULLY"
    msg_diags["atm"] = "Successfully completed generating atmosphere diagnostics"
    case_dict["atm_avg_dates"] = case_dict["atm_diag_dates"] = get_diag_dates(
        "atm", pp_dir
    )

    msg_avg["ice"] = "Successfully completed generating ice climatology averages"
    msg_diags["ice"] = "Successfully completed generating ice diagnostics"
    case_dict["ice_avg_dates"] = case_dict["ice_diag_dates"] = get_diag_dates(
        "ice", pp_dir
    )

    msg_avg["lnd"] = "COMPLETED SUCCESSFULLY"
    msg_diags["lnd"] = "Successfully completed generating land diagnostics"
    case_dict["lnd_avg_dates"] = case_dict["lnd_diag_dates"] = get_diag_dates(
        "lnd", pp_dir
    )

    msg_avg["ocn"] = "Successfully completed generating ocean climatology averages"
    msg_diags["ocn"] = "Successfully completed generating ocean diagnostics"
    case_dict["ocn_avg_dates"] = case_dict["ocn_diag_dates"] = get_diag_dates(
        "ocn", pp_dir
    )

    for comp in diag_comps:
        case_dict[comp + "_avg_status"] = "Unknown"
        case_dict[comp + "_diag_status"] = "Unknown"

        if comp != "ocn":
            case_dict[comp + "_avg_path"] = os.path.join(
                case_dict["DOUT_S_ROOT"], comp, "proc/climo"
            )
            case_dict[comp + "_avg_size"] = get_disk_usage(
                case_dict[comp + "_avg_path"]
            )
            case_dict[comp + "_diag_path"] = os.path.join(
                case_dict["DOUT_S_ROOT"], comp, "proc/diag"
            )
            case_dict[comp + "_diag_size"] = get_disk_usage(
                case_dict[comp + "_diag_path"]
            )
        else:
            case_dict[comp + "_avg_path"] = os.path.join(
                case_dict["DOUT_S_ROOT"], comp, "proc/climo*"
            )
            case_dict[comp + "_avg_size"] = get_ocn_disk_usage(
                case_dict[comp + "_avg_path"]
            )
            case_dict[comp + "_diag_path"] = os.path.join(
                case_dict["DOUT_S_ROOT"], comp, "proc/diag*"
            )
            case_dict[comp + "_diag_size"] = get_ocn_disk_usage(
                case_dict[comp + "_diag_path"]
            )

        avg_logs = list()
        avg_file_pattern = "{0}/{1}_averages.log.*".format(pp_log_dir, comp)
        avg_logs = glob.glob(avg_file_pattern)

        if avg_logs:
            log_file = max(avg_logs, key=os.path.getctime)
            if is_last_process_complete(
                log_file, msg_avg[comp], "Average list complies with standards."
            ):
                case_dict[comp + "_avg_status"] = "Succeeded"
            else:
                case_dict[comp + "_avg_status"] = "Started"

        diag_logs = list()
        diag_file_pattern = "{0}/{1}_diagnostics.log.*".format(pp_log_dir, comp)
        diag_logs = glob.glob(diag_file_pattern)

        if diag_logs:
            log_file = max(diag_logs, key=os.path.getctime)
            if is_last_process_complete(log_file, msg_diags[comp], "ncks version"):
                case_dict[comp + "_diag_status"] = "Succeeded"
            else:
                case_dict[comp + "_diag_status"] = "Started"

    # get overall timeseries status
    case_dict["timeseries_status"] = "Unknown"
    case_dict["timeseries_path"] = get_pp_path(pp_dir, "timeseries")
    case_dict["timeseries_size"] = 0
    case_dict["timeseries_dates"] = "{0}-{1}".format(
        case_dict["RUN_STARTDATE"].replace("-", ""),
        case_dict["RUN_STARTDATE"].replace("-", ""),
    )
    case_dict["timeseries_total_time"] = 0
    tseries_logs = list()
    tseries_file_pattern = "{0}/timeseries.log.*".format(pp_log_dir)
    tseries_logs = glob.glob(tseries_file_pattern)
    if tseries_logs:
        log_file = max(tseries_logs, key=os.path.getctime)
        if is_last_process_complete(
            filepath=log_file, expect_text="Successfully completed", fail_text="opening"
        ):
            case_dict["timeseries_status"] = "Succeeded"
            with open(log_file, "r") as fname:
                log_content = fname.readlines()
            total_time = [line for line in log_content if "Total Time:" in line]
            case_dict["timeseries_total_time"] = " ".join(total_time[0].split())
        else:
            case_dict["timeseries_status"] = "Started"
        sta_dates = case_dict["sta_last_date"].split("-")
        case_dict["timeseries_dates"] = "{0}-{1}".format(
            case_dict["RUN_STARTDATE"].replace("-", ""), "".join(sta_dates[:-1])
        )
        for comp in _pp_tseries_comps:
            tseries_path = "{0}/{1}/proc/tseries".format(
                case_dict["timeseries_path"], comp
            )
            case_dict["timeseries_size"] += get_disk_usage(tseries_path)

    # get iconform status = this initializes files in the POSTPROCESS_PATH
    case_dict["iconform_status"] = "Unknown"
    case_dict["iconform_path"] = ""
    case_dict["iconform_size"] = 0
    case_dict["iconform_dates"] = case_dict["timeseries_dates"]

    iconform_logs = list()
    iconform_file_pattern = "{0}/iconform.log.*".format(pp_log_dir)
    iconform_logs = glob.glob(iconform_file_pattern)
    if iconform_logs:
        log_file = max(iconform_logs, key=os.path.getctime)
        if is_last_process_complete(
            log_file,
            "Successfully created the conform tool",
            "Running createOutputSpecs",
        ):
            case_dict["iconform_status"] = "Succeeded"
        else:
            case_dict["iconform_status"] = "Started"

    # get xconform status
    case_dict["xconform_path"] = ""
    case_dict["xconform_path"] = get_pp_path(pp_dir, "xconform")
    case_dict["xconform_status"] = "Unknown"
    case_dict["xconform_size"] = get_disk_usage(case_dict["xconform_path"])
    case_dict["xconform_dates"] = case_dict["timeseries_dates"]
    case_dict["xconform_total_time"] = 0

    xconform_logs = list()
    xconform_file_pattern = "{0}/xconform.log.*".format(pp_log_dir)
    xconform_logs = glob.glob(xconform_file_pattern)
    if xconform_logs:
        log_file = max(xconform_logs, key=os.path.getctime)
        if is_last_process_complete(
            log_file,
            "Successfully completed converting all files",
            "cesm_conform_generator INFO",
        ):
            case_dict["xconform_status"] = "Succeeded"
            case_dict["xconform_size"] = get_disk_usage(case_dict["xconform_path"])
            with open(log_file, "r") as fname:
                log_content = fname.readlines()
            total_time = [line for line in log_content if "Total Time:" in line]
            if total_time:
                case_dict["xconform_total_time"] = " ".join(total_time[0].split())
        else:
            case_dict["xconform_status"] = "Started"

    return case_dict


# ---------------------------------------------------------------------
def get_run_last_date(casename, run_path):
    # ---------------------------------------------------------------------
    """get_run_last_date
    parse the last cpl.r file in the run_path to retrieve that last date.

    Arguments:
        casename
        run_path - path to run directory
    """
    logger.debug("get_run_last_date")

    pattern = "{0}.cpl.r.*.nc".format(casename)
    cpl_files = sorted(glob.glob(os.path.join(run_path, pattern)))

    if cpl_files:
        _, cpl_file = os.path.split(cpl_files[-1])
        fparts = cpl_file.split(".")
        return fparts[-2]

    return "0000-00-00"


# ---------------------------------------------------------------------
def get_sta_last_date(sta_path):
    # ---------------------------------------------------------------------
    """get_sta_last_date
    parse the last rest directory in the sta_path to retrieve that last date.

    Arguments:
        sta_path - path to run directory
    """
    logger.debug("get_sta_last_date")

    rest_dirs = sorted(glob.glob(os.path.join(sta_path, "rest/*")))

    if rest_dirs:
        _, rest_dir = os.path.split(rest_dirs[-1])
        return rest_dir

    return "0000-00-00"


# ---------------------------------------------------------------------
def get_case_status(case_dict):
    # ---------------------------------------------------------------------
    """get_case_status
    Parse the CaseStatus and postprocessing log files
    looking for status information

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug("get_case_status")

    # initialize status variables
    case_dict["run_status"] = "Unknown"
    case_dict["run_path"] = case_dict["RUNDIR"]
    case_dict["run_size"] = 0
    case_dict["run_last_date"] = case_dict["RUN_STARTDATE"]

    case_dict["sta_status"] = "Unknown"
    case_dict["sta_path"] = case_dict["DOUT_S_ROOT"]
    case_dict["sta_size"] = 0
    case_dict["sta_last_date"] = case_dict["RUN_STARTDATE"]

    cstatus = case_dict["CASEROOT"] + "/CaseStatus"
    if os.path.exists(cstatus):
        # get the run status
        run_status_1 = is_last_process_complete(
            cstatus, "case.run success", "case.run starting"
        )
        run_status_2 = is_last_process_complete(
            cstatus, "model execution success", "model execution starting"
        )
        if run_status_1 is True or run_status_2 is True:
            case_dict["run_status"] = "Succeeded"
        case_dict["run_size"] = get_disk_usage(case_dict["run_path"])
        case_dict["run_last_date"] = get_run_last_date(
            case_dict["CASE"], case_dict["run_path"]
        )

        # get the STA status
        if case_dict["DOUT_S"]:
            # get only the history, rest and logs dir - ignoring the proc subdirs
            sta_status = is_last_process_complete(
                cstatus, "st_archive success", "st_archive starting"
            )
            case_dict["sta_last_date"] = get_sta_last_date(case_dict["DOUT_S_ROOT"])
            if sta_status is True:
                case_dict["sta_status"] = "Succeeded"
            # exclude the proc directories in the sta size estimates
            for subdir in [
                "atm/hist",
                "cpl/hist",
                "esp/hist",
                "ice/hist",
                "glc/hist",
                "lnd/hist",
                "logs",
                "ocn/hist",
                "rest",
                "rof/hist",
                "wav/hist",
                "iac/hist",
            ]:
                path = os.path.join(case_dict["sta_path"], subdir)
                if os.path.isdir(path):
                    case_dict["sta_size"] += get_disk_usage(path)

    # check if the postprocess dir exists in the caseroot
    case_dict["postprocess"] = False
    if os.path.exists(case_dict["CASEROOT"] + "/postprocess"):
        case_dict["postprocess"] = True
        case_dict = get_pp_status(case_dict)

    return case_dict


# ---------------------------------------------------------------------
def check_expdb_case(case_dict, username, password):
    # ---------------------------------------------------------------------
    """check_exp_case
    Cross check the casename with the database for a CMIP6 experiment

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        username (string) - SVN developer's username
        password (string) - SVN developer's password

    Return case_id value; 0 if does not exist or > 0 for exists.

    """
    logger.debug("check_expdb_case")
    data_dict = {
        "casename": case_dict["CASE"],
        "queryType": "checkCaseExists",
        "expType": case_dict["expType"],
    }
    data = json.dumps(data_dict)
    params = urllib.parse.urlencode(
        dict(username=username, password=password, data=data)
    )
    try:
        response = urllib.request.urlopen(
            case_dict["query_expdb_url"], params, context=_context
        )
        output = json.loads(response.read().decode())
    except urllib.error.HTTPError as http_e:
        logger.info('ERROR archive_metadata HTTP post failed "%s"', http_e.code)
        sys.exit(1)
    except urllib.error.URLError as url_e:
        logger.info('ERROR archive_metadata URL failed "%s"', url_e.reason)
        sys.exit(1)

    return int(output["case_id"])


# ---------------------------------------------------------------------
def query_expdb_cmip6(case_dict, username, password):
    # ---------------------------------------------------------------------
    """query_exp_case
    Query the expdb for CMIP6 casename = case_dict['q_casename'] metadata.
    Write out a json file to case_dict['q_outfile'].

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        username (string) - SVN developer's username
        password (string) - SVN developer's password

    """
    logger.debug("query_expdb_cmip6")
    exists = False
    data_dict = {
        "casename": case_dict["q_casename"],
        "queryType": "CMIP6GlobalAtts",
        "expType": "CMIP6",
    }
    data = json.dumps(data_dict)
    params = urllib.parse.urlencode(
        dict(username=username, password=password, data=data)
    )
    try:
        response = urllib.request.urlopen(
            case_dict["query_expdb_url"], params, context=_context
        )
        output = json.load(response)
    except urllib.error.HTTPError as http_e:
        logger.info('ERROR archive_metadata HTTP post failed "%s"', http_e.code)
    except urllib.error.URLError as url_e:
        logger.info('ERROR archive_metadata URL failed "%s"', url_e.reason)

    if output:
        if not os.path.exists("{0}/archive_files".format(case_dict["workdir"])):
            os.makedirs("{0}/archive_files".format(case_dict["workdir"]))

        filename = "{0}/archive_files/{1}".format(
            case_dict["workdir"], case_dict["q_outfile"]
        )
        with io.open(filename, "w+", encoding="utf-8") as fname:
            fname.write(json.dumps(output, ensure_ascii=False))
            fname.close()
        exists = True

    return exists


# ---------------------------------------------------------------------
def create_json(case_dict):
    # ---------------------------------------------------------------------
    """create_json
    Create a JSON file in the caseroot/archive_files dir.

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug("create_json")

    if not os.path.exists("{0}/archive_files".format(case_dict["workdir"])):
        os.makedirs("{0}/archive_files".format(case_dict["workdir"]))

    filename = "{0}/archive_files/json.{1}".format(
        case_dict["workdir"], datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    )
    with io.open(filename, "wb") as fname:
        jstr = str(json.dumps(case_dict, indent=4, sort_keys=True, ensure_ascii=False))
        if isinstance(jstr, str):
            jstr = jstr.decode("utf-8")
        fname.write(jstr)
        fname.close()


# ---------------------------------------------------------------------
def post_json(case_dict, username, password):
    # ---------------------------------------------------------------------
    """post_json
    Post a JSON file in the caseroot/archive_files to the
    remote expdb URL.

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
        username (string) - SVN developers username
        password (string) - SVN developers password
    """
    logger.debug("post_json")

    case_dict["COMPSET"] = urllib.parse.quote(case_dict["COMPSET"])
    case_dict["GRID"] = urllib.parse.quote(case_dict["GRID"])
    data = json.dumps(case_dict)
    params = urllib.parse.urlencode(
        dict(username=username, password=password, data=data)
    )
    try:
        urllib.request.urlopen(case_dict["json_expdb_url"], params, context=_context)
    except urllib.error.HTTPError as http_e:
        logger.info('ERROR archive_metadata HTTP post failed "%s"', http_e.code)
    except urllib.error.URLError as url_e:
        logger.info('ERROR archive_metadata URL failed "%s"', url_e.reason)


# ---------------------------------------------------------------------
def check_svn():
    # ---------------------------------------------------------------------
    """check_svn

    make sure svn client is installed and accessible
    """
    logger.debug("check_svn")

    cmd = ["svn", "--version"]
    svn_exists = True
    result = ""
    try:
        result = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as error:
        msg = _svn_error_template.substitute(
            function="check_svn", cmd=cmd, error=error.returncode, strerror=error.output
        )
        svn_exists = False
        logger.info(msg)
        raise SVNException(msg)

    if "version" not in result:
        msg = "SVN is not available. Ignoring SVN update"
        svn_exists = False
        raise SVNException(msg)

    return svn_exists


# ---------------------------------------------------------------------
def create_temp_archive(case_dict):
    # ---------------------------------------------------------------------
    """create_temp_archive

    Create a temporary SVN sandbox directory in the current caseroot
    """
    archive_temp_dir = "{0}/archive_temp_dir".format(case_dict["workdir"])
    logger.debug("create_temp_archive %s", archive_temp_dir)

    if not os.path.exists(archive_temp_dir):
        os.makedirs(archive_temp_dir)
    else:
        logger.info(
            "ERROR archive_metadata archive_temp_dir already exists. exiting..."
        )
        sys.exit(1)

    return archive_temp_dir


# ---------------------------------------------------------------------
def check_svn_repo(case_dict, username, password):
    # ---------------------------------------------------------------------
    """check_svn_repo

    check if a SVN repo exists for this case
    """
    logger.debug("check_svn_repo")

    repo_exists = False
    svn_repo = "{0}/trunk".format(case_dict["svn_repo_url"])
    cmd = ["svn", "list", svn_repo, "--username", username, "--password", password]
    result = ""
    try:
        result = subprocess.check_output(cmd)
    except subprocess.CalledProcessError:
        msg = "SVN repo does not exist for this case. A new one will be created."
        logger.warning(msg)

    if re.search("README.archive", result):
        repo_exists = True

    return repo_exists


# ---------------------------------------------------------------------
def get_trunk_tag(case_dict, username, password):
    # ---------------------------------------------------------------------
    """get_trunk_tag

    return the most recent trunk tag as an integer
    """
    logger.debug("get_trunk_tag")

    tag = 0
    svn_repo = "{0}/trunk_tags".format(case_dict["svn_repo_url"])
    cmd = ["svn", "list", svn_repo, "--username", username, "--password", password]
    result = ""
    try:
        result = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = [
            "svn",
            "list",
            svn_repo,
            "--username",
            username,
            "--password",
            "******",
        ]
        msg = _call_template.substitute(
            function="get_trunk_tag",
            cmd=cmd_nopasswd,
            error=error.returncode,
            strerror=error.output,
        )
        logger.warning(msg)
        raise SVNException(msg)

    if result:
        last_tag = [i for i in result.split("\n") if i][-1]
        last_tag = last_tag[:-1].split("_")[-1]
        tag = int(last_tag.lstrip("0"))

    return tag


# ---------------------------------------------------------------------
def checkout_repo(case_dict, username, password):
    # ---------------------------------------------------------------------
    """checkout_repo

    checkout the repo into the archive_temp_dir
    """
    logger.debug("checkout_repo")

    os.chdir(case_dict["archive_temp_dir"])
    svn_repo = "{0}/trunk".format(case_dict["svn_repo_url"])
    cmd = ["svn", "co", "--username", username, "--password", password, svn_repo, "."]
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = [
            "svn",
            "co",
            "--username",
            username,
            "--password",
            "******",
            svn_repo,
            ".",
        ]
        msg = _call_template.substitute(
            function="checkout_repo",
            cmd=cmd_nopasswd,
            error=error.returncode,
            strerror=error.output,
        )
        logger.warning(msg)
        raise SVNException(msg)

    os.chdir(case_dict["CASEROOT"])


# ---------------------------------------------------------------------
def create_readme(case_dict):
    # ---------------------------------------------------------------------
    """create_readme

    Create a generic README.archive file
    """
    logger.debug("create_readme")
    os.chdir(case_dict["archive_temp_dir"])

    fname = open("README.archive", "w")
    fname.write("Archived metadata is available for this case at URL:\n")
    fname.write(case_dict["base_expdb_url"])
    fname.close()


# ---------------------------------------------------------------------
def update_repo_add_file(filename, dir1, dir2):
    # ---------------------------------------------------------------------
    """update_repo_add_file

    Add a file to the SVN repository
    """
    src = os.path.join(dir1, filename)
    dest = os.path.join(dir2, filename)
    logger.debug("left_only: " + src + " -> " + dest)
    if not os.path.exists(dest):
        shutil.copy2(src, dest)
        cmd = ["svn", "add", "--parents", dest]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            msg = _call_template.substitute(
                function="update_lcoal_repo",
                cmd=cmd,
                error=error.returncode,
                strerror=error.output,
            )
            logger.warning(msg)
            raise SVNException(msg)


# ---------------------------------------------------------------------
def update_repo_rm_file(filename, dir1, dir2):
    # ---------------------------------------------------------------------
    """update_repo_rm_file

    Remove a file from the SVN repository
    """
    src = os.path.join(dir2, filename)
    dest = os.path.join(dir1, filename)
    logger.debug("right_only: " + src + " -> " + dest)
    if os.path.exists(dest):
        cmd = ["svn", "rm", dest]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            msg = _call_template.substitute(
                function="update_lcoal_repo",
                cmd=cmd,
                error=error.returncode,
                strerror=error.output,
            )
            logger.warning(msg)
            raise SVNException(msg)


# ---------------------------------------------------------------------
def update_repo_copy_file(filename, dir1, dir2):
    # ---------------------------------------------------------------------
    """update_repo_copy_file

    Copy a file into the SVN local repo
    """
    src = os.path.join(dir1, filename)
    dest = os.path.join(dir2, filename)
    shutil.copy2(src, dest)


# ---------------------------------------------------------------------
def compare_dir_trees(dir1, dir2, archive_list):
    # ---------------------------------------------------------------------
    """compare_dir_trees

    Compare two directories recursively. Files in each directory are
    assumed to be equal if their names and contents are equal.
    """
    xml_files = glob.glob(os.path.join(dir1, "*.xml"))
    user_nl_files = glob.glob(os.path.join(dir1, "user_nl_*"))
    dirs_cmp = filecmp.dircmp(dir1, dir2, _ignore_patterns)

    left_only = [
        fn
        for fn in dirs_cmp.left_only
        if not os.path.islink(fn)
        and (fn in xml_files or fn in user_nl_files or fn in archive_list)
    ]
    right_only = [
        fn
        for fn in dirs_cmp.right_only
        if not os.path.islink(fn)
        and (fn in xml_files or fn in user_nl_files or fn in archive_list)
    ]
    funny_files = [
        fn
        for fn in dirs_cmp.funny_files
        if not os.path.islink(fn)
        and (fn in xml_files or fn in user_nl_files or fn in archive_list)
    ]

    # files and directories need to be added to svn repo from the caseroot
    if left_only:
        for filename in left_only:
            if os.path.isfile(os.path.join(dir1, filename)) and filename[-1] != "~":
                update_repo_add_file(filename, dir1, dir2)
            else:
                new_dir1 = os.path.join(dir1, filename)
                new_dir2 = os.path.join(dir2, filename)
                os.makedirs(new_dir2)
                cmd = ["svn", "add", new_dir2]
                try:
                    subprocess.check_call(cmd)
                except subprocess.CalledProcessError as error:
                    msg = _call_template.substitute(
                        function="update_lcoal_repo",
                        cmd=cmd,
                        error=error.returncode,
                        strerror=error.output,
                    )
                    logger.warning(msg)
                    raise SVNException(msg)

                # recurse through this new subdir
                new_archive_list = [filename]
                compare_dir_trees(new_dir1, new_dir2, new_archive_list)

    # files need to be removed from svn repo that are no longer in the caseroot
    if right_only:
        for filename in right_only:
            if os.path.isfile(os.path.join(dir1, filename)) and filename[-1] != "~":
                update_repo_rm_file(filename, dir1, dir2)

    # files are the same but could not be compared so copy the caseroot version
    if funny_files:
        for filename in funny_files:
            if os.path.isfile(os.path.join(dir1, filename)) and filename[-1] != "~":
                update_repo_copy_file(filename, dir1, dir2)

    # common files have changed in the caseroot and need to be copied to the svn repo
    (_, mismatch, errors) = filecmp.cmpfiles(
        dir1, dir2, dirs_cmp.common_files, shallow=False
    )
    if mismatch:
        for filename in mismatch:
            if os.path.isfile(os.path.join(dir1, filename)) and filename[-1] != "~":
                update_repo_copy_file(filename, dir1, dir2)

    # error in file comparison so copy the caseroot file to the svn repo
    if errors:
        for filename in errors:
            if os.path.isfile(os.path.join(dir1, filename)) and filename[-1] != "~":
                update_repo_copy_file(filename, dir1, dir2)

    # recurse through the subdirs
    common_dirs = dirs_cmp.common_dirs
    if common_dirs:
        for common_dir in common_dirs:
            if common_dir in archive_list:
                new_dir1 = os.path.join(dir1, common_dir)
                new_dir2 = os.path.join(dir2, common_dir)
                compare_dir_trees(new_dir1, new_dir2, archive_list)
    else:
        return


# ---------------------------------------------------------------------
def update_local_repo(case_dict, ignore_logs, ignore_timing):
    # ---------------------------------------------------------------------
    """update_local_repo

    Compare and update local SVN sandbox
    """
    logger.debug("update_local_repo")
    from_dir = case_dict["CASEROOT"]
    to_dir = case_dict["archive_temp_dir"]

    compare_dir_trees(from_dir, to_dir, case_dict["archive_list"])

    # check if ignore_logs is specified
    if ignore_logs:
        os.chdir(to_dir)
        if os.path.isdir("./logs"):
            try:
                shutil.rmtree("./logs")
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "logs" in archive dir.'
                )

            cmd = ["svn", "delete", "./logs"]
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(
                    function="update_lcoal_repo",
                    cmd=cmd,
                    error=error.returncode,
                    strerror=error.output,
                )
                logger.warning(msg)
                raise SVNException(msg)

        if os.path.isdir("./postprocess/logs"):
            os.chdir("./postprocess")
            try:
                shutil.rmtree("./logs")
            except OSError:
                logger.warning(
                    'in "update_local_repo" - '
                    'Unable to remove "postprocess/logs" in archive dir.'
                )

            cmd = ["svn", "delete", "./logs"]
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(
                    function="update_lcoal_repo",
                    cmd=cmd,
                    error=error.returncode,
                    strerror=error.output,
                )
                logger.warning(msg)
                raise SVNException(msg)
    else:
        # add log files
        if os.path.exists("{0}/logs".format(from_dir)):
            if not os.path.exists("{0}/logs".format(to_dir)):
                os.makedirs("{0}/logs".format(to_dir))
            os.chdir(os.path.join(from_dir, "logs"))
            for filename in glob.glob("*.*"):
                update_repo_add_file(
                    filename,
                    os.path.join(from_dir, "logs"),
                    os.path.join(to_dir, "logs"),
                )

        if os.path.exists("{0}/postprocess/logs".format(from_dir)):
            if not os.path.exists("{0}/postprocess/logs".format(to_dir)):
                os.makedirs("{0}/postprocess/logs".format(to_dir))
            os.chdir(os.path.join(from_dir, "postprocess/logs"))
            for filename in glob.glob("*.*"):
                update_repo_add_file(
                    filename,
                    os.path.join(from_dir, "postprocess", "logs"),
                    os.path.join(to_dir, "postprocess", "logs"),
                )

    # check if ignore_timing is specified
    if ignore_timing:
        os.chdir(case_dict["archive_temp_dir"])
        if os.path.isdir("./timing"):
            try:
                shutil.rmtree("./timing")
            except OSError:
                logger.warning(
                    'in "update_local_repo" - Unable to remove "timing" in archive dir.'
                )

            cmd = ["svn", "delete", "./timing"]
            try:
                subprocess.check_call(cmd)
            except subprocess.CalledProcessError as error:
                msg = _call_template.substitute(
                    function="update_lcoal_repo",
                    cmd=cmd,
                    error=error.returncode,
                    strerror=error.output,
                )
                logger.warning(msg)
                raise SVNException(msg)
    else:
        # add timing files
        if os.path.exists("{0}/timing".format(from_dir)):
            if not os.path.exists("{0}/timing".format(to_dir)):
                os.makedirs("{0}/timing".format(to_dir))
            os.chdir(os.path.join(from_dir, "timing"))
            for filename in glob.glob("*.*"):
                update_repo_add_file(
                    filename,
                    os.path.join(from_dir, "timing"),
                    os.path.join(to_dir, "timing"),
                )


# ---------------------------------------------------------------------
def populate_local_repo(case_dict, ignore_logs, ignore_timing):
    # ---------------------------------------------------------------------
    """populate_local_repo

    Populate local SVN sandbox
    """
    logger.debug("populate_local_repo")
    os.chdir(case_dict["CASEROOT"])

    # loop through the archive_list and copy to the temp archive dir
    for archive in case_dict["archive_list"]:
        if os.path.exists(archive):
            if os.path.isdir(archive):
                try:
                    target = case_dict["archive_temp_dir"] + "/" + archive
                    shutil.copytree(
                        archive,
                        target,
                        symlinks=False,
                        ignore=shutil.ignore_patterns(*_ignore_patterns),
                    )
                except OSError as error:
                    msg = _copy_template.substitute(
                        function="populate_local_repo",
                        source=archive,
                        dest=case_dict["archive_temp_dir"],
                        error=error.errno,
                        strerror=error.strerror,
                    )
                    logger.warning(msg)
            else:
                try:
                    shutil.copy2(archive, case_dict["archive_temp_dir"])
                except OSError as error:
                    msg = _copy_template.substitute(
                        function="populate_local_repo",
                        source=archive,
                        dest=case_dict["archive_temp_dir"],
                        error=error.errno,
                        strerror=error.strerror,
                    )
                    logger.warning(msg)

    # add files with .xml as the suffix
    xml_files = glob.glob("*.xml")
    for xml_file in xml_files:
        if os.path.isfile(xml_file):
            try:
                shutil.copy2(xml_file, case_dict["archive_temp_dir"])
            except OSError as error:
                msg = _copy_template.substitute(
                    function="populate_local_repo",
                    source=xml_file,
                    dest=case_dict["archive_temp_dir"],
                    error=error.errno,
                    strerror=error.strerror,
                )
                logger.warning(msg)

    # add files with .xml as the suffix from the postprocess directory
    if os.path.isdir("./postprocess"):
        pp_path = "{0}/{1}".format(case_dict["archive_temp_dir"], "postprocess")
        if not os.path.exists(pp_path):
            os.mkdir(pp_path)
        xml_files = glob.glob("./postprocess/*.xml")
        for xml_file in xml_files:
            if os.path.isfile(xml_file):
                try:
                    shutil.copy2(xml_file, pp_path)
                except OSError as error:
                    msg = _copy_template.substitute(
                        function="populate_local_repo",
                        source=xml_file,
                        dest=case_dict["archive_temp_dir"],
                        error=error.errno,
                        strerror=error.strerror,
                    )
                    logger.warning(msg)

    # add files with user_nl_ as the prefix
    user_files = glob.glob("user_nl_*")
    for user_file in user_files:
        if os.path.isfile(user_file):
            try:
                shutil.copy2(user_file, case_dict["archive_temp_dir"])
            except OSError as error:
                msg = _copy_template.substitute(
                    function="populate_local_repo",
                    source=user_file,
                    dest=case_dict["archive_temp_dir"],
                    error=error.errno,
                    strerror=error.strerror,
                )
                logger.warning(msg)

    # add files with Depends as the prefix
    conf_files = glob.glob("Depends.*")
    for conf_file in conf_files:
        if os.path.isfile(conf_file):
            try:
                shutil.copy2(conf_file, case_dict["archive_temp_dir"])
            except OSError as error:
                msg = _copy_template.substitute(
                    function="populate_local_repo",
                    source=conf_file,
                    dest=case_dict["archive_temp_dir"],
                    error=error.errno,
                    strerror=error.strerror,
                )
                logger.warning(msg)

    # check if ignore_logs is specified
    if ignore_logs:
        os.chdir(case_dict["archive_temp_dir"])
        if os.path.isdir("./logs"):
            try:
                shutil.rmtree("./logs")
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "logs" in archive_temp_dir.'
                )
        if os.path.isdir("./postprocess/logs"):
            os.chdir("./postprocess")
            try:
                shutil.rmtree("./logs")
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - '
                    'Unable to remove "postprocess/logs" in archive_temp_dir.'
                )
        os.chdir(case_dict["CASEROOT"])

    # check if ignore_timing is specified
    if ignore_timing:
        os.chdir(case_dict["archive_temp_dir"])
        if os.path.isdir("./timing"):
            try:
                shutil.rmtree("./timing")
            except OSError:
                logger.warning(
                    'in "populate_local_repo" - Unable to remove "timing" in archive_temp_dir.'
                )
        os.chdir(case_dict["CASEROOT"])


# ---------------------------------------------------------------------
def checkin_trunk(case_dict, svn_cmd, message, username, password):
    # ---------------------------------------------------------------------
    """checkin_trunk

    Check in the local SVN sandbox to the remote trunk
    """
    logger.debug("checkin_trunk")

    os.chdir(case_dict["archive_temp_dir"])
    svn_repo = "{0}/trunk".format(case_dict["svn_repo_url"])
    msg = '"{0}"'.format(message)
    cmd = [
        "svn",
        svn_cmd,
        "--username",
        username,
        "--password",
        password,
        ".",
        "--message",
        msg,
    ]

    if svn_cmd in ["import"]:
        # create the trunk dir
        msg = '"create trunk"'
        cmd = [
            "svn",
            "mkdir",
            "--parents",
            svn_repo,
            "--username",
            username,
            "--password",
            password,
            "--message",
            msg,
        ]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            cmd_nopasswd = [
                "svn",
                "mkdir",
                "--parents",
                svn_repo,
                "--username",
                username,
                "--password",
                "******",
                "--message",
                msg,
            ]
            msg = _call_template.substitute(
                function="checkin_trunk",
                cmd=cmd_nopasswd,
                error=error.returncode,
                strerror=error.output,
            )
            logger.warning(msg)
            raise SVNException(msg)

        # create the trunk_tags dir
        tags = "{0}/trunk_tags".format(case_dict["svn_repo_url"])
        msg = '"create trunk_tags"'
        cmd = [
            "svn",
            "mkdir",
            tags,
            "--username",
            username,
            "--password",
            password,
            "--message",
            msg,
        ]
        try:
            subprocess.check_call(cmd)
        except subprocess.CalledProcessError as error:
            cmd_nopasswd = [
                "svn",
                "mkdir",
                tags,
                "--username",
                username,
                "--password",
                "******",
                "--message",
                msg,
            ]
            msg = _call_template.substitute(
                function="checkin_trunk",
                cmd=cmd_nopasswd,
                error=error.returncode,
                strerror=error.output,
            )
            logger.warning(msg)
            raise SVNException(msg)

        msg = '"{0}"'.format(message)
        cmd = [
            "svn",
            svn_cmd,
            "--username",
            username,
            "--password",
            password,
            ".",
            svn_repo,
            "--message",
            msg,
        ]

    # check-in the trunk to svn
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = [
            "svn",
            svn_cmd,
            "--username",
            username,
            "--password",
            "******",
            ".",
            "--message",
            msg,
        ]
        msg = _call_template.substitute(
            function="checkin_trunk",
            cmd=cmd_nopasswd,
            error=error.returncode,
            strerror=error.output,
        )
        logger.warning(msg)
        raise SVNException(msg)


# ---------------------------------------------------------------------
def create_tag(case_dict, new_tag, username, password):
    # ---------------------------------------------------------------------
    """create_tag

    create a new trunk tag in the remote repo
    """
    logger.debug("create_tag")

    # create a new trunk tag
    os.chdir(case_dict["archive_temp_dir"])
    svn_repo = "{0}/trunk".format(case_dict["svn_repo_url"])
    svn_repo_tag = "{0}/trunk_tags/{1}".format(case_dict["svn_repo_url"], new_tag)
    msg = '"create new trunk tag"'
    cmd = [
        "svn",
        "copy",
        "--username",
        username,
        "--password",
        password,
        svn_repo,
        svn_repo_tag,
        "--message",
        msg,
    ]
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as error:
        cmd_nopasswd = [
            "svn",
            "copy",
            "--username",
            username,
            "--password",
            "******",
            svn_repo,
            svn_repo_tag,
            "--message",
            msg,
        ]
        msg = _call_template.substitute(
            function="checkin_trunk",
            cmd=cmd_nopasswd,
            error=error.returncode,
            strerror=error.output,
        )
        logger.warning(msg)
        raise SVNException(msg)


# -------------------------------------------------------------------------
def update_repo(ignore_logs, ignore_timing, case_dict, username, password):
    # -------------------------------------------------------------------------
    """update_repo

    Update SVN repo
    """
    logger.debug("update_repo")

    try:
        # check if svn client is installed
        svn_exists = check_svn()

        if svn_exists:
            # check if the case repo exists
            case_dict["svn_repo_url"] = "{0}/{1}".format(
                _svn_expdb_url, case_dict["CASE"]
            )
            repo_exists = check_svn_repo(case_dict, username, password)
            case_dict["archive_temp_dir"] = create_temp_archive(case_dict)
            case_dict["archive_list"] = _archive_list + case_dict["user_add_files"]

            if repo_exists:
                # update trunk and make a new tag
                last_tag = get_trunk_tag(case_dict, username, password)
                new_tag = "{0}_{1}".format(
                    case_dict["CASE"], str(last_tag + 1).zfill(4)
                )
                checkout_repo(case_dict, username, password)
                update_local_repo(case_dict, ignore_logs, ignore_timing)
                msg = "update case metadata for {0} by {1}".format(
                    case_dict["CASE"], username
                )
                checkin_trunk(case_dict, "ci", msg, username, password)
                create_tag(case_dict, new_tag, username, password)
                logger.info(
                    'SVN repository trunk updated at URL "%s"',
                    case_dict["svn_repo_url"],
                )
                logger.info(' and a new trunk tag created "%s"', new_tag)
            else:
                # create a new case repo
                new_tag = "{0}_0001".format(case_dict["CASE"])
                create_readme(case_dict)
                populate_local_repo(case_dict, ignore_logs, ignore_timing)
                msg = "initial import of case metadata for {0} by {1}".format(
                    case_dict["CASE"], username
                )
                checkin_trunk(case_dict, "import", msg, username, password)
                create_tag(case_dict, new_tag, username, password)
                logger.info(
                    'SVN repository imported to trunk URL "%s"',
                    case_dict["svn_repo_url"],
                )
                logger.info('   and a new trunk tag created for  "%s"', new_tag)

    except SVNException:
        pass

    return case_dict


# ---------------------------------------------------------------------
def get_timing_data(case_dict):
    # ---------------------------------------------------------------------
    """get_timing_data
    parse the timing data file and add information to the case_dict

    Arguments:
        case_dict (dict) - case dictionary to store XML variables
    """
    logger.debug("get_timing_data")

    # initialize the timing values in the dictionary
    case_dict["model_cost"] = "undefined"
    case_dict["model_throughput"] = "undefined"

    timing_dir = case_dict["CASEROOT"] + "/timing"
    last_time = ""
    if os.path.exists(timing_dir):
        # check if timing files exists
        timing_file_pattern = "cesm_timing." + case_dict["CASE"]
        last_time = max(
            glob.glob(timing_dir + "/" + timing_file_pattern + ".*"),
            key=os.path.getctime,
        )
        if last_time:
            if "gz" in last_time:
                # gunzip file first
                with gzip.open(last_time, "rb") as fname:
                    file_content = fname.readlines()
            else:
                with open(last_time, "r") as fname:
                    file_content = fname.readlines()

            # search the file content for matching lines
            model_cost = [line for line in file_content if "Model Cost:" in line]
            model_throughput = [
                line for line in file_content if "Model Throughput:" in line
            ]

            case_dict["model_cost"] = " ".join(model_cost[0].split())
            case_dict["model_throughput"] = " ".join(model_throughput[0].split())

    return case_dict


# ---------------------------------------------------------------------
def initialize_main(options):
    # ---------------------------------------------------------------------
    """initialize_main

    Initialize the case dictionary data structure with command line options
    """
    logger.debug("intialize_main")

    case_dict = dict()

    case_dict["CASEROOT"] = os.getcwd()
    if options.caseroot:
        case_dict["CASEROOT"] = options.caseroot[0]

    case_dict["workdir"] = case_dict["CASEROOT"]
    if options.workdir:
        case_dict["workdir"] = options.workdir[0]

    username = None
    if options.user:
        username = options.user
        case_dict["svnlogin"] = username

    password = None
    if options.password:
        password = options.password

    if options.expType:
        case_dict["expType"] = options.expType[0]

    case_dict["title"] = None
    if options.title:
        case_dict["title"] = options.title[0]

    case_dict["dryrun"] = False
    if options.dryrun:
        case_dict["dryrun"] = True

    case_dict["archive_temp_dir"] = ""

    case_dict["user_add_files"] = list()
    if options.user_add_files:
        case_dict["user_add_files"] = options.user_add_files.split(",")

    case_dict["q_casename"] = ""
    case_dict["q_outfile"] = ""
    if options.query_cmip6:
        case_dict["q_casename"] = options.query_cmip6[0]
        case_dict["q_outfile"] = options.query_cmip6[1]

    case_dict["base_expdb_url"] = "https://csegweb.cgd.ucar.edu/expdb2.0"
    if options.test_post:
        case_dict["base_expdb_url"] = "https://csegwebdev.cgd.ucar.edu/expdb2.0"
    case_dict["json_expdb_url"] = (
        case_dict["base_expdb_url"] + "/cgi-bin/processJSON.cgi"
    )
    case_dict["query_expdb_url"] = case_dict["base_expdb_url"] + "/cgi-bin/query.cgi"

    return case_dict, username, password


# ---------------------------------------------------------------------
def main_func(options):
    # ---------------------------------------------------------------------
    """main function

    Arguments:
        options (list) - input options from command line
    """
    logger.debug("main_func")

    (case_dict, username, password) = initialize_main(options)

    # check if query_cmip6 argument is specified
    if options.query_cmip6:
        if case_dict["dryrun"]:
            logger.info("Dryrun - calling query_expdb_cmip6 for case metadata")
        else:
            if query_expdb_cmip6(case_dict, username, password):
                logger.info(
                    'Casename "%s" CMIP6 global attribute '
                    'metadata written to "%s/archive_files/%s" '
                    'from "%s"',
                    case_dict["workdir"],
                    case_dict["q_casename"],
                    case_dict["q_outfile"],
                    case_dict["query_expdb_url"],
                )
                logger.info("Successful completion of archive_metadata")
                sys.exit(0)
            else:
                logger.info(
                    'ERROR archive_metadata failed to find "%s" '
                    'in experiments database at "%s".',
                    case_dict["q_casename"],
                    case_dict["query_expdb_url"],
                )
                sys.exit(1)

    # loop through the _xml_vars gathering values
    with Case(case_dict["CASEROOT"], read_only=True) as case:
        if case_dict["dryrun"]:
            logger.info("Dryrun - calling get_case_vars")
        else:
            case_dict = get_case_vars(case_dict, case)

    # check reserved casename expdb for CMIP6 experiments
    if case_dict["expType"].lower() == "cmip6":
        if case_dict["dryrun"]:
            logger.info(
                "Dryrun - calling check_expdb_case for CMIP6 experiment reservation"
            )
        else:
            case_dict["case_id"] = check_expdb_case(case_dict, username, password)
            if case_dict["case_id"] < 1:
                logger.info(
                    "Unable to archive CMIP6 metadata. "
                    '"%s" casename does not exist in database. '
                    "All CMIP6 experiments casenames must be "
                    "reserved in the experiments database at URL: "
                    "https://csegweb.cgd.ucar.edu/expdb2.0 "
                    "prior to running archive_metadata.",
                    case_dict["CASE"],
                )
                sys.exit(1)

    # get the case status into the case_dict
    if case_dict["dryrun"]:
        logger.info("Dryrun - calling get_case_status")
    else:
        case_dict = get_case_status(case_dict)

    # create / update the cesm expdb repo with the caseroot files
    if not options.ignore_repo_update:
        if case_dict["dryrun"]:
            logger.info("Dryrun - calling update_repo")
        else:
            case_dict = update_repo(
                options.ignore_logs,
                options.ignore_timing,
                case_dict,
                username,
                password,
            )

    # parse the timing data into the case_dict
    if not options.ignore_timing:
        if case_dict["dryrun"]:
            logger.info("Dryrun - calling get_timing_data")
        else:
            case_dict = get_timing_data(case_dict)

    # Create a JSON file containing the case_dict with the date appended to the filename
    if case_dict["dryrun"]:
        logger.info("Dryrun - calling create_json")
    else:
        create_json(case_dict)

    # post the JSON to the remote DB
    if case_dict["dryrun"]:
        logger.info("Dryrun - calling post_json")
    else:
        post_json(case_dict, username, password)

    # clean-up the temporary archive files dir
    if case_dict["dryrun"]:
        logger.info('Dryrun - deleting "./archive_temp_dir"')
    else:
        if not options.ignore_repo_update and os.path.exists(
            case_dict["archive_temp_dir"]
        ):
            shutil.rmtree(case_dict["archive_temp_dir"])

    logger.info("Successful completion of archive_metadata")

    return 0


# ===================================
if __name__ == "__main__":

    try:
        __status__ = main_func(commandline_options(sys.argv))
        sys.exit(__status__)
    except Exception as error:
        print("{}".format(str(error)))
        sys.exit(1)
