#!/usr/bin/python3
#
# Copyright (C) 2022  Red Hat, Inc.
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# the GNU General Public License v.2, or (at your option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.  You should have received a copy of the
# GNU General Public License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
# source code or documentation are not subject to the GNU General Public
# License and may only be used or replicated with the express permission of
# Red Hat, Inc.
#
# This script generates a weekly report of kickstart-test runs.
# It downloads the daily json summary reports and combines them into a single text
# report.
#
# It requires a github token in GITHUB_TOKEN environmental variable. This token
# needs to have access to the kickstart-tests artifacts via the github API.

import argparse
from datetime import datetime, timedelta
from glob import glob
import io
import json
import os
import shutil
import sys
from subprocess import check_call
import tempfile
import time
from urllib.request import Request, urlopen
import zipfile

import pycurl

URL = "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts?per_page=100"

# Defaults list of artifacts
# For now use the full log artifacts, eventually this will be the json summaries
ARTIFACT_NAMES = ["logs-daily-iso", "logs-rhel9", "logs-rhel8"]

# We should ignore SIGPIPE when using pycurl.NOSIGNAL - see
# the libcurl tutorial for more info.
try:
    import signal
    from signal import SIGPIPE, SIG_IGN
except ImportError:
    pass
else:
    signal.signal(SIGPIPE, SIG_IGN)


def get_artifacts(token, artifact_names, start, end):
    """
    get_artifacts retrieves a list of artifacts from the selected date range
    that are listed in artifact_names

    It returns a list of artifact dicts like this:

    {
      "id": 187278866,
      "node_id": "MDg6QXJ0aWZhY3QxODcyNzg4NjY=",
      "name": "logs-daily-iso",
      "size_in_bytes": 1047408950,
      "url": "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts/187278866",
      "archive_download_url": "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts/187278866/zip",
      "expired": false,
      "created_at": "2022-03-17T03:30:12Z",
      "updated_at": "2022-03-17T03:30:13Z",
      "expires_at": "2022-03-25T03:29:06Z"
    },
    """

    artifacts = []
    page = 1
    retry_limit = 3
    while True:
        req = Request(URL + f"&page={page}")
        req.add_header("Accept", "application/vnd.github.v3+json")
        req.add_header("Authorization", f"token {token}")
        with urlopen(req) as r:
            # Handle hitting the GitHub rate limit
            # If the reset time is < 90s in the future, wait for it (trying 3 times)
            # Otherwise raise an error
            if r.status == 403:
                try:
                    reset = int(r.headers.get("X-RateLimit-Reset"))
                except:
                    raise RuntimeError("Hit GitHub rate limit. Reset header missing.")
                if retry_limit == 0 or time.time() > reset or reset - time.time() > 90:
                    raise RuntimeError("Hit GitHub rate limit. Reset is at %s" % time.ctime(reset))

                # Try waiting until after the reset time
                time.sleep(10 + (reset - time.time()))
                retry_limit = retry_limit - 1
                continue

            if r.status != 200:
                raise RuntimeError("Error (%d) with API request: %s" % (r.status, str(r)))

            data = json.load(r)

        # Only include the artifacts within the date range and names
        for a in data["artifacts"]:
            if a["name"] not in artifact_names:
                continue
            updated_at = datetime.fromisoformat(a["updated_at"][:-1])
            if start <= updated_at <= end:
                artifacts.append(a)

        if len(data["artifacts"]) < 100:
            break

        # There are more results, get the next page
        page = page + 1

        # Avoid hitting per-second rate limits
        time.sleep(2)

    return sorted(artifacts, key=lambda x: x["updated_at"])


def run_curl(token, url, filename):
    """
    run_curl downloads an artifact file

    It returns True if the response was a 200
    If there was an exception is returns False and the error, as well as
    printing it to stderr
    """
    with open(filename, "wb") as f:
        c = pycurl.Curl()
        headers = [
           "Accept: application/vnd.github.v3+json",
           f"Authorization: token {token}",
        ]

        options = {
            pycurl.FOLLOWLOCATION:  1,
            pycurl.MAXREDIRS:       5,
            pycurl.CONNECTTIMEOUT:  30,
            pycurl.TIMEOUT:         300,
            pycurl.NOSIGNAL:        1,
            pycurl.URL:             url,
            pycurl.HTTPHEADER:      headers,
            pycurl.WRITEDATA:       f
        }
        for k, v in options.items():
            c.setopt(k, v)

        try:
            c.perform()
            status = c.getinfo(pycurl.HTTP_CODE)
            ok = (status == 200, None)
        except Exception as e:
            print(f"ERROR: {e}", file=sys.stderr)
            sys.stderr.flush()
            ok = (False, e)
        c.close()

    return ok


def download_artifacts(token, artifacts):
    """
    download_artifacts downloads the artifacts as uniquely named files

    If there is a problem downloading an artifact it is skipped and not
    added to the returned list.

    It returns a list of tuples containing the artifact name, the
    artifact name with the updated date appended (eg. logs-rhel8-2022-04-01),
    and the filename of the zipfile.
    """
    zipfiles = []
    for a in artifacts:
        updated_at = datetime.fromisoformat(a["updated_at"][:-1])
        datename = a["name"]+updated_at.strftime("-%Y-%m-%d")
        filename = datename + ".zip"
        if os.path.exists(filename):
            zipfiles.append((a["name"], datename, filename))
            print(f"{filename} skipped, already downloaded")
            continue

        print(f"Fetching {filename}")
        ok = run_curl(token, a["archive_download_url"], filename)
        if not ok:
            continue

        zipfiles.append((a["name"], datename, filename))

    return zipfiles


def extract_logs(f):
    """
    extract_logs unzips the archive into a temporary directory

    This directory is deleted when the object goes out of scope
    """
    tdir = tempfile.TemporaryDirectory(prefix="kstest-log-", dir="/var/tmp/")
    with zipfile.ZipFile(f) as zf:
        zf.extractall(tdir.name)

    # Return the object so that the temporary directory isn't deleted yet
    return tdir


def generate_test_list(tdir):
    """
    Build kstest-list if it is missing

    This is a list of the expected tests, and it is used to help detect when
    tests are totally missing from the results.

    The original kstest.log has a line:
    Running tests: ./bridge-httpks.sh ./packages-instlangs-1.sh  ...

    Parse this and create kstest-list, one test name per line for later use.
    """

    # Skip this if it already exists
    if os.path.exists(os.path.join(tdir.name, "kstest-list")):
        return

    kstest_log = os.path.join(tdir.name, "kstest.log")
    with open(kstest_log) as f:
        for line in f.readlines():
            if not line.startswith("Running tests: "):
                continue

            tests = [os.path.basename(os.path.splitext(s)[0]) for s in line[15:].split()]
            with open(os.path.join(tdir.name, "kstest-list"), "wt") as klf:
                for t in tests:
                    print(t, file=klf)
            break


def rebuild_logs(tdir):
    """
    rebuild_logs recreates kstest.log with timestamps

    It does this by appending all the individual kstest.log files, which do contain
    timestamps, into a new kstest.log
    """
    # Remove the old log with no timestamps
    kstest_log = os.path.join(tdir.name, "kstest.log")
    os.unlink(kstest_log)

    # Find all the test's kstest.log files and append them to kstest.log
    with open(kstest_log, "w") as ksf:
        for log in glob(os.path.join(tdir.name, "*", "kstest.log")):
            with open(log) as f:
                data = f.read(1024**2)
                while data:
                    ksf.write(data)
                    data = f.read(1024**2)


def check_tests(tests):
    """
    Check the tests for success, failing, missing, or flakes -- success after first failing

    This returns a tuple of:
    - list of the names of successful tests
    - list of the names of the missing tests
    - dict of failed tests, each entry being a list of the failure details dict
    - dict of flaky tests, each entry being a list of the flaky details dict
    """
    success = []
    missing = []
    failed = {}
    flakes = {}

    # The goal is to sort the tests into good/failed and record the ones
    # that passed after first failing in flakes
    for t in tests:
        name = t["name"]
        if t["success"]:
            # Tests should never have more than one success
            if name in success:
                raise RuntimeError(f"{name} already passed, should only be 1 per test")

            success.append(name)
            if name in failed:
                # Previously failed, move that to flakes and remove it from the failed list
                flakes[name] = failed[name]
                del failed[name]
        else:
            if "MISSING" in t["result"]:
                # Test is completely missing, don't count it as failed
                missing.append(name)
            elif name in success:
                # Test was also successful, make sure to record it in flakes list
                if name in flakes:
                    flakes[name].append(t)
                else:
                    flakes[name] = [t]
            else:
                if name in failed:
                    failed[name].append(t)
                else:
                    failed[name] = [t]

    return sorted(success), sorted(missing), failed, flakes


def print_test_details(scenario, days, test_name, buf):
    """Add the details of the named tests (failed-tests or flaky-tests) to the buffer
    """
    for d in days:
        if scenario not in days[d]:
            continue

        for n in days[d][scenario][test_name]:
            print(f"\n{n}:", file=buf)
            for test in days[d][scenario][test_name][n]:
                if "start_time" not in test:
                    start_time = ""
                else:
                    start_time = datetime.fromtimestamp(test["start_time"]).strftime("%m/%d/%Y %H:%M:%S")

                if "elapsed_time" not in test:
                    elapsed_time = 0
                else:
                    elapsed_time = test["elapsed_time"]

                # Get the result message
                msg = test["result"].rsplit("FAILED:")[-1]
                print(f'        {start_time} ({elapsed_time}s): {msg}', file=buf)


def kstest_logdir(tmpdir, test):
    """
    Return the directory of the logs for the test

    The logfile path should start with /var/tmp/kstest-*
    but this finds the kstest-* no matter what the leading
    path elements are.
    """
    logfile = test["logfile"]
    for e in logfile.split(os.path.sep):
        if e.startswith("kstest-"):
            return os.path.join(tmpdir, e)

    raise RuntimeError(f"No kstest-* directory found in {logfile}")


def archive_test_logs(days, archive_path, all_logs):
    """
    Store the logs from the failed/flaky tests for archiving
    and later examination. Paths follow the pattern:

    MM-DD-YYYY / scenario / failed|flakes / TESTNAME / [1...] /

    and each directory contains the logs for that run of the test.
    Currently this is the kstest.log and virt-install.log files
    """
    for day in days.keys():
        daydir = datetime.strptime(day, "%Y%m%d").strftime("%m-%d-%Y")
        for scenario in days[day].keys():
            # temporary log directories are stored by scenario + date
            datename = scenario + "-" + datetime.strptime(day, "%Y%m%d").strftime("%Y-%m-%d")
            if datename not in all_logs:
                raise RuntimeError(f"Missing all_log entry for {datename}")

            if not os.path.exists(all_logs[datename].name):
                raise RuntimeError(f"Missing log directory for {datename}")

            tmpdir = all_logs[datename].name
            failed = days[day][scenario]["failed-tests"]
            flakes = days[day][scenario]["flaky-tests"]

            scenario_archive = os.path.join(archive_path, daydir, scenario)
            os.makedirs(os.path.join(scenario_archive, "failed"))
            os.makedirs(os.path.join(scenario_archive, "flakes"))
            # data is organized by test names as keys with lists of tests
            for name in failed:
                i = 1
                for t in sorted(failed[name], key=lambda x: x["start_time"]):
                    try:
                        logdir = kstest_logdir(tmpdir, t)
                        if not os.path.exists(logdir):
                            raise RuntimeError(f"Missing logdir - {logdir}")
                    except RuntimeError:
                        continue
                    dst = os.path.join(scenario_archive, "failed", name, str(i))
                    shutil.copytree(logdir, dst)
                    i += 1

            for name in flakes:
                i = 1
                for t in sorted(flakes[name], key=lambda x: x["start_time"]):
                    try:
                        logdir = kstest_logdir(tmpdir, t)
                        if not logdir or not os.path.exists(logdir):
                            raise RuntimeError(f"Missing logdir - {logdir}")
                    except RuntimeError:
                        continue
                    dst = os.path.join(scenario_archive, "flakes", name, str(i))
                    shutil.copytree(logdir, dst)
                    i += 1


def process_logs(logs):
    """
    Process the logfiles into a data structure

    Returns a dictionary with each log's data that looks similar to this:

    {"20220401": {"logs-daily-iso": [{test data dict}, ...], ...}}, ...}

    So every day has an entry for the scenarios that were run on that day.
    Note that sometimes a scenario can be missing if there was a problem running it at all.
    """
    all_data = {}
    for log in logs:
        with open(log) as f:
            data = json.load(f)
            scenario = data[0].get("scenario", None)
            if scenario is None:
                # No scenario name, no way to organize the data
                continue

            # Use the log's date as the run identifier
            # This assumes the format is SCENARIO-YYYY-MM-DD.json
            # NOTE: This may not match the GitHub Action run dates due to tests taking
            #       a very long time.
            day = datetime.strptime(log[1+len(scenario):-5], "%Y-%m-%d").strftime("%Y%m%d")
            if day not in all_data:
                all_data[day] = {}

            # Group them by scenario, assume each file is from one scenario per day
            all_data[day][scenario] = data
    return all_data


def summary(args, json_logs, all_logs):
    """
    summary generates a summary of all the tests run in the selected date range

    It returns a string with the summary text
    """
    all_data = process_logs(json_logs)
    if args.debug:
        print(json.dumps(all_data))

    buf = io.StringIO()

    start = args.start.strftime("%m/%d/%Y %H:%M:%S")
    end = args.end.strftime("%m/%d/%Y %H:%M:%S")
    print(f"Test Summary Report: {start} -> {end}\n", file=buf)

    # Calculate test failures per day/scenario
    all_days = {}                   # dict of per-scenario counts
    days = {}                       # dict of per-day -> per-scenario counts and test names
    top_failed = {}                 # dict of per-test failure counts
    top_flakes = {}                 # dict of per-test flake counts
    for day in sorted(all_data.keys()):
        days[day] = {}
        for scenario in sorted(all_data[day].keys()):
            if scenario not in all_days:
                all_days[scenario] = {"success": 0, "missing": 0, "failed": 0, "flakes": 0}

            # Figure out how many were successful, failed, or were flakes
            success, missing, failed, flakes = check_tests(all_data[day][scenario])

            days[day][scenario] = {
                    "success": len(success),
                    "missing": len(missing),
                    "failed": len(failed),
                    "flakes": len(flakes),
                    "missing-tests": missing,
                    "failed-tests": failed,
                    "flaky-tests": flakes}
            all_days[scenario]["success"] += len(success)
            all_days[scenario]["missing"] += len(missing)
            all_days[scenario]["failed"] += len(failed)
            all_days[scenario]["flakes"] += len(flakes)

            for n in failed:
                top_failed[n] = top_failed.get(n, 0) + 1

            for n in flakes:
                top_flakes[n] = top_flakes.get(n, 0) + 1


    # Summary of tests per scenario
    print("Weekly summary", file=buf)
    print("==============", file=buf)
    for scenario in sorted(all_days.keys()):
        success = all_days[scenario]["success"]
        missing = all_days[scenario]["missing"]
        failed = all_days[scenario]["failed"]
        flakes = all_days[scenario]["flakes"]

        print(f"{scenario}: Ran {success+failed+missing} tests. {success} passed, {failed} failed, {missing} missing, {flakes} flakes.", file=buf)
    print("\n", file=buf)

    print("Top 5 failed tests for the week", file=buf)
    for n in sorted((n for n in top_failed), key=lambda x: top_failed[x], reverse=True)[:5]:
        print(f"    {n} - {top_failed[n]}", file=buf)
    print("\n", file=buf)

    print("Top 5 flaky tests for the week", file=buf)
    for n in sorted((n for n in top_flakes), key=lambda x: top_flakes[x], reverse=True)[:5]:
        print(f"    {n} - {top_flakes[n]}", file=buf)
    print("\n", file=buf)

    # Print daily stats
    for day in sorted(days.keys()):
        print(datetime.strptime(day, "%Y%m%d").strftime("%m/%d/%Y"), file=buf)
        for scenario in sorted(days[day].keys()):
            s = days[day][scenario]
            success = s["success"]
            missing = s["missing"]
            failed = s["failed"]
            total = success + failed + missing
            flakes = s["flakes"]
            print(f"    {scenario} (Ran {total}, {success} passed, {failed} failed, {missing} missing. {flakes} flakes) :", file=buf)
            if s["missing-tests"]:
                print("        Missing:", file=buf)
                for n in s["missing-tests"]:
                    print(f"            {n}", file=buf)
            if s["failed-tests"]:
                print("        Failed:", file=buf)
                for n in sorted(s["failed-tests"].keys()):
                    print(f"            {n}", file=buf)
            if s["flaky-tests"]:
                print("        Flakes:", file=buf)
                for n in sorted(s["flaky-tests"].keys()):
                    print(f"            {n}", file=buf)
        print("\n", file=buf)

    # Print the failure details for each scenario, on each day.
    for scenario in sorted(all_days.keys()):
        success = all_days[scenario]["success"]
        failed = all_days[scenario]["failed"]
        flakes = all_days[scenario]["flakes"]

        msg = f"{scenario}: Ran {success+failed} tests. {success} passed, {failed} failed, {flakes} flakes."
        print("=" * len(msg), file=buf)
        print(msg, file=buf)
        print("=" * len(msg), file=buf)

        if args.flake_details:
            print("Failed test details", file=buf)
            print("-------------------", file=buf)
        print_test_details(scenario, days, "failed-tests", buf)

        if args.flake_details:
            print("\nFlake test details", file=buf)
            print("-------------------", file=buf)
            print_test_details(scenario, days, "flaky-tests", buf)

        print("\n", file=buf)


    # Save the logs for the failures and flakes if a path is specified
    try:
        if args.archive_logs:
            archive_test_logs(days, args.archive_logs, all_logs)
    except RuntimeError as e:
        print(f"\nERROR: Problem archiving test logs - {e}", file=buf)

    return buf.getvalue()


def test_summary(args, path):
    """
    Test the weekly summary on a single directory

    Instead of pulling artifacts from github use a single directory with
    kstest.log.json and the log directories.
    """
    log = os.path.join(path, "kstest.log.json")
    with open(log) as f:
        data = json.load(f)
        scenario = data[0].get("scenario", None)
    if not scenario:
        raise RuntimeError("No scenario found in %s" % log)

    # The json log filename needs to be in the form of <scenario>-<YYYY-MM-DD>.json
    datename = f"{scenario}-1990-01-01"
    shutil.copy(log, datename+".json")
    datenames = [datename]
    all_logs = {datename: path}

    report = summary(args, (d+".json" for d in datenames), all_logs)
    if args.output:
        with open(args.output, "w") as f:
            f.write(report)
    else:
        print(report)


def main(args, token):
    artifacts = get_artifacts(token, args.artifact_names, args.start, args.end)
    if args.verbose:
        print(json.dumps(artifacts))
    zipfiles = download_artifacts(token, artifacts)
    if args.debug:
        print(f"zipfiles = {zipfiles}")

    datenames = []          # List of valid logfile names
    all_logs = {}           # Keep track of all the log tempdirs
    for name, datename, f in zipfiles:
        if args.rebuild or not os.path.exists(datename+".json"):
            try:
                all_logs[datename] = extract_logs(f)
            except zipfile.BadZipFile:
                # GitHub can responds with a 200 and a json error instead of a zip
                # so if it isn't a valid zip, just skip it.
                os.unlink(f)
                continue

            # This is needed for logs without timestamps
            if args.rebuild:
                generate_test_list(all_logs[datename])
                rebuild_logs(all_logs[datename])

            # Run summary on kstest.log
            cmd = ["log2json",
                   "--scenario", name,
                   "--output", datename+".json",
                   os.path.join(all_logs[datename].name, "kstest.log")
            ]
            if args.debug:
                print(cmd)
            check_call(cmd)

        # If the summary exists, add it to the list
        if os.path.exists(datename+".json"):
            datenames.append(datename)

    report = summary(args, (d+".json" for d in datenames), all_logs)
    if args.output:
        with open(args.output, "w") as f:
            f.write(report)
    else:
        print(report)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Generate a weekly summary of test results")
    parser.add_argument("--artifacts",
                        type=lambda x: x.split(","),
                        default=ARTIFACT_NAMES,
                        dest="artifact_names",
                        help="Comma separated list of artifact names to summarize")
    parser.add_argument("--start", type=datetime.fromisoformat,
            default=(datetime.now() - timedelta(days=7)),
            help="Start time. eg. 2022-03-03T03:46:11 (default is -7 days)")
    parser.add_argument("--end", type=datetime.fromisoformat,
            default=datetime.now(),
            help="end time. eg. 2022-03-03T03:46:11 (default is now)")
    parser.add_argument("--rebuild",
            default=False, action="store_true",
            help="Rebuild logs with timestamps")
    parser.add_argument("--flake-details",
            default=False, action="store_true",
            help="Include details about flaky tests in summary")
    parser.add_argument("--archive-logs",
            help="Optionally collect the failed/flake logs in an archive directory tree")
    parser.add_argument("--output",
            help="Path and filename to write summary report to")
    parser.add_argument("--test", help=argparse.SUPPRESS)
    parser.add_argument("--debug", default=False, action="store_true")
    parser.add_argument("--verbose", default=False, action="store_true")
    args = parser.parse_args()

    if "GITHUB_TOKEN" not in os.environ:
        print("Set GITHUB_TOKEN environmental variable to github token with access to the artifact api.")
        sys.exit(1)

    if args.debug:
        print(f"args = {args}")

    if args.test:
        test_summary(args, args.test)
    else:
        main(args, os.environ["GITHUB_TOKEN"])

82a6589efc6802b11f26c847b863b41fab209116
<mask token>


def get_artifacts(token, artifact_names, start, end):
    """
    get_artifacts retrieves a list of artifacts from the selected date range
    that are listed in artifact_names

    It returns a list of artifact dicts like this:

    {
      "id": 187278866,
      "node_id": "MDg6QXJ0aWZhY3QxODcyNzg4NjY=",
      "name": "logs-daily-iso",
      "size_in_bytes": 1047408950,
      "url": "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts/187278866",
      "archive_download_url": "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts/187278866/zip",
      "expired": false,
      "created_at": "2022-03-17T03:30:12Z",
      "updated_at": "2022-03-17T03:30:13Z",
      "expires_at": "2022-03-25T03:29:06Z"
    },
    """
    artifacts = []
    page = 1
    retry_limit = 3
    while True:
        req = Request(URL + f'&page={page}')
        req.add_header('Accept', 'application/vnd.github.v3+json')
        req.add_header('Authorization', f'token {token}')
        with urlopen(req) as r:
            if r.status == 403:
                try:
                    reset = int(r.headers.get('X-RateLimit-Reset'))
                except:
                    raise RuntimeError(
                        'Hit GitHub rate limit. Reset header missing.')
                if retry_limit == 0 or time.time(
                    ) > reset or reset - time.time() > 90:
                    raise RuntimeError(
                        'Hit GitHub rate limit. Reset is at %s' % time.
                        ctime(reset))
                time.sleep(10 + (reset - time.time()))
                retry_limit = retry_limit - 1
                continue
            if r.status != 200:
                raise RuntimeError('Error (%d) with API request: %s' % (r.
                    status, str(r)))
            data = json.load(r)
        for a in data['artifacts']:
            if a['name'] not in artifact_names:
                continue
            updated_at = datetime.fromisoformat(a['updated_at'][:-1])
            if start <= updated_at <= end:
                artifacts.append(a)
        if len(data['artifacts']) < 100:
            break
        page = page + 1
        time.sleep(2)
    return sorted(artifacts, key=lambda x: x['updated_at'])


<mask token>


def download_artifacts(token, artifacts):
    """
    download_artifacts downloads the artifacts as uniquely named files

    If there is a problem downloading an artifact it is skipped and not
    added to the returned list.

    It returns a list of tuples containing the artifact name, the
    artifact name with the updated date appended (eg. logs-rhel8-2022-04-01),
    and the filename of the zipfile.
    """
    zipfiles = []
    for a in artifacts:
        updated_at = datetime.fromisoformat(a['updated_at'][:-1])
        datename = a['name'] + updated_at.strftime('-%Y-%m-%d')
        filename = datename + '.zip'
        if os.path.exists(filename):
            zipfiles.append((a['name'], datename, filename))
            print(f'{filename} skipped, already downloaded')
            continue
        print(f'Fetching {filename}')
        ok = run_curl(token, a['archive_download_url'], filename)
        if not ok:
            continue
        zipfiles.append((a['name'], datename, filename))
    return zipfiles


def extract_logs(f):
    """
    extract_logs unzips the archive into a temporary directory

    This directory is deleted when the object goes out of scope
    """
    tdir = tempfile.TemporaryDirectory(prefix='kstest-log-', dir='/var/tmp/')
    with zipfile.ZipFile(f) as zf:
        zf.extractall(tdir.name)
    return tdir


<mask token>


def check_tests(tests):
    """
    Check the tests for success, failing, missing, or flakes -- success after first failing

    This returns a tuple of:
    - list of the names of successful tests
    - list of the names of the missing tests
    - dict of failed tests, each entry being a list of the failure details dict
    - dict of flaky tests, each entry being a list of the flaky details dict
    """
    success = []
    missing = []
    failed = {}
    flakes = {}
    for t in tests:
        name = t['name']
        if t['success']:
            if name in success:
                raise RuntimeError(
                    f'{name} already passed, should only be 1 per test')
            success.append(name)
            if name in failed:
                flakes[name] = failed[name]
                del failed[name]
        elif 'MISSING' in t['result']:
            missing.append(name)
        elif name in success:
            if name in flakes:
                flakes[name].append(t)
            else:
                flakes[name] = [t]
        elif name in failed:
            failed[name].append(t)
        else:
            failed[name] = [t]
    return sorted(success), sorted(missing), failed, flakes


def print_test_details(scenario, days, test_name, buf):
    """Add the details of the named tests (failed-tests or flaky-tests) to the buffer
    """
    for d in days:
        if scenario not in days[d]:
            continue
        for n in days[d][scenario][test_name]:
            print(f'\n{n}:', file=buf)
            for test in days[d][scenario][test_name][n]:
                if 'start_time' not in test:
                    start_time = ''
                else:
                    start_time = datetime.fromtimestamp(test['start_time']
                        ).strftime('%m/%d/%Y %H:%M:%S')
                if 'elapsed_time' not in test:
                    elapsed_time = 0
                else:
                    elapsed_time = test['elapsed_time']
                msg = test['result'].rsplit('FAILED:')[-1]
                print(f'        {start_time} ({elapsed_time}s): {msg}',
                    file=buf)


def kstest_logdir(tmpdir, test):
    """
    Return the directory of the logs for the test

    The logfile path should start with /var/tmp/kstest-*
    but this finds the kstest-* no matter what the leading
    path elements are.
    """
    logfile = test['logfile']
    for e in logfile.split(os.path.sep):
        if e.startswith('kstest-'):
            return os.path.join(tmpdir, e)
    raise RuntimeError(f'No kstest-* directory found in {logfile}')


def archive_test_logs(days, archive_path, all_logs):
    """
    Store the logs from the failed/flaky tests for archiving
    and later examination. Paths follow the pattern:

    MM-DD-YYYY / scenario / failed|flakes / TESTNAME / [1...] /

    and each directory contains the logs for that run of the test.
    Currently this is the kstest.log and virt-install.log files
    """
    for day in days.keys():
        daydir = datetime.strptime(day, '%Y%m%d').strftime('%m-%d-%Y')
        for scenario in days[day].keys():
            datename = scenario + '-' + datetime.strptime(day, '%Y%m%d'
                ).strftime('%Y-%m-%d')
            if datename not in all_logs:
                raise RuntimeError(f'Missing all_log entry for {datename}')
            if not os.path.exists(all_logs[datename].name):
                raise RuntimeError(f'Missing log directory for {datename}')
            tmpdir = all_logs[datename].name
            failed = days[day][scenario]['failed-tests']
            flakes = days[day][scenario]['flaky-tests']
            scenario_archive = os.path.join(archive_path, daydir, scenario)
            os.makedirs(os.path.join(scenario_archive, 'failed'))
            os.makedirs(os.path.join(scenario_archive, 'flakes'))
            for name in failed:
                i = 1
                for t in sorted(failed[name], key=lambda x: x['start_time']):
                    try:
                        logdir = kstest_logdir(tmpdir, t)
                        if not os.path.exists(logdir):
                            raise RuntimeError(f'Missing logdir - {logdir}')
                    except RuntimeError:
                        continue
                    dst = os.path.join(scenario_archive, 'failed', name, str(i)
                        )
                    shutil.copytree(logdir, dst)
                    i += 1
            for name in flakes:
                i = 1
                for t in sorted(flakes[name], key=lambda x: x['start_time']):
                    try:
                        logdir = kstest_logdir(tmpdir, t)
                        if not logdir or not os.path.exists(logdir):
                            raise RuntimeError(f'Missing logdir - {logdir}')
                    except RuntimeError:
                        continue
                    dst = os.path.join(scenario_archive, 'flakes', name, str(i)
                        )
                    shutil.copytree(logdir, dst)
                    i += 1


def process_logs(logs):
    """
    Process the logfiles into a data structure

    Returns a dictionary with each log's data that looks similar to this:

    {"20220401": {"logs-daily-iso": [{test data dict}, ...], ...}}, ...}

    So every day has an entry for the scenarios that were run on that day.
    Note that sometimes a scenario can be missing if there was a problem running it at all.
    """
    all_data = {}
    for log in logs:
        with open(log) as f:
            data = json.load(f)
            scenario = data[0].get('scenario', None)
            if scenario is None:
                continue
            day = datetime.strptime(log[1 + len(scenario):-5], '%Y-%m-%d'
                ).strftime('%Y%m%d')
            if day not in all_data:
                all_data[day] = {}
            all_data[day][scenario] = data
    return all_data


def summary(args, json_logs, all_logs):
    """
    summary generates a summary of all the tests run in the selected date range

    It returns a string with the summary text
    """
    all_data = process_logs(json_logs)
    if args.debug:
        print(json.dumps(all_data))
    buf = io.StringIO()
    start = args.start.strftime('%m/%d/%Y %H:%M:%S')
    end = args.end.strftime('%m/%d/%Y %H:%M:%S')
    print(f'Test Summary Report: {start} -> {end}\n', file=buf)
    all_days = {}
    days = {}
    top_failed = {}
    top_flakes = {}
    for day in sorted(all_data.keys()):
        days[day] = {}
        for scenario in sorted(all_data[day].keys()):
            if scenario not in all_days:
                all_days[scenario] = {'success': 0, 'missing': 0, 'failed':
                    0, 'flakes': 0}
            success, missing, failed, flakes = check_tests(all_data[day][
                scenario])
            days[day][scenario] = {'success': len(success), 'missing': len(
                missing), 'failed': len(failed), 'flakes': len(flakes),
                'missing-tests': missing, 'failed-tests': failed,
                'flaky-tests': flakes}
            all_days[scenario]['success'] += len(success)
            all_days[scenario]['missing'] += len(missing)
            all_days[scenario]['failed'] += len(failed)
            all_days[scenario]['flakes'] += len(flakes)
            for n in failed:
                top_failed[n] = top_failed.get(n, 0) + 1
            for n in flakes:
                top_flakes[n] = top_flakes.get(n, 0) + 1
    print('Weekly summary', file=buf)
    print('==============', file=buf)
    for scenario in sorted(all_days.keys()):
        success = all_days[scenario]['success']
        missing = all_days[scenario]['missing']
        failed = all_days[scenario]['failed']
        flakes = all_days[scenario]['flakes']
        print(
            f'{scenario}: Ran {success + failed + missing} tests. {success} passed, {failed} failed, {missing} missing, {flakes} flakes.'
            , file=buf)
    print('\n', file=buf)
    print('Top 5 failed tests for the week', file=buf)
    for n in sorted((n for n in top_failed), key=lambda x: top_failed[x],
        reverse=True)[:5]:
        print(f'    {n} - {top_failed[n]}', file=buf)
    print('\n', file=buf)
    print('Top 5 flaky tests for the week', file=buf)
    for n in sorted((n for n in top_flakes), key=lambda x: top_flakes[x],
        reverse=True)[:5]:
        print(f'    {n} - {top_flakes[n]}', file=buf)
    print('\n', file=buf)
    for day in sorted(days.keys()):
        print(datetime.strptime(day, '%Y%m%d').strftime('%m/%d/%Y'), file=buf)
        for scenario in sorted(days[day].keys()):
            s = days[day][scenario]
            success = s['success']
            missing = s['missing']
            failed = s['failed']
            total = success + failed + missing
            flakes = s['flakes']
            print(
                f'    {scenario} (Ran {total}, {success} passed, {failed} failed, {missing} missing. {flakes} flakes) :'
                , file=buf)
            if s['missing-tests']:
                print('        Missing:', file=buf)
                for n in s['missing-tests']:
                    print(f'            {n}', file=buf)
            if s['failed-tests']:
                print('        Failed:', file=buf)
                for n in sorted(s['failed-tests'].keys()):
                    print(f'            {n}', file=buf)
            if s['flaky-tests']:
                print('        Flakes:', file=buf)
                for n in sorted(s['flaky-tests'].keys()):
                    print(f'            {n}', file=buf)
        print('\n', file=buf)
    for scenario in sorted(all_days.keys()):
        success = all_days[scenario]['success']
        failed = all_days[scenario]['failed']
        flakes = all_days[scenario]['flakes']
        msg = (
            f'{scenario}: Ran {success + failed} tests. {success} passed, {failed} failed, {flakes} flakes.'
            )
        print('=' * len(msg), file=buf)
        print(msg, file=buf)
        print('=' * len(msg), file=buf)
        if args.flake_details:
            print('Failed test details', file=buf)
            print('-------------------', file=buf)
        print_test_details(scenario, days, 'failed-tests', buf)
        if args.flake_details:
            print('\nFlake test details', file=buf)
            print('-------------------', file=buf)
            print_test_details(scenario, days, 'flaky-tests', buf)
        print('\n', file=buf)
    try:
        if args.archive_logs:
            archive_test_logs(days, args.archive_logs, all_logs)
    except RuntimeError as e:
        print(f'\nERROR: Problem archiving test logs - {e}', file=buf)
    return buf.getvalue()


def test_summary(args, path):
    """
    Test the weekly summary on a single directory

    Instead of pulling artifacts from github use a single directory with
    kstest.log.json and the log directories.
    """
    log = os.path.join(path, 'kstest.log.json')
    with open(log) as f:
        data = json.load(f)
        scenario = data[0].get('scenario', None)
    if not scenario:
        raise RuntimeError('No scenario found in %s' % log)
    datename = f'{scenario}-1990-01-01'
    shutil.copy(log, datename + '.json')
    datenames = [datename]
    all_logs = {datename: path}
    report = summary(args, (d + '.json' for d in datenames), all_logs)
    if args.output:
        with open(args.output, 'w') as f:
            f.write(report)
    else:
        print(report)


def main(args, token):
    artifacts = get_artifacts(token, args.artifact_names, args.start, args.end)
    if args.verbose:
        print(json.dumps(artifacts))
    zipfiles = download_artifacts(token, artifacts)
    if args.debug:
        print(f'zipfiles = {zipfiles}')
    datenames = []
    all_logs = {}
    for name, datename, f in zipfiles:
        if args.rebuild or not os.path.exists(datename + '.json'):
            try:
                all_logs[datename] = extract_logs(f)
            except zipfile.BadZipFile:
                os.unlink(f)
                continue
            if args.rebuild:
                generate_test_list(all_logs[datename])
                rebuild_logs(all_logs[datename])
            cmd = ['log2json', '--scenario', name, '--output', datename +
                '.json', os.path.join(all_logs[datename].name, 'kstest.log')]
            if args.debug:
                print(cmd)
            check_call(cmd)
        if os.path.exists(datename + '.json'):
            datenames.append(datename)
    report = summary(args, (d + '.json' for d in datenames), all_logs)
    if args.output:
        with open(args.output, 'w') as f:
            f.write(report)
    else:
        print(report)


<mask token>

<mask token>


def get_artifacts(token, artifact_names, start, end):
    """
    get_artifacts retrieves a list of artifacts from the selected date range
    that are listed in artifact_names

    It returns a list of artifact dicts like this:

    {
      "id": 187278866,
      "node_id": "MDg6QXJ0aWZhY3QxODcyNzg4NjY=",
      "name": "logs-daily-iso",
      "size_in_bytes": 1047408950,
      "url": "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts/187278866",
      "archive_download_url": "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts/187278866/zip",
      "expired": false,
      "created_at": "2022-03-17T03:30:12Z",
      "updated_at": "2022-03-17T03:30:13Z",
      "expires_at": "2022-03-25T03:29:06Z"
    },
    """
    artifacts = []
    page = 1
    retry_limit = 3
    while True:
        req = Request(URL + f'&page={page}')
        req.add_header('Accept', 'application/vnd.github.v3+json')
        req.add_header('Authorization', f'token {token}')
        with urlopen(req) as r:
            if r.status == 403:
                try:
                    reset = int(r.headers.get('X-RateLimit-Reset'))
                except:
                    raise RuntimeError(
                        'Hit GitHub rate limit. Reset header missing.')
                if retry_limit == 0 or time.time(
                    ) > reset or reset - time.time() > 90:
                    raise RuntimeError(
                        'Hit GitHub rate limit. Reset is at %s' % time.
                        ctime(reset))
                time.sleep(10 + (reset - time.time()))
                retry_limit = retry_limit - 1
                continue
            if r.status != 200:
                raise RuntimeError('Error (%d) with API request: %s' % (r.
                    status, str(r)))
            data = json.load(r)
        for a in data['artifacts']:
            if a['name'] not in artifact_names:
                continue
            updated_at = datetime.fromisoformat(a['updated_at'][:-1])
            if start <= updated_at <= end:
                artifacts.append(a)
        if len(data['artifacts']) < 100:
            break
        page = page + 1
        time.sleep(2)
    return sorted(artifacts, key=lambda x: x['updated_at'])


<mask token>


def download_artifacts(token, artifacts):
    """
    download_artifacts downloads the artifacts as uniquely named files

    If there is a problem downloading an artifact it is skipped and not
    added to the returned list.

    It returns a list of tuples containing the artifact name, the
    artifact name with the updated date appended (eg. logs-rhel8-2022-04-01),
    and the filename of the zipfile.
    """
    zipfiles = []
    for a in artifacts:
        updated_at = datetime.fromisoformat(a['updated_at'][:-1])
        datename = a['name'] + updated_at.strftime('-%Y-%m-%d')
        filename = datename + '.zip'
        if os.path.exists(filename):
            zipfiles.append((a['name'], datename, filename))
            print(f'{filename} skipped, already downloaded')
            continue
        print(f'Fetching {filename}')
        ok = run_curl(token, a['archive_download_url'], filename)
        if not ok:
            continue
        zipfiles.append((a['name'], datename, filename))
    return zipfiles


def extract_logs(f):
    """
    extract_logs unzips the archive into a temporary directory

    This directory is deleted when the object goes out of scope
    """
    tdir = tempfile.TemporaryDirectory(prefix='kstest-log-', dir='/var/tmp/')
    with zipfile.ZipFile(f) as zf:
        zf.extractall(tdir.name)
    return tdir


def generate_test_list(tdir):
    """
    Build kstest-list if it is missing

    This is a list of the expected tests, and it is used to help detect when
    tests are totally missing from the results.

    The original kstest.log has a line:
    Running tests: ./bridge-httpks.sh ./packages-instlangs-1.sh  ...

    Parse this and create kstest-list, one test name per line for later use.
    """
    if os.path.exists(os.path.join(tdir.name, 'kstest-list')):
        return
    kstest_log = os.path.join(tdir.name, 'kstest.log')
    with open(kstest_log) as f:
        for line in f.readlines():
            if not line.startswith('Running tests: '):
                continue
            tests = [os.path.basename(os.path.splitext(s)[0]) for s in line
                [15:].split()]
            with open(os.path.join(tdir.name, 'kstest-list'), 'wt') as klf:
                for t in tests:
                    print(t, file=klf)
            break


<mask token>


def check_tests(tests):
    """
    Check the tests for success, failing, missing, or flakes -- success after first failing

    This returns a tuple of:
    - list of the names of successful tests
    - list of the names of the missing tests
    - dict of failed tests, each entry being a list of the failure details dict
    - dict of flaky tests, each entry being a list of the flaky details dict
    """
    success = []
    missing = []
    failed = {}
    flakes = {}
    for t in tests:
        name = t['name']
        if t['success']:
            if name in success:
                raise RuntimeError(
                    f'{name} already passed, should only be 1 per test')
            success.append(name)
            if name in failed:
                flakes[name] = failed[name]
                del failed[name]
        elif 'MISSING' in t['result']:
            missing.append(name)
        elif name in success:
            if name in flakes:
                flakes[name].append(t)
            else:
                flakes[name] = [t]
        elif name in failed:
            failed[name].append(t)
        else:
            failed[name] = [t]
    return sorted(success), sorted(missing), failed, flakes


def print_test_details(scenario, days, test_name, buf):
    """Add the details of the named tests (failed-tests or flaky-tests) to the buffer
    """
    for d in days:
        if scenario not in days[d]:
            continue
        for n in days[d][scenario][test_name]:
            print(f'\n{n}:', file=buf)
            for test in days[d][scenario][test_name][n]:
                if 'start_time' not in test:
                    start_time = ''
                else:
                    start_time = datetime.fromtimestamp(test['start_time']
                        ).strftime('%m/%d/%Y %H:%M:%S')
                if 'elapsed_time' not in test:
                    elapsed_time = 0
                else:
                    elapsed_time = test['elapsed_time']
                msg = test['result'].rsplit('FAILED:')[-1]
                print(f'        {start_time} ({elapsed_time}s): {msg}',
                    file=buf)


def kstest_logdir(tmpdir, test):
    """
    Return the directory of the logs for the test

    The logfile path should start with /var/tmp/kstest-*
    but this finds the kstest-* no matter what the leading
    path elements are.
    """
    logfile = test['logfile']
    for e in logfile.split(os.path.sep):
        if e.startswith('kstest-'):
            return os.path.join(tmpdir, e)
    raise RuntimeError(f'No kstest-* directory found in {logfile}')


def archive_test_logs(days, archive_path, all_logs):
    """
    Store the logs from the failed/flaky tests for archiving
    and later examination. Paths follow the pattern:

    MM-DD-YYYY / scenario / failed|flakes / TESTNAME / [1...] /

    and each directory contains the logs for that run of the test.
    Currently this is the kstest.log and virt-install.log files
    """
    for day in days.keys():
        daydir = datetime.strptime(day, '%Y%m%d').strftime('%m-%d-%Y')
        for scenario in days[day].keys():
            datename = scenario + '-' + datetime.strptime(day, '%Y%m%d'
                ).strftime('%Y-%m-%d')
            if datename not in all_logs:
                raise RuntimeError(f'Missing all_log entry for {datename}')
            if not os.path.exists(all_logs[datename].name):
                raise RuntimeError(f'Missing log directory for {datename}')
            tmpdir = all_logs[datename].name
            failed = days[day][scenario]['failed-tests']
            flakes = days[day][scenario]['flaky-tests']
            scenario_archive = os.path.join(archive_path, daydir, scenario)
            os.makedirs(os.path.join(scenario_archive, 'failed'))
            os.makedirs(os.path.join(scenario_archive, 'flakes'))
            for name in failed:
                i = 1
                for t in sorted(failed[name], key=lambda x: x['start_time']):
                    try:
                        logdir = kstest_logdir(tmpdir, t)
                        if not os.path.exists(logdir):
                            raise RuntimeError(f'Missing logdir - {logdir}')
                    except RuntimeError:
                        continue
                    dst = os.path.join(scenario_archive, 'failed', name, str(i)
                        )
                    shutil.copytree(logdir, dst)
                    i += 1
            for name in flakes:
                i = 1
                for t in sorted(flakes[name], key=lambda x: x['start_time']):
                    try:
                        logdir = kstest_logdir(tmpdir, t)
                        if not logdir or not os.path.exists(logdir):
                            raise RuntimeError(f'Missing logdir - {logdir}')
                    except RuntimeError:
                        continue
                    dst = os.path.join(scenario_archive, 'flakes', name, str(i)
                        )
                    shutil.copytree(logdir, dst)
                    i += 1


def process_logs(logs):
    """
    Process the logfiles into a data structure

    Returns a dictionary with each log's data that looks similar to this:

    {"20220401": {"logs-daily-iso": [{test data dict}, ...], ...}}, ...}

    So every day has an entry for the scenarios that were run on that day.
    Note that sometimes a scenario can be missing if there was a problem running it at all.
    """
    all_data = {}
    for log in logs:
        with open(log) as f:
            data = json.load(f)
            scenario = data[0].get('scenario', None)
            if scenario is None:
                continue
            day = datetime.strptime(log[1 + len(scenario):-5], '%Y-%m-%d'
                ).strftime('%Y%m%d')
            if day not in all_data:
                all_data[day] = {}
            all_data[day][scenario] = data
    return all_data


def summary(args, json_logs, all_logs):
    """
    summary generates a summary of all the tests run in the selected date range

    It returns a string with the summary text
    """
    all_data = process_logs(json_logs)
    if args.debug:
        print(json.dumps(all_data))
    buf = io.StringIO()
    start = args.start.strftime('%m/%d/%Y %H:%M:%S')
    end = args.end.strftime('%m/%d/%Y %H:%M:%S')
    print(f'Test Summary Report: {start} -> {end}\n', file=buf)
    all_days = {}
    days = {}
    top_failed = {}
    top_flakes = {}
    for day in sorted(all_data.keys()):
        days[day] = {}
        for scenario in sorted(all_data[day].keys()):
            if scenario not in all_days:
                all_days[scenario] = {'success': 0, 'missing': 0, 'failed':
                    0, 'flakes': 0}
            success, missing, failed, flakes = check_tests(all_data[day][
                scenario])
            days[day][scenario] = {'success': len(success), 'missing': len(
                missing), 'failed': len(failed), 'flakes': len(flakes),
                'missing-tests': missing, 'failed-tests': failed,
                'flaky-tests': flakes}
            all_days[scenario]['success'] += len(success)
            all_days[scenario]['missing'] += len(missing)
            all_days[scenario]['failed'] += len(failed)
            all_days[scenario]['flakes'] += len(flakes)
            for n in failed:
                top_failed[n] = top_failed.get(n, 0) + 1
            for n in flakes:
                top_flakes[n] = top_flakes.get(n, 0) + 1
    print('Weekly summary', file=buf)
    print('==============', file=buf)
    for scenario in sorted(all_days.keys()):
        success = all_days[scenario]['success']
        missing = all_days[scenario]['missing']
        failed = all_days[scenario]['failed']
        flakes = all_days[scenario]['flakes']
        print(
            f'{scenario}: Ran {success + failed + missing} tests. {success} passed, {failed} failed, {missing} missing, {flakes} flakes.'
            , file=buf)
    print('\n', file=buf)
    print('Top 5 failed tests for the week', file=buf)
    for n in sorted((n for n in top_failed), key=lambda x: top_failed[x],
        reverse=True)[:5]:
        print(f'    {n} - {top_failed[n]}', file=buf)
    print('\n', file=buf)
    print('Top 5 flaky tests for the week', file=buf)
    for n in sorted((n for n in top_flakes), key=lambda x: top_flakes[x],
        reverse=True)[:5]:
        print(f'    {n} - {top_flakes[n]}', file=buf)
    print('\n', file=buf)
    for day in sorted(days.keys()):
        print(datetime.strptime(day, '%Y%m%d').strftime('%m/%d/%Y'), file=buf)
        for scenario in sorted(days[day].keys()):
            s = days[day][scenario]
            success = s['success']
            missing = s['missing']
            failed = s['failed']
            total = success + failed + missing
            flakes = s['flakes']
            print(
                f'    {scenario} (Ran {total}, {success} passed, {failed} failed, {missing} missing. {flakes} flakes) :'
                , file=buf)
            if s['missing-tests']:
                print('        Missing:', file=buf)
                for n in s['missing-tests']:
                    print(f'            {n}', file=buf)
            if s['failed-tests']:
                print('        Failed:', file=buf)
                for n in sorted(s['failed-tests'].keys()):
                    print(f'            {n}', file=buf)
            if s['flaky-tests']:
                print('        Flakes:', file=buf)
                for n in sorted(s['flaky-tests'].keys()):
                    print(f'            {n}', file=buf)
        print('\n', file=buf)
    for scenario in sorted(all_days.keys()):
        success = all_days[scenario]['success']
        failed = all_days[scenario]['failed']
        flakes = all_days[scenario]['flakes']
        msg = (
            f'{scenario}: Ran {success + failed} tests. {success} passed, {failed} failed, {flakes} flakes.'
            )
        print('=' * len(msg), file=buf)
        print(msg, file=buf)
        print('=' * len(msg), file=buf)
        if args.flake_details:
            print('Failed test details', file=buf)
            print('-------------------', file=buf)
        print_test_details(scenario, days, 'failed-tests', buf)
        if args.flake_details:
            print('\nFlake test details', file=buf)
            print('-------------------', file=buf)
            print_test_details(scenario, days, 'flaky-tests', buf)
        print('\n', file=buf)
    try:
        if args.archive_logs:
            archive_test_logs(days, args.archive_logs, all_logs)
    except RuntimeError as e:
        print(f'\nERROR: Problem archiving test logs - {e}', file=buf)
    return buf.getvalue()


def test_summary(args, path):
    """
    Test the weekly summary on a single directory

    Instead of pulling artifacts from github use a single directory with
    kstest.log.json and the log directories.
    """
    log = os.path.join(path, 'kstest.log.json')
    with open(log) as f:
        data = json.load(f)
        scenario = data[0].get('scenario', None)
    if not scenario:
        raise RuntimeError('No scenario found in %s' % log)
    datename = f'{scenario}-1990-01-01'
    shutil.copy(log, datename + '.json')
    datenames = [datename]
    all_logs = {datename: path}
    report = summary(args, (d + '.json' for d in datenames), all_logs)
    if args.output:
        with open(args.output, 'w') as f:
            f.write(report)
    else:
        print(report)


def main(args, token):
    artifacts = get_artifacts(token, args.artifact_names, args.start, args.end)
    if args.verbose:
        print(json.dumps(artifacts))
    zipfiles = download_artifacts(token, artifacts)
    if args.debug:
        print(f'zipfiles = {zipfiles}')
    datenames = []
    all_logs = {}
    for name, datename, f in zipfiles:
        if args.rebuild or not os.path.exists(datename + '.json'):
            try:
                all_logs[datename] = extract_logs(f)
            except zipfile.BadZipFile:
                os.unlink(f)
                continue
            if args.rebuild:
                generate_test_list(all_logs[datename])
                rebuild_logs(all_logs[datename])
            cmd = ['log2json', '--scenario', name, '--output', datename +
                '.json', os.path.join(all_logs[datename].name, 'kstest.log')]
            if args.debug:
                print(cmd)
            check_call(cmd)
        if os.path.exists(datename + '.json'):
            datenames.append(datename)
    report = summary(args, (d + '.json' for d in datenames), all_logs)
    if args.output:
        with open(args.output, 'w') as f:
            f.write(report)
    else:
        print(report)


<mask token>

<mask token>


def get_artifacts(token, artifact_names, start, end):
    """
    get_artifacts retrieves a list of artifacts from the selected date range
    that are listed in artifact_names

    It returns a list of artifact dicts like this:

    {
      "id": 187278866,
      "node_id": "MDg6QXJ0aWZhY3QxODcyNzg4NjY=",
      "name": "logs-daily-iso",
      "size_in_bytes": 1047408950,
      "url": "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts/187278866",
      "archive_download_url": "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts/187278866/zip",
      "expired": false,
      "created_at": "2022-03-17T03:30:12Z",
      "updated_at": "2022-03-17T03:30:13Z",
      "expires_at": "2022-03-25T03:29:06Z"
    },
    """
    artifacts = []
    page = 1
    retry_limit = 3
    while True:
        req = Request(URL + f'&page={page}')
        req.add_header('Accept', 'application/vnd.github.v3+json')
        req.add_header('Authorization', f'token {token}')
        with urlopen(req) as r:
            if r.status == 403:
                try:
                    reset = int(r.headers.get('X-RateLimit-Reset'))
                except:
                    raise RuntimeError(
                        'Hit GitHub rate limit. Reset header missing.')
                if retry_limit == 0 or time.time(
                    ) > reset or reset - time.time() > 90:
                    raise RuntimeError(
                        'Hit GitHub rate limit. Reset is at %s' % time.
                        ctime(reset))
                time.sleep(10 + (reset - time.time()))
                retry_limit = retry_limit - 1
                continue
            if r.status != 200:
                raise RuntimeError('Error (%d) with API request: %s' % (r.
                    status, str(r)))
            data = json.load(r)
        for a in data['artifacts']:
            if a['name'] not in artifact_names:
                continue
            updated_at = datetime.fromisoformat(a['updated_at'][:-1])
            if start <= updated_at <= end:
                artifacts.append(a)
        if len(data['artifacts']) < 100:
            break
        page = page + 1
        time.sleep(2)
    return sorted(artifacts, key=lambda x: x['updated_at'])


def run_curl(token, url, filename):
    """
    run_curl downloads an artifact file

    It returns True if the response was a 200
    If there was an exception is returns False and the error, as well as
    printing it to stderr
    """
    with open(filename, 'wb') as f:
        c = pycurl.Curl()
        headers = ['Accept: application/vnd.github.v3+json',
            f'Authorization: token {token}']
        options = {pycurl.FOLLOWLOCATION: 1, pycurl.MAXREDIRS: 5, pycurl.
            CONNECTTIMEOUT: 30, pycurl.TIMEOUT: 300, pycurl.NOSIGNAL: 1,
            pycurl.URL: url, pycurl.HTTPHEADER: headers, pycurl.WRITEDATA: f}
        for k, v in options.items():
            c.setopt(k, v)
        try:
            c.perform()
            status = c.getinfo(pycurl.HTTP_CODE)
            ok = status == 200, None
        except Exception as e:
            print(f'ERROR: {e}', file=sys.stderr)
            sys.stderr.flush()
            ok = False, e
        c.close()
    return ok


def download_artifacts(token, artifacts):
    """
    download_artifacts downloads the artifacts as uniquely named files

    If there is a problem downloading an artifact it is skipped and not
    added to the returned list.

    It returns a list of tuples containing the artifact name, the
    artifact name with the updated date appended (eg. logs-rhel8-2022-04-01),
    and the filename of the zipfile.
    """
    zipfiles = []
    for a in artifacts:
        updated_at = datetime.fromisoformat(a['updated_at'][:-1])
        datename = a['name'] + updated_at.strftime('-%Y-%m-%d')
        filename = datename + '.zip'
        if os.path.exists(filename):
            zipfiles.append((a['name'], datename, filename))
            print(f'{filename} skipped, already downloaded')
            continue
        print(f'Fetching {filename}')
        ok = run_curl(token, a['archive_download_url'], filename)
        if not ok:
            continue
        zipfiles.append((a['name'], datename, filename))
    return zipfiles


def extract_logs(f):
    """
    extract_logs unzips the archive into a temporary directory

    This directory is deleted when the object goes out of scope
    """
    tdir = tempfile.TemporaryDirectory(prefix='kstest-log-', dir='/var/tmp/')
    with zipfile.ZipFile(f) as zf:
        zf.extractall(tdir.name)
    return tdir


def generate_test_list(tdir):
    """
    Build kstest-list if it is missing

    This is a list of the expected tests, and it is used to help detect when
    tests are totally missing from the results.

    The original kstest.log has a line:
    Running tests: ./bridge-httpks.sh ./packages-instlangs-1.sh  ...

    Parse this and create kstest-list, one test name per line for later use.
    """
    if os.path.exists(os.path.join(tdir.name, 'kstest-list')):
        return
    kstest_log = os.path.join(tdir.name, 'kstest.log')
    with open(kstest_log) as f:
        for line in f.readlines():
            if not line.startswith('Running tests: '):
                continue
            tests = [os.path.basename(os.path.splitext(s)[0]) for s in line
                [15:].split()]
            with open(os.path.join(tdir.name, 'kstest-list'), 'wt') as klf:
                for t in tests:
                    print(t, file=klf)
            break


def rebuild_logs(tdir):
    """
    rebuild_logs recreates kstest.log with timestamps

    It does this by appending all the individual kstest.log files, which do contain
    timestamps, into a new kstest.log
    """
    kstest_log = os.path.join(tdir.name, 'kstest.log')
    os.unlink(kstest_log)
    with open(kstest_log, 'w') as ksf:
        for log in glob(os.path.join(tdir.name, '*', 'kstest.log')):
            with open(log) as f:
                data = f.read(1024 ** 2)
                while data:
                    ksf.write(data)
                    data = f.read(1024 ** 2)


def check_tests(tests):
    """
    Check the tests for success, failing, missing, or flakes -- success after first failing

    This returns a tuple of:
    - list of the names of successful tests
    - list of the names of the missing tests
    - dict of failed tests, each entry being a list of the failure details dict
    - dict of flaky tests, each entry being a list of the flaky details dict
    """
    success = []
    missing = []
    failed = {}
    flakes = {}
    for t in tests:
        name = t['name']
        if t['success']:
            if name in success:
                raise RuntimeError(
                    f'{name} already passed, should only be 1 per test')
            success.append(name)
            if name in failed:
                flakes[name] = failed[name]
                del failed[name]
        elif 'MISSING' in t['result']:
            missing.append(name)
        elif name in success:
            if name in flakes:
                flakes[name].append(t)
            else:
                flakes[name] = [t]
        elif name in failed:
            failed[name].append(t)
        else:
            failed[name] = [t]
    return sorted(success), sorted(missing), failed, flakes


def print_test_details(scenario, days, test_name, buf):
    """Add the details of the named tests (failed-tests or flaky-tests) to the buffer
    """
    for d in days:
        if scenario not in days[d]:
            continue
        for n in days[d][scenario][test_name]:
            print(f'\n{n}:', file=buf)
            for test in days[d][scenario][test_name][n]:
                if 'start_time' not in test:
                    start_time = ''
                else:
                    start_time = datetime.fromtimestamp(test['start_time']
                        ).strftime('%m/%d/%Y %H:%M:%S')
                if 'elapsed_time' not in test:
                    elapsed_time = 0
                else:
                    elapsed_time = test['elapsed_time']
                msg = test['result'].rsplit('FAILED:')[-1]
                print(f'        {start_time} ({elapsed_time}s): {msg}',
                    file=buf)


def kstest_logdir(tmpdir, test):
    """
    Return the directory of the logs for the test

    The logfile path should start with /var/tmp/kstest-*
    but this finds the kstest-* no matter what the leading
    path elements are.
    """
    logfile = test['logfile']
    for e in logfile.split(os.path.sep):
        if e.startswith('kstest-'):
            return os.path.join(tmpdir, e)
    raise RuntimeError(f'No kstest-* directory found in {logfile}')


def archive_test_logs(days, archive_path, all_logs):
    """
    Store the logs from the failed/flaky tests for archiving
    and later examination. Paths follow the pattern:

    MM-DD-YYYY / scenario / failed|flakes / TESTNAME / [1...] /

    and each directory contains the logs for that run of the test.
    Currently this is the kstest.log and virt-install.log files
    """
    for day in days.keys():
        daydir = datetime.strptime(day, '%Y%m%d').strftime('%m-%d-%Y')
        for scenario in days[day].keys():
            datename = scenario + '-' + datetime.strptime(day, '%Y%m%d'
                ).strftime('%Y-%m-%d')
            if datename not in all_logs:
                raise RuntimeError(f'Missing all_log entry for {datename}')
            if not os.path.exists(all_logs[datename].name):
                raise RuntimeError(f'Missing log directory for {datename}')
            tmpdir = all_logs[datename].name
            failed = days[day][scenario]['failed-tests']
            flakes = days[day][scenario]['flaky-tests']
            scenario_archive = os.path.join(archive_path, daydir, scenario)
            os.makedirs(os.path.join(scenario_archive, 'failed'))
            os.makedirs(os.path.join(scenario_archive, 'flakes'))
            for name in failed:
                i = 1
                for t in sorted(failed[name], key=lambda x: x['start_time']):
                    try:
                        logdir = kstest_logdir(tmpdir, t)
                        if not os.path.exists(logdir):
                            raise RuntimeError(f'Missing logdir - {logdir}')
                    except RuntimeError:
                        continue
                    dst = os.path.join(scenario_archive, 'failed', name, str(i)
                        )
                    shutil.copytree(logdir, dst)
                    i += 1
            for name in flakes:
                i = 1
                for t in sorted(flakes[name], key=lambda x: x['start_time']):
                    try:
                        logdir = kstest_logdir(tmpdir, t)
                        if not logdir or not os.path.exists(logdir):
                            raise RuntimeError(f'Missing logdir - {logdir}')
                    except RuntimeError:
                        continue
                    dst = os.path.join(scenario_archive, 'flakes', name, str(i)
                        )
                    shutil.copytree(logdir, dst)
                    i += 1


def process_logs(logs):
    """
    Process the logfiles into a data structure

    Returns a dictionary with each log's data that looks similar to this:

    {"20220401": {"logs-daily-iso": [{test data dict}, ...], ...}}, ...}

    So every day has an entry for the scenarios that were run on that day.
    Note that sometimes a scenario can be missing if there was a problem running it at all.
    """
    all_data = {}
    for log in logs:
        with open(log) as f:
            data = json.load(f)
            scenario = data[0].get('scenario', None)
            if scenario is None:
                continue
            day = datetime.strptime(log[1 + len(scenario):-5], '%Y-%m-%d'
                ).strftime('%Y%m%d')
            if day not in all_data:
                all_data[day] = {}
            all_data[day][scenario] = data
    return all_data


def summary(args, json_logs, all_logs):
    """
    summary generates a summary of all the tests run in the selected date range

    It returns a string with the summary text
    """
    all_data = process_logs(json_logs)
    if args.debug:
        print(json.dumps(all_data))
    buf = io.StringIO()
    start = args.start.strftime('%m/%d/%Y %H:%M:%S')
    end = args.end.strftime('%m/%d/%Y %H:%M:%S')
    print(f'Test Summary Report: {start} -> {end}\n', file=buf)
    all_days = {}
    days = {}
    top_failed = {}
    top_flakes = {}
    for day in sorted(all_data.keys()):
        days[day] = {}
        for scenario in sorted(all_data[day].keys()):
            if scenario not in all_days:
                all_days[scenario] = {'success': 0, 'missing': 0, 'failed':
                    0, 'flakes': 0}
            success, missing, failed, flakes = check_tests(all_data[day][
                scenario])
            days[day][scenario] = {'success': len(success), 'missing': len(
                missing), 'failed': len(failed), 'flakes': len(flakes),
                'missing-tests': missing, 'failed-tests': failed,
                'flaky-tests': flakes}
            all_days[scenario]['success'] += len(success)
            all_days[scenario]['missing'] += len(missing)
            all_days[scenario]['failed'] += len(failed)
            all_days[scenario]['flakes'] += len(flakes)
            for n in failed:
                top_failed[n] = top_failed.get(n, 0) + 1
            for n in flakes:
                top_flakes[n] = top_flakes.get(n, 0) + 1
    print('Weekly summary', file=buf)
    print('==============', file=buf)
    for scenario in sorted(all_days.keys()):
        success = all_days[scenario]['success']
        missing = all_days[scenario]['missing']
        failed = all_days[scenario]['failed']
        flakes = all_days[scenario]['flakes']
        print(
            f'{scenario}: Ran {success + failed + missing} tests. {success} passed, {failed} failed, {missing} missing, {flakes} flakes.'
            , file=buf)
    print('\n', file=buf)
    print('Top 5 failed tests for the week', file=buf)
    for n in sorted((n for n in top_failed), key=lambda x: top_failed[x],
        reverse=True)[:5]:
        print(f'    {n} - {top_failed[n]}', file=buf)
    print('\n', file=buf)
    print('Top 5 flaky tests for the week', file=buf)
    for n in sorted((n for n in top_flakes), key=lambda x: top_flakes[x],
        reverse=True)[:5]:
        print(f'    {n} - {top_flakes[n]}', file=buf)
    print('\n', file=buf)
    for day in sorted(days.keys()):
        print(datetime.strptime(day, '%Y%m%d').strftime('%m/%d/%Y'), file=buf)
        for scenario in sorted(days[day].keys()):
            s = days[day][scenario]
            success = s['success']
            missing = s['missing']
            failed = s['failed']
            total = success + failed + missing
            flakes = s['flakes']
            print(
                f'    {scenario} (Ran {total}, {success} passed, {failed} failed, {missing} missing. {flakes} flakes) :'
                , file=buf)
            if s['missing-tests']:
                print('        Missing:', file=buf)
                for n in s['missing-tests']:
                    print(f'            {n}', file=buf)
            if s['failed-tests']:
                print('        Failed:', file=buf)
                for n in sorted(s['failed-tests'].keys()):
                    print(f'            {n}', file=buf)
            if s['flaky-tests']:
                print('        Flakes:', file=buf)
                for n in sorted(s['flaky-tests'].keys()):
                    print(f'            {n}', file=buf)
        print('\n', file=buf)
    for scenario in sorted(all_days.keys()):
        success = all_days[scenario]['success']
        failed = all_days[scenario]['failed']
        flakes = all_days[scenario]['flakes']
        msg = (
            f'{scenario}: Ran {success + failed} tests. {success} passed, {failed} failed, {flakes} flakes.'
            )
        print('=' * len(msg), file=buf)
        print(msg, file=buf)
        print('=' * len(msg), file=buf)
        if args.flake_details:
            print('Failed test details', file=buf)
            print('-------------------', file=buf)
        print_test_details(scenario, days, 'failed-tests', buf)
        if args.flake_details:
            print('\nFlake test details', file=buf)
            print('-------------------', file=buf)
            print_test_details(scenario, days, 'flaky-tests', buf)
        print('\n', file=buf)
    try:
        if args.archive_logs:
            archive_test_logs(days, args.archive_logs, all_logs)
    except RuntimeError as e:
        print(f'\nERROR: Problem archiving test logs - {e}', file=buf)
    return buf.getvalue()


def test_summary(args, path):
    """
    Test the weekly summary on a single directory

    Instead of pulling artifacts from github use a single directory with
    kstest.log.json and the log directories.
    """
    log = os.path.join(path, 'kstest.log.json')
    with open(log) as f:
        data = json.load(f)
        scenario = data[0].get('scenario', None)
    if not scenario:
        raise RuntimeError('No scenario found in %s' % log)
    datename = f'{scenario}-1990-01-01'
    shutil.copy(log, datename + '.json')
    datenames = [datename]
    all_logs = {datename: path}
    report = summary(args, (d + '.json' for d in datenames), all_logs)
    if args.output:
        with open(args.output, 'w') as f:
            f.write(report)
    else:
        print(report)


def main(args, token):
    artifacts = get_artifacts(token, args.artifact_names, args.start, args.end)
    if args.verbose:
        print(json.dumps(artifacts))
    zipfiles = download_artifacts(token, artifacts)
    if args.debug:
        print(f'zipfiles = {zipfiles}')
    datenames = []
    all_logs = {}
    for name, datename, f in zipfiles:
        if args.rebuild or not os.path.exists(datename + '.json'):
            try:
                all_logs[datename] = extract_logs(f)
            except zipfile.BadZipFile:
                os.unlink(f)
                continue
            if args.rebuild:
                generate_test_list(all_logs[datename])
                rebuild_logs(all_logs[datename])
            cmd = ['log2json', '--scenario', name, '--output', datename +
                '.json', os.path.join(all_logs[datename].name, 'kstest.log')]
            if args.debug:
                print(cmd)
            check_call(cmd)
        if os.path.exists(datename + '.json'):
            datenames.append(datename)
    report = summary(args, (d + '.json' for d in datenames), all_logs)
    if args.output:
        with open(args.output, 'w') as f:
            f.write(report)
    else:
        print(report)


<mask token>

import argparse
from datetime import datetime, timedelta
from glob import glob
import io
import json
import os
import shutil
import sys
from subprocess import check_call
import tempfile
import time
from urllib.request import Request, urlopen
import zipfile
import pycurl
URL = (
    'https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts?per_page=100'
    )
ARTIFACT_NAMES = ['logs-daily-iso', 'logs-rhel9', 'logs-rhel8']
try:
    import signal
    from signal import SIGPIPE, SIG_IGN
except ImportError:
    pass
else:
    signal.signal(SIGPIPE, SIG_IGN)


def get_artifacts(token, artifact_names, start, end):
    """
    get_artifacts retrieves a list of artifacts from the selected date range
    that are listed in artifact_names

    It returns a list of artifact dicts like this:

    {
      "id": 187278866,
      "node_id": "MDg6QXJ0aWZhY3QxODcyNzg4NjY=",
      "name": "logs-daily-iso",
      "size_in_bytes": 1047408950,
      "url": "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts/187278866",
      "archive_download_url": "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts/187278866/zip",
      "expired": false,
      "created_at": "2022-03-17T03:30:12Z",
      "updated_at": "2022-03-17T03:30:13Z",
      "expires_at": "2022-03-25T03:29:06Z"
    },
    """
    artifacts = []
    page = 1
    retry_limit = 3
    while True:
        req = Request(URL + f'&page={page}')
        req.add_header('Accept', 'application/vnd.github.v3+json')
        req.add_header('Authorization', f'token {token}')
        with urlopen(req) as r:
            if r.status == 403:
                try:
                    reset = int(r.headers.get('X-RateLimit-Reset'))
                except:
                    raise RuntimeError(
                        'Hit GitHub rate limit. Reset header missing.')
                if retry_limit == 0 or time.time(
                    ) > reset or reset - time.time() > 90:
                    raise RuntimeError(
                        'Hit GitHub rate limit. Reset is at %s' % time.
                        ctime(reset))
                time.sleep(10 + (reset - time.time()))
                retry_limit = retry_limit - 1
                continue
            if r.status != 200:
                raise RuntimeError('Error (%d) with API request: %s' % (r.
                    status, str(r)))
            data = json.load(r)
        for a in data['artifacts']:
            if a['name'] not in artifact_names:
                continue
            updated_at = datetime.fromisoformat(a['updated_at'][:-1])
            if start <= updated_at <= end:
                artifacts.append(a)
        if len(data['artifacts']) < 100:
            break
        page = page + 1
        time.sleep(2)
    return sorted(artifacts, key=lambda x: x['updated_at'])


def run_curl(token, url, filename):
    """
    run_curl downloads an artifact file

    It returns True if the response was a 200
    If there was an exception is returns False and the error, as well as
    printing it to stderr
    """
    with open(filename, 'wb') as f:
        c = pycurl.Curl()
        headers = ['Accept: application/vnd.github.v3+json',
            f'Authorization: token {token}']
        options = {pycurl.FOLLOWLOCATION: 1, pycurl.MAXREDIRS: 5, pycurl.
            CONNECTTIMEOUT: 30, pycurl.TIMEOUT: 300, pycurl.NOSIGNAL: 1,
            pycurl.URL: url, pycurl.HTTPHEADER: headers, pycurl.WRITEDATA: f}
        for k, v in options.items():
            c.setopt(k, v)
        try:
            c.perform()
            status = c.getinfo(pycurl.HTTP_CODE)
            ok = status == 200, None
        except Exception as e:
            print(f'ERROR: {e}', file=sys.stderr)
            sys.stderr.flush()
            ok = False, e
        c.close()
    return ok


def download_artifacts(token, artifacts):
    """
    download_artifacts downloads the artifacts as uniquely named files

    If there is a problem downloading an artifact it is skipped and not
    added to the returned list.

    It returns a list of tuples containing the artifact name, the
    artifact name with the updated date appended (eg. logs-rhel8-2022-04-01),
    and the filename of the zipfile.
    """
    zipfiles = []
    for a in artifacts:
        updated_at = datetime.fromisoformat(a['updated_at'][:-1])
        datename = a['name'] + updated_at.strftime('-%Y-%m-%d')
        filename = datename + '.zip'
        if os.path.exists(filename):
            zipfiles.append((a['name'], datename, filename))
            print(f'{filename} skipped, already downloaded')
            continue
        print(f'Fetching {filename}')
        ok = run_curl(token, a['archive_download_url'], filename)
        if not ok:
            continue
        zipfiles.append((a['name'], datename, filename))
    return zipfiles


def extract_logs(f):
    """
    extract_logs unzips the archive into a temporary directory

    This directory is deleted when the object goes out of scope
    """
    tdir = tempfile.TemporaryDirectory(prefix='kstest-log-', dir='/var/tmp/')
    with zipfile.ZipFile(f) as zf:
        zf.extractall(tdir.name)
    return tdir


def generate_test_list(tdir):
    """
    Build kstest-list if it is missing

    This is a list of the expected tests, and it is used to help detect when
    tests are totally missing from the results.

    The original kstest.log has a line:
    Running tests: ./bridge-httpks.sh ./packages-instlangs-1.sh  ...

    Parse this and create kstest-list, one test name per line for later use.
    """
    if os.path.exists(os.path.join(tdir.name, 'kstest-list')):
        return
    kstest_log = os.path.join(tdir.name, 'kstest.log')
    with open(kstest_log) as f:
        for line in f.readlines():
            if not line.startswith('Running tests: '):
                continue
            tests = [os.path.basename(os.path.splitext(s)[0]) for s in line
                [15:].split()]
            with open(os.path.join(tdir.name, 'kstest-list'), 'wt') as klf:
                for t in tests:
                    print(t, file=klf)
            break


def rebuild_logs(tdir):
    """
    rebuild_logs recreates kstest.log with timestamps

    It does this by appending all the individual kstest.log files, which do contain
    timestamps, into a new kstest.log
    """
    kstest_log = os.path.join(tdir.name, 'kstest.log')
    os.unlink(kstest_log)
    with open(kstest_log, 'w') as ksf:
        for log in glob(os.path.join(tdir.name, '*', 'kstest.log')):
            with open(log) as f:
                data = f.read(1024 ** 2)
                while data:
                    ksf.write(data)
                    data = f.read(1024 ** 2)


def check_tests(tests):
    """
    Check the tests for success, failing, missing, or flakes -- success after first failing

    This returns a tuple of:
    - list of the names of successful tests
    - list of the names of the missing tests
    - dict of failed tests, each entry being a list of the failure details dict
    - dict of flaky tests, each entry being a list of the flaky details dict
    """
    success = []
    missing = []
    failed = {}
    flakes = {}
    for t in tests:
        name = t['name']
        if t['success']:
            if name in success:
                raise RuntimeError(
                    f'{name} already passed, should only be 1 per test')
            success.append(name)
            if name in failed:
                flakes[name] = failed[name]
                del failed[name]
        elif 'MISSING' in t['result']:
            missing.append(name)
        elif name in success:
            if name in flakes:
                flakes[name].append(t)
            else:
                flakes[name] = [t]
        elif name in failed:
            failed[name].append(t)
        else:
            failed[name] = [t]
    return sorted(success), sorted(missing), failed, flakes


def print_test_details(scenario, days, test_name, buf):
    """Add the details of the named tests (failed-tests or flaky-tests) to the buffer
    """
    for d in days:
        if scenario not in days[d]:
            continue
        for n in days[d][scenario][test_name]:
            print(f'\n{n}:', file=buf)
            for test in days[d][scenario][test_name][n]:
                if 'start_time' not in test:
                    start_time = ''
                else:
                    start_time = datetime.fromtimestamp(test['start_time']
                        ).strftime('%m/%d/%Y %H:%M:%S')
                if 'elapsed_time' not in test:
                    elapsed_time = 0
                else:
                    elapsed_time = test['elapsed_time']
                msg = test['result'].rsplit('FAILED:')[-1]
                print(f'        {start_time} ({elapsed_time}s): {msg}',
                    file=buf)


def kstest_logdir(tmpdir, test):
    """
    Return the directory of the logs for the test

    The logfile path should start with /var/tmp/kstest-*
    but this finds the kstest-* no matter what the leading
    path elements are.
    """
    logfile = test['logfile']
    for e in logfile.split(os.path.sep):
        if e.startswith('kstest-'):
            return os.path.join(tmpdir, e)
    raise RuntimeError(f'No kstest-* directory found in {logfile}')


def archive_test_logs(days, archive_path, all_logs):
    """
    Store the logs from the failed/flaky tests for archiving
    and later examination. Paths follow the pattern:

    MM-DD-YYYY / scenario / failed|flakes / TESTNAME / [1...] /

    and each directory contains the logs for that run of the test.
    Currently this is the kstest.log and virt-install.log files
    """
    for day in days.keys():
        daydir = datetime.strptime(day, '%Y%m%d').strftime('%m-%d-%Y')
        for scenario in days[day].keys():
            datename = scenario + '-' + datetime.strptime(day, '%Y%m%d'
                ).strftime('%Y-%m-%d')
            if datename not in all_logs:
                raise RuntimeError(f'Missing all_log entry for {datename}')
            if not os.path.exists(all_logs[datename].name):
                raise RuntimeError(f'Missing log directory for {datename}')
            tmpdir = all_logs[datename].name
            failed = days[day][scenario]['failed-tests']
            flakes = days[day][scenario]['flaky-tests']
            scenario_archive = os.path.join(archive_path, daydir, scenario)
            os.makedirs(os.path.join(scenario_archive, 'failed'))
            os.makedirs(os.path.join(scenario_archive, 'flakes'))
            for name in failed:
                i = 1
                for t in sorted(failed[name], key=lambda x: x['start_time']):
                    try:
                        logdir = kstest_logdir(tmpdir, t)
                        if not os.path.exists(logdir):
                            raise RuntimeError(f'Missing logdir - {logdir}')
                    except RuntimeError:
                        continue
                    dst = os.path.join(scenario_archive, 'failed', name, str(i)
                        )
                    shutil.copytree(logdir, dst)
                    i += 1
            for name in flakes:
                i = 1
                for t in sorted(flakes[name], key=lambda x: x['start_time']):
                    try:
                        logdir = kstest_logdir(tmpdir, t)
                        if not logdir or not os.path.exists(logdir):
                            raise RuntimeError(f'Missing logdir - {logdir}')
                    except RuntimeError:
                        continue
                    dst = os.path.join(scenario_archive, 'flakes', name, str(i)
                        )
                    shutil.copytree(logdir, dst)
                    i += 1


def process_logs(logs):
    """
    Process the logfiles into a data structure

    Returns a dictionary with each log's data that looks similar to this:

    {"20220401": {"logs-daily-iso": [{test data dict}, ...], ...}}, ...}

    So every day has an entry for the scenarios that were run on that day.
    Note that sometimes a scenario can be missing if there was a problem running it at all.
    """
    all_data = {}
    for log in logs:
        with open(log) as f:
            data = json.load(f)
            scenario = data[0].get('scenario', None)
            if scenario is None:
                continue
            day = datetime.strptime(log[1 + len(scenario):-5], '%Y-%m-%d'
                ).strftime('%Y%m%d')
            if day not in all_data:
                all_data[day] = {}
            all_data[day][scenario] = data
    return all_data


def summary(args, json_logs, all_logs):
    """
    summary generates a summary of all the tests run in the selected date range

    It returns a string with the summary text
    """
    all_data = process_logs(json_logs)
    if args.debug:
        print(json.dumps(all_data))
    buf = io.StringIO()
    start = args.start.strftime('%m/%d/%Y %H:%M:%S')
    end = args.end.strftime('%m/%d/%Y %H:%M:%S')
    print(f'Test Summary Report: {start} -> {end}\n', file=buf)
    all_days = {}
    days = {}
    top_failed = {}
    top_flakes = {}
    for day in sorted(all_data.keys()):
        days[day] = {}
        for scenario in sorted(all_data[day].keys()):
            if scenario not in all_days:
                all_days[scenario] = {'success': 0, 'missing': 0, 'failed':
                    0, 'flakes': 0}
            success, missing, failed, flakes = check_tests(all_data[day][
                scenario])
            days[day][scenario] = {'success': len(success), 'missing': len(
                missing), 'failed': len(failed), 'flakes': len(flakes),
                'missing-tests': missing, 'failed-tests': failed,
                'flaky-tests': flakes}
            all_days[scenario]['success'] += len(success)
            all_days[scenario]['missing'] += len(missing)
            all_days[scenario]['failed'] += len(failed)
            all_days[scenario]['flakes'] += len(flakes)
            for n in failed:
                top_failed[n] = top_failed.get(n, 0) + 1
            for n in flakes:
                top_flakes[n] = top_flakes.get(n, 0) + 1
    print('Weekly summary', file=buf)
    print('==============', file=buf)
    for scenario in sorted(all_days.keys()):
        success = all_days[scenario]['success']
        missing = all_days[scenario]['missing']
        failed = all_days[scenario]['failed']
        flakes = all_days[scenario]['flakes']
        print(
            f'{scenario}: Ran {success + failed + missing} tests. {success} passed, {failed} failed, {missing} missing, {flakes} flakes.'
            , file=buf)
    print('\n', file=buf)
    print('Top 5 failed tests for the week', file=buf)
    for n in sorted((n for n in top_failed), key=lambda x: top_failed[x],
        reverse=True)[:5]:
        print(f'    {n} - {top_failed[n]}', file=buf)
    print('\n', file=buf)
    print('Top 5 flaky tests for the week', file=buf)
    for n in sorted((n for n in top_flakes), key=lambda x: top_flakes[x],
        reverse=True)[:5]:
        print(f'    {n} - {top_flakes[n]}', file=buf)
    print('\n', file=buf)
    for day in sorted(days.keys()):
        print(datetime.strptime(day, '%Y%m%d').strftime('%m/%d/%Y'), file=buf)
        for scenario in sorted(days[day].keys()):
            s = days[day][scenario]
            success = s['success']
            missing = s['missing']
            failed = s['failed']
            total = success + failed + missing
            flakes = s['flakes']
            print(
                f'    {scenario} (Ran {total}, {success} passed, {failed} failed, {missing} missing. {flakes} flakes) :'
                , file=buf)
            if s['missing-tests']:
                print('        Missing:', file=buf)
                for n in s['missing-tests']:
                    print(f'            {n}', file=buf)
            if s['failed-tests']:
                print('        Failed:', file=buf)
                for n in sorted(s['failed-tests'].keys()):
                    print(f'            {n}', file=buf)
            if s['flaky-tests']:
                print('        Flakes:', file=buf)
                for n in sorted(s['flaky-tests'].keys()):
                    print(f'            {n}', file=buf)
        print('\n', file=buf)
    for scenario in sorted(all_days.keys()):
        success = all_days[scenario]['success']
        failed = all_days[scenario]['failed']
        flakes = all_days[scenario]['flakes']
        msg = (
            f'{scenario}: Ran {success + failed} tests. {success} passed, {failed} failed, {flakes} flakes.'
            )
        print('=' * len(msg), file=buf)
        print(msg, file=buf)
        print('=' * len(msg), file=buf)
        if args.flake_details:
            print('Failed test details', file=buf)
            print('-------------------', file=buf)
        print_test_details(scenario, days, 'failed-tests', buf)
        if args.flake_details:
            print('\nFlake test details', file=buf)
            print('-------------------', file=buf)
            print_test_details(scenario, days, 'flaky-tests', buf)
        print('\n', file=buf)
    try:
        if args.archive_logs:
            archive_test_logs(days, args.archive_logs, all_logs)
    except RuntimeError as e:
        print(f'\nERROR: Problem archiving test logs - {e}', file=buf)
    return buf.getvalue()


def test_summary(args, path):
    """
    Test the weekly summary on a single directory

    Instead of pulling artifacts from github use a single directory with
    kstest.log.json and the log directories.
    """
    log = os.path.join(path, 'kstest.log.json')
    with open(log) as f:
        data = json.load(f)
        scenario = data[0].get('scenario', None)
    if not scenario:
        raise RuntimeError('No scenario found in %s' % log)
    datename = f'{scenario}-1990-01-01'
    shutil.copy(log, datename + '.json')
    datenames = [datename]
    all_logs = {datename: path}
    report = summary(args, (d + '.json' for d in datenames), all_logs)
    if args.output:
        with open(args.output, 'w') as f:
            f.write(report)
    else:
        print(report)


def main(args, token):
    artifacts = get_artifacts(token, args.artifact_names, args.start, args.end)
    if args.verbose:
        print(json.dumps(artifacts))
    zipfiles = download_artifacts(token, artifacts)
    if args.debug:
        print(f'zipfiles = {zipfiles}')
    datenames = []
    all_logs = {}
    for name, datename, f in zipfiles:
        if args.rebuild or not os.path.exists(datename + '.json'):
            try:
                all_logs[datename] = extract_logs(f)
            except zipfile.BadZipFile:
                os.unlink(f)
                continue
            if args.rebuild:
                generate_test_list(all_logs[datename])
                rebuild_logs(all_logs[datename])
            cmd = ['log2json', '--scenario', name, '--output', datename +
                '.json', os.path.join(all_logs[datename].name, 'kstest.log')]
            if args.debug:
                print(cmd)
            check_call(cmd)
        if os.path.exists(datename + '.json'):
            datenames.append(datename)
    report = summary(args, (d + '.json' for d in datenames), all_logs)
    if args.output:
        with open(args.output, 'w') as f:
            f.write(report)
    else:
        print(report)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=
        'Generate a weekly summary of test results')
    parser.add_argument('--artifacts', type=lambda x: x.split(','), default
        =ARTIFACT_NAMES, dest='artifact_names', help=
        'Comma separated list of artifact names to summarize')
    parser.add_argument('--start', type=datetime.fromisoformat, default=
        datetime.now() - timedelta(days=7), help=
        'Start time. eg. 2022-03-03T03:46:11 (default is -7 days)')
    parser.add_argument('--end', type=datetime.fromisoformat, default=
        datetime.now(), help=
        'end time. eg. 2022-03-03T03:46:11 (default is now)')
    parser.add_argument('--rebuild', default=False, action='store_true',
        help='Rebuild logs with timestamps')
    parser.add_argument('--flake-details', default=False, action=
        'store_true', help='Include details about flaky tests in summary')
    parser.add_argument('--archive-logs', help=
        'Optionally collect the failed/flake logs in an archive directory tree'
        )
    parser.add_argument('--output', help=
        'Path and filename to write summary report to')
    parser.add_argument('--test', help=argparse.SUPPRESS)
    parser.add_argument('--debug', default=False, action='store_true')
    parser.add_argument('--verbose', default=False, action='store_true')
    args = parser.parse_args()
    if 'GITHUB_TOKEN' not in os.environ:
        print(
            'Set GITHUB_TOKEN environmental variable to github token with access to the artifact api.'
            )
        sys.exit(1)
    if args.debug:
        print(f'args = {args}')
    if args.test:
        test_summary(args, args.test)
    else:
        main(args, os.environ['GITHUB_TOKEN'])

#!/usr/bin/python3
#
# Copyright (C) 2022  Red Hat, Inc.
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# the GNU General Public License v.2, or (at your option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.  You should have received a copy of the
# GNU General Public License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
# source code or documentation are not subject to the GNU General Public
# License and may only be used or replicated with the express permission of
# Red Hat, Inc.
#
# This script generates a weekly report of kickstart-test runs.
# It downloads the daily json summary reports and combines them into a single text
# report.
#
# It requires a github token in GITHUB_TOKEN environmental variable. This token
# needs to have access to the kickstart-tests artifacts via the github API.

import argparse
from datetime import datetime, timedelta
from glob import glob
import io
import json
import os
import shutil
import sys
from subprocess import check_call
import tempfile
import time
from urllib.request import Request, urlopen
import zipfile

import pycurl

URL = "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts?per_page=100"

# Defaults list of artifacts
# For now use the full log artifacts, eventually this will be the json summaries
ARTIFACT_NAMES = ["logs-daily-iso", "logs-rhel9", "logs-rhel8"]

# We should ignore SIGPIPE when using pycurl.NOSIGNAL - see
# the libcurl tutorial for more info.
try:
    import signal
    from signal import SIGPIPE, SIG_IGN
except ImportError:
    pass
else:
    signal.signal(SIGPIPE, SIG_IGN)


def get_artifacts(token, artifact_names, start, end):
    """
    get_artifacts retrieves a list of artifacts from the selected date range
    that are listed in artifact_names

    It returns a list of artifact dicts like this:

    {
      "id": 187278866,
      "node_id": "MDg6QXJ0aWZhY3QxODcyNzg4NjY=",
      "name": "logs-daily-iso",
      "size_in_bytes": 1047408950,
      "url": "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts/187278866",
      "archive_download_url": "https://api.github.com/repos/rhinstaller/kickstart-tests/actions/artifacts/187278866/zip",
      "expired": false,
      "created_at": "2022-03-17T03:30:12Z",
      "updated_at": "2022-03-17T03:30:13Z",
      "expires_at": "2022-03-25T03:29:06Z"
    },
    """

    artifacts = []
    page = 1
    retry_limit = 3
    while True:
        req = Request(URL + f"&page={page}")
        req.add_header("Accept", "application/vnd.github.v3+json")
        req.add_header("Authorization", f"token {token}")
        with urlopen(req) as r:
            # Handle hitting the GitHub rate limit
            # If the reset time is < 90s in the future, wait for it (trying 3 times)
            # Otherwise raise an error
            if r.status == 403:
                try:
                    reset = int(r.headers.get("X-RateLimit-Reset"))
                except:
                    raise RuntimeError("Hit GitHub rate limit. Reset header missing.")
                if retry_limit == 0 or time.time() > reset or reset - time.time() > 90:
                    raise RuntimeError("Hit GitHub rate limit. Reset is at %s" % time.ctime(reset))

                # Try waiting until after the reset time
                time.sleep(10 + (reset - time.time()))
                retry_limit = retry_limit - 1
                continue

            if r.status != 200:
                raise RuntimeError("Error (%d) with API request: %s" % (r.status, str(r)))

            data = json.load(r)

        # Only include the artifacts within the date range and names
        for a in data["artifacts"]:
            if a["name"] not in artifact_names:
                continue
            updated_at = datetime.fromisoformat(a["updated_at"][:-1])
            if start <= updated_at <= end:
                artifacts.append(a)

        if len(data["artifacts"]) < 100:
            break

        # There are more results, get the next page
        page = page + 1

        # Avoid hitting per-second rate limits
        time.sleep(2)

    return sorted(artifacts, key=lambda x: x["updated_at"])


def run_curl(token, url, filename):
    """
    run_curl downloads an artifact file

    It returns True if the response was a 200
    If there was an exception is returns False and the error, as well as
    printing it to stderr
    """
    with open(filename, "wb") as f:
        c = pycurl.Curl()
        headers = [
           "Accept: application/vnd.github.v3+json",
           f"Authorization: token {token}",
        ]

        options = {
            pycurl.FOLLOWLOCATION:  1,
            pycurl.MAXREDIRS:       5,
            pycurl.CONNECTTIMEOUT:  30,
            pycurl.TIMEOUT:         300,
            pycurl.NOSIGNAL:        1,
            pycurl.URL:             url,
            pycurl.HTTPHEADER:      headers,
            pycurl.WRITEDATA:       f
        }
        for k, v in options.items():
            c.setopt(k, v)

        try:
            c.perform()
            status = c.getinfo(pycurl.HTTP_CODE)
            ok = (status == 200, None)
        except Exception as e:
            print(f"ERROR: {e}", file=sys.stderr)
            sys.stderr.flush()
            ok = (False, e)
        c.close()

    return ok


def download_artifacts(token, artifacts):
    """
    download_artifacts downloads the artifacts as uniquely named files

    If there is a problem downloading an artifact it is skipped and not
    added to the returned list.

    It returns a list of tuples containing the artifact name, the
    artifact name with the updated date appended (eg. logs-rhel8-2022-04-01),
    and the filename of the zipfile.
    """
    zipfiles = []
    for a in artifacts:
        updated_at = datetime.fromisoformat(a["updated_at"][:-1])
        datename = a["name"]+updated_at.strftime("-%Y-%m-%d")
        filename = datename + ".zip"
        if os.path.exists(filename):
            zipfiles.append((a["name"], datename, filename))
            print(f"{filename} skipped, already downloaded")
            continue

        print(f"Fetching {filename}")
        ok = run_curl(token, a["archive_download_url"], filename)
        if not ok:
            continue

        zipfiles.append((a["name"], datename, filename))

    return zipfiles


def extract_logs(f):
    """
    extract_logs unzips the archive into a temporary directory

    This directory is deleted when the object goes out of scope
    """
    tdir = tempfile.TemporaryDirectory(prefix="kstest-log-", dir="/var/tmp/")
    with zipfile.ZipFile(f) as zf:
        zf.extractall(tdir.name)

    # Return the object so that the temporary directory isn't deleted yet
    return tdir


def generate_test_list(tdir):
    """
    Build kstest-list if it is missing

    This is a list of the expected tests, and it is used to help detect when
    tests are totally missing from the results.

    The original kstest.log has a line:
    Running tests: ./bridge-httpks.sh ./packages-instlangs-1.sh  ...

    Parse this and create kstest-list, one test name per line for later use.
    """

    # Skip this if it already exists
    if os.path.exists(os.path.join(tdir.name, "kstest-list")):
        return

    kstest_log = os.path.join(tdir.name, "kstest.log")
    with open(kstest_log) as f:
        for line in f.readlines():
            if not line.startswith("Running tests: "):
                continue

            tests = [os.path.basename(os.path.splitext(s)[0]) for s in line[15:].split()]
            with open(os.path.join(tdir.name, "kstest-list"), "wt") as klf:
                for t in tests:
                    print(t, file=klf)
            break


def rebuild_logs(tdir):
    """
    rebuild_logs recreates kstest.log with timestamps

    It does this by appending all the individual kstest.log files, which do contain
    timestamps, into a new kstest.log
    """
    # Remove the old log with no timestamps
    kstest_log = os.path.join(tdir.name, "kstest.log")
    os.unlink(kstest_log)

    # Find all the test's kstest.log files and append them to kstest.log
    with open(kstest_log, "w") as ksf:
        for log in glob(os.path.join(tdir.name, "*", "kstest.log")):
            with open(log) as f:
                data = f.read(1024**2)
                while data:
                    ksf.write(data)
                    data = f.read(1024**2)


def check_tests(tests):
    """
    Check the tests for success, failing, missing, or flakes -- success after first failing

    This returns a tuple of:
    - list of the names of successful tests
    - list of the names of the missing tests
    - dict of failed tests, each entry being a list of the failure details dict
    - dict of flaky tests, each entry being a list of the flaky details dict
    """
    success = []
    missing = []
    failed = {}
    flakes = {}

    # The goal is to sort the tests into good/failed and record the ones
    # that passed after first failing in flakes
    for t in tests:
        name = t["name"]
        if t["success"]:
            # Tests should never have more than one success
            if name in success:
                raise RuntimeError(f"{name} already passed, should only be 1 per test")

            success.append(name)
            if name in failed:
                # Previously failed, move that to flakes and remove it from the failed list
                flakes[name] = failed[name]
                del failed[name]
        else:
            if "MISSING" in t["result"]:
                # Test is completely missing, don't count it as failed
                missing.append(name)
            elif name in success:
                # Test was also successful, make sure to record it in flakes list
                if name in flakes:
                    flakes[name].append(t)
                else:
                    flakes[name] = [t]
            else:
                if name in failed:
                    failed[name].append(t)
                else:
                    failed[name] = [t]

    return sorted(success), sorted(missing), failed, flakes


def print_test_details(scenario, days, test_name, buf):
    """Add the details of the named tests (failed-tests or flaky-tests) to the buffer
    """
    for d in days:
        if scenario not in days[d]:
            continue

        for n in days[d][scenario][test_name]:
            print(f"\n{n}:", file=buf)
            for test in days[d][scenario][test_name][n]:
                if "start_time" not in test:
                    start_time = ""
                else:
                    start_time = datetime.fromtimestamp(test["start_time"]).strftime("%m/%d/%Y %H:%M:%S")

                if "elapsed_time" not in test:
                    elapsed_time = 0
                else:
                    elapsed_time = test["elapsed_time"]

                # Get the result message
                msg = test["result"].rsplit("FAILED:")[-1]
                print(f'        {start_time} ({elapsed_time}s): {msg}', file=buf)


def kstest_logdir(tmpdir, test):
    """
    Return the directory of the logs for the test

    The logfile path should start with /var/tmp/kstest-*
    but this finds the kstest-* no matter what the leading
    path elements are.
    """
    logfile = test["logfile"]
    for e in logfile.split(os.path.sep):
        if e.startswith("kstest-"):
            return os.path.join(tmpdir, e)

    raise RuntimeError(f"No kstest-* directory found in {logfile}")


def archive_test_logs(days, archive_path, all_logs):
    """
    Store the logs from the failed/flaky tests for archiving
    and later examination. Paths follow the pattern:

    MM-DD-YYYY / scenario / failed|flakes / TESTNAME / [1...] /

    and each directory contains the logs for that run of the test.
    Currently this is the kstest.log and virt-install.log files
    """
    for day in days.keys():
        daydir = datetime.strptime(day, "%Y%m%d").strftime("%m-%d-%Y")
        for scenario in days[day].keys():
            # temporary log directories are stored by scenario + date
            datename = scenario + "-" + datetime.strptime(day, "%Y%m%d").strftime("%Y-%m-%d")
            if datename not in all_logs:
                raise RuntimeError(f"Missing all_log entry for {datename}")

            if not os.path.exists(all_logs[datename].name):
                raise RuntimeError(f"Missing log directory for {datename}")

            tmpdir = all_logs[datename].name
            failed = days[day][scenario]["failed-tests"]
            flakes = days[day][scenario]["flaky-tests"]

            scenario_archive = os.path.join(archive_path, daydir, scenario)
            os.makedirs(os.path.join(scenario_archive, "failed"))
            os.makedirs(os.path.join(scenario_archive, "flakes"))
            # data is organized by test names as keys with lists of tests
            for name in failed:
                i = 1
                for t in sorted(failed[name], key=lambda x: x["start_time"]):
                    try:
                        logdir = kstest_logdir(tmpdir, t)
                        if not os.path.exists(logdir):
                            raise RuntimeError(f"Missing logdir - {logdir}")
                    except RuntimeError:
                        continue
                    dst = os.path.join(scenario_archive, "failed", name, str(i))
                    shutil.copytree(logdir, dst)
                    i += 1

            for name in flakes:
                i = 1
                for t in sorted(flakes[name], key=lambda x: x["start_time"]):
                    try:
                        logdir = kstest_logdir(tmpdir, t)
                        if not logdir or not os.path.exists(logdir):
                            raise RuntimeError(f"Missing logdir - {logdir}")
                    except RuntimeError:
                        continue
                    dst = os.path.join(scenario_archive, "flakes", name, str(i))
                    shutil.copytree(logdir, dst)
                    i += 1


def process_logs(logs):
    """
    Process the logfiles into a data structure

    Returns a dictionary with each log's data that looks similar to this:

    {"20220401": {"logs-daily-iso": [{test data dict}, ...], ...}}, ...}

    So every day has an entry for the scenarios that were run on that day.
    Note that sometimes a scenario can be missing if there was a problem running it at all.
    """
    all_data = {}
    for log in logs:
        with open(log) as f:
            data = json.load(f)
            scenario = data[0].get("scenario", None)
            if scenario is None:
                # No scenario name, no way to organize the data
                continue

            # Use the log's date as the run identifier
            # This assumes the format is SCENARIO-YYYY-MM-DD.json
            # NOTE: This may not match the GitHub Action run dates due to tests taking
            #       a very long time.
            day = datetime.strptime(log[1+len(scenario):-5], "%Y-%m-%d").strftime("%Y%m%d")
            if day not in all_data:
                all_data[day] = {}

            # Group them by scenario, assume each file is from one scenario per day
            all_data[day][scenario] = data
    return all_data


def summary(args, json_logs, all_logs):
    """
    summary generates a summary of all the tests run in the selected date range

    It returns a string with the summary text
    """
    all_data = process_logs(json_logs)
    if args.debug:
        print(json.dumps(all_data))

    buf = io.StringIO()

    start = args.start.strftime("%m/%d/%Y %H:%M:%S")
    end = args.end.strftime("%m/%d/%Y %H:%M:%S")
    print(f"Test Summary Report: {start} -> {end}\n", file=buf)

    # Calculate test failures per day/scenario
    all_days = {}                   # dict of per-scenario counts
    days = {}                       # dict of per-day -> per-scenario counts and test names
    top_failed = {}                 # dict of per-test failure counts
    top_flakes = {}                 # dict of per-test flake counts
    for day in sorted(all_data.keys()):
        days[day] = {}
        for scenario in sorted(all_data[day].keys()):
            if scenario not in all_days:
                all_days[scenario] = {"success": 0, "missing": 0, "failed": 0, "flakes": 0}

            # Figure out how many were successful, failed, or were flakes
            success, missing, failed, flakes = check_tests(all_data[day][scenario])

            days[day][scenario] = {
                    "success": len(success),
                    "missing": len(missing),
                    "failed": len(failed),
                    "flakes": len(flakes),
                    "missing-tests": missing,
                    "failed-tests": failed,
                    "flaky-tests": flakes}
            all_days[scenario]["success"] += len(success)
            all_days[scenario]["missing"] += len(missing)
            all_days[scenario]["failed"] += len(failed)
            all_days[scenario]["flakes"] += len(flakes)

            for n in failed:
                top_failed[n] = top_failed.get(n, 0) + 1

            for n in flakes:
                top_flakes[n] = top_flakes.get(n, 0) + 1


    # Summary of tests per scenario
    print("Weekly summary", file=buf)
    print("==============", file=buf)
    for scenario in sorted(all_days.keys()):
        success = all_days[scenario]["success"]
        missing = all_days[scenario]["missing"]
        failed = all_days[scenario]["failed"]
        flakes = all_days[scenario]["flakes"]

        print(f"{scenario}: Ran {success+failed+missing} tests. {success} passed, {failed} failed, {missing} missing, {flakes} flakes.", file=buf)
    print("\n", file=buf)

    print("Top 5 failed tests for the week", file=buf)
    for n in sorted((n for n in top_failed), key=lambda x: top_failed[x], reverse=True)[:5]:
        print(f"    {n} - {top_failed[n]}", file=buf)
    print("\n", file=buf)

    print("Top 5 flaky tests for the week", file=buf)
    for n in sorted((n for n in top_flakes), key=lambda x: top_flakes[x], reverse=True)[:5]:
        print(f"    {n} - {top_flakes[n]}", file=buf)
    print("\n", file=buf)

    # Print daily stats
    for day in sorted(days.keys()):
        print(datetime.strptime(day, "%Y%m%d").strftime("%m/%d/%Y"), file=buf)
        for scenario in sorted(days[day].keys()):
            s = days[day][scenario]
            success = s["success"]
            missing = s["missing"]
            failed = s["failed"]
            total = success + failed + missing
            flakes = s["flakes"]
            print(f"    {scenario} (Ran {total}, {success} passed, {failed} failed, {missing} missing. {flakes} flakes) :", file=buf)
            if s["missing-tests"]:
                print("        Missing:", file=buf)
                for n in s["missing-tests"]:
                    print(f"            {n}", file=buf)
            if s["failed-tests"]:
                print("        Failed:", file=buf)
                for n in sorted(s["failed-tests"].keys()):
                    print(f"            {n}", file=buf)
            if s["flaky-tests"]:
                print("        Flakes:", file=buf)
                for n in sorted(s["flaky-tests"].keys()):
                    print(f"            {n}", file=buf)
        print("\n", file=buf)

    # Print the failure details for each scenario, on each day.
    for scenario in sorted(all_days.keys()):
        success = all_days[scenario]["success"]
        failed = all_days[scenario]["failed"]
        flakes = all_days[scenario]["flakes"]

        msg = f"{scenario}: Ran {success+failed} tests. {success} passed, {failed} failed, {flakes} flakes."
        print("=" * len(msg), file=buf)
        print(msg, file=buf)
        print("=" * len(msg), file=buf)

        if args.flake_details:
            print("Failed test details", file=buf)
            print("-------------------", file=buf)
        print_test_details(scenario, days, "failed-tests", buf)

        if args.flake_details:
            print("\nFlake test details", file=buf)
            print("-------------------", file=buf)
            print_test_details(scenario, days, "flaky-tests", buf)

        print("\n", file=buf)


    # Save the logs for the failures and flakes if a path is specified
    try:
        if args.archive_logs:
            archive_test_logs(days, args.archive_logs, all_logs)
    except RuntimeError as e:
        print(f"\nERROR: Problem archiving test logs - {e}", file=buf)

    return buf.getvalue()


def test_summary(args, path):
    """
    Test the weekly summary on a single directory

    Instead of pulling artifacts from github use a single directory with
    kstest.log.json and the log directories.
    """
    log = os.path.join(path, "kstest.log.json")
    with open(log) as f:
        data = json.load(f)
        scenario = data[0].get("scenario", None)
    if not scenario:
        raise RuntimeError("No scenario found in %s" % log)

    # The json log filename needs to be in the form of <scenario>-<YYYY-MM-DD>.json
    datename = f"{scenario}-1990-01-01"
    shutil.copy(log, datename+".json")
    datenames = [datename]
    all_logs = {datename: path}

    report = summary(args, (d+".json" for d in datenames), all_logs)
    if args.output:
        with open(args.output, "w") as f:
            f.write(report)
    else:
        print(report)


def main(args, token):
    artifacts = get_artifacts(token, args.artifact_names, args.start, args.end)
    if args.verbose:
        print(json.dumps(artifacts))
    zipfiles = download_artifacts(token, artifacts)
    if args.debug:
        print(f"zipfiles = {zipfiles}")

    datenames = []          # List of valid logfile names
    all_logs = {}           # Keep track of all the log tempdirs
    for name, datename, f in zipfiles:
        if args.rebuild or not os.path.exists(datename+".json"):
            try:
                all_logs[datename] = extract_logs(f)
            except zipfile.BadZipFile:
                # GitHub can responds with a 200 and a json error instead of a zip
                # so if it isn't a valid zip, just skip it.
                os.unlink(f)
                continue

            # This is needed for logs without timestamps
            if args.rebuild:
                generate_test_list(all_logs[datename])
                rebuild_logs(all_logs[datename])

            # Run summary on kstest.log
            cmd = ["log2json",
                   "--scenario", name,
                   "--output", datename+".json",
                   os.path.join(all_logs[datename].name, "kstest.log")
            ]
            if args.debug:
                print(cmd)
            check_call(cmd)

        # If the summary exists, add it to the list
        if os.path.exists(datename+".json"):
            datenames.append(datename)

    report = summary(args, (d+".json" for d in datenames), all_logs)
    if args.output:
        with open(args.output, "w") as f:
            f.write(report)
    else:
        print(report)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Generate a weekly summary of test results")
    parser.add_argument("--artifacts",
                        type=lambda x: x.split(","),
                        default=ARTIFACT_NAMES,
                        dest="artifact_names",
                        help="Comma separated list of artifact names to summarize")
    parser.add_argument("--start", type=datetime.fromisoformat,
            default=(datetime.now() - timedelta(days=7)),
            help="Start time. eg. 2022-03-03T03:46:11 (default is -7 days)")
    parser.add_argument("--end", type=datetime.fromisoformat,
            default=datetime.now(),
            help="end time. eg. 2022-03-03T03:46:11 (default is now)")
    parser.add_argument("--rebuild",
            default=False, action="store_true",
            help="Rebuild logs with timestamps")
    parser.add_argument("--flake-details",
            default=False, action="store_true",
            help="Include details about flaky tests in summary")
    parser.add_argument("--archive-logs",
            help="Optionally collect the failed/flake logs in an archive directory tree")
    parser.add_argument("--output",
            help="Path and filename to write summary report to")
    parser.add_argument("--test", help=argparse.SUPPRESS)
    parser.add_argument("--debug", default=False, action="store_true")
    parser.add_argument("--verbose", default=False, action="store_true")
    args = parser.parse_args()

    if "GITHUB_TOKEN" not in os.environ:
        print("Set GITHUB_TOKEN environmental variable to github token with access to the artifact api.")
        sys.exit(1)

    if args.debug:
        print(f"args = {args}")

    if args.test:
        test_summary(args, args.test)
    else:
        main(args, os.environ["GITHUB_TOKEN"])
