#! /usr/bin/env python3.3
"""Code for checking a 'git push' before writing anything to Perforce."""
import functools
import logging
import re
import sys

import P4

import p4gf_branch
from   p4gf_case_conflict_checker   import CaseConflictChecker
import p4gf_config
# no   p4gf_copy_to_p4              avoid circular import
import p4gf_const
from   p4gf_filemode                import FileModeStr
import p4gf_g2p_job                 as     G2PJob
import p4gf_git
from   p4gf_lfs_row                 import LFSRow
from   p4gf_l10n                    import _, NTR
import p4gf_path_convert
from   p4gf_profiler                import Timer
import p4gf_progress_reporter       as     ProgressReporter
import p4gf_protect
import p4gf_pygit2
import p4gf_util

LOG = logging.getLogger(__name__)

# Assume the cached objects will not be mapped into the repository, and
# skip permission checks on those paths since they're submitted as the
# git-fusion-user anyway.
P4GF_DEPOT_OBJECTS_RE = re.compile('//' + p4gf_const.P4GF_DEPOT + '/objects/')
P4GF_DEPOT_BRANCHES_RE = re.compile('//' + p4gf_const.P4GF_DEPOT + '/branches/')

# timer names
CHECK_PROTECTS  = NTR('Check Protects')
CHECK_OVERLAP   = NTR('Check Overlap')


class PreflightChecker:
    """A class that knows how to check a 'git push' for things that
    should not be copied into Perforce.

    Things you might not expect from a preflight checker:
    - user lookup:
        PreflightChecker has to see if the author has permission
        to write files, so PreflightChecker needs a way to look up
        a single Git commit's author.
        Could move to a separate class, passed in as part of __init__.
    - branch definition:
        A side effect of rolling through commits is that we
        have to fully define depot branches to hold those commits,
        and branch views to map the Git work tree to those depot branches.
        Could move elsewhere.
        Not sure if we want to require this as something done BEFORE
        preflight, or done DURING preflight.

    Extracted and decoupled from "classic push" p4gf_copy_to_p4.G2P and reused
    by "fast push" p4gf_fast_push.FastPush. If you change this for classic
    push, test your changes against fast push, too. Such is the price of
    maintaining two parallel code paths.
    """

    def __init__(self, *
            , ctx
            , g2p_user
            , assigner
            , gsreview_coll                 = None
            , already_copied_commit_runner  = None
            , finish_branch_definition      = None
            ):

        self.ctx                = ctx
        self.g2p_user           = g2p_user
        self.assigner           = assigner
        self.gsreview_coll      = gsreview_coll

        # ------------- # Explicitly listing each callback into G2P so I can
                        # find the API between these two snotballs.
                        # These are all "an instace (of G2P) that implements X:

                        # already_copied_commit(self, commit_sha1, branch_id)
        self.already_copied_commit_runner = already_copied_commit_runner

                        # finish_branch_definition(self, commit, branch)
        self._finish_branch_definition = finish_branch_definition

        # ------------- # End callbacks into G2P

                        # set() of depot names of type 'stream'
        self.stream_depots      = set()

                        # PreReceiveTuple whose commits we're checking.
                        # Used when permitting Swarm review merges, but
                        # prohibiting all other merge commits.
                        #
                        # OK to leave None if you don't care about
                        # Swarm reviews.
        self._current_prt       = None

                        # p4gf_fastexport_marks.Marks instance
                        # of all known git-fast-export marks.
        self.fast_export_marks  = None

        self._current_branch    = None
        self._curr_fe_commit    = None   # Current git-fast-export 'commit'.

                        # Behavior control for fast push: what to pass to
                        # Context.switched_to_branch() for set_client.
                        #
                        # When set to False, the Context.p4 client is
                        # untouched, which can save a huge amount of time
                        # in check_commit_for_branch(): 45 seconds drops
                        # to 9 seconds, but works only if you promist to make
                        # no calls to Perforce through that ctx.p4 connection.
                        #
        self.set_client_on_branch_switch = True

                        # list of fully populated Branch instances that overlap
                        # other fully populated branches.
                        #
                        # See _overlapping_branch_list() for calc and cache,
                        # call _invalidate_branch_cache() if you change any
                        # branch definitions after most recent call to
                        # _overlapping_branch_list().
                        #
        self._cached_overlapping_branch_list = None # list[] of FP Branch instances

                        # LFSRow text pointers that are part of this push.
        self.lfs_row_list = []


    #
    # -- callbacks into G2P --------------------------------------------------
    #    Optional. NOP if not running with a full G2P.
    #

    def _already_copied_commit(self, commit_sha1, branch_id):
        """Do we already have a Perforce changelist for this commit,
        this branch_id?
        """
        if not self.already_copied_commit_runner:
            return False
        return self.already_copied_commit_runner.already_copied_commit(
                    commit_sha1, branch_id)

    def finish_branch_definition(self, commit, branch):
        """Inflate a partially-created Branch with a proper depot branch to
        hold its files, and a branch view to map that depot branch to
        the Git work tree.
        """
        assert self._finish_branch_definition
        self._finish_branch_definition.finish_branch_definition(commit, branch)
        self._invalidate_branch_cache()

    #
    # -- end callbacks into G2P ----------------------------------------------
    #

    def check_prt_and_commits(self, prt, commits, marks):
        """Run a preflight check on a set of commits from Git.

        :param prt: pre-receive tuple
        :param commits: FastExport.commits list

        :type marks: :class:`p4gf_fastexport_marks.Marks`
        :param marks: all known fast-export marks

        Raises a PreflightException if anything is out of sorts.

        """
        self._current_prt = prt
        self.fast_export_marks = marks
        self.fast_export_marks.set_head(prt.ref)
        self.check_p4gf_user_write_permission()
        self.check_commits(commits)

    def check_p4gf_user_write_permission(self):
        """Ensure git-fusion-user has permissions to write to depot."""
        gf_client_map = P4.Map()
        gf_client_map.insert("//...", "//client/...")
        utp = p4gf_protect.UserToProtect(self.ctx.p4)
        prot = utp.user_to_protect(p4gf_const.P4GF_USER)
        gf_write_filter = prot.map_for_perm(p4gf_protect.WRITE)
        gf_write_filter = P4.Map.join(gf_write_filter, gf_client_map)
        if not gf_write_filter.includes('//{depot}/...'.format(depot=p4gf_const.P4GF_DEPOT)):
            raise RuntimeError(_('permission denied'))

    def _find_locked_by(self):
        """Return a dict of depot_path => user of any locked files."""
        fstat_flags = NTR('otherLock | otherOpen0 & headType=*+l')
        any_locked_files = {}  # depot_path : user
        for branch_chunk in self.ctx.iter_writable_branch_chunks():
            # Skip any newly defined branches: they're new, won't contain any
            # files yet, and won't get a view definition until later at per-
            # commit preflight time.
            bvl = [b for b in branch_chunk if b.view_lines]
            if not bvl:
                continue
            with self.ctx.switched_to_union(bvl):
                r = self.ctx.p4run('fstat', '-F', fstat_flags, '-m1',
                                   '//{}/...'.format(self.ctx.p4.client),
                                   log_warnings=logging.DEBUG)
                # Collect a dictionary of the locked files from the writable union of branch views
                for lf in r:
                    user = lf['otherOpen'][0] if 'otherOpen' in lf else NTR('<unknown>')
                    any_locked_files[lf['depotFile']] = user
        return any_locked_files

    def check_commits(self, commits):
        """Ensure the entire sequence of commits will (likely) go through
        without any errors related to permissions or locks. Raises an
        exception if anything goes wrong.

        Arguments:
            commits -- commits from FastExport class
        """
        LOG.info('Checking Perforce permissions and locks')
        self.ctx.checkpoint("copy_to_p4._preflight_check")

        # Stop if files are opened in our repo client
        # We expect this to be none, since we have the view lock
        opened = self.ctx.p4.run(['opened', '-m1'])
        if opened:
            raise PreflightException(_('There are files opened by Git Fusion for this repo.'))

        # fetch the repo setting only, without cascading to global config
        is_read_only = self.ctx.repo_config.getboolean(p4gf_config.SECTION_REPO,
                                                       p4gf_config.KEY_READ_ONLY,
                                                       fallback=False)
        if is_read_only:
            raise PreflightException(_("Push to repo {repo_name} prohibited.")
                                     .format(repo_name=self.ctx.config.repo_name))

        # get a list of stream depots for later checks for read-only paths
        depots = self.ctx.p4.run(['depots'])
        self.stream_depots = set([d['name'] for d in depots if d['type'] == 'stream'])
        any_locked_files = self._find_locked_by()
        LOG.debug("any_locked_files {0}".format(any_locked_files))
        case_conflict_checker = None
        if not self.ctx.server_is_case_sensitive:
            case_conflict_checker = CaseConflictChecker(self.ctx)
            case_conflict_checker.read_perforce_paths()

        ui_name = self._curr_ref_ui_name()
        if ui_name:
            progress_msg = _('Checking commits for {ref}...').format(ref=ui_name)
        else:
            progress_msg = _('Checking commits...')

        with ProgressReporter.Determinate(len(commits)):
            for commit in commits:
                ProgressReporter.increment(progress_msg)

                self.g2p_user.get_author_pusher_owner(commit)

                rev = commit['sha1']
                if not self.assigner.is_assigned(commit['sha1']):
                    continue

                self.check_commit(commit)

                for branch_id in self.assigner.branch_id_list(rev):
                    self.check_commit_for_branch(
                                                   commit
                                                 , branch_id
                                                 , any_locked_files
                                                 , case_conflict_checker )

        if case_conflict_checker:
            cc_text = case_conflict_checker.conflict_text()
            if cc_text:
                raise PreflightException(cc_text)

    def check_commit(self, commit):
        """Prior to copying a commit, perform a set of checks to ensure the commit
        will (likely) go through successfully.
        This includes:
            * verifying permission to commit for author p4 user
            * screening for merge commits
            * screening for submodules
            * checking valid filenames
            * checking write permissions for each file
        """
        # pylint: disable=too-many-branches
        if LOG.isEnabledFor(logging.DEBUG):
            LOG.debug('check_commit() Checking mark={} sha1={} file-ct={} -- {}'
                      .format(  commit['mark']
                              , p4gf_util.abbrev(commit['sha1'])
                              , len(commit['files'])
                              , repr(commit['data'])[:20].splitlines()[0]))

        if not commit['author_p4user']:
            raise PreflightException(_("User '{user}' not permitted to commit")
                                     .format(user=commit['author']['email'].strip('<>')))

        if 'merge' in commit:
            ref_is_review = (self.gsreview_coll and
                             self.gsreview_coll.ref_in_review_list(self._current_prt.ref))
            if not ref_is_review and not self.ctx.merge_commits:
                raise PreflightException(_('Merge commits are not enabled for this repo.'))
            if (not ref_is_review and
                    not self.ctx.branch_creation and self.assigner.have_anonymous_branches):
                msg = _('Git branch creation is prohibited for this repo.')
                p4_branch_names_non_lw = [b.git_branch_name for b in self.ctx.branch_dict().values()
                                          if b.git_branch_name and not b.is_lightweight]
                if len(p4_branch_names_non_lw) > 1:
                    msg += _('\nThis repo has more than one named branch.'
                             '\nTry altering the push order - '
                             'pushing branches with merge ancestors first.')
                raise PreflightException(msg)
            if LOG.isEnabledFor(logging.DEBUG):
                for parent_mark in commit['merge']:
                    parent_sha1 = self.fast_export_marks.get_commit(parent_mark)[:7]
                    LOG.debug("check_commit() merge mark={} sha1={}"
                              .format(parent_mark, parent_sha1))

        if not self.ctx.submodules and 'files' in commit:
            for f in commit['files']:
                if f.get('mode') == '160000':
                    if 'first_commit' in commit and not self._path_added(f.get('path'), commit):
                        LOG.debug2('check_commit() passed {} in {}'.format(
                            f.get('path'), p4gf_util.abbrev(commit['sha1'])))
                        continue
                    raise PreflightException(
                        _('Git submodules not permitted: path={path} commit={commit_sha1}')
                        .format(path=f.get('path'), commit_sha1=p4gf_util.abbrev(commit['sha1'])))

        for f in commit['files']:
            LOG.debug3("check_commit : commit files: " + _log_fe_file(f))
            err = check_valid_filename(f['path'], self.ctx)
            if err:
                raise PreflightException(err)
            if self.ctx.is_lfs_enabled:
                self._check_lfs(commit, f)

        # Warn user about any jobs that appear to not exist
        jobs = G2PJob.lookup_jobs(self.ctx, G2PJob.extract_jobs(commit['data']))
        if jobs:
            for job_id in jobs:
                r = self.ctx.p4run('jobs', '-e', 'job={}'.format(job_id))
                if not r:
                    _print_error(_("Job '{job_id}' doesn't exist").format(job_id=job_id))
        # Create pending changes for any Git-Swarm reviews

    def check_commit_for_branch( self
                               , commit
                               , branch_id
                               , any_locked_files
                               , case_conflict_checker ):
        """
        Prior to copying a commit, perform a set of checks for a specific branch
        to ensure the commit will (likely) go through successfully.
        """
        rev = commit['sha1']
        if LOG.isEnabledFor(logging.DEBUG):
            LOG.debug("check_commit_for_branch() "
                      "Checking branch={} mark={} sha1={} file-ct={} -- {}"
                      .format(  branch_id
                              , commit['mark']
                              , p4gf_util.abbrev(rev)
                              , len(commit['files'])
                              , repr(commit['data'])[:20].splitlines()[0]))

        if self._already_copied_commit(rev, branch_id):
            return

        # following checks assume client has been set for branch
        self.ensure_branch_preflight(commit, branch_id)
        with self.ctx.switched_to_branch(
                  self._current_branch
                , set_client=self.set_client_on_branch_switch
                ):
            if case_conflict_checker:
                case_conflict_checker.read_fast_export_commit(
                    commit, self._current_branch)

            # Empty commits require root-level .p4gf_placeholder to be mapped
            # in the current branch view.
            if not commit['files'] and not self._is_placeholder_mapped():
                raise PreflightException(
                    _("Empty commit {sha1} not permitted. Git Fusion branch views"
                      " must include root to permit empty commits.")
                    .format(sha1=p4gf_util.abbrev(rev)))

            with Timer(CHECK_PROTECTS):
                self._check_protects(commit['author_p4user'], commit['files'])

            with Timer(CHECK_OVERLAP):
                self._check_overlap(commit)

            # fetch the branch setting only, without cascading to repo/global config
            if self._current_branch.is_read_only:
                raise PreflightException(_("Push to branch {branch} prohibited.")
                                         .format(branch=self._current_branch.git_branch_name))
            self._check_stream_writable(commit)
            self._check_stream_in_classic(commit)

            LOG.debug('checking locked files under //{}/...'.format(self.ctx.p4.client))
            if any_locked_files:
                # Convert the git commit paths to depotPaths
                files_in_commit = [self.ctx.gwt_path(f['path']).to_depot()
                                   for f in commit['files']]
                LOG.debug("files_in_commit {0}".format(files_in_commit))
                for f in files_in_commit:
                    if f in any_locked_files:
                        # Collect the names (and clients) of users with locked files.
                        # Report back to the pusher so they can take appropriate action.
                        msg = _('{file} - locked by {user}').format(file=f,
                                                                    user=any_locked_files[f])
                        LOG.info(msg)
                        raise PreflightException(msg)

                    # +++ Spend time extracting Jobs and P4Changelist owner
                    #     here if we actually do need to call
                    #     the preflight-commit hook.
            if self.ctx.preflight_hook.is_callable():
                jobs = G2PJob.extract_jobs(commit['data'])
                jobs2 = G2PJob.lookup_jobs(self.ctx, jobs)
                self.ctx.preflight_hook(
                     ctx                 = self.ctx
                   , fe_commit           = commit
                   , branch_id           = branch_id
                   , jobs                = jobs2
                   )

    def ensure_branch_preflight(self, commit, branch_id):
        """If not already switched to and synced to the correct branch for the
        given commit, do so.

        If this is a new lightweight branch, perform whatever creation we can do
        at preflight time. We don't have commits/marks for any not-yet-submitted
        parent commits, so the depot_branch_info will often lack a correct
        parent or fully populated basis.

        * depot tree, along with a branch-info file
        * branch mapping, along with entry in p4gf_config2 (if not anonymous)

        Return requested branch
        """
        log = LOG.getChild('ensure_branch_preflight')
        branch = self.ctx.branch_dict().get(branch_id)
        # branch should never be None here. p4gf_branch_id.Assigner() must
        # create Branch objects for each assignment.

        if      self._current_branch \
            and self._current_branch.branch_id == branch_id:
            log.debug("sha={} want branch_id={} curr branch_id={} NOP"
                      .format( commit['sha1'][:7]
                             , branch_id[:7]
                             , self._current_branch.branch_id[:7]))
            log.debug("staying on  branch {}"
                      .format(self.ctx.branch_dict().get(branch_id)))

            return branch

        cbid = self._current_branch.branch_id if self._current_branch else 'None'
        log.debug("sha={} want branch_id={} curr branch_id={} switch"
                  .format(commit['sha1'][:7], branch_id[:7], cbid[:7]))

        if not branch.view_lines:
            self.finish_branch_definition(commit, branch)

        elif branch.view_p4map:
            # if this is a stream branch, check for mutation of the stream's
            # view by comparing with the original view saved in p4gf_config2
            if branch.original_view_lines:
                original_view_lines = '\n'.join(branch.original_view_lines)
                view_lines = p4gf_path_convert.convert_view_to_no_client_name(branch.view_lines)
                if not view_lines == original_view_lines:
                    raise PreflightException(
                        _('Unable to push.  Stream view changed from:\n'
                          '{old_view}\nto:\n{new_view}')
                        .format(old_view=original_view_lines, new_view=view_lines))
            # Find existing depot branch for branch view's LHS.
            lhs = branch.view_p4map.lhs()
            branch.depot_branch = self.ctx.depot_branch_info_index()    \
                .find_depot_path(lhs[0])

        log.debug("switching to branch {}".format(branch))

        # By now we should have a branch and a branch.view_lines.
        # First remove current branch's files from workspace
        # Client spec is set to normdir
        self._current_branch = branch
        return branch

    def _is_placeholder_mapped(self):
        """Does this branch map our placeholder file?

        Returns non-False if mapped, None or empty string if not.
        """
        return self.ctx.gwt_path(
            p4gf_const.P4GF_EMPTY_CHANGELIST_PLACEHOLDER).to_depot()

    def _check_protects(self, p4user, blobs):
        """check if author is authorized to submit files."""
        pc = ProtectsChecker(self.ctx, p4user, self.ctx.authenticated_p4user,
                             self.ctx.foruser)
        pc.filter_paths(blobs)
        if pc.has_error():
            raise PreflightException(pc.error_message())

    def _check_overlap(self, fe_commit):
        """If any of the files in this commit intersect any fully populated branch
        (other than the current branch), then reject this commit.

        Shared/common/overlapping paths in branch views must be read-only from
        Git. Otherwise you end up with a Git push of commit on one Git branch
        inserting  changes into other Git branches behind Git's back.

        To modify shared paths, either do so from Perforce, or create a Git
        Fusion repo with no more than one branch that maps that shared path.
        """
                        # +++ Avoid O(b branches * r rev) checks when
                        #     overlap is impossible because current branch
                        #     overlaps no other branch.
        if self._current_branch not in self._overlapping_branch_list():
            return

        for fe_file in fe_commit['files']:
            gwt_path   = fe_file['path']
            depot_path = self.ctx.gwt_to_depot_path(gwt_path)

            for branch in self._overlapping_branch_list():
                if branch == self._current_branch:
                    continue
                if not branch.intersects_depot_path(depot_path):
                    continue

                LOG.debug("_check_overlap() branch {br1} <> {br2}"
                          " gwt={gwt:<40}   {dp}\n{view}"
                          .format(
                              br1  = p4gf_util.abbrev(self._current_branch.branch_id)
                            , br2  = p4gf_util.abbrev(branch.branch_id)
                            , gwt  = gwt_path
                            , dp   = depot_path
                            , view = "\n".join(branch.view_p4map.as_array())
                            ))

                if self._current_branch.is_new_fp_from_push or branch.is_new_fp_from_push:
                    current_branch_name = self._current_branch.git_branch_name
                    if self._current_branch.is_new_fp_from_push:
                        current_branch_name += '(new)'
                    other_branch_name = branch.git_branch_name
                    if branch.is_new_fp_from_push:
                        other_branch_name += '(new)'
                    human_msg = (_(
                        "Perforce: Cannot commit {sha1} '{gwt_path}' to '{depot_path}'.\n"
                        " You are attempting to push and create a new fully populated branch\n"
                        " with paths which overlap another branch. Contact your admin\n"
                        " to configure non-conflicting destination branch paths.\n"
                        " Branches: '{b1}', '{b2}'")
                        .format( sha1       = p4gf_util.abbrev(fe_commit['sha1'])
                               , gwt_path   = gwt_path
                               , depot_path = depot_path
                               , b1 = current_branch_name
                               , b2 = other_branch_name ))
                else:
                    human_msg = (_(
                        "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'."
                        " Paths that overlap multiple Git Fusion branches are read-only."
                        " Branches: '{b1}', '{b2}'")
                        .format( sha1       = p4gf_util.abbrev(fe_commit['sha1'])
                               , gwt_path   = gwt_path
                               , depot_path = depot_path
                               , b1 = self._current_branch.branch_id
                               , b2 = branch.branch_id ))
                raise PreflightException(human_msg)

    def _check_stream_writable(self, fe_commit):
        """If this is a stream branch, check that all files in the commit are
        writable.  If any of the files is not writable then reject this commit.
        """
        if not self._current_branch.stream_name:
            return
        prefix = self._current_branch.writable_stream_name + '/'
        for fe_file in fe_commit['files']:
            gwt_path   = fe_file['path']
            depot_path = self.ctx.gwt_path(gwt_path).to_depot()
            if depot_path.startswith(prefix):
                continue

            human_msg = (_(
                "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'."
                " Paths not in stream '{stream}' are read-only for branch '{b}'.")
                .format( sha1       = p4gf_util.abbrev(fe_commit['sha1'])
                       , gwt_path   = gwt_path
                       , depot_path = depot_path
                       , stream     = self._current_branch.writable_stream_name
                       , b          = self._current_branch.branch_id ))
            raise PreflightException(human_msg)

    def _check_stream_in_classic(self, fe_commit):
        """If this is a classic branch, check that none of the files in the commit
        are in stream depots and thus not writable.  If any of the files is not
        writable then reject this commit.
        """
        if self._current_branch.stream_name:
            return

        depot_re = re.compile(r'^//([^/]+)/([^/]+)/.*$')
        for fe_file in fe_commit['files']:
            gwt_path   = fe_file['path']
            depot_path = self.ctx.gwt_path(gwt_path).to_depot()
            m          = depot_re.match(depot_path)
            if m:
                depot = m.group(1)
                if depot in self.stream_depots:
                    stream = '//{}/{}'.format(m.group(1), m.group(2))
                    human_msg = (
                        _("Cannot commit {sha1} '{gwt_path}' to '{depot_path}'."
                          " Paths in stream '{stream}' are read-only for branch '{b}'.")
                        .format( sha1       = p4gf_util.abbrev(fe_commit['sha1'])
                               , gwt_path   = gwt_path
                               , depot_path = depot_path
                               , stream     = stream
                               , b          = self._current_branch.branch_id ))
                    raise PreflightException(human_msg)

    def _path_added(self, path, fecommit):
        """Return True if the named path was introduced in the HEAD commit.

        :param self: this object
        :param path: repo path to be evaluated.
        :param fecommit: commit object from fast-export parser.

        """
        # Because git-fast-export includes the entire tree in its output,
        # regardless of whether the requested commit is the first in the
        # branch or not, we need to check the repo itself to be certain if
        # this path was truly introduced in this commit, or simply existed
        # in the tree prior to the "first" commit.
        commit = self.ctx.repo.get(fecommit['sha1'])
        if commit is None:
            # empty repository?
            LOG.debug2("_path_added() commit {} is missing".format(fecommit['sha1']))
            return True
        for parent in commit.parents:
            if p4gf_git.exists_in_tree(self.ctx.repo, path, parent.tree):
                LOG.debug2("_path_added() {} exists in parent tree {}".format(
                    path, p4gf_util.abbrev(p4gf_pygit2.object_to_sha1(parent))))
                return False
        return True

    def _invalidate_branch_cache(self):
        """We have changed our branch_dict (or more likely
        finish_branch_definition()ed a branch within that dict) in a way that
        invalidates any cached calculations that consumed the branch dict.
        """
        self._cached_overlapping_branch_list = None

    def _overlapping_branch_list(self):
        """Return a list of fully populated branches that overlap
        other fully populated branches.

        Caches the result because we check every file revision
        path for overlap, and for huge repos with thousands of
        non-overlapping LW branches, just iterating through the
        branch list starts to waste measurable CPU time.
        """
        if self._cached_overlapping_branch_list is not None:
            return self._cached_overlapping_branch_list

        have_overlap = set()
        for outer in p4gf_branch.iter_fp_non_deleted(self.ctx.branch_dict()):
            outer_lhs = P4.Map()
            outer_lhs.insert(outer.view_p4map.lhs())
            for inner in p4gf_branch.iter_fp_non_deleted(self.ctx.branch_dict()):
                if outer == inner:
                    continue
                overlap = P4.Map.join(outer_lhs, inner.view_p4map)
                        # Any non-exclusionary lines shared between branches?
                for line in overlap.as_array():
                    if line.startswith('-') or line.startswith('"-'):
                        continue
                    # Yep. Non-exclusionary line implies overlap
                    have_overlap.add(outer)
                    have_overlap.add(inner)
                    break

        self._cached_overlapping_branch_list = have_overlap
        return self._cached_overlapping_branch_list

    def _curr_ref_ui_name(self):
        """Return a string suitable for use in progress messages as a
        short hint to what we're processing.

        If we have a PreReceiveTuple with a Git branch/tag name, return
        that name (minus any refs/heads/ prefix). If not, return None.
        """
        if not (self._current_prt and self._current_prt.ref):
            return None
        s = self._current_prt.ref
        prefixes = ["refs/heads/", "refs/tags/"]
        for p in prefixes:
            if s.startswith(p):
                return s[len(p):]
        return s

    def _check_lfs(self, fe_commit, fe_file):
        """If gfe_file is under Git LFS control, require that its
        large file content exist somewhere, either in our upload
        cache (it's new!) or in depot de-dupe storage (already got it).
        """
                        # Deleted files carry no LFS pointer.
        if "sha1" not in fe_file:
            return
                        # Symlinks and non-files carry no LFS pointer.
        if fe_file.get("mode") not in [ FileModeStr.PLAIN
                                      , FileModeStr.EXECUTABLE ]:
            return

                        # Files not under Git LFS control should not carry LFS
                        # pointer information. While legal and permissible,
                        # this is usually a mistake (misconfigured Git client)
                        # and something most users want caught before the push
                        # gets into Helix.
        is_tracked = self.ctx.lfs_tracker.is_tracked_git(
                      commit_sha1 = fe_commit["sha1"]
                    , gwt_path    = fe_file["path"])

        LOG.debug3("_check_lfs() tracked {lfs}  commit {commit_sha1}  gwt {gwt}"
            .format( commit_sha1 = p4gf_util.abbrev(fe_commit["sha1"])
                   , lfs         = 1 if is_tracked else 0
                   , gwt         = fe_file["path"]
                   ))

        if not is_tracked:
            lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file)
            if lfs_row:
                raise PreflightException(
                        _("Push of Git LFS text pointer not tracked by LFS:"
                          "\ncommit {commit_sha1} path {gwt_path}")
                        .format( commit_sha1 = p4gf_util.abbrev(fe_commit["sha1"])
                               , gwt_path    = fe_file["path"] ))
            return

                        # Files under Git LFS control should carry LFS pointer
                        # information, but sometimes might not, and that's
                        # okay.
        lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file)
        if not lfs_row:
            return
                        # But if they DO carry LFS pointer information, that
                        # pointer needs to point to a valid LFS large file
                        # either already in Perforce or recently uploaded.
        if not lfs_row.large_file_source:
            LOG.error("LFS text pointer missing content.")
            LOG.error("LFS   commit {}".format(p4gf_util.abbrev(fe_commit["sha1"])))
            LOG.error("LFS   lfs oid {}".format(lfs_row.large_file_oid))
            LOG.error("LFS   ptr {blob_sha1} {blob_mode} {gwt}"
                      .format( blob_sha1 = p4gf_util.abbrev(fe_file["sha1"])
                             , blob_mode = p4gf_util.mode_str(fe_file["mode"])
                             , gwt       = fe_file["path"]))
            LOG.error("LFS   upload  {}".format(lfs_row.to_lfsfs().cache_path(self.ctx)))
            LOG.error("LFS   de-dupe {}".format(lfs_row.to_lfsfs().depot_path(self.ctx)))

            raise PreflightException(_("Push of Git LFS text pointer missing content:"
                        "\ncommit {commit_sha1} path {gwt_path}")
                        .format( commit_sha1 = p4gf_util.abbrev(fe_commit["sha1"])
                               , gwt_path    = fe_file["path"] ))

                        # We have an acceptable LFS text pointer.
                        # Remember it for later.
        self.lfs_row_list.append(lfs_row)

# end class PreflightChecker
# ----------------------------------------------------------------------------


class ProtectsChecker:

    """Handle filtering a list of paths against view and protections."""

    def __init__(self, ctx, author, pusher, foruser):
        """Init P4.Map objects for author, pusher, view and combination."""
        self.ctx = ctx
        self.author = author
        self.pusher = pusher
        self.foruser = foruser

        self.ignore_author_perms = ctx.repo_config.getboolean(p4gf_config.SECTION_GIT_TO_PERFORCE,
                                                              p4gf_config.KEY_IGNORE_AUTHOR_PERMS)

        self.view_map = None
        self.write_protect_author = None
        self.write_protect_pusher = None
        self.write_protect_foruser = None
        self.write_protect_fusion = None
        self.write_filter = None

        self.init_view()
        self.init_write_filter()

        self.author_denied = []
        self.pusher_denied = []
        self.foruser_denied = []
        self.fusion_denied = []
        self.unmapped = []

    def init_view(self):
        """Init view map for client."""
        self.view_map = self.ctx.clientmap

    def init_write_filter(self):
        """Init write filter."""
        self.write_protect_author = self.ctx.user_to_protect(self.author).map_for_perm(
            p4gf_protect.WRITE)
        if self.author != self.pusher or self.ignore_author_perms:
            # If the author and pusher differ, _or_ if we are ignoring
            # author permissions, then we must also included the pusher
            # protections, as well as set write_protect_pusher.
            self.write_protect_pusher = self.ctx.user_to_protect(self.pusher).map_for_perm(
                p4gf_protect.WRITE)
        if self.foruser:
            self.write_protect_foruser = self.ctx.user_to_protect(self.foruser).map_for_perm(
                p4gf_protect.WRITE)
        # Ensure that git-fusion-user can write to the depot
        self.write_protect_fusion = self.ctx.user_to_protect(
            p4gf_const.P4GF_USER).map_for_perm(p4gf_protect.WRITE)
        self.write_filter = functools.reduce(P4.Map.join, [
            vmap for vmap in [
                self.write_protect_author,
                self.write_protect_pusher,
                self.write_protect_foruser,
                self.write_protect_fusion,
                self.view_map,  # view_map has rhs, must come last in join
            ] if vmap])

    def filter_paths(self, blobs):
        """Run list of paths through filter and set list of paths that don't pass."""
        # check against one map for read, one for write
        # if check fails, figure out if it was the view map or the protects
        # that caused the problem and report accordingly
        self.author_denied = []
        self.pusher_denied = []
        self.foruser_denied = []
        self.fusion_denied = []
        self.unmapped = []
        c2d = P4.Map.RIGHT2LEFT

        LOG.debug('filter_paths() write_filter: %s', self.write_filter)
        for blob in blobs:
            gwt_path = self.ctx.gwt_path(blob['path'])
            topath_c = gwt_path.to_client()
            topath_d = gwt_path.to_depot()

            LOG.debug('filter_paths() topath_d: %s', topath_d)
            # for all actions, need to check write access for dest path
            result = "  "   # zum loggen
            if topath_d and P4GF_DEPOT_OBJECTS_RE.match(topath_d):
                LOG.debug('filter_paths() topath_d in //.git-fusion/objects')
                continue
            # do not require user write access to //.git-fusion/branches
            if topath_d and P4GF_DEPOT_BRANCHES_RE.match(topath_d):
                LOG.debug('filter_paths() topath_d in //.git-fusion/branches')
                continue
            if not self.write_filter.includes(topath_c, c2d):
                if not self.view_map.includes(topath_c, c2d):
                    self.unmapped.append(topath_c)
                    result = NTR('unmapped')
                elif not (self.ignore_author_perms or
                          self.write_protect_author.includes(topath_d)):
                    self.author_denied.append(topath_c)
                    result = NTR('author denied')
                elif (self.write_protect_pusher and
                      not self.write_protect_pusher.includes(topath_d)):
                    self.pusher_denied.append(topath_c)
                    result = NTR('pusher denied')
                elif (self.write_protect_foruser and
                      not self.write_protect_foruser.includes(topath_d)):
                    self.foruser_denied.append(topath_c)
                    result = NTR('foruser denied')
                elif not self.write_protect_fusion.includes(topath_d):
                    self.fusion_denied.append(topath_c)
                    result = NTR('Git Fusion denied')
                else:
                    result = "?"
                LOG.error('filter_paths() {:<13} {}, {}, {}'
                          .format(result, blob['path'], topath_d, topath_c))
            elif LOG.isEnabledFor(logging.DEBUG):
                LOG.debug('filter_paths() topath_c in write_filter: %s', topath_c)

    def has_error(self):
        """Return True if any paths not passed by filters."""
        return len(self.unmapped) or len(self.author_denied) \
            or len(self.pusher_denied) or len(self.foruser_denied) \
            or len(self.fusion_denied)

    def error_message(self):
        """Return message indicating what's blocking the push."""
        if len(self.unmapped):
            return _('file(s) not in client view')
        if len(self.author_denied):
            restricted_user = self.author
        elif len(self.pusher_denied):
            restricted_user = self.pusher
        elif len(self.foruser_denied):
            restricted_user = self.foruser
        elif len(self.fusion_denied):
            restricted_user = p4gf_const.P4GF_USER
        else:
            restricted_user = _('<unknown>')
        return _("user '{user}' not authorized to submit file(s) in git commit").format(
            user=restricted_user)

# end ProtectsChecker
# ----------------------------------------------------------------------------


class PreflightException(Exception):

    """This exception is raised when a push was rejected during preflight
    checks.
    """

    pass

# -- module-wide -------------------------------------------------------------

def _log_fe_file(fe_file):
    """Return loggable string for a single fe_commit['files'] element."""
    mode = '      '
    if 'mode' in fe_file:
        mode = fe_file['mode']
    sha1 = '       '
    if 'sha1' in fe_file:
        sha1 = p4gf_util.abbrev(fe_file['sha1'])

    return NTR('{mode} {action} {sha1} {path}') \
           .format( mode   = mode
                  , action = fe_file['action']
                  , sha1   = sha1
                  , path   = fe_file['path'])


def _print_error(msg):
    """Print the given message to the error stream, as well as to the log."""
    sys.stderr.write(msg + '\n')
    LOG.error(msg)


def is_p4d_printable(c):
    """Check if c will be rejected by P4D as non-printable.

    P4D rejects "non-printable" characters with
      ErrorId MsgDm::IdNonPrint = { ErrorOf( ES_DM, 6, E_FAILED, EV_USAGE, 1 )
      "Non-printable characters not allowed in '%id%'." } ;

    Where "character" here means C "char" aka "byte"
    and non-printable means 0x00-0x1f or 0x7f
    """
    if ord(c) < 0x20:
        return False
    if ord(c) == 0x7F:
        return False
    return True


def check_valid_filename(name, ctx):
    """Test the given name for illegal characters.

    Return None if okay, otherwise an error message.
    Illegal characters and sequences include: [...]
    """
    for idx, c in enumerate(name):
        if not is_p4d_printable(c):
            fullname = name[:idx] + "x{ch:02X}".format(ch=ord(c)) + name[idx:]
            return _("Perforce: Non-printable characters not allowed in Perforce: "
                     "character x{ch:02X} in filepath: {filename}").format(
                     filename=fullname, ch=ord(c))
    if '...' in name:
        return _("Perforce: bad filename (...): '{filename}'").format(filename=name)
    if 'P4D/NT' in ctx.server_version:
        if ':' in name:
            return _("Perforce: unsupported filename on windows: {filename}").format(filename=name)
    # This should usually be en_US.UTF-8 which also needs to be defined
    # on the os
    encoding = sys.getfilesystemencoding()
    try:
        name.encode(encoding, "strict")
    except UnicodeEncodeError:
        return _("Perforce: Cannot convert filename to '{encoding}': {filename}").format(
            encoding=encoding, filename=name)
    return None

c5c9d59ad4cb491a9845db3c4f0f6afff836b148
<mask token>


class PreflightChecker:
    <mask token>
    <mask token>

    def _already_copied_commit(self, commit_sha1, branch_id):
        """Do we already have a Perforce changelist for this commit,
        this branch_id?
        """
        if not self.already_copied_commit_runner:
            return False
        return self.already_copied_commit_runner.already_copied_commit(
            commit_sha1, branch_id)
    <mask token>

    def check_prt_and_commits(self, prt, commits, marks):
        """Run a preflight check on a set of commits from Git.

        :param prt: pre-receive tuple
        :param commits: FastExport.commits list

        :type marks: :class:`p4gf_fastexport_marks.Marks`
        :param marks: all known fast-export marks

        Raises a PreflightException if anything is out of sorts.

        """
        self._current_prt = prt
        self.fast_export_marks = marks
        self.fast_export_marks.set_head(prt.ref)
        self.check_p4gf_user_write_permission()
        self.check_commits(commits)
    <mask token>
    <mask token>

    def check_commits(self, commits):
        """Ensure the entire sequence of commits will (likely) go through
        without any errors related to permissions or locks. Raises an
        exception if anything goes wrong.

        Arguments:
            commits -- commits from FastExport class
        """
        LOG.info('Checking Perforce permissions and locks')
        self.ctx.checkpoint('copy_to_p4._preflight_check')
        opened = self.ctx.p4.run(['opened', '-m1'])
        if opened:
            raise PreflightException(_(
                'There are files opened by Git Fusion for this repo.'))
        is_read_only = self.ctx.repo_config.getboolean(p4gf_config.
            SECTION_REPO, p4gf_config.KEY_READ_ONLY, fallback=False)
        if is_read_only:
            raise PreflightException(_(
                'Push to repo {repo_name} prohibited.').format(repo_name=
                self.ctx.config.repo_name))
        depots = self.ctx.p4.run(['depots'])
        self.stream_depots = set([d['name'] for d in depots if d['type'] ==
            'stream'])
        any_locked_files = self._find_locked_by()
        LOG.debug('any_locked_files {0}'.format(any_locked_files))
        case_conflict_checker = None
        if not self.ctx.server_is_case_sensitive:
            case_conflict_checker = CaseConflictChecker(self.ctx)
            case_conflict_checker.read_perforce_paths()
        ui_name = self._curr_ref_ui_name()
        if ui_name:
            progress_msg = _('Checking commits for {ref}...').format(ref=
                ui_name)
        else:
            progress_msg = _('Checking commits...')
        with ProgressReporter.Determinate(len(commits)):
            for commit in commits:
                ProgressReporter.increment(progress_msg)
                self.g2p_user.get_author_pusher_owner(commit)
                rev = commit['sha1']
                if not self.assigner.is_assigned(commit['sha1']):
                    continue
                self.check_commit(commit)
                for branch_id in self.assigner.branch_id_list(rev):
                    self.check_commit_for_branch(commit, branch_id,
                        any_locked_files, case_conflict_checker)
        if case_conflict_checker:
            cc_text = case_conflict_checker.conflict_text()
            if cc_text:
                raise PreflightException(cc_text)
    <mask token>

    def check_commit_for_branch(self, commit, branch_id, any_locked_files,
        case_conflict_checker):
        """
        Prior to copying a commit, perform a set of checks for a specific branch
        to ensure the commit will (likely) go through successfully.
        """
        rev = commit['sha1']
        if LOG.isEnabledFor(logging.DEBUG):
            LOG.debug(
                'check_commit_for_branch() Checking branch={} mark={} sha1={} file-ct={} -- {}'
                .format(branch_id, commit['mark'], p4gf_util.abbrev(rev),
                len(commit['files']), repr(commit['data'])[:20].splitlines(
                )[0]))
        if self._already_copied_commit(rev, branch_id):
            return
        self.ensure_branch_preflight(commit, branch_id)
        with self.ctx.switched_to_branch(self._current_branch, set_client=
            self.set_client_on_branch_switch):
            if case_conflict_checker:
                case_conflict_checker.read_fast_export_commit(commit, self.
                    _current_branch)
            if not commit['files'] and not self._is_placeholder_mapped():
                raise PreflightException(_(
                    'Empty commit {sha1} not permitted. Git Fusion branch views must include root to permit empty commits.'
                    ).format(sha1=p4gf_util.abbrev(rev)))
            with Timer(CHECK_PROTECTS):
                self._check_protects(commit['author_p4user'], commit['files'])
            with Timer(CHECK_OVERLAP):
                self._check_overlap(commit)
            if self._current_branch.is_read_only:
                raise PreflightException(_(
                    'Push to branch {branch} prohibited.').format(branch=
                    self._current_branch.git_branch_name))
            self._check_stream_writable(commit)
            self._check_stream_in_classic(commit)
            LOG.debug('checking locked files under //{}/...'.format(self.
                ctx.p4.client))
            if any_locked_files:
                files_in_commit = [self.ctx.gwt_path(f['path']).to_depot() for
                    f in commit['files']]
                LOG.debug('files_in_commit {0}'.format(files_in_commit))
                for f in files_in_commit:
                    if f in any_locked_files:
                        msg = _('{file} - locked by {user}').format(file=f,
                            user=any_locked_files[f])
                        LOG.info(msg)
                        raise PreflightException(msg)
            if self.ctx.preflight_hook.is_callable():
                jobs = G2PJob.extract_jobs(commit['data'])
                jobs2 = G2PJob.lookup_jobs(self.ctx, jobs)
                self.ctx.preflight_hook(ctx=self.ctx, fe_commit=commit,
                    branch_id=branch_id, jobs=jobs2)

    def ensure_branch_preflight(self, commit, branch_id):
        """If not already switched to and synced to the correct branch for the
        given commit, do so.

        If this is a new lightweight branch, perform whatever creation we can do
        at preflight time. We don't have commits/marks for any not-yet-submitted
        parent commits, so the depot_branch_info will often lack a correct
        parent or fully populated basis.

        * depot tree, along with a branch-info file
        * branch mapping, along with entry in p4gf_config2 (if not anonymous)

        Return requested branch
        """
        log = LOG.getChild('ensure_branch_preflight')
        branch = self.ctx.branch_dict().get(branch_id)
        if (self._current_branch and self._current_branch.branch_id ==
            branch_id):
            log.debug('sha={} want branch_id={} curr branch_id={} NOP'.
                format(commit['sha1'][:7], branch_id[:7], self.
                _current_branch.branch_id[:7]))
            log.debug('staying on  branch {}'.format(self.ctx.branch_dict()
                .get(branch_id)))
            return branch
        cbid = (self._current_branch.branch_id if self._current_branch else
            'None')
        log.debug('sha={} want branch_id={} curr branch_id={} switch'.
            format(commit['sha1'][:7], branch_id[:7], cbid[:7]))
        if not branch.view_lines:
            self.finish_branch_definition(commit, branch)
        elif branch.view_p4map:
            if branch.original_view_lines:
                original_view_lines = '\n'.join(branch.original_view_lines)
                view_lines = p4gf_path_convert.convert_view_to_no_client_name(
                    branch.view_lines)
                if not view_lines == original_view_lines:
                    raise PreflightException(_(
                        """Unable to push.  Stream view changed from:
{old_view}
to:
{new_view}"""
                        ).format(old_view=original_view_lines, new_view=
                        view_lines))
            lhs = branch.view_p4map.lhs()
            branch.depot_branch = self.ctx.depot_branch_info_index(
                ).find_depot_path(lhs[0])
        log.debug('switching to branch {}'.format(branch))
        self._current_branch = branch
        return branch

    def _is_placeholder_mapped(self):
        """Does this branch map our placeholder file?

        Returns non-False if mapped, None or empty string if not.
        """
        return self.ctx.gwt_path(p4gf_const.P4GF_EMPTY_CHANGELIST_PLACEHOLDER
            ).to_depot()

    def _check_protects(self, p4user, blobs):
        """check if author is authorized to submit files."""
        pc = ProtectsChecker(self.ctx, p4user, self.ctx.
            authenticated_p4user, self.ctx.foruser)
        pc.filter_paths(blobs)
        if pc.has_error():
            raise PreflightException(pc.error_message())

    def _check_overlap(self, fe_commit):
        """If any of the files in this commit intersect any fully populated branch
        (other than the current branch), then reject this commit.

        Shared/common/overlapping paths in branch views must be read-only from
        Git. Otherwise you end up with a Git push of commit on one Git branch
        inserting  changes into other Git branches behind Git's back.

        To modify shared paths, either do so from Perforce, or create a Git
        Fusion repo with no more than one branch that maps that shared path.
        """
        if self._current_branch not in self._overlapping_branch_list():
            return
        for fe_file in fe_commit['files']:
            gwt_path = fe_file['path']
            depot_path = self.ctx.gwt_to_depot_path(gwt_path)
            for branch in self._overlapping_branch_list():
                if branch == self._current_branch:
                    continue
                if not branch.intersects_depot_path(depot_path):
                    continue
                LOG.debug(
                    '_check_overlap() branch {br1} <> {br2} gwt={gwt:<40}   {dp}\n{view}'
                    .format(br1=p4gf_util.abbrev(self._current_branch.
                    branch_id), br2=p4gf_util.abbrev(branch.branch_id), gwt
                    =gwt_path, dp=depot_path, view='\n'.join(branch.
                    view_p4map.as_array())))
                if (self._current_branch.is_new_fp_from_push or branch.
                    is_new_fp_from_push):
                    current_branch_name = self._current_branch.git_branch_name
                    if self._current_branch.is_new_fp_from_push:
                        current_branch_name += '(new)'
                    other_branch_name = branch.git_branch_name
                    if branch.is_new_fp_from_push:
                        other_branch_name += '(new)'
                    human_msg = _(
                        """Perforce: Cannot commit {sha1} '{gwt_path}' to '{depot_path}'.
 You are attempting to push and create a new fully populated branch
 with paths which overlap another branch. Contact your admin
 to configure non-conflicting destination branch paths.
 Branches: '{b1}', '{b2}'"""
                        ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']),
                        gwt_path=gwt_path, depot_path=depot_path, b1=
                        current_branch_name, b2=other_branch_name)
                else:
                    human_msg = _(
                        "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'. Paths that overlap multiple Git Fusion branches are read-only. Branches: '{b1}', '{b2}'"
                        ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']),
                        gwt_path=gwt_path, depot_path=depot_path, b1=self.
                        _current_branch.branch_id, b2=branch.branch_id)
                raise PreflightException(human_msg)

    def _check_stream_writable(self, fe_commit):
        """If this is a stream branch, check that all files in the commit are
        writable.  If any of the files is not writable then reject this commit.
        """
        if not self._current_branch.stream_name:
            return
        prefix = self._current_branch.writable_stream_name + '/'
        for fe_file in fe_commit['files']:
            gwt_path = fe_file['path']
            depot_path = self.ctx.gwt_path(gwt_path).to_depot()
            if depot_path.startswith(prefix):
                continue
            human_msg = _(
                "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'. Paths not in stream '{stream}' are read-only for branch '{b}'."
                ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']), gwt_path
                =gwt_path, depot_path=depot_path, stream=self.
                _current_branch.writable_stream_name, b=self.
                _current_branch.branch_id)
            raise PreflightException(human_msg)

    def _check_stream_in_classic(self, fe_commit):
        """If this is a classic branch, check that none of the files in the commit
        are in stream depots and thus not writable.  If any of the files is not
        writable then reject this commit.
        """
        if self._current_branch.stream_name:
            return
        depot_re = re.compile('^//([^/]+)/([^/]+)/.*$')
        for fe_file in fe_commit['files']:
            gwt_path = fe_file['path']
            depot_path = self.ctx.gwt_path(gwt_path).to_depot()
            m = depot_re.match(depot_path)
            if m:
                depot = m.group(1)
                if depot in self.stream_depots:
                    stream = '//{}/{}'.format(m.group(1), m.group(2))
                    human_msg = _(
                        "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'. Paths in stream '{stream}' are read-only for branch '{b}'."
                        ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']),
                        gwt_path=gwt_path, depot_path=depot_path, stream=
                        stream, b=self._current_branch.branch_id)
                    raise PreflightException(human_msg)
    <mask token>

    def _invalidate_branch_cache(self):
        """We have changed our branch_dict (or more likely
        finish_branch_definition()ed a branch within that dict) in a way that
        invalidates any cached calculations that consumed the branch dict.
        """
        self._cached_overlapping_branch_list = None
    <mask token>

    def _curr_ref_ui_name(self):
        """Return a string suitable for use in progress messages as a
        short hint to what we're processing.

        If we have a PreReceiveTuple with a Git branch/tag name, return
        that name (minus any refs/heads/ prefix). If not, return None.
        """
        if not (self._current_prt and self._current_prt.ref):
            return None
        s = self._current_prt.ref
        prefixes = ['refs/heads/', 'refs/tags/']
        for p in prefixes:
            if s.startswith(p):
                return s[len(p):]
        return s

    def _check_lfs(self, fe_commit, fe_file):
        """If gfe_file is under Git LFS control, require that its
        large file content exist somewhere, either in our upload
        cache (it's new!) or in depot de-dupe storage (already got it).
        """
        if 'sha1' not in fe_file:
            return
        if fe_file.get('mode') not in [FileModeStr.PLAIN, FileModeStr.
            EXECUTABLE]:
            return
        is_tracked = self.ctx.lfs_tracker.is_tracked_git(commit_sha1=
            fe_commit['sha1'], gwt_path=fe_file['path'])
        LOG.debug3(
            '_check_lfs() tracked {lfs}  commit {commit_sha1}  gwt {gwt}'.
            format(commit_sha1=p4gf_util.abbrev(fe_commit['sha1']), lfs=1 if
            is_tracked else 0, gwt=fe_file['path']))
        if not is_tracked:
            lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file)
            if lfs_row:
                raise PreflightException(_(
                    """Push of Git LFS text pointer not tracked by LFS:
commit {commit_sha1} path {gwt_path}"""
                    ).format(commit_sha1=p4gf_util.abbrev(fe_commit['sha1']
                    ), gwt_path=fe_file['path']))
            return
        lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file)
        if not lfs_row:
            return
        if not lfs_row.large_file_source:
            LOG.error('LFS text pointer missing content.')
            LOG.error('LFS   commit {}'.format(p4gf_util.abbrev(fe_commit[
                'sha1'])))
            LOG.error('LFS   lfs oid {}'.format(lfs_row.large_file_oid))
            LOG.error('LFS   ptr {blob_sha1} {blob_mode} {gwt}'.format(
                blob_sha1=p4gf_util.abbrev(fe_file['sha1']), blob_mode=
                p4gf_util.mode_str(fe_file['mode']), gwt=fe_file['path']))
            LOG.error('LFS   upload  {}'.format(lfs_row.to_lfsfs().
                cache_path(self.ctx)))
            LOG.error('LFS   de-dupe {}'.format(lfs_row.to_lfsfs().
                depot_path(self.ctx)))
            raise PreflightException(_(
                """Push of Git LFS text pointer missing content:
commit {commit_sha1} path {gwt_path}"""
                ).format(commit_sha1=p4gf_util.abbrev(fe_commit['sha1']),
                gwt_path=fe_file['path']))
        self.lfs_row_list.append(lfs_row)


class ProtectsChecker:
    """Handle filtering a list of paths against view and protections."""

    def __init__(self, ctx, author, pusher, foruser):
        """Init P4.Map objects for author, pusher, view and combination."""
        self.ctx = ctx
        self.author = author
        self.pusher = pusher
        self.foruser = foruser
        self.ignore_author_perms = ctx.repo_config.getboolean(p4gf_config.
            SECTION_GIT_TO_PERFORCE, p4gf_config.KEY_IGNORE_AUTHOR_PERMS)
        self.view_map = None
        self.write_protect_author = None
        self.write_protect_pusher = None
        self.write_protect_foruser = None
        self.write_protect_fusion = None
        self.write_filter = None
        self.init_view()
        self.init_write_filter()
        self.author_denied = []
        self.pusher_denied = []
        self.foruser_denied = []
        self.fusion_denied = []
        self.unmapped = []

    def init_view(self):
        """Init view map for client."""
        self.view_map = self.ctx.clientmap

    def init_write_filter(self):
        """Init write filter."""
        self.write_protect_author = self.ctx.user_to_protect(self.author
            ).map_for_perm(p4gf_protect.WRITE)
        if self.author != self.pusher or self.ignore_author_perms:
            self.write_protect_pusher = self.ctx.user_to_protect(self.pusher
                ).map_for_perm(p4gf_protect.WRITE)
        if self.foruser:
            self.write_protect_foruser = self.ctx.user_to_protect(self.foruser
                ).map_for_perm(p4gf_protect.WRITE)
        self.write_protect_fusion = self.ctx.user_to_protect(p4gf_const.
            P4GF_USER).map_for_perm(p4gf_protect.WRITE)
        self.write_filter = functools.reduce(P4.Map.join, [vmap for vmap in
            [self.write_protect_author, self.write_protect_pusher, self.
            write_protect_foruser, self.write_protect_fusion, self.view_map
            ] if vmap])

    def filter_paths(self, blobs):
        """Run list of paths through filter and set list of paths that don't pass."""
        self.author_denied = []
        self.pusher_denied = []
        self.foruser_denied = []
        self.fusion_denied = []
        self.unmapped = []
        c2d = P4.Map.RIGHT2LEFT
        LOG.debug('filter_paths() write_filter: %s', self.write_filter)
        for blob in blobs:
            gwt_path = self.ctx.gwt_path(blob['path'])
            topath_c = gwt_path.to_client()
            topath_d = gwt_path.to_depot()
            LOG.debug('filter_paths() topath_d: %s', topath_d)
            result = '  '
            if topath_d and P4GF_DEPOT_OBJECTS_RE.match(topath_d):
                LOG.debug('filter_paths() topath_d in //.git-fusion/objects')
                continue
            if topath_d and P4GF_DEPOT_BRANCHES_RE.match(topath_d):
                LOG.debug('filter_paths() topath_d in //.git-fusion/branches')
                continue
            if not self.write_filter.includes(topath_c, c2d):
                if not self.view_map.includes(topath_c, c2d):
                    self.unmapped.append(topath_c)
                    result = NTR('unmapped')
                elif not (self.ignore_author_perms or self.
                    write_protect_author.includes(topath_d)):
                    self.author_denied.append(topath_c)
                    result = NTR('author denied')
                elif self.write_protect_pusher and not self.write_protect_pusher.includes(
                    topath_d):
                    self.pusher_denied.append(topath_c)
                    result = NTR('pusher denied')
                elif self.write_protect_foruser and not self.write_protect_foruser.includes(
                    topath_d):
                    self.foruser_denied.append(topath_c)
                    result = NTR('foruser denied')
                elif not self.write_protect_fusion.includes(topath_d):
                    self.fusion_denied.append(topath_c)
                    result = NTR('Git Fusion denied')
                else:
                    result = '?'
                LOG.error('filter_paths() {:<13} {}, {}, {}'.format(result,
                    blob['path'], topath_d, topath_c))
            elif LOG.isEnabledFor(logging.DEBUG):
                LOG.debug('filter_paths() topath_c in write_filter: %s',
                    topath_c)

    def has_error(self):
        """Return True if any paths not passed by filters."""
        return len(self.unmapped) or len(self.author_denied) or len(self.
            pusher_denied) or len(self.foruser_denied) or len(self.
            fusion_denied)

    def error_message(self):
        """Return message indicating what's blocking the push."""
        if len(self.unmapped):
            return _('file(s) not in client view')
        if len(self.author_denied):
            restricted_user = self.author
        elif len(self.pusher_denied):
            restricted_user = self.pusher
        elif len(self.foruser_denied):
            restricted_user = self.foruser
        elif len(self.fusion_denied):
            restricted_user = p4gf_const.P4GF_USER
        else:
            restricted_user = _('<unknown>')
        return _("user '{user}' not authorized to submit file(s) in git commit"
            ).format(user=restricted_user)


class PreflightException(Exception):
    """This exception is raised when a push was rejected during preflight
    checks.
    """
    pass


<mask token>

<mask token>


class PreflightChecker:
    <mask token>

    def __init__(self, *, ctx, g2p_user, assigner, gsreview_coll=None,
        already_copied_commit_runner=None, finish_branch_definition=None):
        self.ctx = ctx
        self.g2p_user = g2p_user
        self.assigner = assigner
        self.gsreview_coll = gsreview_coll
        self.already_copied_commit_runner = already_copied_commit_runner
        self._finish_branch_definition = finish_branch_definition
        self.stream_depots = set()
        self._current_prt = None
        self.fast_export_marks = None
        self._current_branch = None
        self._curr_fe_commit = None
        self.set_client_on_branch_switch = True
        self._cached_overlapping_branch_list = None
        self.lfs_row_list = []

    def _already_copied_commit(self, commit_sha1, branch_id):
        """Do we already have a Perforce changelist for this commit,
        this branch_id?
        """
        if not self.already_copied_commit_runner:
            return False
        return self.already_copied_commit_runner.already_copied_commit(
            commit_sha1, branch_id)
    <mask token>

    def check_prt_and_commits(self, prt, commits, marks):
        """Run a preflight check on a set of commits from Git.

        :param prt: pre-receive tuple
        :param commits: FastExport.commits list

        :type marks: :class:`p4gf_fastexport_marks.Marks`
        :param marks: all known fast-export marks

        Raises a PreflightException if anything is out of sorts.

        """
        self._current_prt = prt
        self.fast_export_marks = marks
        self.fast_export_marks.set_head(prt.ref)
        self.check_p4gf_user_write_permission()
        self.check_commits(commits)
    <mask token>
    <mask token>

    def check_commits(self, commits):
        """Ensure the entire sequence of commits will (likely) go through
        without any errors related to permissions or locks. Raises an
        exception if anything goes wrong.

        Arguments:
            commits -- commits from FastExport class
        """
        LOG.info('Checking Perforce permissions and locks')
        self.ctx.checkpoint('copy_to_p4._preflight_check')
        opened = self.ctx.p4.run(['opened', '-m1'])
        if opened:
            raise PreflightException(_(
                'There are files opened by Git Fusion for this repo.'))
        is_read_only = self.ctx.repo_config.getboolean(p4gf_config.
            SECTION_REPO, p4gf_config.KEY_READ_ONLY, fallback=False)
        if is_read_only:
            raise PreflightException(_(
                'Push to repo {repo_name} prohibited.').format(repo_name=
                self.ctx.config.repo_name))
        depots = self.ctx.p4.run(['depots'])
        self.stream_depots = set([d['name'] for d in depots if d['type'] ==
            'stream'])
        any_locked_files = self._find_locked_by()
        LOG.debug('any_locked_files {0}'.format(any_locked_files))
        case_conflict_checker = None
        if not self.ctx.server_is_case_sensitive:
            case_conflict_checker = CaseConflictChecker(self.ctx)
            case_conflict_checker.read_perforce_paths()
        ui_name = self._curr_ref_ui_name()
        if ui_name:
            progress_msg = _('Checking commits for {ref}...').format(ref=
                ui_name)
        else:
            progress_msg = _('Checking commits...')
        with ProgressReporter.Determinate(len(commits)):
            for commit in commits:
                ProgressReporter.increment(progress_msg)
                self.g2p_user.get_author_pusher_owner(commit)
                rev = commit['sha1']
                if not self.assigner.is_assigned(commit['sha1']):
                    continue
                self.check_commit(commit)
                for branch_id in self.assigner.branch_id_list(rev):
                    self.check_commit_for_branch(commit, branch_id,
                        any_locked_files, case_conflict_checker)
        if case_conflict_checker:
            cc_text = case_conflict_checker.conflict_text()
            if cc_text:
                raise PreflightException(cc_text)
    <mask token>

    def check_commit_for_branch(self, commit, branch_id, any_locked_files,
        case_conflict_checker):
        """
        Prior to copying a commit, perform a set of checks for a specific branch
        to ensure the commit will (likely) go through successfully.
        """
        rev = commit['sha1']
        if LOG.isEnabledFor(logging.DEBUG):
            LOG.debug(
                'check_commit_for_branch() Checking branch={} mark={} sha1={} file-ct={} -- {}'
                .format(branch_id, commit['mark'], p4gf_util.abbrev(rev),
                len(commit['files']), repr(commit['data'])[:20].splitlines(
                )[0]))
        if self._already_copied_commit(rev, branch_id):
            return
        self.ensure_branch_preflight(commit, branch_id)
        with self.ctx.switched_to_branch(self._current_branch, set_client=
            self.set_client_on_branch_switch):
            if case_conflict_checker:
                case_conflict_checker.read_fast_export_commit(commit, self.
                    _current_branch)
            if not commit['files'] and not self._is_placeholder_mapped():
                raise PreflightException(_(
                    'Empty commit {sha1} not permitted. Git Fusion branch views must include root to permit empty commits.'
                    ).format(sha1=p4gf_util.abbrev(rev)))
            with Timer(CHECK_PROTECTS):
                self._check_protects(commit['author_p4user'], commit['files'])
            with Timer(CHECK_OVERLAP):
                self._check_overlap(commit)
            if self._current_branch.is_read_only:
                raise PreflightException(_(
                    'Push to branch {branch} prohibited.').format(branch=
                    self._current_branch.git_branch_name))
            self._check_stream_writable(commit)
            self._check_stream_in_classic(commit)
            LOG.debug('checking locked files under //{}/...'.format(self.
                ctx.p4.client))
            if any_locked_files:
                files_in_commit = [self.ctx.gwt_path(f['path']).to_depot() for
                    f in commit['files']]
                LOG.debug('files_in_commit {0}'.format(files_in_commit))
                for f in files_in_commit:
                    if f in any_locked_files:
                        msg = _('{file} - locked by {user}').format(file=f,
                            user=any_locked_files[f])
                        LOG.info(msg)
                        raise PreflightException(msg)
            if self.ctx.preflight_hook.is_callable():
                jobs = G2PJob.extract_jobs(commit['data'])
                jobs2 = G2PJob.lookup_jobs(self.ctx, jobs)
                self.ctx.preflight_hook(ctx=self.ctx, fe_commit=commit,
                    branch_id=branch_id, jobs=jobs2)

    def ensure_branch_preflight(self, commit, branch_id):
        """If not already switched to and synced to the correct branch for the
        given commit, do so.

        If this is a new lightweight branch, perform whatever creation we can do
        at preflight time. We don't have commits/marks for any not-yet-submitted
        parent commits, so the depot_branch_info will often lack a correct
        parent or fully populated basis.

        * depot tree, along with a branch-info file
        * branch mapping, along with entry in p4gf_config2 (if not anonymous)

        Return requested branch
        """
        log = LOG.getChild('ensure_branch_preflight')
        branch = self.ctx.branch_dict().get(branch_id)
        if (self._current_branch and self._current_branch.branch_id ==
            branch_id):
            log.debug('sha={} want branch_id={} curr branch_id={} NOP'.
                format(commit['sha1'][:7], branch_id[:7], self.
                _current_branch.branch_id[:7]))
            log.debug('staying on  branch {}'.format(self.ctx.branch_dict()
                .get(branch_id)))
            return branch
        cbid = (self._current_branch.branch_id if self._current_branch else
            'None')
        log.debug('sha={} want branch_id={} curr branch_id={} switch'.
            format(commit['sha1'][:7], branch_id[:7], cbid[:7]))
        if not branch.view_lines:
            self.finish_branch_definition(commit, branch)
        elif branch.view_p4map:
            if branch.original_view_lines:
                original_view_lines = '\n'.join(branch.original_view_lines)
                view_lines = p4gf_path_convert.convert_view_to_no_client_name(
                    branch.view_lines)
                if not view_lines == original_view_lines:
                    raise PreflightException(_(
                        """Unable to push.  Stream view changed from:
{old_view}
to:
{new_view}"""
                        ).format(old_view=original_view_lines, new_view=
                        view_lines))
            lhs = branch.view_p4map.lhs()
            branch.depot_branch = self.ctx.depot_branch_info_index(
                ).find_depot_path(lhs[0])
        log.debug('switching to branch {}'.format(branch))
        self._current_branch = branch
        return branch

    def _is_placeholder_mapped(self):
        """Does this branch map our placeholder file?

        Returns non-False if mapped, None or empty string if not.
        """
        return self.ctx.gwt_path(p4gf_const.P4GF_EMPTY_CHANGELIST_PLACEHOLDER
            ).to_depot()

    def _check_protects(self, p4user, blobs):
        """check if author is authorized to submit files."""
        pc = ProtectsChecker(self.ctx, p4user, self.ctx.
            authenticated_p4user, self.ctx.foruser)
        pc.filter_paths(blobs)
        if pc.has_error():
            raise PreflightException(pc.error_message())

    def _check_overlap(self, fe_commit):
        """If any of the files in this commit intersect any fully populated branch
        (other than the current branch), then reject this commit.

        Shared/common/overlapping paths in branch views must be read-only from
        Git. Otherwise you end up with a Git push of commit on one Git branch
        inserting  changes into other Git branches behind Git's back.

        To modify shared paths, either do so from Perforce, or create a Git
        Fusion repo with no more than one branch that maps that shared path.
        """
        if self._current_branch not in self._overlapping_branch_list():
            return
        for fe_file in fe_commit['files']:
            gwt_path = fe_file['path']
            depot_path = self.ctx.gwt_to_depot_path(gwt_path)
            for branch in self._overlapping_branch_list():
                if branch == self._current_branch:
                    continue
                if not branch.intersects_depot_path(depot_path):
                    continue
                LOG.debug(
                    '_check_overlap() branch {br1} <> {br2} gwt={gwt:<40}   {dp}\n{view}'
                    .format(br1=p4gf_util.abbrev(self._current_branch.
                    branch_id), br2=p4gf_util.abbrev(branch.branch_id), gwt
                    =gwt_path, dp=depot_path, view='\n'.join(branch.
                    view_p4map.as_array())))
                if (self._current_branch.is_new_fp_from_push or branch.
                    is_new_fp_from_push):
                    current_branch_name = self._current_branch.git_branch_name
                    if self._current_branch.is_new_fp_from_push:
                        current_branch_name += '(new)'
                    other_branch_name = branch.git_branch_name
                    if branch.is_new_fp_from_push:
                        other_branch_name += '(new)'
                    human_msg = _(
                        """Perforce: Cannot commit {sha1} '{gwt_path}' to '{depot_path}'.
 You are attempting to push and create a new fully populated branch
 with paths which overlap another branch. Contact your admin
 to configure non-conflicting destination branch paths.
 Branches: '{b1}', '{b2}'"""
                        ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']),
                        gwt_path=gwt_path, depot_path=depot_path, b1=
                        current_branch_name, b2=other_branch_name)
                else:
                    human_msg = _(
                        "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'. Paths that overlap multiple Git Fusion branches are read-only. Branches: '{b1}', '{b2}'"
                        ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']),
                        gwt_path=gwt_path, depot_path=depot_path, b1=self.
                        _current_branch.branch_id, b2=branch.branch_id)
                raise PreflightException(human_msg)

    def _check_stream_writable(self, fe_commit):
        """If this is a stream branch, check that all files in the commit are
        writable.  If any of the files is not writable then reject this commit.
        """
        if not self._current_branch.stream_name:
            return
        prefix = self._current_branch.writable_stream_name + '/'
        for fe_file in fe_commit['files']:
            gwt_path = fe_file['path']
            depot_path = self.ctx.gwt_path(gwt_path).to_depot()
            if depot_path.startswith(prefix):
                continue
            human_msg = _(
                "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'. Paths not in stream '{stream}' are read-only for branch '{b}'."
                ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']), gwt_path
                =gwt_path, depot_path=depot_path, stream=self.
                _current_branch.writable_stream_name, b=self.
                _current_branch.branch_id)
            raise PreflightException(human_msg)

    def _check_stream_in_classic(self, fe_commit):
        """If this is a classic branch, check that none of the files in the commit
        are in stream depots and thus not writable.  If any of the files is not
        writable then reject this commit.
        """
        if self._current_branch.stream_name:
            return
        depot_re = re.compile('^//([^/]+)/([^/]+)/.*$')
        for fe_file in fe_commit['files']:
            gwt_path = fe_file['path']
            depot_path = self.ctx.gwt_path(gwt_path).to_depot()
            m = depot_re.match(depot_path)
            if m:
                depot = m.group(1)
                if depot in self.stream_depots:
                    stream = '//{}/{}'.format(m.group(1), m.group(2))
                    human_msg = _(
                        "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'. Paths in stream '{stream}' are read-only for branch '{b}'."
                        ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']),
                        gwt_path=gwt_path, depot_path=depot_path, stream=
                        stream, b=self._current_branch.branch_id)
                    raise PreflightException(human_msg)

    def _path_added(self, path, fecommit):
        """Return True if the named path was introduced in the HEAD commit.

        :param self: this object
        :param path: repo path to be evaluated.
        :param fecommit: commit object from fast-export parser.

        """
        commit = self.ctx.repo.get(fecommit['sha1'])
        if commit is None:
            LOG.debug2('_path_added() commit {} is missing'.format(fecommit
                ['sha1']))
            return True
        for parent in commit.parents:
            if p4gf_git.exists_in_tree(self.ctx.repo, path, parent.tree):
                LOG.debug2('_path_added() {} exists in parent tree {}'.
                    format(path, p4gf_util.abbrev(p4gf_pygit2.
                    object_to_sha1(parent))))
                return False
        return True

    def _invalidate_branch_cache(self):
        """We have changed our branch_dict (or more likely
        finish_branch_definition()ed a branch within that dict) in a way that
        invalidates any cached calculations that consumed the branch dict.
        """
        self._cached_overlapping_branch_list = None
    <mask token>

    def _curr_ref_ui_name(self):
        """Return a string suitable for use in progress messages as a
        short hint to what we're processing.

        If we have a PreReceiveTuple with a Git branch/tag name, return
        that name (minus any refs/heads/ prefix). If not, return None.
        """
        if not (self._current_prt and self._current_prt.ref):
            return None
        s = self._current_prt.ref
        prefixes = ['refs/heads/', 'refs/tags/']
        for p in prefixes:
            if s.startswith(p):
                return s[len(p):]
        return s

    def _check_lfs(self, fe_commit, fe_file):
        """If gfe_file is under Git LFS control, require that its
        large file content exist somewhere, either in our upload
        cache (it's new!) or in depot de-dupe storage (already got it).
        """
        if 'sha1' not in fe_file:
            return
        if fe_file.get('mode') not in [FileModeStr.PLAIN, FileModeStr.
            EXECUTABLE]:
            return
        is_tracked = self.ctx.lfs_tracker.is_tracked_git(commit_sha1=
            fe_commit['sha1'], gwt_path=fe_file['path'])
        LOG.debug3(
            '_check_lfs() tracked {lfs}  commit {commit_sha1}  gwt {gwt}'.
            format(commit_sha1=p4gf_util.abbrev(fe_commit['sha1']), lfs=1 if
            is_tracked else 0, gwt=fe_file['path']))
        if not is_tracked:
            lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file)
            if lfs_row:
                raise PreflightException(_(
                    """Push of Git LFS text pointer not tracked by LFS:
commit {commit_sha1} path {gwt_path}"""
                    ).format(commit_sha1=p4gf_util.abbrev(fe_commit['sha1']
                    ), gwt_path=fe_file['path']))
            return
        lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file)
        if not lfs_row:
            return
        if not lfs_row.large_file_source:
            LOG.error('LFS text pointer missing content.')
            LOG.error('LFS   commit {}'.format(p4gf_util.abbrev(fe_commit[
                'sha1'])))
            LOG.error('LFS   lfs oid {}'.format(lfs_row.large_file_oid))
            LOG.error('LFS   ptr {blob_sha1} {blob_mode} {gwt}'.format(
                blob_sha1=p4gf_util.abbrev(fe_file['sha1']), blob_mode=
                p4gf_util.mode_str(fe_file['mode']), gwt=fe_file['path']))
            LOG.error('LFS   upload  {}'.format(lfs_row.to_lfsfs().
                cache_path(self.ctx)))
            LOG.error('LFS   de-dupe {}'.format(lfs_row.to_lfsfs().
                depot_path(self.ctx)))
            raise PreflightException(_(
                """Push of Git LFS text pointer missing content:
commit {commit_sha1} path {gwt_path}"""
                ).format(commit_sha1=p4gf_util.abbrev(fe_commit['sha1']),
                gwt_path=fe_file['path']))
        self.lfs_row_list.append(lfs_row)


class ProtectsChecker:
    """Handle filtering a list of paths against view and protections."""

    def __init__(self, ctx, author, pusher, foruser):
        """Init P4.Map objects for author, pusher, view and combination."""
        self.ctx = ctx
        self.author = author
        self.pusher = pusher
        self.foruser = foruser
        self.ignore_author_perms = ctx.repo_config.getboolean(p4gf_config.
            SECTION_GIT_TO_PERFORCE, p4gf_config.KEY_IGNORE_AUTHOR_PERMS)
        self.view_map = None
        self.write_protect_author = None
        self.write_protect_pusher = None
        self.write_protect_foruser = None
        self.write_protect_fusion = None
        self.write_filter = None
        self.init_view()
        self.init_write_filter()
        self.author_denied = []
        self.pusher_denied = []
        self.foruser_denied = []
        self.fusion_denied = []
        self.unmapped = []

    def init_view(self):
        """Init view map for client."""
        self.view_map = self.ctx.clientmap

    def init_write_filter(self):
        """Init write filter."""
        self.write_protect_author = self.ctx.user_to_protect(self.author
            ).map_for_perm(p4gf_protect.WRITE)
        if self.author != self.pusher or self.ignore_author_perms:
            self.write_protect_pusher = self.ctx.user_to_protect(self.pusher
                ).map_for_perm(p4gf_protect.WRITE)
        if self.foruser:
            self.write_protect_foruser = self.ctx.user_to_protect(self.foruser
                ).map_for_perm(p4gf_protect.WRITE)
        self.write_protect_fusion = self.ctx.user_to_protect(p4gf_const.
            P4GF_USER).map_for_perm(p4gf_protect.WRITE)
        self.write_filter = functools.reduce(P4.Map.join, [vmap for vmap in
            [self.write_protect_author, self.write_protect_pusher, self.
            write_protect_foruser, self.write_protect_fusion, self.view_map
            ] if vmap])

    def filter_paths(self, blobs):
        """Run list of paths through filter and set list of paths that don't pass."""
        self.author_denied = []
        self.pusher_denied = []
        self.foruser_denied = []
        self.fusion_denied = []
        self.unmapped = []
        c2d = P4.Map.RIGHT2LEFT
        LOG.debug('filter_paths() write_filter: %s', self.write_filter)
        for blob in blobs:
            gwt_path = self.ctx.gwt_path(blob['path'])
            topath_c = gwt_path.to_client()
            topath_d = gwt_path.to_depot()
            LOG.debug('filter_paths() topath_d: %s', topath_d)
            result = '  '
            if topath_d and P4GF_DEPOT_OBJECTS_RE.match(topath_d):
                LOG.debug('filter_paths() topath_d in //.git-fusion/objects')
                continue
            if topath_d and P4GF_DEPOT_BRANCHES_RE.match(topath_d):
                LOG.debug('filter_paths() topath_d in //.git-fusion/branches')
                continue
            if not self.write_filter.includes(topath_c, c2d):
                if not self.view_map.includes(topath_c, c2d):
                    self.unmapped.append(topath_c)
                    result = NTR('unmapped')
                elif not (self.ignore_author_perms or self.
                    write_protect_author.includes(topath_d)):
                    self.author_denied.append(topath_c)
                    result = NTR('author denied')
                elif self.write_protect_pusher and not self.write_protect_pusher.includes(
                    topath_d):
                    self.pusher_denied.append(topath_c)
                    result = NTR('pusher denied')
                elif self.write_protect_foruser and not self.write_protect_foruser.includes(
                    topath_d):
                    self.foruser_denied.append(topath_c)
                    result = NTR('foruser denied')
                elif not self.write_protect_fusion.includes(topath_d):
                    self.fusion_denied.append(topath_c)
                    result = NTR('Git Fusion denied')
                else:
                    result = '?'
                LOG.error('filter_paths() {:<13} {}, {}, {}'.format(result,
                    blob['path'], topath_d, topath_c))
            elif LOG.isEnabledFor(logging.DEBUG):
                LOG.debug('filter_paths() topath_c in write_filter: %s',
                    topath_c)

    def has_error(self):
        """Return True if any paths not passed by filters."""
        return len(self.unmapped) or len(self.author_denied) or len(self.
            pusher_denied) or len(self.foruser_denied) or len(self.
            fusion_denied)

    def error_message(self):
        """Return message indicating what's blocking the push."""
        if len(self.unmapped):
            return _('file(s) not in client view')
        if len(self.author_denied):
            restricted_user = self.author
        elif len(self.pusher_denied):
            restricted_user = self.pusher
        elif len(self.foruser_denied):
            restricted_user = self.foruser
        elif len(self.fusion_denied):
            restricted_user = p4gf_const.P4GF_USER
        else:
            restricted_user = _('<unknown>')
        return _("user '{user}' not authorized to submit file(s) in git commit"
            ).format(user=restricted_user)


class PreflightException(Exception):
    """This exception is raised when a push was rejected during preflight
    checks.
    """
    pass


<mask token>

<mask token>


class PreflightChecker:
    <mask token>

    def __init__(self, *, ctx, g2p_user, assigner, gsreview_coll=None,
        already_copied_commit_runner=None, finish_branch_definition=None):
        self.ctx = ctx
        self.g2p_user = g2p_user
        self.assigner = assigner
        self.gsreview_coll = gsreview_coll
        self.already_copied_commit_runner = already_copied_commit_runner
        self._finish_branch_definition = finish_branch_definition
        self.stream_depots = set()
        self._current_prt = None
        self.fast_export_marks = None
        self._current_branch = None
        self._curr_fe_commit = None
        self.set_client_on_branch_switch = True
        self._cached_overlapping_branch_list = None
        self.lfs_row_list = []

    def _already_copied_commit(self, commit_sha1, branch_id):
        """Do we already have a Perforce changelist for this commit,
        this branch_id?
        """
        if not self.already_copied_commit_runner:
            return False
        return self.already_copied_commit_runner.already_copied_commit(
            commit_sha1, branch_id)

    def finish_branch_definition(self, commit, branch):
        """Inflate a partially-created Branch with a proper depot branch to
        hold its files, and a branch view to map that depot branch to
        the Git work tree.
        """
        assert self._finish_branch_definition
        self._finish_branch_definition.finish_branch_definition(commit, branch)
        self._invalidate_branch_cache()

    def check_prt_and_commits(self, prt, commits, marks):
        """Run a preflight check on a set of commits from Git.

        :param prt: pre-receive tuple
        :param commits: FastExport.commits list

        :type marks: :class:`p4gf_fastexport_marks.Marks`
        :param marks: all known fast-export marks

        Raises a PreflightException if anything is out of sorts.

        """
        self._current_prt = prt
        self.fast_export_marks = marks
        self.fast_export_marks.set_head(prt.ref)
        self.check_p4gf_user_write_permission()
        self.check_commits(commits)

    def check_p4gf_user_write_permission(self):
        """Ensure git-fusion-user has permissions to write to depot."""
        gf_client_map = P4.Map()
        gf_client_map.insert('//...', '//client/...')
        utp = p4gf_protect.UserToProtect(self.ctx.p4)
        prot = utp.user_to_protect(p4gf_const.P4GF_USER)
        gf_write_filter = prot.map_for_perm(p4gf_protect.WRITE)
        gf_write_filter = P4.Map.join(gf_write_filter, gf_client_map)
        if not gf_write_filter.includes('//{depot}/...'.format(depot=
            p4gf_const.P4GF_DEPOT)):
            raise RuntimeError(_('permission denied'))

    def _find_locked_by(self):
        """Return a dict of depot_path => user of any locked files."""
        fstat_flags = NTR('otherLock | otherOpen0 & headType=*+l')
        any_locked_files = {}
        for branch_chunk in self.ctx.iter_writable_branch_chunks():
            bvl = [b for b in branch_chunk if b.view_lines]
            if not bvl:
                continue
            with self.ctx.switched_to_union(bvl):
                r = self.ctx.p4run('fstat', '-F', fstat_flags, '-m1',
                    '//{}/...'.format(self.ctx.p4.client), log_warnings=
                    logging.DEBUG)
                for lf in r:
                    user = lf['otherOpen'][0] if 'otherOpen' in lf else NTR(
                        '<unknown>')
                    any_locked_files[lf['depotFile']] = user
        return any_locked_files

    def check_commits(self, commits):
        """Ensure the entire sequence of commits will (likely) go through
        without any errors related to permissions or locks. Raises an
        exception if anything goes wrong.

        Arguments:
            commits -- commits from FastExport class
        """
        LOG.info('Checking Perforce permissions and locks')
        self.ctx.checkpoint('copy_to_p4._preflight_check')
        opened = self.ctx.p4.run(['opened', '-m1'])
        if opened:
            raise PreflightException(_(
                'There are files opened by Git Fusion for this repo.'))
        is_read_only = self.ctx.repo_config.getboolean(p4gf_config.
            SECTION_REPO, p4gf_config.KEY_READ_ONLY, fallback=False)
        if is_read_only:
            raise PreflightException(_(
                'Push to repo {repo_name} prohibited.').format(repo_name=
                self.ctx.config.repo_name))
        depots = self.ctx.p4.run(['depots'])
        self.stream_depots = set([d['name'] for d in depots if d['type'] ==
            'stream'])
        any_locked_files = self._find_locked_by()
        LOG.debug('any_locked_files {0}'.format(any_locked_files))
        case_conflict_checker = None
        if not self.ctx.server_is_case_sensitive:
            case_conflict_checker = CaseConflictChecker(self.ctx)
            case_conflict_checker.read_perforce_paths()
        ui_name = self._curr_ref_ui_name()
        if ui_name:
            progress_msg = _('Checking commits for {ref}...').format(ref=
                ui_name)
        else:
            progress_msg = _('Checking commits...')
        with ProgressReporter.Determinate(len(commits)):
            for commit in commits:
                ProgressReporter.increment(progress_msg)
                self.g2p_user.get_author_pusher_owner(commit)
                rev = commit['sha1']
                if not self.assigner.is_assigned(commit['sha1']):
                    continue
                self.check_commit(commit)
                for branch_id in self.assigner.branch_id_list(rev):
                    self.check_commit_for_branch(commit, branch_id,
                        any_locked_files, case_conflict_checker)
        if case_conflict_checker:
            cc_text = case_conflict_checker.conflict_text()
            if cc_text:
                raise PreflightException(cc_text)

    def check_commit(self, commit):
        """Prior to copying a commit, perform a set of checks to ensure the commit
        will (likely) go through successfully.
        This includes:
            * verifying permission to commit for author p4 user
            * screening for merge commits
            * screening for submodules
            * checking valid filenames
            * checking write permissions for each file
        """
        if LOG.isEnabledFor(logging.DEBUG):
            LOG.debug(
                'check_commit() Checking mark={} sha1={} file-ct={} -- {}'.
                format(commit['mark'], p4gf_util.abbrev(commit['sha1']),
                len(commit['files']), repr(commit['data'])[:20].splitlines(
                )[0]))
        if not commit['author_p4user']:
            raise PreflightException(_(
                "User '{user}' not permitted to commit").format(user=commit
                ['author']['email'].strip('<>')))
        if 'merge' in commit:
            ref_is_review = (self.gsreview_coll and self.gsreview_coll.
                ref_in_review_list(self._current_prt.ref))
            if not ref_is_review and not self.ctx.merge_commits:
                raise PreflightException(_(
                    'Merge commits are not enabled for this repo.'))
            if (not ref_is_review and not self.ctx.branch_creation and self
                .assigner.have_anonymous_branches):
                msg = _('Git branch creation is prohibited for this repo.')
                p4_branch_names_non_lw = [b.git_branch_name for b in self.
                    ctx.branch_dict().values() if b.git_branch_name and not
                    b.is_lightweight]
                if len(p4_branch_names_non_lw) > 1:
                    msg += _(
                        """
This repo has more than one named branch.
Try altering the push order - pushing branches with merge ancestors first."""
                        )
                raise PreflightException(msg)
            if LOG.isEnabledFor(logging.DEBUG):
                for parent_mark in commit['merge']:
                    parent_sha1 = self.fast_export_marks.get_commit(parent_mark
                        )[:7]
                    LOG.debug('check_commit() merge mark={} sha1={}'.format
                        (parent_mark, parent_sha1))
        if not self.ctx.submodules and 'files' in commit:
            for f in commit['files']:
                if f.get('mode') == '160000':
                    if 'first_commit' in commit and not self._path_added(f.
                        get('path'), commit):
                        LOG.debug2('check_commit() passed {} in {}'.format(
                            f.get('path'), p4gf_util.abbrev(commit['sha1'])))
                        continue
                    raise PreflightException(_(
                        'Git submodules not permitted: path={path} commit={commit_sha1}'
                        ).format(path=f.get('path'), commit_sha1=p4gf_util.
                        abbrev(commit['sha1'])))
        for f in commit['files']:
            LOG.debug3('check_commit : commit files: ' + _log_fe_file(f))
            err = check_valid_filename(f['path'], self.ctx)
            if err:
                raise PreflightException(err)
            if self.ctx.is_lfs_enabled:
                self._check_lfs(commit, f)
        jobs = G2PJob.lookup_jobs(self.ctx, G2PJob.extract_jobs(commit['data'])
            )
        if jobs:
            for job_id in jobs:
                r = self.ctx.p4run('jobs', '-e', 'job={}'.format(job_id))
                if not r:
                    _print_error(_("Job '{job_id}' doesn't exist").format(
                        job_id=job_id))

    def check_commit_for_branch(self, commit, branch_id, any_locked_files,
        case_conflict_checker):
        """
        Prior to copying a commit, perform a set of checks for a specific branch
        to ensure the commit will (likely) go through successfully.
        """
        rev = commit['sha1']
        if LOG.isEnabledFor(logging.DEBUG):
            LOG.debug(
                'check_commit_for_branch() Checking branch={} mark={} sha1={} file-ct={} -- {}'
                .format(branch_id, commit['mark'], p4gf_util.abbrev(rev),
                len(commit['files']), repr(commit['data'])[:20].splitlines(
                )[0]))
        if self._already_copied_commit(rev, branch_id):
            return
        self.ensure_branch_preflight(commit, branch_id)
        with self.ctx.switched_to_branch(self._current_branch, set_client=
            self.set_client_on_branch_switch):
            if case_conflict_checker:
                case_conflict_checker.read_fast_export_commit(commit, self.
                    _current_branch)
            if not commit['files'] and not self._is_placeholder_mapped():
                raise PreflightException(_(
                    'Empty commit {sha1} not permitted. Git Fusion branch views must include root to permit empty commits.'
                    ).format(sha1=p4gf_util.abbrev(rev)))
            with Timer(CHECK_PROTECTS):
                self._check_protects(commit['author_p4user'], commit['files'])
            with Timer(CHECK_OVERLAP):
                self._check_overlap(commit)
            if self._current_branch.is_read_only:
                raise PreflightException(_(
                    'Push to branch {branch} prohibited.').format(branch=
                    self._current_branch.git_branch_name))
            self._check_stream_writable(commit)
            self._check_stream_in_classic(commit)
            LOG.debug('checking locked files under //{}/...'.format(self.
                ctx.p4.client))
            if any_locked_files:
                files_in_commit = [self.ctx.gwt_path(f['path']).to_depot() for
                    f in commit['files']]
                LOG.debug('files_in_commit {0}'.format(files_in_commit))
                for f in files_in_commit:
                    if f in any_locked_files:
                        msg = _('{file} - locked by {user}').format(file=f,
                            user=any_locked_files[f])
                        LOG.info(msg)
                        raise PreflightException(msg)
            if self.ctx.preflight_hook.is_callable():
                jobs = G2PJob.extract_jobs(commit['data'])
                jobs2 = G2PJob.lookup_jobs(self.ctx, jobs)
                self.ctx.preflight_hook(ctx=self.ctx, fe_commit=commit,
                    branch_id=branch_id, jobs=jobs2)

    def ensure_branch_preflight(self, commit, branch_id):
        """If not already switched to and synced to the correct branch for the
        given commit, do so.

        If this is a new lightweight branch, perform whatever creation we can do
        at preflight time. We don't have commits/marks for any not-yet-submitted
        parent commits, so the depot_branch_info will often lack a correct
        parent or fully populated basis.

        * depot tree, along with a branch-info file
        * branch mapping, along with entry in p4gf_config2 (if not anonymous)

        Return requested branch
        """
        log = LOG.getChild('ensure_branch_preflight')
        branch = self.ctx.branch_dict().get(branch_id)
        if (self._current_branch and self._current_branch.branch_id ==
            branch_id):
            log.debug('sha={} want branch_id={} curr branch_id={} NOP'.
                format(commit['sha1'][:7], branch_id[:7], self.
                _current_branch.branch_id[:7]))
            log.debug('staying on  branch {}'.format(self.ctx.branch_dict()
                .get(branch_id)))
            return branch
        cbid = (self._current_branch.branch_id if self._current_branch else
            'None')
        log.debug('sha={} want branch_id={} curr branch_id={} switch'.
            format(commit['sha1'][:7], branch_id[:7], cbid[:7]))
        if not branch.view_lines:
            self.finish_branch_definition(commit, branch)
        elif branch.view_p4map:
            if branch.original_view_lines:
                original_view_lines = '\n'.join(branch.original_view_lines)
                view_lines = p4gf_path_convert.convert_view_to_no_client_name(
                    branch.view_lines)
                if not view_lines == original_view_lines:
                    raise PreflightException(_(
                        """Unable to push.  Stream view changed from:
{old_view}
to:
{new_view}"""
                        ).format(old_view=original_view_lines, new_view=
                        view_lines))
            lhs = branch.view_p4map.lhs()
            branch.depot_branch = self.ctx.depot_branch_info_index(
                ).find_depot_path(lhs[0])
        log.debug('switching to branch {}'.format(branch))
        self._current_branch = branch
        return branch

    def _is_placeholder_mapped(self):
        """Does this branch map our placeholder file?

        Returns non-False if mapped, None or empty string if not.
        """
        return self.ctx.gwt_path(p4gf_const.P4GF_EMPTY_CHANGELIST_PLACEHOLDER
            ).to_depot()

    def _check_protects(self, p4user, blobs):
        """check if author is authorized to submit files."""
        pc = ProtectsChecker(self.ctx, p4user, self.ctx.
            authenticated_p4user, self.ctx.foruser)
        pc.filter_paths(blobs)
        if pc.has_error():
            raise PreflightException(pc.error_message())

    def _check_overlap(self, fe_commit):
        """If any of the files in this commit intersect any fully populated branch
        (other than the current branch), then reject this commit.

        Shared/common/overlapping paths in branch views must be read-only from
        Git. Otherwise you end up with a Git push of commit on one Git branch
        inserting  changes into other Git branches behind Git's back.

        To modify shared paths, either do so from Perforce, or create a Git
        Fusion repo with no more than one branch that maps that shared path.
        """
        if self._current_branch not in self._overlapping_branch_list():
            return
        for fe_file in fe_commit['files']:
            gwt_path = fe_file['path']
            depot_path = self.ctx.gwt_to_depot_path(gwt_path)
            for branch in self._overlapping_branch_list():
                if branch == self._current_branch:
                    continue
                if not branch.intersects_depot_path(depot_path):
                    continue
                LOG.debug(
                    '_check_overlap() branch {br1} <> {br2} gwt={gwt:<40}   {dp}\n{view}'
                    .format(br1=p4gf_util.abbrev(self._current_branch.
                    branch_id), br2=p4gf_util.abbrev(branch.branch_id), gwt
                    =gwt_path, dp=depot_path, view='\n'.join(branch.
                    view_p4map.as_array())))
                if (self._current_branch.is_new_fp_from_push or branch.
                    is_new_fp_from_push):
                    current_branch_name = self._current_branch.git_branch_name
                    if self._current_branch.is_new_fp_from_push:
                        current_branch_name += '(new)'
                    other_branch_name = branch.git_branch_name
                    if branch.is_new_fp_from_push:
                        other_branch_name += '(new)'
                    human_msg = _(
                        """Perforce: Cannot commit {sha1} '{gwt_path}' to '{depot_path}'.
 You are attempting to push and create a new fully populated branch
 with paths which overlap another branch. Contact your admin
 to configure non-conflicting destination branch paths.
 Branches: '{b1}', '{b2}'"""
                        ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']),
                        gwt_path=gwt_path, depot_path=depot_path, b1=
                        current_branch_name, b2=other_branch_name)
                else:
                    human_msg = _(
                        "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'. Paths that overlap multiple Git Fusion branches are read-only. Branches: '{b1}', '{b2}'"
                        ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']),
                        gwt_path=gwt_path, depot_path=depot_path, b1=self.
                        _current_branch.branch_id, b2=branch.branch_id)
                raise PreflightException(human_msg)

    def _check_stream_writable(self, fe_commit):
        """If this is a stream branch, check that all files in the commit are
        writable.  If any of the files is not writable then reject this commit.
        """
        if not self._current_branch.stream_name:
            return
        prefix = self._current_branch.writable_stream_name + '/'
        for fe_file in fe_commit['files']:
            gwt_path = fe_file['path']
            depot_path = self.ctx.gwt_path(gwt_path).to_depot()
            if depot_path.startswith(prefix):
                continue
            human_msg = _(
                "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'. Paths not in stream '{stream}' are read-only for branch '{b}'."
                ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']), gwt_path
                =gwt_path, depot_path=depot_path, stream=self.
                _current_branch.writable_stream_name, b=self.
                _current_branch.branch_id)
            raise PreflightException(human_msg)

    def _check_stream_in_classic(self, fe_commit):
        """If this is a classic branch, check that none of the files in the commit
        are in stream depots and thus not writable.  If any of the files is not
        writable then reject this commit.
        """
        if self._current_branch.stream_name:
            return
        depot_re = re.compile('^//([^/]+)/([^/]+)/.*$')
        for fe_file in fe_commit['files']:
            gwt_path = fe_file['path']
            depot_path = self.ctx.gwt_path(gwt_path).to_depot()
            m = depot_re.match(depot_path)
            if m:
                depot = m.group(1)
                if depot in self.stream_depots:
                    stream = '//{}/{}'.format(m.group(1), m.group(2))
                    human_msg = _(
                        "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'. Paths in stream '{stream}' are read-only for branch '{b}'."
                        ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']),
                        gwt_path=gwt_path, depot_path=depot_path, stream=
                        stream, b=self._current_branch.branch_id)
                    raise PreflightException(human_msg)

    def _path_added(self, path, fecommit):
        """Return True if the named path was introduced in the HEAD commit.

        :param self: this object
        :param path: repo path to be evaluated.
        :param fecommit: commit object from fast-export parser.

        """
        commit = self.ctx.repo.get(fecommit['sha1'])
        if commit is None:
            LOG.debug2('_path_added() commit {} is missing'.format(fecommit
                ['sha1']))
            return True
        for parent in commit.parents:
            if p4gf_git.exists_in_tree(self.ctx.repo, path, parent.tree):
                LOG.debug2('_path_added() {} exists in parent tree {}'.
                    format(path, p4gf_util.abbrev(p4gf_pygit2.
                    object_to_sha1(parent))))
                return False
        return True

    def _invalidate_branch_cache(self):
        """We have changed our branch_dict (or more likely
        finish_branch_definition()ed a branch within that dict) in a way that
        invalidates any cached calculations that consumed the branch dict.
        """
        self._cached_overlapping_branch_list = None

    def _overlapping_branch_list(self):
        """Return a list of fully populated branches that overlap
        other fully populated branches.

        Caches the result because we check every file revision
        path for overlap, and for huge repos with thousands of
        non-overlapping LW branches, just iterating through the
        branch list starts to waste measurable CPU time.
        """
        if self._cached_overlapping_branch_list is not None:
            return self._cached_overlapping_branch_list
        have_overlap = set()
        for outer in p4gf_branch.iter_fp_non_deleted(self.ctx.branch_dict()):
            outer_lhs = P4.Map()
            outer_lhs.insert(outer.view_p4map.lhs())
            for inner in p4gf_branch.iter_fp_non_deleted(self.ctx.branch_dict()
                ):
                if outer == inner:
                    continue
                overlap = P4.Map.join(outer_lhs, inner.view_p4map)
                for line in overlap.as_array():
                    if line.startswith('-') or line.startswith('"-'):
                        continue
                    have_overlap.add(outer)
                    have_overlap.add(inner)
                    break
        self._cached_overlapping_branch_list = have_overlap
        return self._cached_overlapping_branch_list

    def _curr_ref_ui_name(self):
        """Return a string suitable for use in progress messages as a
        short hint to what we're processing.

        If we have a PreReceiveTuple with a Git branch/tag name, return
        that name (minus any refs/heads/ prefix). If not, return None.
        """
        if not (self._current_prt and self._current_prt.ref):
            return None
        s = self._current_prt.ref
        prefixes = ['refs/heads/', 'refs/tags/']
        for p in prefixes:
            if s.startswith(p):
                return s[len(p):]
        return s

    def _check_lfs(self, fe_commit, fe_file):
        """If gfe_file is under Git LFS control, require that its
        large file content exist somewhere, either in our upload
        cache (it's new!) or in depot de-dupe storage (already got it).
        """
        if 'sha1' not in fe_file:
            return
        if fe_file.get('mode') not in [FileModeStr.PLAIN, FileModeStr.
            EXECUTABLE]:
            return
        is_tracked = self.ctx.lfs_tracker.is_tracked_git(commit_sha1=
            fe_commit['sha1'], gwt_path=fe_file['path'])
        LOG.debug3(
            '_check_lfs() tracked {lfs}  commit {commit_sha1}  gwt {gwt}'.
            format(commit_sha1=p4gf_util.abbrev(fe_commit['sha1']), lfs=1 if
            is_tracked else 0, gwt=fe_file['path']))
        if not is_tracked:
            lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file)
            if lfs_row:
                raise PreflightException(_(
                    """Push of Git LFS text pointer not tracked by LFS:
commit {commit_sha1} path {gwt_path}"""
                    ).format(commit_sha1=p4gf_util.abbrev(fe_commit['sha1']
                    ), gwt_path=fe_file['path']))
            return
        lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file)
        if not lfs_row:
            return
        if not lfs_row.large_file_source:
            LOG.error('LFS text pointer missing content.')
            LOG.error('LFS   commit {}'.format(p4gf_util.abbrev(fe_commit[
                'sha1'])))
            LOG.error('LFS   lfs oid {}'.format(lfs_row.large_file_oid))
            LOG.error('LFS   ptr {blob_sha1} {blob_mode} {gwt}'.format(
                blob_sha1=p4gf_util.abbrev(fe_file['sha1']), blob_mode=
                p4gf_util.mode_str(fe_file['mode']), gwt=fe_file['path']))
            LOG.error('LFS   upload  {}'.format(lfs_row.to_lfsfs().
                cache_path(self.ctx)))
            LOG.error('LFS   de-dupe {}'.format(lfs_row.to_lfsfs().
                depot_path(self.ctx)))
            raise PreflightException(_(
                """Push of Git LFS text pointer missing content:
commit {commit_sha1} path {gwt_path}"""
                ).format(commit_sha1=p4gf_util.abbrev(fe_commit['sha1']),
                gwt_path=fe_file['path']))
        self.lfs_row_list.append(lfs_row)


class ProtectsChecker:
    """Handle filtering a list of paths against view and protections."""

    def __init__(self, ctx, author, pusher, foruser):
        """Init P4.Map objects for author, pusher, view and combination."""
        self.ctx = ctx
        self.author = author
        self.pusher = pusher
        self.foruser = foruser
        self.ignore_author_perms = ctx.repo_config.getboolean(p4gf_config.
            SECTION_GIT_TO_PERFORCE, p4gf_config.KEY_IGNORE_AUTHOR_PERMS)
        self.view_map = None
        self.write_protect_author = None
        self.write_protect_pusher = None
        self.write_protect_foruser = None
        self.write_protect_fusion = None
        self.write_filter = None
        self.init_view()
        self.init_write_filter()
        self.author_denied = []
        self.pusher_denied = []
        self.foruser_denied = []
        self.fusion_denied = []
        self.unmapped = []

    def init_view(self):
        """Init view map for client."""
        self.view_map = self.ctx.clientmap

    def init_write_filter(self):
        """Init write filter."""
        self.write_protect_author = self.ctx.user_to_protect(self.author
            ).map_for_perm(p4gf_protect.WRITE)
        if self.author != self.pusher or self.ignore_author_perms:
            self.write_protect_pusher = self.ctx.user_to_protect(self.pusher
                ).map_for_perm(p4gf_protect.WRITE)
        if self.foruser:
            self.write_protect_foruser = self.ctx.user_to_protect(self.foruser
                ).map_for_perm(p4gf_protect.WRITE)
        self.write_protect_fusion = self.ctx.user_to_protect(p4gf_const.
            P4GF_USER).map_for_perm(p4gf_protect.WRITE)
        self.write_filter = functools.reduce(P4.Map.join, [vmap for vmap in
            [self.write_protect_author, self.write_protect_pusher, self.
            write_protect_foruser, self.write_protect_fusion, self.view_map
            ] if vmap])

    def filter_paths(self, blobs):
        """Run list of paths through filter and set list of paths that don't pass."""
        self.author_denied = []
        self.pusher_denied = []
        self.foruser_denied = []
        self.fusion_denied = []
        self.unmapped = []
        c2d = P4.Map.RIGHT2LEFT
        LOG.debug('filter_paths() write_filter: %s', self.write_filter)
        for blob in blobs:
            gwt_path = self.ctx.gwt_path(blob['path'])
            topath_c = gwt_path.to_client()
            topath_d = gwt_path.to_depot()
            LOG.debug('filter_paths() topath_d: %s', topath_d)
            result = '  '
            if topath_d and P4GF_DEPOT_OBJECTS_RE.match(topath_d):
                LOG.debug('filter_paths() topath_d in //.git-fusion/objects')
                continue
            if topath_d and P4GF_DEPOT_BRANCHES_RE.match(topath_d):
                LOG.debug('filter_paths() topath_d in //.git-fusion/branches')
                continue
            if not self.write_filter.includes(topath_c, c2d):
                if not self.view_map.includes(topath_c, c2d):
                    self.unmapped.append(topath_c)
                    result = NTR('unmapped')
                elif not (self.ignore_author_perms or self.
                    write_protect_author.includes(topath_d)):
                    self.author_denied.append(topath_c)
                    result = NTR('author denied')
                elif self.write_protect_pusher and not self.write_protect_pusher.includes(
                    topath_d):
                    self.pusher_denied.append(topath_c)
                    result = NTR('pusher denied')
                elif self.write_protect_foruser and not self.write_protect_foruser.includes(
                    topath_d):
                    self.foruser_denied.append(topath_c)
                    result = NTR('foruser denied')
                elif not self.write_protect_fusion.includes(topath_d):
                    self.fusion_denied.append(topath_c)
                    result = NTR('Git Fusion denied')
                else:
                    result = '?'
                LOG.error('filter_paths() {:<13} {}, {}, {}'.format(result,
                    blob['path'], topath_d, topath_c))
            elif LOG.isEnabledFor(logging.DEBUG):
                LOG.debug('filter_paths() topath_c in write_filter: %s',
                    topath_c)

    def has_error(self):
        """Return True if any paths not passed by filters."""
        return len(self.unmapped) or len(self.author_denied) or len(self.
            pusher_denied) or len(self.foruser_denied) or len(self.
            fusion_denied)

    def error_message(self):
        """Return message indicating what's blocking the push."""
        if len(self.unmapped):
            return _('file(s) not in client view')
        if len(self.author_denied):
            restricted_user = self.author
        elif len(self.pusher_denied):
            restricted_user = self.pusher
        elif len(self.foruser_denied):
            restricted_user = self.foruser
        elif len(self.fusion_denied):
            restricted_user = p4gf_const.P4GF_USER
        else:
            restricted_user = _('<unknown>')
        return _("user '{user}' not authorized to submit file(s) in git commit"
            ).format(user=restricted_user)


class PreflightException(Exception):
    """This exception is raised when a push was rejected during preflight
    checks.
    """
    pass


<mask token>

<mask token>


class PreflightChecker:
    """A class that knows how to check a 'git push' for things that
    should not be copied into Perforce.

    Things you might not expect from a preflight checker:
    - user lookup:
        PreflightChecker has to see if the author has permission
        to write files, so PreflightChecker needs a way to look up
        a single Git commit's author.
        Could move to a separate class, passed in as part of __init__.
    - branch definition:
        A side effect of rolling through commits is that we
        have to fully define depot branches to hold those commits,
        and branch views to map the Git work tree to those depot branches.
        Could move elsewhere.
        Not sure if we want to require this as something done BEFORE
        preflight, or done DURING preflight.

    Extracted and decoupled from "classic push" p4gf_copy_to_p4.G2P and reused
    by "fast push" p4gf_fast_push.FastPush. If you change this for classic
    push, test your changes against fast push, too. Such is the price of
    maintaining two parallel code paths.
    """

    def __init__(self, *, ctx, g2p_user, assigner, gsreview_coll=None,
        already_copied_commit_runner=None, finish_branch_definition=None):
        self.ctx = ctx
        self.g2p_user = g2p_user
        self.assigner = assigner
        self.gsreview_coll = gsreview_coll
        self.already_copied_commit_runner = already_copied_commit_runner
        self._finish_branch_definition = finish_branch_definition
        self.stream_depots = set()
        self._current_prt = None
        self.fast_export_marks = None
        self._current_branch = None
        self._curr_fe_commit = None
        self.set_client_on_branch_switch = True
        self._cached_overlapping_branch_list = None
        self.lfs_row_list = []

    def _already_copied_commit(self, commit_sha1, branch_id):
        """Do we already have a Perforce changelist for this commit,
        this branch_id?
        """
        if not self.already_copied_commit_runner:
            return False
        return self.already_copied_commit_runner.already_copied_commit(
            commit_sha1, branch_id)

    def finish_branch_definition(self, commit, branch):
        """Inflate a partially-created Branch with a proper depot branch to
        hold its files, and a branch view to map that depot branch to
        the Git work tree.
        """
        assert self._finish_branch_definition
        self._finish_branch_definition.finish_branch_definition(commit, branch)
        self._invalidate_branch_cache()

    def check_prt_and_commits(self, prt, commits, marks):
        """Run a preflight check on a set of commits from Git.

        :param prt: pre-receive tuple
        :param commits: FastExport.commits list

        :type marks: :class:`p4gf_fastexport_marks.Marks`
        :param marks: all known fast-export marks

        Raises a PreflightException if anything is out of sorts.

        """
        self._current_prt = prt
        self.fast_export_marks = marks
        self.fast_export_marks.set_head(prt.ref)
        self.check_p4gf_user_write_permission()
        self.check_commits(commits)

    def check_p4gf_user_write_permission(self):
        """Ensure git-fusion-user has permissions to write to depot."""
        gf_client_map = P4.Map()
        gf_client_map.insert('//...', '//client/...')
        utp = p4gf_protect.UserToProtect(self.ctx.p4)
        prot = utp.user_to_protect(p4gf_const.P4GF_USER)
        gf_write_filter = prot.map_for_perm(p4gf_protect.WRITE)
        gf_write_filter = P4.Map.join(gf_write_filter, gf_client_map)
        if not gf_write_filter.includes('//{depot}/...'.format(depot=
            p4gf_const.P4GF_DEPOT)):
            raise RuntimeError(_('permission denied'))

    def _find_locked_by(self):
        """Return a dict of depot_path => user of any locked files."""
        fstat_flags = NTR('otherLock | otherOpen0 & headType=*+l')
        any_locked_files = {}
        for branch_chunk in self.ctx.iter_writable_branch_chunks():
            bvl = [b for b in branch_chunk if b.view_lines]
            if not bvl:
                continue
            with self.ctx.switched_to_union(bvl):
                r = self.ctx.p4run('fstat', '-F', fstat_flags, '-m1',
                    '//{}/...'.format(self.ctx.p4.client), log_warnings=
                    logging.DEBUG)
                for lf in r:
                    user = lf['otherOpen'][0] if 'otherOpen' in lf else NTR(
                        '<unknown>')
                    any_locked_files[lf['depotFile']] = user
        return any_locked_files

    def check_commits(self, commits):
        """Ensure the entire sequence of commits will (likely) go through
        without any errors related to permissions or locks. Raises an
        exception if anything goes wrong.

        Arguments:
            commits -- commits from FastExport class
        """
        LOG.info('Checking Perforce permissions and locks')
        self.ctx.checkpoint('copy_to_p4._preflight_check')
        opened = self.ctx.p4.run(['opened', '-m1'])
        if opened:
            raise PreflightException(_(
                'There are files opened by Git Fusion for this repo.'))
        is_read_only = self.ctx.repo_config.getboolean(p4gf_config.
            SECTION_REPO, p4gf_config.KEY_READ_ONLY, fallback=False)
        if is_read_only:
            raise PreflightException(_(
                'Push to repo {repo_name} prohibited.').format(repo_name=
                self.ctx.config.repo_name))
        depots = self.ctx.p4.run(['depots'])
        self.stream_depots = set([d['name'] for d in depots if d['type'] ==
            'stream'])
        any_locked_files = self._find_locked_by()
        LOG.debug('any_locked_files {0}'.format(any_locked_files))
        case_conflict_checker = None
        if not self.ctx.server_is_case_sensitive:
            case_conflict_checker = CaseConflictChecker(self.ctx)
            case_conflict_checker.read_perforce_paths()
        ui_name = self._curr_ref_ui_name()
        if ui_name:
            progress_msg = _('Checking commits for {ref}...').format(ref=
                ui_name)
        else:
            progress_msg = _('Checking commits...')
        with ProgressReporter.Determinate(len(commits)):
            for commit in commits:
                ProgressReporter.increment(progress_msg)
                self.g2p_user.get_author_pusher_owner(commit)
                rev = commit['sha1']
                if not self.assigner.is_assigned(commit['sha1']):
                    continue
                self.check_commit(commit)
                for branch_id in self.assigner.branch_id_list(rev):
                    self.check_commit_for_branch(commit, branch_id,
                        any_locked_files, case_conflict_checker)
        if case_conflict_checker:
            cc_text = case_conflict_checker.conflict_text()
            if cc_text:
                raise PreflightException(cc_text)

    def check_commit(self, commit):
        """Prior to copying a commit, perform a set of checks to ensure the commit
        will (likely) go through successfully.
        This includes:
            * verifying permission to commit for author p4 user
            * screening for merge commits
            * screening for submodules
            * checking valid filenames
            * checking write permissions for each file
        """
        if LOG.isEnabledFor(logging.DEBUG):
            LOG.debug(
                'check_commit() Checking mark={} sha1={} file-ct={} -- {}'.
                format(commit['mark'], p4gf_util.abbrev(commit['sha1']),
                len(commit['files']), repr(commit['data'])[:20].splitlines(
                )[0]))
        if not commit['author_p4user']:
            raise PreflightException(_(
                "User '{user}' not permitted to commit").format(user=commit
                ['author']['email'].strip('<>')))
        if 'merge' in commit:
            ref_is_review = (self.gsreview_coll and self.gsreview_coll.
                ref_in_review_list(self._current_prt.ref))
            if not ref_is_review and not self.ctx.merge_commits:
                raise PreflightException(_(
                    'Merge commits are not enabled for this repo.'))
            if (not ref_is_review and not self.ctx.branch_creation and self
                .assigner.have_anonymous_branches):
                msg = _('Git branch creation is prohibited for this repo.')
                p4_branch_names_non_lw = [b.git_branch_name for b in self.
                    ctx.branch_dict().values() if b.git_branch_name and not
                    b.is_lightweight]
                if len(p4_branch_names_non_lw) > 1:
                    msg += _(
                        """
This repo has more than one named branch.
Try altering the push order - pushing branches with merge ancestors first."""
                        )
                raise PreflightException(msg)
            if LOG.isEnabledFor(logging.DEBUG):
                for parent_mark in commit['merge']:
                    parent_sha1 = self.fast_export_marks.get_commit(parent_mark
                        )[:7]
                    LOG.debug('check_commit() merge mark={} sha1={}'.format
                        (parent_mark, parent_sha1))
        if not self.ctx.submodules and 'files' in commit:
            for f in commit['files']:
                if f.get('mode') == '160000':
                    if 'first_commit' in commit and not self._path_added(f.
                        get('path'), commit):
                        LOG.debug2('check_commit() passed {} in {}'.format(
                            f.get('path'), p4gf_util.abbrev(commit['sha1'])))
                        continue
                    raise PreflightException(_(
                        'Git submodules not permitted: path={path} commit={commit_sha1}'
                        ).format(path=f.get('path'), commit_sha1=p4gf_util.
                        abbrev(commit['sha1'])))
        for f in commit['files']:
            LOG.debug3('check_commit : commit files: ' + _log_fe_file(f))
            err = check_valid_filename(f['path'], self.ctx)
            if err:
                raise PreflightException(err)
            if self.ctx.is_lfs_enabled:
                self._check_lfs(commit, f)
        jobs = G2PJob.lookup_jobs(self.ctx, G2PJob.extract_jobs(commit['data'])
            )
        if jobs:
            for job_id in jobs:
                r = self.ctx.p4run('jobs', '-e', 'job={}'.format(job_id))
                if not r:
                    _print_error(_("Job '{job_id}' doesn't exist").format(
                        job_id=job_id))

    def check_commit_for_branch(self, commit, branch_id, any_locked_files,
        case_conflict_checker):
        """
        Prior to copying a commit, perform a set of checks for a specific branch
        to ensure the commit will (likely) go through successfully.
        """
        rev = commit['sha1']
        if LOG.isEnabledFor(logging.DEBUG):
            LOG.debug(
                'check_commit_for_branch() Checking branch={} mark={} sha1={} file-ct={} -- {}'
                .format(branch_id, commit['mark'], p4gf_util.abbrev(rev),
                len(commit['files']), repr(commit['data'])[:20].splitlines(
                )[0]))
        if self._already_copied_commit(rev, branch_id):
            return
        self.ensure_branch_preflight(commit, branch_id)
        with self.ctx.switched_to_branch(self._current_branch, set_client=
            self.set_client_on_branch_switch):
            if case_conflict_checker:
                case_conflict_checker.read_fast_export_commit(commit, self.
                    _current_branch)
            if not commit['files'] and not self._is_placeholder_mapped():
                raise PreflightException(_(
                    'Empty commit {sha1} not permitted. Git Fusion branch views must include root to permit empty commits.'
                    ).format(sha1=p4gf_util.abbrev(rev)))
            with Timer(CHECK_PROTECTS):
                self._check_protects(commit['author_p4user'], commit['files'])
            with Timer(CHECK_OVERLAP):
                self._check_overlap(commit)
            if self._current_branch.is_read_only:
                raise PreflightException(_(
                    'Push to branch {branch} prohibited.').format(branch=
                    self._current_branch.git_branch_name))
            self._check_stream_writable(commit)
            self._check_stream_in_classic(commit)
            LOG.debug('checking locked files under //{}/...'.format(self.
                ctx.p4.client))
            if any_locked_files:
                files_in_commit = [self.ctx.gwt_path(f['path']).to_depot() for
                    f in commit['files']]
                LOG.debug('files_in_commit {0}'.format(files_in_commit))
                for f in files_in_commit:
                    if f in any_locked_files:
                        msg = _('{file} - locked by {user}').format(file=f,
                            user=any_locked_files[f])
                        LOG.info(msg)
                        raise PreflightException(msg)
            if self.ctx.preflight_hook.is_callable():
                jobs = G2PJob.extract_jobs(commit['data'])
                jobs2 = G2PJob.lookup_jobs(self.ctx, jobs)
                self.ctx.preflight_hook(ctx=self.ctx, fe_commit=commit,
                    branch_id=branch_id, jobs=jobs2)

    def ensure_branch_preflight(self, commit, branch_id):
        """If not already switched to and synced to the correct branch for the
        given commit, do so.

        If this is a new lightweight branch, perform whatever creation we can do
        at preflight time. We don't have commits/marks for any not-yet-submitted
        parent commits, so the depot_branch_info will often lack a correct
        parent or fully populated basis.

        * depot tree, along with a branch-info file
        * branch mapping, along with entry in p4gf_config2 (if not anonymous)

        Return requested branch
        """
        log = LOG.getChild('ensure_branch_preflight')
        branch = self.ctx.branch_dict().get(branch_id)
        if (self._current_branch and self._current_branch.branch_id ==
            branch_id):
            log.debug('sha={} want branch_id={} curr branch_id={} NOP'.
                format(commit['sha1'][:7], branch_id[:7], self.
                _current_branch.branch_id[:7]))
            log.debug('staying on  branch {}'.format(self.ctx.branch_dict()
                .get(branch_id)))
            return branch
        cbid = (self._current_branch.branch_id if self._current_branch else
            'None')
        log.debug('sha={} want branch_id={} curr branch_id={} switch'.
            format(commit['sha1'][:7], branch_id[:7], cbid[:7]))
        if not branch.view_lines:
            self.finish_branch_definition(commit, branch)
        elif branch.view_p4map:
            if branch.original_view_lines:
                original_view_lines = '\n'.join(branch.original_view_lines)
                view_lines = p4gf_path_convert.convert_view_to_no_client_name(
                    branch.view_lines)
                if not view_lines == original_view_lines:
                    raise PreflightException(_(
                        """Unable to push.  Stream view changed from:
{old_view}
to:
{new_view}"""
                        ).format(old_view=original_view_lines, new_view=
                        view_lines))
            lhs = branch.view_p4map.lhs()
            branch.depot_branch = self.ctx.depot_branch_info_index(
                ).find_depot_path(lhs[0])
        log.debug('switching to branch {}'.format(branch))
        self._current_branch = branch
        return branch

    def _is_placeholder_mapped(self):
        """Does this branch map our placeholder file?

        Returns non-False if mapped, None or empty string if not.
        """
        return self.ctx.gwt_path(p4gf_const.P4GF_EMPTY_CHANGELIST_PLACEHOLDER
            ).to_depot()

    def _check_protects(self, p4user, blobs):
        """check if author is authorized to submit files."""
        pc = ProtectsChecker(self.ctx, p4user, self.ctx.
            authenticated_p4user, self.ctx.foruser)
        pc.filter_paths(blobs)
        if pc.has_error():
            raise PreflightException(pc.error_message())

    def _check_overlap(self, fe_commit):
        """If any of the files in this commit intersect any fully populated branch
        (other than the current branch), then reject this commit.

        Shared/common/overlapping paths in branch views must be read-only from
        Git. Otherwise you end up with a Git push of commit on one Git branch
        inserting  changes into other Git branches behind Git's back.

        To modify shared paths, either do so from Perforce, or create a Git
        Fusion repo with no more than one branch that maps that shared path.
        """
        if self._current_branch not in self._overlapping_branch_list():
            return
        for fe_file in fe_commit['files']:
            gwt_path = fe_file['path']
            depot_path = self.ctx.gwt_to_depot_path(gwt_path)
            for branch in self._overlapping_branch_list():
                if branch == self._current_branch:
                    continue
                if not branch.intersects_depot_path(depot_path):
                    continue
                LOG.debug(
                    '_check_overlap() branch {br1} <> {br2} gwt={gwt:<40}   {dp}\n{view}'
                    .format(br1=p4gf_util.abbrev(self._current_branch.
                    branch_id), br2=p4gf_util.abbrev(branch.branch_id), gwt
                    =gwt_path, dp=depot_path, view='\n'.join(branch.
                    view_p4map.as_array())))
                if (self._current_branch.is_new_fp_from_push or branch.
                    is_new_fp_from_push):
                    current_branch_name = self._current_branch.git_branch_name
                    if self._current_branch.is_new_fp_from_push:
                        current_branch_name += '(new)'
                    other_branch_name = branch.git_branch_name
                    if branch.is_new_fp_from_push:
                        other_branch_name += '(new)'
                    human_msg = _(
                        """Perforce: Cannot commit {sha1} '{gwt_path}' to '{depot_path}'.
 You are attempting to push and create a new fully populated branch
 with paths which overlap another branch. Contact your admin
 to configure non-conflicting destination branch paths.
 Branches: '{b1}', '{b2}'"""
                        ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']),
                        gwt_path=gwt_path, depot_path=depot_path, b1=
                        current_branch_name, b2=other_branch_name)
                else:
                    human_msg = _(
                        "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'. Paths that overlap multiple Git Fusion branches are read-only. Branches: '{b1}', '{b2}'"
                        ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']),
                        gwt_path=gwt_path, depot_path=depot_path, b1=self.
                        _current_branch.branch_id, b2=branch.branch_id)
                raise PreflightException(human_msg)

    def _check_stream_writable(self, fe_commit):
        """If this is a stream branch, check that all files in the commit are
        writable.  If any of the files is not writable then reject this commit.
        """
        if not self._current_branch.stream_name:
            return
        prefix = self._current_branch.writable_stream_name + '/'
        for fe_file in fe_commit['files']:
            gwt_path = fe_file['path']
            depot_path = self.ctx.gwt_path(gwt_path).to_depot()
            if depot_path.startswith(prefix):
                continue
            human_msg = _(
                "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'. Paths not in stream '{stream}' are read-only for branch '{b}'."
                ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']), gwt_path
                =gwt_path, depot_path=depot_path, stream=self.
                _current_branch.writable_stream_name, b=self.
                _current_branch.branch_id)
            raise PreflightException(human_msg)

    def _check_stream_in_classic(self, fe_commit):
        """If this is a classic branch, check that none of the files in the commit
        are in stream depots and thus not writable.  If any of the files is not
        writable then reject this commit.
        """
        if self._current_branch.stream_name:
            return
        depot_re = re.compile('^//([^/]+)/([^/]+)/.*$')
        for fe_file in fe_commit['files']:
            gwt_path = fe_file['path']
            depot_path = self.ctx.gwt_path(gwt_path).to_depot()
            m = depot_re.match(depot_path)
            if m:
                depot = m.group(1)
                if depot in self.stream_depots:
                    stream = '//{}/{}'.format(m.group(1), m.group(2))
                    human_msg = _(
                        "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'. Paths in stream '{stream}' are read-only for branch '{b}'."
                        ).format(sha1=p4gf_util.abbrev(fe_commit['sha1']),
                        gwt_path=gwt_path, depot_path=depot_path, stream=
                        stream, b=self._current_branch.branch_id)
                    raise PreflightException(human_msg)

    def _path_added(self, path, fecommit):
        """Return True if the named path was introduced in the HEAD commit.

        :param self: this object
        :param path: repo path to be evaluated.
        :param fecommit: commit object from fast-export parser.

        """
        commit = self.ctx.repo.get(fecommit['sha1'])
        if commit is None:
            LOG.debug2('_path_added() commit {} is missing'.format(fecommit
                ['sha1']))
            return True
        for parent in commit.parents:
            if p4gf_git.exists_in_tree(self.ctx.repo, path, parent.tree):
                LOG.debug2('_path_added() {} exists in parent tree {}'.
                    format(path, p4gf_util.abbrev(p4gf_pygit2.
                    object_to_sha1(parent))))
                return False
        return True

    def _invalidate_branch_cache(self):
        """We have changed our branch_dict (or more likely
        finish_branch_definition()ed a branch within that dict) in a way that
        invalidates any cached calculations that consumed the branch dict.
        """
        self._cached_overlapping_branch_list = None

    def _overlapping_branch_list(self):
        """Return a list of fully populated branches that overlap
        other fully populated branches.

        Caches the result because we check every file revision
        path for overlap, and for huge repos with thousands of
        non-overlapping LW branches, just iterating through the
        branch list starts to waste measurable CPU time.
        """
        if self._cached_overlapping_branch_list is not None:
            return self._cached_overlapping_branch_list
        have_overlap = set()
        for outer in p4gf_branch.iter_fp_non_deleted(self.ctx.branch_dict()):
            outer_lhs = P4.Map()
            outer_lhs.insert(outer.view_p4map.lhs())
            for inner in p4gf_branch.iter_fp_non_deleted(self.ctx.branch_dict()
                ):
                if outer == inner:
                    continue
                overlap = P4.Map.join(outer_lhs, inner.view_p4map)
                for line in overlap.as_array():
                    if line.startswith('-') or line.startswith('"-'):
                        continue
                    have_overlap.add(outer)
                    have_overlap.add(inner)
                    break
        self._cached_overlapping_branch_list = have_overlap
        return self._cached_overlapping_branch_list

    def _curr_ref_ui_name(self):
        """Return a string suitable for use in progress messages as a
        short hint to what we're processing.

        If we have a PreReceiveTuple with a Git branch/tag name, return
        that name (minus any refs/heads/ prefix). If not, return None.
        """
        if not (self._current_prt and self._current_prt.ref):
            return None
        s = self._current_prt.ref
        prefixes = ['refs/heads/', 'refs/tags/']
        for p in prefixes:
            if s.startswith(p):
                return s[len(p):]
        return s

    def _check_lfs(self, fe_commit, fe_file):
        """If gfe_file is under Git LFS control, require that its
        large file content exist somewhere, either in our upload
        cache (it's new!) or in depot de-dupe storage (already got it).
        """
        if 'sha1' not in fe_file:
            return
        if fe_file.get('mode') not in [FileModeStr.PLAIN, FileModeStr.
            EXECUTABLE]:
            return
        is_tracked = self.ctx.lfs_tracker.is_tracked_git(commit_sha1=
            fe_commit['sha1'], gwt_path=fe_file['path'])
        LOG.debug3(
            '_check_lfs() tracked {lfs}  commit {commit_sha1}  gwt {gwt}'.
            format(commit_sha1=p4gf_util.abbrev(fe_commit['sha1']), lfs=1 if
            is_tracked else 0, gwt=fe_file['path']))
        if not is_tracked:
            lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file)
            if lfs_row:
                raise PreflightException(_(
                    """Push of Git LFS text pointer not tracked by LFS:
commit {commit_sha1} path {gwt_path}"""
                    ).format(commit_sha1=p4gf_util.abbrev(fe_commit['sha1']
                    ), gwt_path=fe_file['path']))
            return
        lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file)
        if not lfs_row:
            return
        if not lfs_row.large_file_source:
            LOG.error('LFS text pointer missing content.')
            LOG.error('LFS   commit {}'.format(p4gf_util.abbrev(fe_commit[
                'sha1'])))
            LOG.error('LFS   lfs oid {}'.format(lfs_row.large_file_oid))
            LOG.error('LFS   ptr {blob_sha1} {blob_mode} {gwt}'.format(
                blob_sha1=p4gf_util.abbrev(fe_file['sha1']), blob_mode=
                p4gf_util.mode_str(fe_file['mode']), gwt=fe_file['path']))
            LOG.error('LFS   upload  {}'.format(lfs_row.to_lfsfs().
                cache_path(self.ctx)))
            LOG.error('LFS   de-dupe {}'.format(lfs_row.to_lfsfs().
                depot_path(self.ctx)))
            raise PreflightException(_(
                """Push of Git LFS text pointer missing content:
commit {commit_sha1} path {gwt_path}"""
                ).format(commit_sha1=p4gf_util.abbrev(fe_commit['sha1']),
                gwt_path=fe_file['path']))
        self.lfs_row_list.append(lfs_row)


class ProtectsChecker:
    """Handle filtering a list of paths against view and protections."""

    def __init__(self, ctx, author, pusher, foruser):
        """Init P4.Map objects for author, pusher, view and combination."""
        self.ctx = ctx
        self.author = author
        self.pusher = pusher
        self.foruser = foruser
        self.ignore_author_perms = ctx.repo_config.getboolean(p4gf_config.
            SECTION_GIT_TO_PERFORCE, p4gf_config.KEY_IGNORE_AUTHOR_PERMS)
        self.view_map = None
        self.write_protect_author = None
        self.write_protect_pusher = None
        self.write_protect_foruser = None
        self.write_protect_fusion = None
        self.write_filter = None
        self.init_view()
        self.init_write_filter()
        self.author_denied = []
        self.pusher_denied = []
        self.foruser_denied = []
        self.fusion_denied = []
        self.unmapped = []

    def init_view(self):
        """Init view map for client."""
        self.view_map = self.ctx.clientmap

    def init_write_filter(self):
        """Init write filter."""
        self.write_protect_author = self.ctx.user_to_protect(self.author
            ).map_for_perm(p4gf_protect.WRITE)
        if self.author != self.pusher or self.ignore_author_perms:
            self.write_protect_pusher = self.ctx.user_to_protect(self.pusher
                ).map_for_perm(p4gf_protect.WRITE)
        if self.foruser:
            self.write_protect_foruser = self.ctx.user_to_protect(self.foruser
                ).map_for_perm(p4gf_protect.WRITE)
        self.write_protect_fusion = self.ctx.user_to_protect(p4gf_const.
            P4GF_USER).map_for_perm(p4gf_protect.WRITE)
        self.write_filter = functools.reduce(P4.Map.join, [vmap for vmap in
            [self.write_protect_author, self.write_protect_pusher, self.
            write_protect_foruser, self.write_protect_fusion, self.view_map
            ] if vmap])

    def filter_paths(self, blobs):
        """Run list of paths through filter and set list of paths that don't pass."""
        self.author_denied = []
        self.pusher_denied = []
        self.foruser_denied = []
        self.fusion_denied = []
        self.unmapped = []
        c2d = P4.Map.RIGHT2LEFT
        LOG.debug('filter_paths() write_filter: %s', self.write_filter)
        for blob in blobs:
            gwt_path = self.ctx.gwt_path(blob['path'])
            topath_c = gwt_path.to_client()
            topath_d = gwt_path.to_depot()
            LOG.debug('filter_paths() topath_d: %s', topath_d)
            result = '  '
            if topath_d and P4GF_DEPOT_OBJECTS_RE.match(topath_d):
                LOG.debug('filter_paths() topath_d in //.git-fusion/objects')
                continue
            if topath_d and P4GF_DEPOT_BRANCHES_RE.match(topath_d):
                LOG.debug('filter_paths() topath_d in //.git-fusion/branches')
                continue
            if not self.write_filter.includes(topath_c, c2d):
                if not self.view_map.includes(topath_c, c2d):
                    self.unmapped.append(topath_c)
                    result = NTR('unmapped')
                elif not (self.ignore_author_perms or self.
                    write_protect_author.includes(topath_d)):
                    self.author_denied.append(topath_c)
                    result = NTR('author denied')
                elif self.write_protect_pusher and not self.write_protect_pusher.includes(
                    topath_d):
                    self.pusher_denied.append(topath_c)
                    result = NTR('pusher denied')
                elif self.write_protect_foruser and not self.write_protect_foruser.includes(
                    topath_d):
                    self.foruser_denied.append(topath_c)
                    result = NTR('foruser denied')
                elif not self.write_protect_fusion.includes(topath_d):
                    self.fusion_denied.append(topath_c)
                    result = NTR('Git Fusion denied')
                else:
                    result = '?'
                LOG.error('filter_paths() {:<13} {}, {}, {}'.format(result,
                    blob['path'], topath_d, topath_c))
            elif LOG.isEnabledFor(logging.DEBUG):
                LOG.debug('filter_paths() topath_c in write_filter: %s',
                    topath_c)

    def has_error(self):
        """Return True if any paths not passed by filters."""
        return len(self.unmapped) or len(self.author_denied) or len(self.
            pusher_denied) or len(self.foruser_denied) or len(self.
            fusion_denied)

    def error_message(self):
        """Return message indicating what's blocking the push."""
        if len(self.unmapped):
            return _('file(s) not in client view')
        if len(self.author_denied):
            restricted_user = self.author
        elif len(self.pusher_denied):
            restricted_user = self.pusher
        elif len(self.foruser_denied):
            restricted_user = self.foruser
        elif len(self.fusion_denied):
            restricted_user = p4gf_const.P4GF_USER
        else:
            restricted_user = _('<unknown>')
        return _("user '{user}' not authorized to submit file(s) in git commit"
            ).format(user=restricted_user)


class PreflightException(Exception):
    """This exception is raised when a push was rejected during preflight
    checks.
    """
    pass


<mask token>


def is_p4d_printable(c):
    """Check if c will be rejected by P4D as non-printable.

    P4D rejects "non-printable" characters with
      ErrorId MsgDm::IdNonPrint = { ErrorOf( ES_DM, 6, E_FAILED, EV_USAGE, 1 )
      "Non-printable characters not allowed in '%id%'." } ;

    Where "character" here means C "char" aka "byte"
    and non-printable means 0x00-0x1f or 0x7f
    """
    if ord(c) < 32:
        return False
    if ord(c) == 127:
        return False
    return True


<mask token>

#! /usr/bin/env python3.3
"""Code for checking a 'git push' before writing anything to Perforce."""
import functools
import logging
import re
import sys

import P4

import p4gf_branch
from   p4gf_case_conflict_checker   import CaseConflictChecker
import p4gf_config
# no   p4gf_copy_to_p4              avoid circular import
import p4gf_const
from   p4gf_filemode                import FileModeStr
import p4gf_g2p_job                 as     G2PJob
import p4gf_git
from   p4gf_lfs_row                 import LFSRow
from   p4gf_l10n                    import _, NTR
import p4gf_path_convert
from   p4gf_profiler                import Timer
import p4gf_progress_reporter       as     ProgressReporter
import p4gf_protect
import p4gf_pygit2
import p4gf_util

LOG = logging.getLogger(__name__)

# Assume the cached objects will not be mapped into the repository, and
# skip permission checks on those paths since they're submitted as the
# git-fusion-user anyway.
P4GF_DEPOT_OBJECTS_RE = re.compile('//' + p4gf_const.P4GF_DEPOT + '/objects/')
P4GF_DEPOT_BRANCHES_RE = re.compile('//' + p4gf_const.P4GF_DEPOT + '/branches/')

# timer names
CHECK_PROTECTS  = NTR('Check Protects')
CHECK_OVERLAP   = NTR('Check Overlap')


class PreflightChecker:
    """A class that knows how to check a 'git push' for things that
    should not be copied into Perforce.

    Things you might not expect from a preflight checker:
    - user lookup:
        PreflightChecker has to see if the author has permission
        to write files, so PreflightChecker needs a way to look up
        a single Git commit's author.
        Could move to a separate class, passed in as part of __init__.
    - branch definition:
        A side effect of rolling through commits is that we
        have to fully define depot branches to hold those commits,
        and branch views to map the Git work tree to those depot branches.
        Could move elsewhere.
        Not sure if we want to require this as something done BEFORE
        preflight, or done DURING preflight.

    Extracted and decoupled from "classic push" p4gf_copy_to_p4.G2P and reused
    by "fast push" p4gf_fast_push.FastPush. If you change this for classic
    push, test your changes against fast push, too. Such is the price of
    maintaining two parallel code paths.
    """

    def __init__(self, *
            , ctx
            , g2p_user
            , assigner
            , gsreview_coll                 = None
            , already_copied_commit_runner  = None
            , finish_branch_definition      = None
            ):

        self.ctx                = ctx
        self.g2p_user           = g2p_user
        self.assigner           = assigner
        self.gsreview_coll      = gsreview_coll

        # ------------- # Explicitly listing each callback into G2P so I can
                        # find the API between these two snotballs.
                        # These are all "an instace (of G2P) that implements X:

                        # already_copied_commit(self, commit_sha1, branch_id)
        self.already_copied_commit_runner = already_copied_commit_runner

                        # finish_branch_definition(self, commit, branch)
        self._finish_branch_definition = finish_branch_definition

        # ------------- # End callbacks into G2P

                        # set() of depot names of type 'stream'
        self.stream_depots      = set()

                        # PreReceiveTuple whose commits we're checking.
                        # Used when permitting Swarm review merges, but
                        # prohibiting all other merge commits.
                        #
                        # OK to leave None if you don't care about
                        # Swarm reviews.
        self._current_prt       = None

                        # p4gf_fastexport_marks.Marks instance
                        # of all known git-fast-export marks.
        self.fast_export_marks  = None

        self._current_branch    = None
        self._curr_fe_commit    = None   # Current git-fast-export 'commit'.

                        # Behavior control for fast push: what to pass to
                        # Context.switched_to_branch() for set_client.
                        #
                        # When set to False, the Context.p4 client is
                        # untouched, which can save a huge amount of time
                        # in check_commit_for_branch(): 45 seconds drops
                        # to 9 seconds, but works only if you promist to make
                        # no calls to Perforce through that ctx.p4 connection.
                        #
        self.set_client_on_branch_switch = True

                        # list of fully populated Branch instances that overlap
                        # other fully populated branches.
                        #
                        # See _overlapping_branch_list() for calc and cache,
                        # call _invalidate_branch_cache() if you change any
                        # branch definitions after most recent call to
                        # _overlapping_branch_list().
                        #
        self._cached_overlapping_branch_list = None # list[] of FP Branch instances

                        # LFSRow text pointers that are part of this push.
        self.lfs_row_list = []


    #
    # -- callbacks into G2P --------------------------------------------------
    #    Optional. NOP if not running with a full G2P.
    #

    def _already_copied_commit(self, commit_sha1, branch_id):
        """Do we already have a Perforce changelist for this commit,
        this branch_id?
        """
        if not self.already_copied_commit_runner:
            return False
        return self.already_copied_commit_runner.already_copied_commit(
                    commit_sha1, branch_id)

    def finish_branch_definition(self, commit, branch):
        """Inflate a partially-created Branch with a proper depot branch to
        hold its files, and a branch view to map that depot branch to
        the Git work tree.
        """
        assert self._finish_branch_definition
        self._finish_branch_definition.finish_branch_definition(commit, branch)
        self._invalidate_branch_cache()

    #
    # -- end callbacks into G2P ----------------------------------------------
    #

    def check_prt_and_commits(self, prt, commits, marks):
        """Run a preflight check on a set of commits from Git.

        :param prt: pre-receive tuple
        :param commits: FastExport.commits list

        :type marks: :class:`p4gf_fastexport_marks.Marks`
        :param marks: all known fast-export marks

        Raises a PreflightException if anything is out of sorts.

        """
        self._current_prt = prt
        self.fast_export_marks = marks
        self.fast_export_marks.set_head(prt.ref)
        self.check_p4gf_user_write_permission()
        self.check_commits(commits)

    def check_p4gf_user_write_permission(self):
        """Ensure git-fusion-user has permissions to write to depot."""
        gf_client_map = P4.Map()
        gf_client_map.insert("//...", "//client/...")
        utp = p4gf_protect.UserToProtect(self.ctx.p4)
        prot = utp.user_to_protect(p4gf_const.P4GF_USER)
        gf_write_filter = prot.map_for_perm(p4gf_protect.WRITE)
        gf_write_filter = P4.Map.join(gf_write_filter, gf_client_map)
        if not gf_write_filter.includes('//{depot}/...'.format(depot=p4gf_const.P4GF_DEPOT)):
            raise RuntimeError(_('permission denied'))

    def _find_locked_by(self):
        """Return a dict of depot_path => user of any locked files."""
        fstat_flags = NTR('otherLock | otherOpen0 & headType=*+l')
        any_locked_files = {}  # depot_path : user
        for branch_chunk in self.ctx.iter_writable_branch_chunks():
            # Skip any newly defined branches: they're new, won't contain any
            # files yet, and won't get a view definition until later at per-
            # commit preflight time.
            bvl = [b for b in branch_chunk if b.view_lines]
            if not bvl:
                continue
            with self.ctx.switched_to_union(bvl):
                r = self.ctx.p4run('fstat', '-F', fstat_flags, '-m1',
                                   '//{}/...'.format(self.ctx.p4.client),
                                   log_warnings=logging.DEBUG)
                # Collect a dictionary of the locked files from the writable union of branch views
                for lf in r:
                    user = lf['otherOpen'][0] if 'otherOpen' in lf else NTR('<unknown>')
                    any_locked_files[lf['depotFile']] = user
        return any_locked_files

    def check_commits(self, commits):
        """Ensure the entire sequence of commits will (likely) go through
        without any errors related to permissions or locks. Raises an
        exception if anything goes wrong.

        Arguments:
            commits -- commits from FastExport class
        """
        LOG.info('Checking Perforce permissions and locks')
        self.ctx.checkpoint("copy_to_p4._preflight_check")

        # Stop if files are opened in our repo client
        # We expect this to be none, since we have the view lock
        opened = self.ctx.p4.run(['opened', '-m1'])
        if opened:
            raise PreflightException(_('There are files opened by Git Fusion for this repo.'))

        # fetch the repo setting only, without cascading to global config
        is_read_only = self.ctx.repo_config.getboolean(p4gf_config.SECTION_REPO,
                                                       p4gf_config.KEY_READ_ONLY,
                                                       fallback=False)
        if is_read_only:
            raise PreflightException(_("Push to repo {repo_name} prohibited.")
                                     .format(repo_name=self.ctx.config.repo_name))

        # get a list of stream depots for later checks for read-only paths
        depots = self.ctx.p4.run(['depots'])
        self.stream_depots = set([d['name'] for d in depots if d['type'] == 'stream'])
        any_locked_files = self._find_locked_by()
        LOG.debug("any_locked_files {0}".format(any_locked_files))
        case_conflict_checker = None
        if not self.ctx.server_is_case_sensitive:
            case_conflict_checker = CaseConflictChecker(self.ctx)
            case_conflict_checker.read_perforce_paths()

        ui_name = self._curr_ref_ui_name()
        if ui_name:
            progress_msg = _('Checking commits for {ref}...').format(ref=ui_name)
        else:
            progress_msg = _('Checking commits...')

        with ProgressReporter.Determinate(len(commits)):
            for commit in commits:
                ProgressReporter.increment(progress_msg)

                self.g2p_user.get_author_pusher_owner(commit)

                rev = commit['sha1']
                if not self.assigner.is_assigned(commit['sha1']):
                    continue

                self.check_commit(commit)

                for branch_id in self.assigner.branch_id_list(rev):
                    self.check_commit_for_branch(
                                                   commit
                                                 , branch_id
                                                 , any_locked_files
                                                 , case_conflict_checker )

        if case_conflict_checker:
            cc_text = case_conflict_checker.conflict_text()
            if cc_text:
                raise PreflightException(cc_text)

    def check_commit(self, commit):
        """Prior to copying a commit, perform a set of checks to ensure the commit
        will (likely) go through successfully.
        This includes:
            * verifying permission to commit for author p4 user
            * screening for merge commits
            * screening for submodules
            * checking valid filenames
            * checking write permissions for each file
        """
        # pylint: disable=too-many-branches
        if LOG.isEnabledFor(logging.DEBUG):
            LOG.debug('check_commit() Checking mark={} sha1={} file-ct={} -- {}'
                      .format(  commit['mark']
                              , p4gf_util.abbrev(commit['sha1'])
                              , len(commit['files'])
                              , repr(commit['data'])[:20].splitlines()[0]))

        if not commit['author_p4user']:
            raise PreflightException(_("User '{user}' not permitted to commit")
                                     .format(user=commit['author']['email'].strip('<>')))

        if 'merge' in commit:
            ref_is_review = (self.gsreview_coll and
                             self.gsreview_coll.ref_in_review_list(self._current_prt.ref))
            if not ref_is_review and not self.ctx.merge_commits:
                raise PreflightException(_('Merge commits are not enabled for this repo.'))
            if (not ref_is_review and
                    not self.ctx.branch_creation and self.assigner.have_anonymous_branches):
                msg = _('Git branch creation is prohibited for this repo.')
                p4_branch_names_non_lw = [b.git_branch_name for b in self.ctx.branch_dict().values()
                                          if b.git_branch_name and not b.is_lightweight]
                if len(p4_branch_names_non_lw) > 1:
                    msg += _('\nThis repo has more than one named branch.'
                             '\nTry altering the push order - '
                             'pushing branches with merge ancestors first.')
                raise PreflightException(msg)
            if LOG.isEnabledFor(logging.DEBUG):
                for parent_mark in commit['merge']:
                    parent_sha1 = self.fast_export_marks.get_commit(parent_mark)[:7]
                    LOG.debug("check_commit() merge mark={} sha1={}"
                              .format(parent_mark, parent_sha1))

        if not self.ctx.submodules and 'files' in commit:
            for f in commit['files']:
                if f.get('mode') == '160000':
                    if 'first_commit' in commit and not self._path_added(f.get('path'), commit):
                        LOG.debug2('check_commit() passed {} in {}'.format(
                            f.get('path'), p4gf_util.abbrev(commit['sha1'])))
                        continue
                    raise PreflightException(
                        _('Git submodules not permitted: path={path} commit={commit_sha1}')
                        .format(path=f.get('path'), commit_sha1=p4gf_util.abbrev(commit['sha1'])))

        for f in commit['files']:
            LOG.debug3("check_commit : commit files: " + _log_fe_file(f))
            err = check_valid_filename(f['path'], self.ctx)
            if err:
                raise PreflightException(err)
            if self.ctx.is_lfs_enabled:
                self._check_lfs(commit, f)

        # Warn user about any jobs that appear to not exist
        jobs = G2PJob.lookup_jobs(self.ctx, G2PJob.extract_jobs(commit['data']))
        if jobs:
            for job_id in jobs:
                r = self.ctx.p4run('jobs', '-e', 'job={}'.format(job_id))
                if not r:
                    _print_error(_("Job '{job_id}' doesn't exist").format(job_id=job_id))
        # Create pending changes for any Git-Swarm reviews

    def check_commit_for_branch( self
                               , commit
                               , branch_id
                               , any_locked_files
                               , case_conflict_checker ):
        """
        Prior to copying a commit, perform a set of checks for a specific branch
        to ensure the commit will (likely) go through successfully.
        """
        rev = commit['sha1']
        if LOG.isEnabledFor(logging.DEBUG):
            LOG.debug("check_commit_for_branch() "
                      "Checking branch={} mark={} sha1={} file-ct={} -- {}"
                      .format(  branch_id
                              , commit['mark']
                              , p4gf_util.abbrev(rev)
                              , len(commit['files'])
                              , repr(commit['data'])[:20].splitlines()[0]))

        if self._already_copied_commit(rev, branch_id):
            return

        # following checks assume client has been set for branch
        self.ensure_branch_preflight(commit, branch_id)
        with self.ctx.switched_to_branch(
                  self._current_branch
                , set_client=self.set_client_on_branch_switch
                ):
            if case_conflict_checker:
                case_conflict_checker.read_fast_export_commit(
                    commit, self._current_branch)

            # Empty commits require root-level .p4gf_placeholder to be mapped
            # in the current branch view.
            if not commit['files'] and not self._is_placeholder_mapped():
                raise PreflightException(
                    _("Empty commit {sha1} not permitted. Git Fusion branch views"
                      " must include root to permit empty commits.")
                    .format(sha1=p4gf_util.abbrev(rev)))

            with Timer(CHECK_PROTECTS):
                self._check_protects(commit['author_p4user'], commit['files'])

            with Timer(CHECK_OVERLAP):
                self._check_overlap(commit)

            # fetch the branch setting only, without cascading to repo/global config
            if self._current_branch.is_read_only:
                raise PreflightException(_("Push to branch {branch} prohibited.")
                                         .format(branch=self._current_branch.git_branch_name))
            self._check_stream_writable(commit)
            self._check_stream_in_classic(commit)

            LOG.debug('checking locked files under //{}/...'.format(self.ctx.p4.client))
            if any_locked_files:
                # Convert the git commit paths to depotPaths
                files_in_commit = [self.ctx.gwt_path(f['path']).to_depot()
                                   for f in commit['files']]
                LOG.debug("files_in_commit {0}".format(files_in_commit))
                for f in files_in_commit:
                    if f in any_locked_files:
                        # Collect the names (and clients) of users with locked files.
                        # Report back to the pusher so they can take appropriate action.
                        msg = _('{file} - locked by {user}').format(file=f,
                                                                    user=any_locked_files[f])
                        LOG.info(msg)
                        raise PreflightException(msg)

                    # +++ Spend time extracting Jobs and P4Changelist owner
                    #     here if we actually do need to call
                    #     the preflight-commit hook.
            if self.ctx.preflight_hook.is_callable():
                jobs = G2PJob.extract_jobs(commit['data'])
                jobs2 = G2PJob.lookup_jobs(self.ctx, jobs)
                self.ctx.preflight_hook(
                     ctx                 = self.ctx
                   , fe_commit           = commit
                   , branch_id           = branch_id
                   , jobs                = jobs2
                   )

    def ensure_branch_preflight(self, commit, branch_id):
        """If not already switched to and synced to the correct branch for the
        given commit, do so.

        If this is a new lightweight branch, perform whatever creation we can do
        at preflight time. We don't have commits/marks for any not-yet-submitted
        parent commits, so the depot_branch_info will often lack a correct
        parent or fully populated basis.

        * depot tree, along with a branch-info file
        * branch mapping, along with entry in p4gf_config2 (if not anonymous)

        Return requested branch
        """
        log = LOG.getChild('ensure_branch_preflight')
        branch = self.ctx.branch_dict().get(branch_id)
        # branch should never be None here. p4gf_branch_id.Assigner() must
        # create Branch objects for each assignment.

        if      self._current_branch \
            and self._current_branch.branch_id == branch_id:
            log.debug("sha={} want branch_id={} curr branch_id={} NOP"
                      .format( commit['sha1'][:7]
                             , branch_id[:7]
                             , self._current_branch.branch_id[:7]))
            log.debug("staying on  branch {}"
                      .format(self.ctx.branch_dict().get(branch_id)))

            return branch

        cbid = self._current_branch.branch_id if self._current_branch else 'None'
        log.debug("sha={} want branch_id={} curr branch_id={} switch"
                  .format(commit['sha1'][:7], branch_id[:7], cbid[:7]))

        if not branch.view_lines:
            self.finish_branch_definition(commit, branch)

        elif branch.view_p4map:
            # if this is a stream branch, check for mutation of the stream's
            # view by comparing with the original view saved in p4gf_config2
            if branch.original_view_lines:
                original_view_lines = '\n'.join(branch.original_view_lines)
                view_lines = p4gf_path_convert.convert_view_to_no_client_name(branch.view_lines)
                if not view_lines == original_view_lines:
                    raise PreflightException(
                        _('Unable to push.  Stream view changed from:\n'
                          '{old_view}\nto:\n{new_view}')
                        .format(old_view=original_view_lines, new_view=view_lines))
            # Find existing depot branch for branch view's LHS.
            lhs = branch.view_p4map.lhs()
            branch.depot_branch = self.ctx.depot_branch_info_index()    \
                .find_depot_path(lhs[0])

        log.debug("switching to branch {}".format(branch))

        # By now we should have a branch and a branch.view_lines.
        # First remove current branch's files from workspace
        # Client spec is set to normdir
        self._current_branch = branch
        return branch

    def _is_placeholder_mapped(self):
        """Does this branch map our placeholder file?

        Returns non-False if mapped, None or empty string if not.
        """
        return self.ctx.gwt_path(
            p4gf_const.P4GF_EMPTY_CHANGELIST_PLACEHOLDER).to_depot()

    def _check_protects(self, p4user, blobs):
        """check if author is authorized to submit files."""
        pc = ProtectsChecker(self.ctx, p4user, self.ctx.authenticated_p4user,
                             self.ctx.foruser)
        pc.filter_paths(blobs)
        if pc.has_error():
            raise PreflightException(pc.error_message())

    def _check_overlap(self, fe_commit):
        """If any of the files in this commit intersect any fully populated branch
        (other than the current branch), then reject this commit.

        Shared/common/overlapping paths in branch views must be read-only from
        Git. Otherwise you end up with a Git push of commit on one Git branch
        inserting  changes into other Git branches behind Git's back.

        To modify shared paths, either do so from Perforce, or create a Git
        Fusion repo with no more than one branch that maps that shared path.
        """
                        # +++ Avoid O(b branches * r rev) checks when
                        #     overlap is impossible because current branch
                        #     overlaps no other branch.
        if self._current_branch not in self._overlapping_branch_list():
            return

        for fe_file in fe_commit['files']:
            gwt_path   = fe_file['path']
            depot_path = self.ctx.gwt_to_depot_path(gwt_path)

            for branch in self._overlapping_branch_list():
                if branch == self._current_branch:
                    continue
                if not branch.intersects_depot_path(depot_path):
                    continue

                LOG.debug("_check_overlap() branch {br1} <> {br2}"
                          " gwt={gwt:<40}   {dp}\n{view}"
                          .format(
                              br1  = p4gf_util.abbrev(self._current_branch.branch_id)
                            , br2  = p4gf_util.abbrev(branch.branch_id)
                            , gwt  = gwt_path
                            , dp   = depot_path
                            , view = "\n".join(branch.view_p4map.as_array())
                            ))

                if self._current_branch.is_new_fp_from_push or branch.is_new_fp_from_push:
                    current_branch_name = self._current_branch.git_branch_name
                    if self._current_branch.is_new_fp_from_push:
                        current_branch_name += '(new)'
                    other_branch_name = branch.git_branch_name
                    if branch.is_new_fp_from_push:
                        other_branch_name += '(new)'
                    human_msg = (_(
                        "Perforce: Cannot commit {sha1} '{gwt_path}' to '{depot_path}'.\n"
                        " You are attempting to push and create a new fully populated branch\n"
                        " with paths which overlap another branch. Contact your admin\n"
                        " to configure non-conflicting destination branch paths.\n"
                        " Branches: '{b1}', '{b2}'")
                        .format( sha1       = p4gf_util.abbrev(fe_commit['sha1'])
                               , gwt_path   = gwt_path
                               , depot_path = depot_path
                               , b1 = current_branch_name
                               , b2 = other_branch_name ))
                else:
                    human_msg = (_(
                        "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'."
                        " Paths that overlap multiple Git Fusion branches are read-only."
                        " Branches: '{b1}', '{b2}'")
                        .format( sha1       = p4gf_util.abbrev(fe_commit['sha1'])
                               , gwt_path   = gwt_path
                               , depot_path = depot_path
                               , b1 = self._current_branch.branch_id
                               , b2 = branch.branch_id ))
                raise PreflightException(human_msg)

    def _check_stream_writable(self, fe_commit):
        """If this is a stream branch, check that all files in the commit are
        writable.  If any of the files is not writable then reject this commit.
        """
        if not self._current_branch.stream_name:
            return
        prefix = self._current_branch.writable_stream_name + '/'
        for fe_file in fe_commit['files']:
            gwt_path   = fe_file['path']
            depot_path = self.ctx.gwt_path(gwt_path).to_depot()
            if depot_path.startswith(prefix):
                continue

            human_msg = (_(
                "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'."
                " Paths not in stream '{stream}' are read-only for branch '{b}'.")
                .format( sha1       = p4gf_util.abbrev(fe_commit['sha1'])
                       , gwt_path   = gwt_path
                       , depot_path = depot_path
                       , stream     = self._current_branch.writable_stream_name
                       , b          = self._current_branch.branch_id ))
            raise PreflightException(human_msg)

    def _check_stream_in_classic(self, fe_commit):
        """If this is a classic branch, check that none of the files in the commit
        are in stream depots and thus not writable.  If any of the files is not
        writable then reject this commit.
        """
        if self._current_branch.stream_name:
            return

        depot_re = re.compile(r'^//([^/]+)/([^/]+)/.*$')
        for fe_file in fe_commit['files']:
            gwt_path   = fe_file['path']
            depot_path = self.ctx.gwt_path(gwt_path).to_depot()
            m          = depot_re.match(depot_path)
            if m:
                depot = m.group(1)
                if depot in self.stream_depots:
                    stream = '//{}/{}'.format(m.group(1), m.group(2))
                    human_msg = (
                        _("Cannot commit {sha1} '{gwt_path}' to '{depot_path}'."
                          " Paths in stream '{stream}' are read-only for branch '{b}'.")
                        .format( sha1       = p4gf_util.abbrev(fe_commit['sha1'])
                               , gwt_path   = gwt_path
                               , depot_path = depot_path
                               , stream     = stream
                               , b          = self._current_branch.branch_id ))
                    raise PreflightException(human_msg)

    def _path_added(self, path, fecommit):
        """Return True if the named path was introduced in the HEAD commit.

        :param self: this object
        :param path: repo path to be evaluated.
        :param fecommit: commit object from fast-export parser.

        """
        # Because git-fast-export includes the entire tree in its output,
        # regardless of whether the requested commit is the first in the
        # branch or not, we need to check the repo itself to be certain if
        # this path was truly introduced in this commit, or simply existed
        # in the tree prior to the "first" commit.
        commit = self.ctx.repo.get(fecommit['sha1'])
        if commit is None:
            # empty repository?
            LOG.debug2("_path_added() commit {} is missing".format(fecommit['sha1']))
            return True
        for parent in commit.parents:
            if p4gf_git.exists_in_tree(self.ctx.repo, path, parent.tree):
                LOG.debug2("_path_added() {} exists in parent tree {}".format(
                    path, p4gf_util.abbrev(p4gf_pygit2.object_to_sha1(parent))))
                return False
        return True

    def _invalidate_branch_cache(self):
        """We have changed our branch_dict (or more likely
        finish_branch_definition()ed a branch within that dict) in a way that
        invalidates any cached calculations that consumed the branch dict.
        """
        self._cached_overlapping_branch_list = None

    def _overlapping_branch_list(self):
        """Return a list of fully populated branches that overlap
        other fully populated branches.

        Caches the result because we check every file revision
        path for overlap, and for huge repos with thousands of
        non-overlapping LW branches, just iterating through the
        branch list starts to waste measurable CPU time.
        """
        if self._cached_overlapping_branch_list is not None:
            return self._cached_overlapping_branch_list

        have_overlap = set()
        for outer in p4gf_branch.iter_fp_non_deleted(self.ctx.branch_dict()):
            outer_lhs = P4.Map()
            outer_lhs.insert(outer.view_p4map.lhs())
            for inner in p4gf_branch.iter_fp_non_deleted(self.ctx.branch_dict()):
                if outer == inner:
                    continue
                overlap = P4.Map.join(outer_lhs, inner.view_p4map)
                        # Any non-exclusionary lines shared between branches?
                for line in overlap.as_array():
                    if line.startswith('-') or line.startswith('"-'):
                        continue
                    # Yep. Non-exclusionary line implies overlap
                    have_overlap.add(outer)
                    have_overlap.add(inner)
                    break

        self._cached_overlapping_branch_list = have_overlap
        return self._cached_overlapping_branch_list

    def _curr_ref_ui_name(self):
        """Return a string suitable for use in progress messages as a
        short hint to what we're processing.

        If we have a PreReceiveTuple with a Git branch/tag name, return
        that name (minus any refs/heads/ prefix). If not, return None.
        """
        if not (self._current_prt and self._current_prt.ref):
            return None
        s = self._current_prt.ref
        prefixes = ["refs/heads/", "refs/tags/"]
        for p in prefixes:
            if s.startswith(p):
                return s[len(p):]
        return s

    def _check_lfs(self, fe_commit, fe_file):
        """If gfe_file is under Git LFS control, require that its
        large file content exist somewhere, either in our upload
        cache (it's new!) or in depot de-dupe storage (already got it).
        """
                        # Deleted files carry no LFS pointer.
        if "sha1" not in fe_file:
            return
                        # Symlinks and non-files carry no LFS pointer.
        if fe_file.get("mode") not in [ FileModeStr.PLAIN
                                      , FileModeStr.EXECUTABLE ]:
            return

                        # Files not under Git LFS control should not carry LFS
                        # pointer information. While legal and permissible,
                        # this is usually a mistake (misconfigured Git client)
                        # and something most users want caught before the push
                        # gets into Helix.
        is_tracked = self.ctx.lfs_tracker.is_tracked_git(
                      commit_sha1 = fe_commit["sha1"]
                    , gwt_path    = fe_file["path"])

        LOG.debug3("_check_lfs() tracked {lfs}  commit {commit_sha1}  gwt {gwt}"
            .format( commit_sha1 = p4gf_util.abbrev(fe_commit["sha1"])
                   , lfs         = 1 if is_tracked else 0
                   , gwt         = fe_file["path"]
                   ))

        if not is_tracked:
            lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file)
            if lfs_row:
                raise PreflightException(
                        _("Push of Git LFS text pointer not tracked by LFS:"
                          "\ncommit {commit_sha1} path {gwt_path}")
                        .format( commit_sha1 = p4gf_util.abbrev(fe_commit["sha1"])
                               , gwt_path    = fe_file["path"] ))
            return

                        # Files under Git LFS control should carry LFS pointer
                        # information, but sometimes might not, and that's
                        # okay.
        lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file)
        if not lfs_row:
            return
                        # But if they DO carry LFS pointer information, that
                        # pointer needs to point to a valid LFS large file
                        # either already in Perforce or recently uploaded.
        if not lfs_row.large_file_source:
            LOG.error("LFS text pointer missing content.")
            LOG.error("LFS   commit {}".format(p4gf_util.abbrev(fe_commit["sha1"])))
            LOG.error("LFS   lfs oid {}".format(lfs_row.large_file_oid))
            LOG.error("LFS   ptr {blob_sha1} {blob_mode} {gwt}"
                      .format( blob_sha1 = p4gf_util.abbrev(fe_file["sha1"])
                             , blob_mode = p4gf_util.mode_str(fe_file["mode"])
                             , gwt       = fe_file["path"]))
            LOG.error("LFS   upload  {}".format(lfs_row.to_lfsfs().cache_path(self.ctx)))
            LOG.error("LFS   de-dupe {}".format(lfs_row.to_lfsfs().depot_path(self.ctx)))

            raise PreflightException(_("Push of Git LFS text pointer missing content:"
                        "\ncommit {commit_sha1} path {gwt_path}")
                        .format( commit_sha1 = p4gf_util.abbrev(fe_commit["sha1"])
                               , gwt_path    = fe_file["path"] ))

                        # We have an acceptable LFS text pointer.
                        # Remember it for later.
        self.lfs_row_list.append(lfs_row)

# end class PreflightChecker
# ----------------------------------------------------------------------------


class ProtectsChecker:

    """Handle filtering a list of paths against view and protections."""

    def __init__(self, ctx, author, pusher, foruser):
        """Init P4.Map objects for author, pusher, view and combination."""
        self.ctx = ctx
        self.author = author
        self.pusher = pusher
        self.foruser = foruser

        self.ignore_author_perms = ctx.repo_config.getboolean(p4gf_config.SECTION_GIT_TO_PERFORCE,
                                                              p4gf_config.KEY_IGNORE_AUTHOR_PERMS)

        self.view_map = None
        self.write_protect_author = None
        self.write_protect_pusher = None
        self.write_protect_foruser = None
        self.write_protect_fusion = None
        self.write_filter = None

        self.init_view()
        self.init_write_filter()

        self.author_denied = []
        self.pusher_denied = []
        self.foruser_denied = []
        self.fusion_denied = []
        self.unmapped = []

    def init_view(self):
        """Init view map for client."""
        self.view_map = self.ctx.clientmap

    def init_write_filter(self):
        """Init write filter."""
        self.write_protect_author = self.ctx.user_to_protect(self.author).map_for_perm(
            p4gf_protect.WRITE)
        if self.author != self.pusher or self.ignore_author_perms:
            # If the author and pusher differ, _or_ if we are ignoring
            # author permissions, then we must also included the pusher
            # protections, as well as set write_protect_pusher.
            self.write_protect_pusher = self.ctx.user_to_protect(self.pusher).map_for_perm(
                p4gf_protect.WRITE)
        if self.foruser:
            self.write_protect_foruser = self.ctx.user_to_protect(self.foruser).map_for_perm(
                p4gf_protect.WRITE)
        # Ensure that git-fusion-user can write to the depot
        self.write_protect_fusion = self.ctx.user_to_protect(
            p4gf_const.P4GF_USER).map_for_perm(p4gf_protect.WRITE)
        self.write_filter = functools.reduce(P4.Map.join, [
            vmap for vmap in [
                self.write_protect_author,
                self.write_protect_pusher,
                self.write_protect_foruser,
                self.write_protect_fusion,
                self.view_map,  # view_map has rhs, must come last in join
            ] if vmap])

    def filter_paths(self, blobs):
        """Run list of paths through filter and set list of paths that don't pass."""
        # check against one map for read, one for write
        # if check fails, figure out if it was the view map or the protects
        # that caused the problem and report accordingly
        self.author_denied = []
        self.pusher_denied = []
        self.foruser_denied = []
        self.fusion_denied = []
        self.unmapped = []
        c2d = P4.Map.RIGHT2LEFT

        LOG.debug('filter_paths() write_filter: %s', self.write_filter)
        for blob in blobs:
            gwt_path = self.ctx.gwt_path(blob['path'])
            topath_c = gwt_path.to_client()
            topath_d = gwt_path.to_depot()

            LOG.debug('filter_paths() topath_d: %s', topath_d)
            # for all actions, need to check write access for dest path
            result = "  "   # zum loggen
            if topath_d and P4GF_DEPOT_OBJECTS_RE.match(topath_d):
                LOG.debug('filter_paths() topath_d in //.git-fusion/objects')
                continue
            # do not require user write access to //.git-fusion/branches
            if topath_d and P4GF_DEPOT_BRANCHES_RE.match(topath_d):
                LOG.debug('filter_paths() topath_d in //.git-fusion/branches')
                continue
            if not self.write_filter.includes(topath_c, c2d):
                if not self.view_map.includes(topath_c, c2d):
                    self.unmapped.append(topath_c)
                    result = NTR('unmapped')
                elif not (self.ignore_author_perms or
                          self.write_protect_author.includes(topath_d)):
                    self.author_denied.append(topath_c)
                    result = NTR('author denied')
                elif (self.write_protect_pusher and
                      not self.write_protect_pusher.includes(topath_d)):
                    self.pusher_denied.append(topath_c)
                    result = NTR('pusher denied')
                elif (self.write_protect_foruser and
                      not self.write_protect_foruser.includes(topath_d)):
                    self.foruser_denied.append(topath_c)
                    result = NTR('foruser denied')
                elif not self.write_protect_fusion.includes(topath_d):
                    self.fusion_denied.append(topath_c)
                    result = NTR('Git Fusion denied')
                else:
                    result = "?"
                LOG.error('filter_paths() {:<13} {}, {}, {}'
                          .format(result, blob['path'], topath_d, topath_c))
            elif LOG.isEnabledFor(logging.DEBUG):
                LOG.debug('filter_paths() topath_c in write_filter: %s', topath_c)

    def has_error(self):
        """Return True if any paths not passed by filters."""
        return len(self.unmapped) or len(self.author_denied) \
            or len(self.pusher_denied) or len(self.foruser_denied) \
            or len(self.fusion_denied)

    def error_message(self):
        """Return message indicating what's blocking the push."""
        if len(self.unmapped):
            return _('file(s) not in client view')
        if len(self.author_denied):
            restricted_user = self.author
        elif len(self.pusher_denied):
            restricted_user = self.pusher
        elif len(self.foruser_denied):
            restricted_user = self.foruser
        elif len(self.fusion_denied):
            restricted_user = p4gf_const.P4GF_USER
        else:
            restricted_user = _('<unknown>')
        return _("user '{user}' not authorized to submit file(s) in git commit").format(
            user=restricted_user)

# end ProtectsChecker
# ----------------------------------------------------------------------------


class PreflightException(Exception):

    """This exception is raised when a push was rejected during preflight
    checks.
    """

    pass

# -- module-wide -------------------------------------------------------------

def _log_fe_file(fe_file):
    """Return loggable string for a single fe_commit['files'] element."""
    mode = '      '
    if 'mode' in fe_file:
        mode = fe_file['mode']
    sha1 = '       '
    if 'sha1' in fe_file:
        sha1 = p4gf_util.abbrev(fe_file['sha1'])

    return NTR('{mode} {action} {sha1} {path}') \
           .format( mode   = mode
                  , action = fe_file['action']
                  , sha1   = sha1
                  , path   = fe_file['path'])


def _print_error(msg):
    """Print the given message to the error stream, as well as to the log."""
    sys.stderr.write(msg + '\n')
    LOG.error(msg)


def is_p4d_printable(c):
    """Check if c will be rejected by P4D as non-printable.

    P4D rejects "non-printable" characters with
      ErrorId MsgDm::IdNonPrint = { ErrorOf( ES_DM, 6, E_FAILED, EV_USAGE, 1 )
      "Non-printable characters not allowed in '%id%'." } ;

    Where "character" here means C "char" aka "byte"
    and non-printable means 0x00-0x1f or 0x7f
    """
    if ord(c) < 0x20:
        return False
    if ord(c) == 0x7F:
        return False
    return True


def check_valid_filename(name, ctx):
    """Test the given name for illegal characters.

    Return None if okay, otherwise an error message.
    Illegal characters and sequences include: [...]
    """
    for idx, c in enumerate(name):
        if not is_p4d_printable(c):
            fullname = name[:idx] + "x{ch:02X}".format(ch=ord(c)) + name[idx:]
            return _("Perforce: Non-printable characters not allowed in Perforce: "
                     "character x{ch:02X} in filepath: {filename}").format(
                     filename=fullname, ch=ord(c))
    if '...' in name:
        return _("Perforce: bad filename (...): '{filename}'").format(filename=name)
    if 'P4D/NT' in ctx.server_version:
        if ':' in name:
            return _("Perforce: unsupported filename on windows: {filename}").format(filename=name)
    # This should usually be en_US.UTF-8 which also needs to be defined
    # on the os
    encoding = sys.getfilesystemencoding()
    try:
        name.encode(encoding, "strict")
    except UnicodeEncodeError:
        return _("Perforce: Cannot convert filename to '{encoding}': {filename}").format(
            encoding=encoding, filename=name)
    return None
