#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file

"""Python wrapper around gcc to make it behave a little
more like cl.exe WRT to parallel building.
"""

import multiprocessing
import os
import Queue
import shlex
import subprocess
import sys
import threading
import time

verbose = int(os.environ.get('NACL_GCC_VERBOSE', '0'))
show_commands = int(os.environ.get('NACL_GCC_SHOW_COMMANDS', '0'))
stop_on_error = False


def RunGCC(cmd, basename):
  """Run gcc and return the result along will the stdout/stderr."""
  cmdstring = subprocess.list2cmdline(cmd)
  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  stdout, stderr = p.communicate()
  p.wait()
  if show_commands:
    logmsg = cmdstring
  else:
    logmsg = basename
  stderr = logmsg + '\n' + stderr
  return (p.returncode, stdout, stderr)


def BuildSerial(base_cmd, outpath, files):
  final_result = 0

  for filename in files:
    cmd, basename = MakeCommand(base_cmd, outpath, filename)
    rtn, stdout, stderr = RunGCC(cmd, basename)
    sys.stdout.write(stdout)
    sys.stdout.flush()
    sys.stderr.write(stderr)
    sys.stderr.flush()
    if rtn:
      final_result = rtn
      if stop_on_error:
        break

  return final_result


def Worker(queue, out_queue):
  """Entry point got worker threads.

  Each thread will compiler jobs from the queue until
  there are no jobs left or until the main thread signals
  for the work to stop.
  """
  while not queue.empty() and Worker.running:
    item = queue.get(False)
    if not item:
      break
    results = RunGCC(item[0], item[1])
    out_queue.put(results)


def MakeCommand(base_cmd, outpath, filename):
  """Build the full commandline given that output root
  and the intput filename.

  If VS passes an existing directory to -o, then we derive the
  actual object name by combining he directory name with the
  basename of the source file and andding ".obj"
  """
  basename = os.path.basename(filename)
  out = os.path.join(outpath, os.path.splitext(basename)[0] + '.obj')
  return (base_cmd + ['-c', filename, '-o', out], basename)


def BuildParallel(cores, base_cmd, outpath, files):
  Worker.running = True
  pool = []
  job_queue = Queue.Queue()
  out_queue = Queue.Queue()

  for filename in files:
    cmd, basename = MakeCommand(base_cmd, outpath, filename)
    job_queue.put((cmd, basename))

  # Create worker thread pool, passing job queue
  # and output queue to each worker.
  args = (job_queue, out_queue)
  for i in xrange(cores):
    t = threading.Thread(target=Worker, args=args)
    t.start()

  results = 0
  Trace("waiting for %d results" % len(files))
  final_result = 0
  while results < len(files):
    results += 1
    rtn, stdout, stderr = out_queue.get()
    # stdout seem to be completely ignored by visual studio
    # but GCC should output all useful information on stderr
    # anyway.
    sys.stdout.write(stdout)
    sys.stdout.flush()
    sys.stderr.write(stderr)
    sys.stderr.flush()
    if rtn:
      final_result = rtn
      if stop_on_error:
        # stop all workers
        Worker.running = False
        break

  return final_result


def Log(msg):
  """Log message to stderr."""
  # Since Visual Studio basically seems to completely ignore the stdout
  # of the compiler and only echo stderr we print everythign to stderr.
  sys.stderr.write(str(msg) + '\n')
  sys.stderr.flush()


def Trace(msg):
  if verbose:
    Log("nacl_compiler:" + str(msg))


def main(args):
  if args[0][0] == '@':
    rspfile = args[0][1:]
    args = shlex.split(open(rspfile).read())

  # find the last occurrence of '--' in the argument
  # list and use that to signify the start of the
  # list of sources
  index = list(reversed(args)).index('--')
  index = len(args) - index
  base_cmd = args[:index-1]
  files = args[index:]

  # remove -o <path> from base_cmd
  index = base_cmd.index('-o')
  outpath = base_cmd[index+1]
  del base_cmd[index+1]
  del base_cmd[index]

  cores = int(os.environ.get('NACL_GCC_CORES', '0'))
  if not cores:
    cores = multiprocessing.cpu_count()
  cores = min(cores, len(files))

  Trace("compiling %d sources using %d threads" % (len(files), cores))
  rtn = BuildParallel(cores, base_cmd, outpath, files)
  Trace("returning %d" % rtn)
  return rtn


if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))

eee9841021105bd39c7e43fe8e33292447dc3e48
<mask token>


def BuildSerial(base_cmd, outpath, files):
    final_result = 0
    for filename in files:
        cmd, basename = MakeCommand(base_cmd, outpath, filename)
        rtn, stdout, stderr = RunGCC(cmd, basename)
        sys.stdout.write(stdout)
        sys.stdout.flush()
        sys.stderr.write(stderr)
        sys.stderr.flush()
        if rtn:
            final_result = rtn
            if stop_on_error:
                break
    return final_result


def Worker(queue, out_queue):
    """Entry point got worker threads.

  Each thread will compiler jobs from the queue until
  there are no jobs left or until the main thread signals
  for the work to stop.
  """
    while not queue.empty() and Worker.running:
        item = queue.get(False)
        if not item:
            break
        results = RunGCC(item[0], item[1])
        out_queue.put(results)


def MakeCommand(base_cmd, outpath, filename):
    """Build the full commandline given that output root
  and the intput filename.

  If VS passes an existing directory to -o, then we derive the
  actual object name by combining he directory name with the
  basename of the source file and andding ".obj"
  """
    basename = os.path.basename(filename)
    out = os.path.join(outpath, os.path.splitext(basename)[0] + '.obj')
    return base_cmd + ['-c', filename, '-o', out], basename


def BuildParallel(cores, base_cmd, outpath, files):
    Worker.running = True
    pool = []
    job_queue = Queue.Queue()
    out_queue = Queue.Queue()
    for filename in files:
        cmd, basename = MakeCommand(base_cmd, outpath, filename)
        job_queue.put((cmd, basename))
    args = job_queue, out_queue
    for i in xrange(cores):
        t = threading.Thread(target=Worker, args=args)
        t.start()
    results = 0
    Trace('waiting for %d results' % len(files))
    final_result = 0
    while results < len(files):
        results += 1
        rtn, stdout, stderr = out_queue.get()
        sys.stdout.write(stdout)
        sys.stdout.flush()
        sys.stderr.write(stderr)
        sys.stderr.flush()
        if rtn:
            final_result = rtn
            if stop_on_error:
                Worker.running = False
                break
    return final_result


<mask token>


def main(args):
    if args[0][0] == '@':
        rspfile = args[0][1:]
        args = shlex.split(open(rspfile).read())
    index = list(reversed(args)).index('--')
    index = len(args) - index
    base_cmd = args[:index - 1]
    files = args[index:]
    index = base_cmd.index('-o')
    outpath = base_cmd[index + 1]
    del base_cmd[index + 1]
    del base_cmd[index]
    cores = int(os.environ.get('NACL_GCC_CORES', '0'))
    if not cores:
        cores = multiprocessing.cpu_count()
    cores = min(cores, len(files))
    Trace('compiling %d sources using %d threads' % (len(files), cores))
    rtn = BuildParallel(cores, base_cmd, outpath, files)
    Trace('returning %d' % rtn)
    return rtn


<mask token>

<mask token>


def RunGCC(cmd, basename):
    """Run gcc and return the result along will the stdout/stderr."""
    cmdstring = subprocess.list2cmdline(cmd)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = p.communicate()
    p.wait()
    if show_commands:
        logmsg = cmdstring
    else:
        logmsg = basename
    stderr = logmsg + '\n' + stderr
    return p.returncode, stdout, stderr


def BuildSerial(base_cmd, outpath, files):
    final_result = 0
    for filename in files:
        cmd, basename = MakeCommand(base_cmd, outpath, filename)
        rtn, stdout, stderr = RunGCC(cmd, basename)
        sys.stdout.write(stdout)
        sys.stdout.flush()
        sys.stderr.write(stderr)
        sys.stderr.flush()
        if rtn:
            final_result = rtn
            if stop_on_error:
                break
    return final_result


def Worker(queue, out_queue):
    """Entry point got worker threads.

  Each thread will compiler jobs from the queue until
  there are no jobs left or until the main thread signals
  for the work to stop.
  """
    while not queue.empty() and Worker.running:
        item = queue.get(False)
        if not item:
            break
        results = RunGCC(item[0], item[1])
        out_queue.put(results)


def MakeCommand(base_cmd, outpath, filename):
    """Build the full commandline given that output root
  and the intput filename.

  If VS passes an existing directory to -o, then we derive the
  actual object name by combining he directory name with the
  basename of the source file and andding ".obj"
  """
    basename = os.path.basename(filename)
    out = os.path.join(outpath, os.path.splitext(basename)[0] + '.obj')
    return base_cmd + ['-c', filename, '-o', out], basename


def BuildParallel(cores, base_cmd, outpath, files):
    Worker.running = True
    pool = []
    job_queue = Queue.Queue()
    out_queue = Queue.Queue()
    for filename in files:
        cmd, basename = MakeCommand(base_cmd, outpath, filename)
        job_queue.put((cmd, basename))
    args = job_queue, out_queue
    for i in xrange(cores):
        t = threading.Thread(target=Worker, args=args)
        t.start()
    results = 0
    Trace('waiting for %d results' % len(files))
    final_result = 0
    while results < len(files):
        results += 1
        rtn, stdout, stderr = out_queue.get()
        sys.stdout.write(stdout)
        sys.stdout.flush()
        sys.stderr.write(stderr)
        sys.stderr.flush()
        if rtn:
            final_result = rtn
            if stop_on_error:
                Worker.running = False
                break
    return final_result


def Log(msg):
    """Log message to stderr."""
    sys.stderr.write(str(msg) + '\n')
    sys.stderr.flush()


def Trace(msg):
    if verbose:
        Log('nacl_compiler:' + str(msg))


def main(args):
    if args[0][0] == '@':
        rspfile = args[0][1:]
        args = shlex.split(open(rspfile).read())
    index = list(reversed(args)).index('--')
    index = len(args) - index
    base_cmd = args[:index - 1]
    files = args[index:]
    index = base_cmd.index('-o')
    outpath = base_cmd[index + 1]
    del base_cmd[index + 1]
    del base_cmd[index]
    cores = int(os.environ.get('NACL_GCC_CORES', '0'))
    if not cores:
        cores = multiprocessing.cpu_count()
    cores = min(cores, len(files))
    Trace('compiling %d sources using %d threads' % (len(files), cores))
    rtn = BuildParallel(cores, base_cmd, outpath, files)
    Trace('returning %d' % rtn)
    return rtn


<mask token>

<mask token>


def RunGCC(cmd, basename):
    """Run gcc and return the result along will the stdout/stderr."""
    cmdstring = subprocess.list2cmdline(cmd)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = p.communicate()
    p.wait()
    if show_commands:
        logmsg = cmdstring
    else:
        logmsg = basename
    stderr = logmsg + '\n' + stderr
    return p.returncode, stdout, stderr


def BuildSerial(base_cmd, outpath, files):
    final_result = 0
    for filename in files:
        cmd, basename = MakeCommand(base_cmd, outpath, filename)
        rtn, stdout, stderr = RunGCC(cmd, basename)
        sys.stdout.write(stdout)
        sys.stdout.flush()
        sys.stderr.write(stderr)
        sys.stderr.flush()
        if rtn:
            final_result = rtn
            if stop_on_error:
                break
    return final_result


def Worker(queue, out_queue):
    """Entry point got worker threads.

  Each thread will compiler jobs from the queue until
  there are no jobs left or until the main thread signals
  for the work to stop.
  """
    while not queue.empty() and Worker.running:
        item = queue.get(False)
        if not item:
            break
        results = RunGCC(item[0], item[1])
        out_queue.put(results)


def MakeCommand(base_cmd, outpath, filename):
    """Build the full commandline given that output root
  and the intput filename.

  If VS passes an existing directory to -o, then we derive the
  actual object name by combining he directory name with the
  basename of the source file and andding ".obj"
  """
    basename = os.path.basename(filename)
    out = os.path.join(outpath, os.path.splitext(basename)[0] + '.obj')
    return base_cmd + ['-c', filename, '-o', out], basename


def BuildParallel(cores, base_cmd, outpath, files):
    Worker.running = True
    pool = []
    job_queue = Queue.Queue()
    out_queue = Queue.Queue()
    for filename in files:
        cmd, basename = MakeCommand(base_cmd, outpath, filename)
        job_queue.put((cmd, basename))
    args = job_queue, out_queue
    for i in xrange(cores):
        t = threading.Thread(target=Worker, args=args)
        t.start()
    results = 0
    Trace('waiting for %d results' % len(files))
    final_result = 0
    while results < len(files):
        results += 1
        rtn, stdout, stderr = out_queue.get()
        sys.stdout.write(stdout)
        sys.stdout.flush()
        sys.stderr.write(stderr)
        sys.stderr.flush()
        if rtn:
            final_result = rtn
            if stop_on_error:
                Worker.running = False
                break
    return final_result


def Log(msg):
    """Log message to stderr."""
    sys.stderr.write(str(msg) + '\n')
    sys.stderr.flush()


def Trace(msg):
    if verbose:
        Log('nacl_compiler:' + str(msg))


def main(args):
    if args[0][0] == '@':
        rspfile = args[0][1:]
        args = shlex.split(open(rspfile).read())
    index = list(reversed(args)).index('--')
    index = len(args) - index
    base_cmd = args[:index - 1]
    files = args[index:]
    index = base_cmd.index('-o')
    outpath = base_cmd[index + 1]
    del base_cmd[index + 1]
    del base_cmd[index]
    cores = int(os.environ.get('NACL_GCC_CORES', '0'))
    if not cores:
        cores = multiprocessing.cpu_count()
    cores = min(cores, len(files))
    Trace('compiling %d sources using %d threads' % (len(files), cores))
    rtn = BuildParallel(cores, base_cmd, outpath, files)
    Trace('returning %d' % rtn)
    return rtn


if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))

<mask token>
verbose = int(os.environ.get('NACL_GCC_VERBOSE', '0'))
show_commands = int(os.environ.get('NACL_GCC_SHOW_COMMANDS', '0'))
stop_on_error = False


def RunGCC(cmd, basename):
    """Run gcc and return the result along will the stdout/stderr."""
    cmdstring = subprocess.list2cmdline(cmd)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = p.communicate()
    p.wait()
    if show_commands:
        logmsg = cmdstring
    else:
        logmsg = basename
    stderr = logmsg + '\n' + stderr
    return p.returncode, stdout, stderr


def BuildSerial(base_cmd, outpath, files):
    final_result = 0
    for filename in files:
        cmd, basename = MakeCommand(base_cmd, outpath, filename)
        rtn, stdout, stderr = RunGCC(cmd, basename)
        sys.stdout.write(stdout)
        sys.stdout.flush()
        sys.stderr.write(stderr)
        sys.stderr.flush()
        if rtn:
            final_result = rtn
            if stop_on_error:
                break
    return final_result


def Worker(queue, out_queue):
    """Entry point got worker threads.

  Each thread will compiler jobs from the queue until
  there are no jobs left or until the main thread signals
  for the work to stop.
  """
    while not queue.empty() and Worker.running:
        item = queue.get(False)
        if not item:
            break
        results = RunGCC(item[0], item[1])
        out_queue.put(results)


def MakeCommand(base_cmd, outpath, filename):
    """Build the full commandline given that output root
  and the intput filename.

  If VS passes an existing directory to -o, then we derive the
  actual object name by combining he directory name with the
  basename of the source file and andding ".obj"
  """
    basename = os.path.basename(filename)
    out = os.path.join(outpath, os.path.splitext(basename)[0] + '.obj')
    return base_cmd + ['-c', filename, '-o', out], basename


def BuildParallel(cores, base_cmd, outpath, files):
    Worker.running = True
    pool = []
    job_queue = Queue.Queue()
    out_queue = Queue.Queue()
    for filename in files:
        cmd, basename = MakeCommand(base_cmd, outpath, filename)
        job_queue.put((cmd, basename))
    args = job_queue, out_queue
    for i in xrange(cores):
        t = threading.Thread(target=Worker, args=args)
        t.start()
    results = 0
    Trace('waiting for %d results' % len(files))
    final_result = 0
    while results < len(files):
        results += 1
        rtn, stdout, stderr = out_queue.get()
        sys.stdout.write(stdout)
        sys.stdout.flush()
        sys.stderr.write(stderr)
        sys.stderr.flush()
        if rtn:
            final_result = rtn
            if stop_on_error:
                Worker.running = False
                break
    return final_result


def Log(msg):
    """Log message to stderr."""
    sys.stderr.write(str(msg) + '\n')
    sys.stderr.flush()


def Trace(msg):
    if verbose:
        Log('nacl_compiler:' + str(msg))


def main(args):
    if args[0][0] == '@':
        rspfile = args[0][1:]
        args = shlex.split(open(rspfile).read())
    index = list(reversed(args)).index('--')
    index = len(args) - index
    base_cmd = args[:index - 1]
    files = args[index:]
    index = base_cmd.index('-o')
    outpath = base_cmd[index + 1]
    del base_cmd[index + 1]
    del base_cmd[index]
    cores = int(os.environ.get('NACL_GCC_CORES', '0'))
    if not cores:
        cores = multiprocessing.cpu_count()
    cores = min(cores, len(files))
    Trace('compiling %d sources using %d threads' % (len(files), cores))
    rtn = BuildParallel(cores, base_cmd, outpath, files)
    Trace('returning %d' % rtn)
    return rtn


if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))

#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file

"""Python wrapper around gcc to make it behave a little
more like cl.exe WRT to parallel building.
"""

import multiprocessing
import os
import Queue
import shlex
import subprocess
import sys
import threading
import time

verbose = int(os.environ.get('NACL_GCC_VERBOSE', '0'))
show_commands = int(os.environ.get('NACL_GCC_SHOW_COMMANDS', '0'))
stop_on_error = False


def RunGCC(cmd, basename):
  """Run gcc and return the result along will the stdout/stderr."""
  cmdstring = subprocess.list2cmdline(cmd)
  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  stdout, stderr = p.communicate()
  p.wait()
  if show_commands:
    logmsg = cmdstring
  else:
    logmsg = basename
  stderr = logmsg + '\n' + stderr
  return (p.returncode, stdout, stderr)


def BuildSerial(base_cmd, outpath, files):
  final_result = 0

  for filename in files:
    cmd, basename = MakeCommand(base_cmd, outpath, filename)
    rtn, stdout, stderr = RunGCC(cmd, basename)
    sys.stdout.write(stdout)
    sys.stdout.flush()
    sys.stderr.write(stderr)
    sys.stderr.flush()
    if rtn:
      final_result = rtn
      if stop_on_error:
        break

  return final_result


def Worker(queue, out_queue):
  """Entry point got worker threads.

  Each thread will compiler jobs from the queue until
  there are no jobs left or until the main thread signals
  for the work to stop.
  """
  while not queue.empty() and Worker.running:
    item = queue.get(False)
    if not item:
      break
    results = RunGCC(item[0], item[1])
    out_queue.put(results)


def MakeCommand(base_cmd, outpath, filename):
  """Build the full commandline given that output root
  and the intput filename.

  If VS passes an existing directory to -o, then we derive the
  actual object name by combining he directory name with the
  basename of the source file and andding ".obj"
  """
  basename = os.path.basename(filename)
  out = os.path.join(outpath, os.path.splitext(basename)[0] + '.obj')
  return (base_cmd + ['-c', filename, '-o', out], basename)


def BuildParallel(cores, base_cmd, outpath, files):
  Worker.running = True
  pool = []
  job_queue = Queue.Queue()
  out_queue = Queue.Queue()

  for filename in files:
    cmd, basename = MakeCommand(base_cmd, outpath, filename)
    job_queue.put((cmd, basename))

  # Create worker thread pool, passing job queue
  # and output queue to each worker.
  args = (job_queue, out_queue)
  for i in xrange(cores):
    t = threading.Thread(target=Worker, args=args)
    t.start()

  results = 0
  Trace("waiting for %d results" % len(files))
  final_result = 0
  while results < len(files):
    results += 1
    rtn, stdout, stderr = out_queue.get()
    # stdout seem to be completely ignored by visual studio
    # but GCC should output all useful information on stderr
    # anyway.
    sys.stdout.write(stdout)
    sys.stdout.flush()
    sys.stderr.write(stderr)
    sys.stderr.flush()
    if rtn:
      final_result = rtn
      if stop_on_error:
        # stop all workers
        Worker.running = False
        break

  return final_result


def Log(msg):
  """Log message to stderr."""
  # Since Visual Studio basically seems to completely ignore the stdout
  # of the compiler and only echo stderr we print everythign to stderr.
  sys.stderr.write(str(msg) + '\n')
  sys.stderr.flush()


def Trace(msg):
  if verbose:
    Log("nacl_compiler:" + str(msg))


def main(args):
  if args[0][0] == '@':
    rspfile = args[0][1:]
    args = shlex.split(open(rspfile).read())

  # find the last occurrence of '--' in the argument
  # list and use that to signify the start of the
  # list of sources
  index = list(reversed(args)).index('--')
  index = len(args) - index
  base_cmd = args[:index-1]
  files = args[index:]

  # remove -o <path> from base_cmd
  index = base_cmd.index('-o')
  outpath = base_cmd[index+1]
  del base_cmd[index+1]
  del base_cmd[index]

  cores = int(os.environ.get('NACL_GCC_CORES', '0'))
  if not cores:
    cores = multiprocessing.cpu_count()
  cores = min(cores, len(files))

  Trace("compiling %d sources using %d threads" % (len(files), cores))
  rtn = BuildParallel(cores, base_cmd, outpath, files)
  Trace("returning %d" % rtn)
  return rtn


if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
