#!/usr/bin/python

import sys
import os
import argparse
import subprocess
import cx_Oracle

qa_types = {'MR structural (FSPGR)': 'structural', 
            'MR structural (MPRAGE)': 'structural', 
            'MR structural (T1)': 'structural', 
            'fMRI': 'time series'}

progname = os.path.basename(sys.argv[0])

description = 'Queue NDAR QA runs.'
parser = argparse.ArgumentParser(description=description)

parser.add_argument('--bogus', '-b', 
                    action='store_true', 
                    default=False, 
                    help='queue bogus runs')
parser.add_argument('--file', '-f', 
                    type=file, 
                    help='file containing S3 files to process')
parser.add_argument('--all', '-a', 
                    action='store_true', 
                    default=False, 
                    help='queue all files')
parser.add_argument('--check-only', '-c', 
                    action='store_true', 
                    default=False, 
                    help='check only, don\'t actually queue')
parser.add_argument('--type', '-t', 
                    dest='types', 
                    action='append', 
                    help='QA types to queue (one -t per type)')
parser.add_argument('files', 
                    help='S3 files to process', 
                    nargs='*')

args = parser.parse_args()

if args.all:
    if args.files or args.file:
        parser.print_usage(sys.stderr)
        fmt = '%s: error: S3 files (file or positional arguments) given with --all\n'
        sys.stderr.write(fmt % progname)
        sys.exit(2)
else:
    if not args.files and not args.file:
        parser.print_usage(sys.stderr)
        fmt = '%s: error: S3 files (file or positional arguments) or --all required\n'
        sys.stderr.write(fmt % progname)
        sys.exit(2)

print 'connecting to database...'

for var in ('DB_HOST', 'DB_SERVICE', 'DB_USER', 'DB_PASSWORD'):
    if var not in os.environ:
        sys.stderr.write('%s: %s not set\n' % (progname, var))
        sys.exit(1)

dsn = cx_Oracle.makedsn(os.environ['DB_HOST'], 1521, os.environ['DB_SERVICE'])
db = cx_Oracle.connect(os.environ['DB_USER'], os.environ['DB_PASSWORD'], dsn)

c = db.cursor()

if args.all:
    print 'getting files...'
    files = set()
    query = """SELECT DISTINCT image_file 
                 FROM image03 
                WHERE scan_type = :st 
                  AND image_file NOT IN (SELECT file_source 
                                           FROM imaging_qa01)"""
    for scan_type in qa_types.keys():
        c.execute(query, {'st': scan_type})
        for row in c:
            files.add(row[0])
    files = list(files)
else:
    files = []
    if args.file:
        files.extend([ line.rstrip('\n') for line in args.file ])
    if args.files:
        files.extend(args.files)

print '%d files to process' % len(files)

print 'reading imaging_qa01...'

query = """SELECT file_source FROM imaging_qa01"""
c.execute(query)
qa_done_files = [ row[0] for row in c ]

print 'reading image03...'

scan_types = {}
c.execute("SELECT DISTINCT image_file, scan_type FROM image03")
for (image_file, scan_type) in c:
    scan_types.setdefault(image_file, {})
    try:
        qa_type = qa_types[scan_type]
    except KeyError:
        qa_type = None
    scan_types[image_file][scan_type] = qa_type

subject_info = {}
query = """SELECT DISTINCT image_file, subjectkey, interview_age, image03_id 
             FROM image03"""
c.execute(query)
for (image_file, subjectkey, interview_age, image03_id) in c:
    subject_info.setdefault(image_file, [])
    subject_info[image_file].append((subjectkey, interview_age, image03_id))

c.close()
db.close()

print 'checking files...'

error = False

qa_types = {}
for f in files:
    if f in qa_done_files:
        print 'ERROR: %s is in imaging_qa01' % f
        error = True
        continue
    try:
        distinct_qa_types = set(scan_types[f].values())
    except KeyError:
        print 'ERROR: %s is not in image03' % f
        error = True
        continue
    if len(distinct_qa_types) > 1:
        print 'ERROR: multiple QA types for %s' % f
        error = True
    qa_type = distinct_qa_types.pop()
    if not qa_type:
        print 'ERROR: unset or unsupported scan type for %s' % f
        error = True
    qa_types[f] = qa_type
    if len(subject_info[f]) > 1:
        fmt = 'ERROR: multiple subject key, interview age, image03 id for %s'
        print fmt % f
        error = True

if error:
    sys.exit(1)

if args.check_only:
    print 'done checks'
    sys.exit(0)

if args.bogus:
    report = '    %s (BOGUS): %s'
else:
    report = '    %s: %s'

for f in files:
    print f
    qa_type = qa_types[f]
    if args.types is not None and qa_type not in args.types:
        continue
    if qa_type == 'structural':
        cmd_args = ['qsub', '/ndar/sge/launch_structural_qa']
        if args.bogus:
            cmd_args.append('--bogus')
        cmd_args.append(f)
    if qa_type == 'time series':
        cmd_args = ['qsub', '/ndar/sge/launch_time_series_qa']
        if args.bogus:
            cmd_args.append('--bogus')
        # subjectkey, interview_age, image03_id
        cmd_args.append(subject_info[f][0][0])
        cmd_args.append(str(subject_info[f][0][1]))
        cmd_args.append(str(subject_info[f][0][2]))
        cmd_args.append(f)
    po = subprocess.Popen(cmd_args, 
                          stdout=subprocess.PIPE, 
                          stderr=subprocess.PIPE)
    if po.wait() != 0:
        print 'ERROR in qsub:'
        print po.stderr.read()
        sys.exit(1)
    print report % (qa_type, po.stdout.read().strip())

sys.exit(0)

# eof

99a40de66cc78e730d62a389a734da3afeb6c386
#!/usr/bin/python

import sys
import os
import argparse
import subprocess
import cx_Oracle

qa_types = {'MR structural (FSPGR)': 'structural', 
            'MR structural (MPRAGE)': 'structural', 
            'MR structural (T1)': 'structural', 
            'fMRI': 'time series'}

progname = os.path.basename(sys.argv[0])

description = 'Queue NDAR QA runs.'
parser = argparse.ArgumentParser(description=description)

parser.add_argument('--bogus', '-b', 
                    action='store_true', 
                    default=False, 
                    help='queue bogus runs')
parser.add_argument('--file', '-f', 
                    type=file, 
                    help='file containing S3 files to process')
parser.add_argument('--all', '-a', 
                    action='store_true', 
                    default=False, 
                    help='queue all files')
parser.add_argument('--check-only', '-c', 
                    action='store_true', 
                    default=False, 
                    help='check only, don\'t actually queue')
parser.add_argument('--type', '-t', 
                    dest='types', 
                    action='append', 
                    help='QA types to queue (one -t per type)')
parser.add_argument('files', 
                    help='S3 files to process', 
                    nargs='*')

args = parser.parse_args()

if args.all:
    if args.files or args.file:
        parser.print_usage(sys.stderr)
        fmt = '%s: error: S3 files (file or positional arguments) given with --all\n'
        sys.stderr.write(fmt % progname)
        sys.exit(2)
else:
    if not args.files and not args.file:
        parser.print_usage(sys.stderr)
        fmt = '%s: error: S3 files (file or positional arguments) or --all required\n'
        sys.stderr.write(fmt % progname)
        sys.exit(2)

print 'connecting to database...'

for var in ('DB_HOST', 'DB_SERVICE', 'DB_USER', 'DB_PASSWORD'):
    if var not in os.environ:
        sys.stderr.write('%s: %s not set\n' % (progname, var))
        sys.exit(1)

dsn = cx_Oracle.makedsn(os.environ['DB_HOST'], 1521, os.environ['DB_SERVICE'])
db = cx_Oracle.connect(os.environ['DB_USER'], os.environ['DB_PASSWORD'], dsn)

c = db.cursor()

if args.all:
    print 'getting files...'
    files = set()
    query = """SELECT DISTINCT image_file 
                 FROM image03 
                WHERE scan_type = :st 
                  AND image_file NOT IN (SELECT file_source 
                                           FROM imaging_qa01)"""
    for scan_type in qa_types.keys():
        c.execute(query, {'st': scan_type})
        for row in c:
            files.add(row[0])
    files = list(files)
else:
    files = []
    if args.file:
        files.extend([ line.rstrip('\n') for line in args.file ])
    if args.files:
        files.extend(args.files)

print '%d files to process' % len(files)

print 'reading imaging_qa01...'

query = """SELECT file_source FROM imaging_qa01"""
c.execute(query)
qa_done_files = [ row[0] for row in c ]

print 'reading image03...'

scan_types = {}
c.execute("SELECT DISTINCT image_file, scan_type FROM image03")
for (image_file, scan_type) in c:
    scan_types.setdefault(image_file, {})
    try:
        qa_type = qa_types[scan_type]
    except KeyError:
        qa_type = None
    scan_types[image_file][scan_type] = qa_type

subject_info = {}
query = """SELECT DISTINCT image_file, subjectkey, interview_age, image03_id 
             FROM image03"""
c.execute(query)
for (image_file, subjectkey, interview_age, image03_id) in c:
    subject_info.setdefault(image_file, [])
    subject_info[image_file].append((subjectkey, interview_age, image03_id))

c.close()
db.close()

print 'checking files...'

error = False

qa_types = {}
for f in files:
    if f in qa_done_files:
        print 'ERROR: %s is in imaging_qa01' % f
        error = True
        continue
    try:
        distinct_qa_types = set(scan_types[f].values())
    except KeyError:
        print 'ERROR: %s is not in image03' % f
        error = True
        continue
    if len(distinct_qa_types) > 1:
        print 'ERROR: multiple QA types for %s' % f
        error = True
    qa_type = distinct_qa_types.pop()
    if not qa_type:
        print 'ERROR: unset or unsupported scan type for %s' % f
        error = True
    qa_types[f] = qa_type
    if len(subject_info[f]) > 1:
        fmt = 'ERROR: multiple subject key, interview age, image03 id for %s'
        print fmt % f
        error = True

if error:
    sys.exit(1)

if args.check_only:
    print 'done checks'
    sys.exit(0)

if args.bogus:
    report = '    %s (BOGUS): %s'
else:
    report = '    %s: %s'

for f in files:
    print f
    qa_type = qa_types[f]
    if args.types is not None and qa_type not in args.types:
        continue
    if qa_type == 'structural':
        cmd_args = ['qsub', '/ndar/sge/launch_structural_qa']
        if args.bogus:
            cmd_args.append('--bogus')
        cmd_args.append(f)
    if qa_type == 'time series':
        cmd_args = ['qsub', '/ndar/sge/launch_time_series_qa']
        if args.bogus:
            cmd_args.append('--bogus')
        # subjectkey, interview_age, image03_id
        cmd_args.append(subject_info[f][0][0])
        cmd_args.append(str(subject_info[f][0][1]))
        cmd_args.append(str(subject_info[f][0][2]))
        cmd_args.append(f)
    po = subprocess.Popen(cmd_args, 
                          stdout=subprocess.PIPE, 
                          stderr=subprocess.PIPE)
    if po.wait() != 0:
        print 'ERROR in qsub:'
        print po.stderr.read()
        sys.exit(1)
    print report % (qa_type, po.stdout.read().strip())

sys.exit(0)

# eof
