"""
Script to Query ArangoDB. See README.md.

Author: Volker Hoffman <volker.hoffmann@sintef.no>
Update: 06 June 2018
"""

from __future__ import print_function
import pyArango.connection
import json
import time
import argparse


def connect_to_database():
    conn = pyArango.connection.Connection(arangoURL='http://192.168.1.45:8529')
    db = conn['_system']
    return db


def read_campaign_keys_from_json(fname):
    with open(fname, 'r') as f:
        json_of_keywords = json.load(f)
    campaign_keys = []
    for json_of_keyword in json_of_keywords:
        campaign_keys.append(int(json_of_keyword['key']))
    return campaign_keys


def read_query_from_file(fname):
    with open(fname, 'r') as f:
        lines = f.readlines()
    qry = ''.join(lines)
    return qry


# run query against database
# for Bind Parameters (aka bindVars), see
# https://github.com/tariqdaouda/pyArango#queries--aql
# https://docs.arangodb.com/3.3/AQL/Fundamentals/BindParameters.html
# do note the peculiarities about string processing
# also see query.aql
def run_query(db, qry, campaign_key=546426629):
    queryResult = db.AQLQuery(qry, rawResults=True, batchSize=64, count=True, \
                              bindVars={'campaign_key': campaign_key})
    return queryResult


def query_to_result_list(queryResult):
    t0 = time.time()
    result_list = []
    for ii, qq in enumerate(queryResult):
        if ii % 5000 == 0:
            t1 = time.time()
            print("** Fetched %07d/%07d Records (%.1fs Elapsed)" % \
                  (ii, queryResult.count, t1-t0))
        result_list.append(qq)
    t1 = time.time()
    print("** Fetched %06d/%06d Records (%.1fs Elapsed)" % \
          (ii+1, queryResult.count, t1-t0))
    return result_list


# write query output to file
def result_list_to_file(result_list, fname):
    with open(fname, 'w') as f:
        json.dump(result_list, f)


print('// Parsing Arguments')
parser = argparse.ArgumentParser()
parser.add_argument('--keyfile', default='1m10small.json', \
                    help='Name of JSON File w/ Campaign Keys.')
args = parser.parse_args()

print('// Connecting to Database')
db = connect_to_database()

print("// Loading Campaign Keys (%s)" % args.keyfile)
campaign_keys = read_campaign_keys_from_json(args.keyfile)
print('** List of Requested Keys is:')
print(campaign_keys)

print('// Looping Over Campaign Keys')
print('')
for campaign_key in campaign_keys:
    t00 = time.time()

    print("**** Running Query/Fetch/Save for Campaign Key %i" % campaign_key)
    print(">>>> %s UTC" % time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))

    print('// Loading Query')
    qry = read_query_from_file('query.aql')
    print('** Query:')
    print(qry)

    print("// Excecuting Query for Campaign Key %i" % campaign_key)
    queryResult = run_query(db, qry, campaign_key=campaign_key)

    print("** Query Returned %i Records" % queryResult.count)
    if queryResult.response['cached'] == True:
        print("** Query Was Cached")
    else:
        print("** Query Took %.2f Seconds" % \
              queryResult.response['extra']['stats']['executionTime'])
    print('// Fetching Resulting. Wallclock Ticking.')
    result_list = query_to_result_list(queryResult)

    fname = "result_%i.json" % campaign_key
    print("// Writing Result to File (%s)" % fname)
    t0 = time.time()
    result_list_to_file(result_list, fname)
    t1 = time.time()
    print("** Took %.1fs" % (t1-t0))

    print("**** Finished for Campaign Key %i" % campaign_key)
    print("**** Round-Trip Time %.1fs" % (time.time() - t00))
    print(">>>> %s UTC" % time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))
    print('')

print('!! Done')

b6bc2cc268adeb29cf8d916661c13141d4b1f8e9
<mask token>


def connect_to_database():
    conn = pyArango.connection.Connection(arangoURL='http://192.168.1.45:8529')
    db = conn['_system']
    return db


def read_campaign_keys_from_json(fname):
    with open(fname, 'r') as f:
        json_of_keywords = json.load(f)
    campaign_keys = []
    for json_of_keyword in json_of_keywords:
        campaign_keys.append(int(json_of_keyword['key']))
    return campaign_keys


def read_query_from_file(fname):
    with open(fname, 'r') as f:
        lines = f.readlines()
    qry = ''.join(lines)
    return qry


def run_query(db, qry, campaign_key=546426629):
    queryResult = db.AQLQuery(qry, rawResults=True, batchSize=64, count=
        True, bindVars={'campaign_key': campaign_key})
    return queryResult


def query_to_result_list(queryResult):
    t0 = time.time()
    result_list = []
    for ii, qq in enumerate(queryResult):
        if ii % 5000 == 0:
            t1 = time.time()
            print('** Fetched %07d/%07d Records (%.1fs Elapsed)' % (ii,
                queryResult.count, t1 - t0))
        result_list.append(qq)
    t1 = time.time()
    print('** Fetched %06d/%06d Records (%.1fs Elapsed)' % (ii + 1,
        queryResult.count, t1 - t0))
    return result_list


def result_list_to_file(result_list, fname):
    with open(fname, 'w') as f:
        json.dump(result_list, f)


<mask token>

<mask token>


def connect_to_database():
    conn = pyArango.connection.Connection(arangoURL='http://192.168.1.45:8529')
    db = conn['_system']
    return db


def read_campaign_keys_from_json(fname):
    with open(fname, 'r') as f:
        json_of_keywords = json.load(f)
    campaign_keys = []
    for json_of_keyword in json_of_keywords:
        campaign_keys.append(int(json_of_keyword['key']))
    return campaign_keys


def read_query_from_file(fname):
    with open(fname, 'r') as f:
        lines = f.readlines()
    qry = ''.join(lines)
    return qry


def run_query(db, qry, campaign_key=546426629):
    queryResult = db.AQLQuery(qry, rawResults=True, batchSize=64, count=
        True, bindVars={'campaign_key': campaign_key})
    return queryResult


def query_to_result_list(queryResult):
    t0 = time.time()
    result_list = []
    for ii, qq in enumerate(queryResult):
        if ii % 5000 == 0:
            t1 = time.time()
            print('** Fetched %07d/%07d Records (%.1fs Elapsed)' % (ii,
                queryResult.count, t1 - t0))
        result_list.append(qq)
    t1 = time.time()
    print('** Fetched %06d/%06d Records (%.1fs Elapsed)' % (ii + 1,
        queryResult.count, t1 - t0))
    return result_list


def result_list_to_file(result_list, fname):
    with open(fname, 'w') as f:
        json.dump(result_list, f)


print('// Parsing Arguments')
<mask token>
parser.add_argument('--keyfile', default='1m10small.json', help=
    'Name of JSON File w/ Campaign Keys.')
<mask token>
print('// Connecting to Database')
<mask token>
print('// Loading Campaign Keys (%s)' % args.keyfile)
<mask token>
print('** List of Requested Keys is:')
print(campaign_keys)
print('// Looping Over Campaign Keys')
print('')
for campaign_key in campaign_keys:
    t00 = time.time()
    print('**** Running Query/Fetch/Save for Campaign Key %i' % campaign_key)
    print('>>>> %s UTC' % time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()))
    print('// Loading Query')
    qry = read_query_from_file('query.aql')
    print('** Query:')
    print(qry)
    print('// Excecuting Query for Campaign Key %i' % campaign_key)
    queryResult = run_query(db, qry, campaign_key=campaign_key)
    print('** Query Returned %i Records' % queryResult.count)
    if queryResult.response['cached'] == True:
        print('** Query Was Cached')
    else:
        print('** Query Took %.2f Seconds' % queryResult.response['extra'][
            'stats']['executionTime'])
    print('// Fetching Resulting. Wallclock Ticking.')
    result_list = query_to_result_list(queryResult)
    fname = 'result_%i.json' % campaign_key
    print('// Writing Result to File (%s)' % fname)
    t0 = time.time()
    result_list_to_file(result_list, fname)
    t1 = time.time()
    print('** Took %.1fs' % (t1 - t0))
    print('**** Finished for Campaign Key %i' % campaign_key)
    print('**** Round-Trip Time %.1fs' % (time.time() - t00))
    print('>>>> %s UTC' % time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()))
    print('')
print('!! Done')

<mask token>


def connect_to_database():
    conn = pyArango.connection.Connection(arangoURL='http://192.168.1.45:8529')
    db = conn['_system']
    return db


def read_campaign_keys_from_json(fname):
    with open(fname, 'r') as f:
        json_of_keywords = json.load(f)
    campaign_keys = []
    for json_of_keyword in json_of_keywords:
        campaign_keys.append(int(json_of_keyword['key']))
    return campaign_keys


def read_query_from_file(fname):
    with open(fname, 'r') as f:
        lines = f.readlines()
    qry = ''.join(lines)
    return qry


def run_query(db, qry, campaign_key=546426629):
    queryResult = db.AQLQuery(qry, rawResults=True, batchSize=64, count=
        True, bindVars={'campaign_key': campaign_key})
    return queryResult


def query_to_result_list(queryResult):
    t0 = time.time()
    result_list = []
    for ii, qq in enumerate(queryResult):
        if ii % 5000 == 0:
            t1 = time.time()
            print('** Fetched %07d/%07d Records (%.1fs Elapsed)' % (ii,
                queryResult.count, t1 - t0))
        result_list.append(qq)
    t1 = time.time()
    print('** Fetched %06d/%06d Records (%.1fs Elapsed)' % (ii + 1,
        queryResult.count, t1 - t0))
    return result_list


def result_list_to_file(result_list, fname):
    with open(fname, 'w') as f:
        json.dump(result_list, f)


print('// Parsing Arguments')
parser = argparse.ArgumentParser()
parser.add_argument('--keyfile', default='1m10small.json', help=
    'Name of JSON File w/ Campaign Keys.')
args = parser.parse_args()
print('// Connecting to Database')
db = connect_to_database()
print('// Loading Campaign Keys (%s)' % args.keyfile)
campaign_keys = read_campaign_keys_from_json(args.keyfile)
print('** List of Requested Keys is:')
print(campaign_keys)
print('// Looping Over Campaign Keys')
print('')
for campaign_key in campaign_keys:
    t00 = time.time()
    print('**** Running Query/Fetch/Save for Campaign Key %i' % campaign_key)
    print('>>>> %s UTC' % time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()))
    print('// Loading Query')
    qry = read_query_from_file('query.aql')
    print('** Query:')
    print(qry)
    print('// Excecuting Query for Campaign Key %i' % campaign_key)
    queryResult = run_query(db, qry, campaign_key=campaign_key)
    print('** Query Returned %i Records' % queryResult.count)
    if queryResult.response['cached'] == True:
        print('** Query Was Cached')
    else:
        print('** Query Took %.2f Seconds' % queryResult.response['extra'][
            'stats']['executionTime'])
    print('// Fetching Resulting. Wallclock Ticking.')
    result_list = query_to_result_list(queryResult)
    fname = 'result_%i.json' % campaign_key
    print('// Writing Result to File (%s)' % fname)
    t0 = time.time()
    result_list_to_file(result_list, fname)
    t1 = time.time()
    print('** Took %.1fs' % (t1 - t0))
    print('**** Finished for Campaign Key %i' % campaign_key)
    print('**** Round-Trip Time %.1fs' % (time.time() - t00))
    print('>>>> %s UTC' % time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()))
    print('')
print('!! Done')

<mask token>
from __future__ import print_function
import pyArango.connection
import json
import time
import argparse


def connect_to_database():
    conn = pyArango.connection.Connection(arangoURL='http://192.168.1.45:8529')
    db = conn['_system']
    return db


def read_campaign_keys_from_json(fname):
    with open(fname, 'r') as f:
        json_of_keywords = json.load(f)
    campaign_keys = []
    for json_of_keyword in json_of_keywords:
        campaign_keys.append(int(json_of_keyword['key']))
    return campaign_keys


def read_query_from_file(fname):
    with open(fname, 'r') as f:
        lines = f.readlines()
    qry = ''.join(lines)
    return qry


def run_query(db, qry, campaign_key=546426629):
    queryResult = db.AQLQuery(qry, rawResults=True, batchSize=64, count=
        True, bindVars={'campaign_key': campaign_key})
    return queryResult


def query_to_result_list(queryResult):
    t0 = time.time()
    result_list = []
    for ii, qq in enumerate(queryResult):
        if ii % 5000 == 0:
            t1 = time.time()
            print('** Fetched %07d/%07d Records (%.1fs Elapsed)' % (ii,
                queryResult.count, t1 - t0))
        result_list.append(qq)
    t1 = time.time()
    print('** Fetched %06d/%06d Records (%.1fs Elapsed)' % (ii + 1,
        queryResult.count, t1 - t0))
    return result_list


def result_list_to_file(result_list, fname):
    with open(fname, 'w') as f:
        json.dump(result_list, f)


print('// Parsing Arguments')
parser = argparse.ArgumentParser()
parser.add_argument('--keyfile', default='1m10small.json', help=
    'Name of JSON File w/ Campaign Keys.')
args = parser.parse_args()
print('// Connecting to Database')
db = connect_to_database()
print('// Loading Campaign Keys (%s)' % args.keyfile)
campaign_keys = read_campaign_keys_from_json(args.keyfile)
print('** List of Requested Keys is:')
print(campaign_keys)
print('// Looping Over Campaign Keys')
print('')
for campaign_key in campaign_keys:
    t00 = time.time()
    print('**** Running Query/Fetch/Save for Campaign Key %i' % campaign_key)
    print('>>>> %s UTC' % time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()))
    print('// Loading Query')
    qry = read_query_from_file('query.aql')
    print('** Query:')
    print(qry)
    print('// Excecuting Query for Campaign Key %i' % campaign_key)
    queryResult = run_query(db, qry, campaign_key=campaign_key)
    print('** Query Returned %i Records' % queryResult.count)
    if queryResult.response['cached'] == True:
        print('** Query Was Cached')
    else:
        print('** Query Took %.2f Seconds' % queryResult.response['extra'][
            'stats']['executionTime'])
    print('// Fetching Resulting. Wallclock Ticking.')
    result_list = query_to_result_list(queryResult)
    fname = 'result_%i.json' % campaign_key
    print('// Writing Result to File (%s)' % fname)
    t0 = time.time()
    result_list_to_file(result_list, fname)
    t1 = time.time()
    print('** Took %.1fs' % (t1 - t0))
    print('**** Finished for Campaign Key %i' % campaign_key)
    print('**** Round-Trip Time %.1fs' % (time.time() - t00))
    print('>>>> %s UTC' % time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()))
    print('')
print('!! Done')

"""
Script to Query ArangoDB. See README.md.

Author: Volker Hoffman <volker.hoffmann@sintef.no>
Update: 06 June 2018
"""

from __future__ import print_function
import pyArango.connection
import json
import time
import argparse


def connect_to_database():
    conn = pyArango.connection.Connection(arangoURL='http://192.168.1.45:8529')
    db = conn['_system']
    return db


def read_campaign_keys_from_json(fname):
    with open(fname, 'r') as f:
        json_of_keywords = json.load(f)
    campaign_keys = []
    for json_of_keyword in json_of_keywords:
        campaign_keys.append(int(json_of_keyword['key']))
    return campaign_keys


def read_query_from_file(fname):
    with open(fname, 'r') as f:
        lines = f.readlines()
    qry = ''.join(lines)
    return qry


# run query against database
# for Bind Parameters (aka bindVars), see
# https://github.com/tariqdaouda/pyArango#queries--aql
# https://docs.arangodb.com/3.3/AQL/Fundamentals/BindParameters.html
# do note the peculiarities about string processing
# also see query.aql
def run_query(db, qry, campaign_key=546426629):
    queryResult = db.AQLQuery(qry, rawResults=True, batchSize=64, count=True, \
                              bindVars={'campaign_key': campaign_key})
    return queryResult


def query_to_result_list(queryResult):
    t0 = time.time()
    result_list = []
    for ii, qq in enumerate(queryResult):
        if ii % 5000 == 0:
            t1 = time.time()
            print("** Fetched %07d/%07d Records (%.1fs Elapsed)" % \
                  (ii, queryResult.count, t1-t0))
        result_list.append(qq)
    t1 = time.time()
    print("** Fetched %06d/%06d Records (%.1fs Elapsed)" % \
          (ii+1, queryResult.count, t1-t0))
    return result_list


# write query output to file
def result_list_to_file(result_list, fname):
    with open(fname, 'w') as f:
        json.dump(result_list, f)


print('// Parsing Arguments')
parser = argparse.ArgumentParser()
parser.add_argument('--keyfile', default='1m10small.json', \
                    help='Name of JSON File w/ Campaign Keys.')
args = parser.parse_args()

print('// Connecting to Database')
db = connect_to_database()

print("// Loading Campaign Keys (%s)" % args.keyfile)
campaign_keys = read_campaign_keys_from_json(args.keyfile)
print('** List of Requested Keys is:')
print(campaign_keys)

print('// Looping Over Campaign Keys')
print('')
for campaign_key in campaign_keys:
    t00 = time.time()

    print("**** Running Query/Fetch/Save for Campaign Key %i" % campaign_key)
    print(">>>> %s UTC" % time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))

    print('// Loading Query')
    qry = read_query_from_file('query.aql')
    print('** Query:')
    print(qry)

    print("// Excecuting Query for Campaign Key %i" % campaign_key)
    queryResult = run_query(db, qry, campaign_key=campaign_key)

    print("** Query Returned %i Records" % queryResult.count)
    if queryResult.response['cached'] == True:
        print("** Query Was Cached")
    else:
        print("** Query Took %.2f Seconds" % \
              queryResult.response['extra']['stats']['executionTime'])
    print('// Fetching Resulting. Wallclock Ticking.')
    result_list = query_to_result_list(queryResult)

    fname = "result_%i.json" % campaign_key
    print("// Writing Result to File (%s)" % fname)
    t0 = time.time()
    result_list_to_file(result_list, fname)
    t1 = time.time()
    print("** Took %.1fs" % (t1-t0))

    print("**** Finished for Campaign Key %i" % campaign_key)
    print("**** Round-Trip Time %.1fs" % (time.time() - t00))
    print(">>>> %s UTC" % time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))
    print('')

print('!! Done')
