#!/usr/bin/python
import pickle
import string
import numpy
import getopt
import sys
import random
import time
import re
import pprint
import codecs
import datetime, os
import scipy.io
import nltk
import numpy
import optparse


# bash scrip to terminate all sub-processes
# kill $(ps aux | grep 'python infag' | awk '{print $2}')

def parse_args():
    parser = optparse.OptionParser()
    parser.set_defaults(
        # parameter set 1
        input_directory=None,
        model_directory=None,
        non_terminal_symbol="Word",
        number_of_samples=10,
        number_of_processes=0,
    )
    # parameter set 1
    parser.add_option("--input_directory", type="string", dest="input_directory",
                      help="input directory [None]")
    parser.add_option("--model_directory", type="string", dest="model_directory",
                      help="model directory [None]")
    parser.add_option("--non_terminal_symbol", type="string", dest="non_terminal_symbol",
                      help="non-terminal symbol [Word]")
    parser.add_option("--number_of_samples", type="int", dest="number_of_samples",
                      help="number of samples [10]")
    parser.add_option("--number_of_processes", type="int", dest="number_of_processes",
                      help="number of processes [0]")

    (options, args) = parser.parse_args()
    return options


def main():
    options = parse_args()

    # parameter set 1
    assert (options.input_directory is not None)
    input_directory = options.input_directory
    assert (options.model_directory is not None)
    model_directory = options.model_directory
    assert (options.non_terminal_symbol is not None)
    non_terminal_symbol = options.non_terminal_symbol

    assert (options.number_of_samples > 0)
    number_of_samples = options.number_of_samples
    assert (options.number_of_processes >= 0)
    number_of_processes = options.number_of_processes

    print("========== ========== ========== ========== ==========")
    # parameter set 1
    print("input_directory=" + input_directory)
    print("model_directory=" + model_directory)
    print("non_terminal_symbol=" + non_terminal_symbol)

    print("number_of_samples=" + str(number_of_samples))
    print("number_of_processes=" + str(number_of_processes))
    print("========== ========== ========== ========== ==========")

    # Documents
    train_docs = []
    input_stream = open(os.path.join(input_directory, 'train.dat'), 'r')
    for line in input_stream:
        train_docs.append(line.strip())
    input_stream.close()
    print("successfully load %d training documents..." % (len(train_docs)))

    refer_docs = []
    input_stream = open(os.path.join(input_directory, 'truth.dat'), 'r')
    for line in input_stream:
        refer_docs.append(line.strip())
    input_stream.close()
    print("successfully load %d testing documents..." % (len(refer_docs)))

    for model_file in os.listdir(model_directory):
        if not model_file.startswith("model-"):
            continue

        model_file_path = os.path.join(model_directory, model_file)

        try:
            cpickle_file = open(model_file_path, 'rb')
            infinite_adaptor_grammar = pickle.load(cpickle_file)
            print("successfully load model from %s" % (model_file_path))
            cpickle_file.close()
        except ValueError:
            print("warning: unsuccessfully load model from %s due to value error..." % (model_file_path))
            continue
        except EOFError:
            print("warning: unsuccessfully load model from %s due to EOF error..." % (model_file_path))
            continue

        non_terminal = nltk.grammar.Nonterminal(non_terminal_symbol)
        # assert non_terminal in infinite_adaptor_grammar._adapted_non_terminals, (
        #     non_terminal, infinite_adaptor_grammar._adapted_non_terminals)

        inference_parameter = (refer_docs, non_terminal, model_directory, model_file)
        infinite_adaptor_grammar.inference(train_docs, inference_parameter, number_of_samples, number_of_processes)

        '''
        #from launch_train import shuffle_lists
        #shuffle_lists(train_docs, refer_docs)
        if number_of_processes==0:
            infinite_adaptor_grammar.inference(train_docs, refer_docs, non_terminal, model_directory, model_file, number_of_samples)
        else:
            from hybrid_process import inference_process
            inference_process(infinite_adaptor_grammar, train_docs, refer_docs, non_terminal, model_directory, number_of_samples, number_of_processes)
        '''


if __name__ == '__main__':
    main()

d59daf52c42fd10026059b5c7d5e57de225c8aab
<mask token>


def parse_args():
    parser = optparse.OptionParser()
    parser.set_defaults(input_directory=None, model_directory=None,
        non_terminal_symbol='Word', number_of_samples=10, number_of_processes=0
        )
    parser.add_option('--input_directory', type='string', dest=
        'input_directory', help='input directory [None]')
    parser.add_option('--model_directory', type='string', dest=
        'model_directory', help='model directory [None]')
    parser.add_option('--non_terminal_symbol', type='string', dest=
        'non_terminal_symbol', help='non-terminal symbol [Word]')
    parser.add_option('--number_of_samples', type='int', dest=
        'number_of_samples', help='number of samples [10]')
    parser.add_option('--number_of_processes', type='int', dest=
        'number_of_processes', help='number of processes [0]')
    options, args = parser.parse_args()
    return options


<mask token>

<mask token>


def parse_args():
    parser = optparse.OptionParser()
    parser.set_defaults(input_directory=None, model_directory=None,
        non_terminal_symbol='Word', number_of_samples=10, number_of_processes=0
        )
    parser.add_option('--input_directory', type='string', dest=
        'input_directory', help='input directory [None]')
    parser.add_option('--model_directory', type='string', dest=
        'model_directory', help='model directory [None]')
    parser.add_option('--non_terminal_symbol', type='string', dest=
        'non_terminal_symbol', help='non-terminal symbol [Word]')
    parser.add_option('--number_of_samples', type='int', dest=
        'number_of_samples', help='number of samples [10]')
    parser.add_option('--number_of_processes', type='int', dest=
        'number_of_processes', help='number of processes [0]')
    options, args = parser.parse_args()
    return options


def main():
    options = parse_args()
    assert options.input_directory is not None
    input_directory = options.input_directory
    assert options.model_directory is not None
    model_directory = options.model_directory
    assert options.non_terminal_symbol is not None
    non_terminal_symbol = options.non_terminal_symbol
    assert options.number_of_samples > 0
    number_of_samples = options.number_of_samples
    assert options.number_of_processes >= 0
    number_of_processes = options.number_of_processes
    print('========== ========== ========== ========== ==========')
    print('input_directory=' + input_directory)
    print('model_directory=' + model_directory)
    print('non_terminal_symbol=' + non_terminal_symbol)
    print('number_of_samples=' + str(number_of_samples))
    print('number_of_processes=' + str(number_of_processes))
    print('========== ========== ========== ========== ==========')
    train_docs = []
    input_stream = open(os.path.join(input_directory, 'train.dat'), 'r')
    for line in input_stream:
        train_docs.append(line.strip())
    input_stream.close()
    print('successfully load %d training documents...' % len(train_docs))
    refer_docs = []
    input_stream = open(os.path.join(input_directory, 'truth.dat'), 'r')
    for line in input_stream:
        refer_docs.append(line.strip())
    input_stream.close()
    print('successfully load %d testing documents...' % len(refer_docs))
    for model_file in os.listdir(model_directory):
        if not model_file.startswith('model-'):
            continue
        model_file_path = os.path.join(model_directory, model_file)
        try:
            cpickle_file = open(model_file_path, 'rb')
            infinite_adaptor_grammar = pickle.load(cpickle_file)
            print('successfully load model from %s' % model_file_path)
            cpickle_file.close()
        except ValueError:
            print(
                'warning: unsuccessfully load model from %s due to value error...'
                 % model_file_path)
            continue
        except EOFError:
            print(
                'warning: unsuccessfully load model from %s due to EOF error...'
                 % model_file_path)
            continue
        non_terminal = nltk.grammar.Nonterminal(non_terminal_symbol)
        inference_parameter = (refer_docs, non_terminal, model_directory,
            model_file)
        infinite_adaptor_grammar.inference(train_docs, inference_parameter,
            number_of_samples, number_of_processes)
        """
        #from launch_train import shuffle_lists
        #shuffle_lists(train_docs, refer_docs)
        if number_of_processes==0:
            infinite_adaptor_grammar.inference(train_docs, refer_docs, non_terminal, model_directory, model_file, number_of_samples)
        else:
            from hybrid_process import inference_process
            inference_process(infinite_adaptor_grammar, train_docs, refer_docs, non_terminal, model_directory, number_of_samples, number_of_processes)
        """


<mask token>

<mask token>


def parse_args():
    parser = optparse.OptionParser()
    parser.set_defaults(input_directory=None, model_directory=None,
        non_terminal_symbol='Word', number_of_samples=10, number_of_processes=0
        )
    parser.add_option('--input_directory', type='string', dest=
        'input_directory', help='input directory [None]')
    parser.add_option('--model_directory', type='string', dest=
        'model_directory', help='model directory [None]')
    parser.add_option('--non_terminal_symbol', type='string', dest=
        'non_terminal_symbol', help='non-terminal symbol [Word]')
    parser.add_option('--number_of_samples', type='int', dest=
        'number_of_samples', help='number of samples [10]')
    parser.add_option('--number_of_processes', type='int', dest=
        'number_of_processes', help='number of processes [0]')
    options, args = parser.parse_args()
    return options


def main():
    options = parse_args()
    assert options.input_directory is not None
    input_directory = options.input_directory
    assert options.model_directory is not None
    model_directory = options.model_directory
    assert options.non_terminal_symbol is not None
    non_terminal_symbol = options.non_terminal_symbol
    assert options.number_of_samples > 0
    number_of_samples = options.number_of_samples
    assert options.number_of_processes >= 0
    number_of_processes = options.number_of_processes
    print('========== ========== ========== ========== ==========')
    print('input_directory=' + input_directory)
    print('model_directory=' + model_directory)
    print('non_terminal_symbol=' + non_terminal_symbol)
    print('number_of_samples=' + str(number_of_samples))
    print('number_of_processes=' + str(number_of_processes))
    print('========== ========== ========== ========== ==========')
    train_docs = []
    input_stream = open(os.path.join(input_directory, 'train.dat'), 'r')
    for line in input_stream:
        train_docs.append(line.strip())
    input_stream.close()
    print('successfully load %d training documents...' % len(train_docs))
    refer_docs = []
    input_stream = open(os.path.join(input_directory, 'truth.dat'), 'r')
    for line in input_stream:
        refer_docs.append(line.strip())
    input_stream.close()
    print('successfully load %d testing documents...' % len(refer_docs))
    for model_file in os.listdir(model_directory):
        if not model_file.startswith('model-'):
            continue
        model_file_path = os.path.join(model_directory, model_file)
        try:
            cpickle_file = open(model_file_path, 'rb')
            infinite_adaptor_grammar = pickle.load(cpickle_file)
            print('successfully load model from %s' % model_file_path)
            cpickle_file.close()
        except ValueError:
            print(
                'warning: unsuccessfully load model from %s due to value error...'
                 % model_file_path)
            continue
        except EOFError:
            print(
                'warning: unsuccessfully load model from %s due to EOF error...'
                 % model_file_path)
            continue
        non_terminal = nltk.grammar.Nonterminal(non_terminal_symbol)
        inference_parameter = (refer_docs, non_terminal, model_directory,
            model_file)
        infinite_adaptor_grammar.inference(train_docs, inference_parameter,
            number_of_samples, number_of_processes)
        """
        #from launch_train import shuffle_lists
        #shuffle_lists(train_docs, refer_docs)
        if number_of_processes==0:
            infinite_adaptor_grammar.inference(train_docs, refer_docs, non_terminal, model_directory, model_file, number_of_samples)
        else:
            from hybrid_process import inference_process
            inference_process(infinite_adaptor_grammar, train_docs, refer_docs, non_terminal, model_directory, number_of_samples, number_of_processes)
        """


if __name__ == '__main__':
    main()

import pickle
import string
import numpy
import getopt
import sys
import random
import time
import re
import pprint
import codecs
import datetime, os
import scipy.io
import nltk
import numpy
import optparse


def parse_args():
    parser = optparse.OptionParser()
    parser.set_defaults(input_directory=None, model_directory=None,
        non_terminal_symbol='Word', number_of_samples=10, number_of_processes=0
        )
    parser.add_option('--input_directory', type='string', dest=
        'input_directory', help='input directory [None]')
    parser.add_option('--model_directory', type='string', dest=
        'model_directory', help='model directory [None]')
    parser.add_option('--non_terminal_symbol', type='string', dest=
        'non_terminal_symbol', help='non-terminal symbol [Word]')
    parser.add_option('--number_of_samples', type='int', dest=
        'number_of_samples', help='number of samples [10]')
    parser.add_option('--number_of_processes', type='int', dest=
        'number_of_processes', help='number of processes [0]')
    options, args = parser.parse_args()
    return options


def main():
    options = parse_args()
    assert options.input_directory is not None
    input_directory = options.input_directory
    assert options.model_directory is not None
    model_directory = options.model_directory
    assert options.non_terminal_symbol is not None
    non_terminal_symbol = options.non_terminal_symbol
    assert options.number_of_samples > 0
    number_of_samples = options.number_of_samples
    assert options.number_of_processes >= 0
    number_of_processes = options.number_of_processes
    print('========== ========== ========== ========== ==========')
    print('input_directory=' + input_directory)
    print('model_directory=' + model_directory)
    print('non_terminal_symbol=' + non_terminal_symbol)
    print('number_of_samples=' + str(number_of_samples))
    print('number_of_processes=' + str(number_of_processes))
    print('========== ========== ========== ========== ==========')
    train_docs = []
    input_stream = open(os.path.join(input_directory, 'train.dat'), 'r')
    for line in input_stream:
        train_docs.append(line.strip())
    input_stream.close()
    print('successfully load %d training documents...' % len(train_docs))
    refer_docs = []
    input_stream = open(os.path.join(input_directory, 'truth.dat'), 'r')
    for line in input_stream:
        refer_docs.append(line.strip())
    input_stream.close()
    print('successfully load %d testing documents...' % len(refer_docs))
    for model_file in os.listdir(model_directory):
        if not model_file.startswith('model-'):
            continue
        model_file_path = os.path.join(model_directory, model_file)
        try:
            cpickle_file = open(model_file_path, 'rb')
            infinite_adaptor_grammar = pickle.load(cpickle_file)
            print('successfully load model from %s' % model_file_path)
            cpickle_file.close()
        except ValueError:
            print(
                'warning: unsuccessfully load model from %s due to value error...'
                 % model_file_path)
            continue
        except EOFError:
            print(
                'warning: unsuccessfully load model from %s due to EOF error...'
                 % model_file_path)
            continue
        non_terminal = nltk.grammar.Nonterminal(non_terminal_symbol)
        inference_parameter = (refer_docs, non_terminal, model_directory,
            model_file)
        infinite_adaptor_grammar.inference(train_docs, inference_parameter,
            number_of_samples, number_of_processes)
        """
        #from launch_train import shuffle_lists
        #shuffle_lists(train_docs, refer_docs)
        if number_of_processes==0:
            infinite_adaptor_grammar.inference(train_docs, refer_docs, non_terminal, model_directory, model_file, number_of_samples)
        else:
            from hybrid_process import inference_process
            inference_process(infinite_adaptor_grammar, train_docs, refer_docs, non_terminal, model_directory, number_of_samples, number_of_processes)
        """


if __name__ == '__main__':
    main()

#!/usr/bin/python
import pickle
import string
import numpy
import getopt
import sys
import random
import time
import re
import pprint
import codecs
import datetime, os
import scipy.io
import nltk
import numpy
import optparse


# bash scrip to terminate all sub-processes
# kill $(ps aux | grep 'python infag' | awk '{print $2}')

def parse_args():
    parser = optparse.OptionParser()
    parser.set_defaults(
        # parameter set 1
        input_directory=None,
        model_directory=None,
        non_terminal_symbol="Word",
        number_of_samples=10,
        number_of_processes=0,
    )
    # parameter set 1
    parser.add_option("--input_directory", type="string", dest="input_directory",
                      help="input directory [None]")
    parser.add_option("--model_directory", type="string", dest="model_directory",
                      help="model directory [None]")
    parser.add_option("--non_terminal_symbol", type="string", dest="non_terminal_symbol",
                      help="non-terminal symbol [Word]")
    parser.add_option("--number_of_samples", type="int", dest="number_of_samples",
                      help="number of samples [10]")
    parser.add_option("--number_of_processes", type="int", dest="number_of_processes",
                      help="number of processes [0]")

    (options, args) = parser.parse_args()
    return options


def main():
    options = parse_args()

    # parameter set 1
    assert (options.input_directory is not None)
    input_directory = options.input_directory
    assert (options.model_directory is not None)
    model_directory = options.model_directory
    assert (options.non_terminal_symbol is not None)
    non_terminal_symbol = options.non_terminal_symbol

    assert (options.number_of_samples > 0)
    number_of_samples = options.number_of_samples
    assert (options.number_of_processes >= 0)
    number_of_processes = options.number_of_processes

    print("========== ========== ========== ========== ==========")
    # parameter set 1
    print("input_directory=" + input_directory)
    print("model_directory=" + model_directory)
    print("non_terminal_symbol=" + non_terminal_symbol)

    print("number_of_samples=" + str(number_of_samples))
    print("number_of_processes=" + str(number_of_processes))
    print("========== ========== ========== ========== ==========")

    # Documents
    train_docs = []
    input_stream = open(os.path.join(input_directory, 'train.dat'), 'r')
    for line in input_stream:
        train_docs.append(line.strip())
    input_stream.close()
    print("successfully load %d training documents..." % (len(train_docs)))

    refer_docs = []
    input_stream = open(os.path.join(input_directory, 'truth.dat'), 'r')
    for line in input_stream:
        refer_docs.append(line.strip())
    input_stream.close()
    print("successfully load %d testing documents..." % (len(refer_docs)))

    for model_file in os.listdir(model_directory):
        if not model_file.startswith("model-"):
            continue

        model_file_path = os.path.join(model_directory, model_file)

        try:
            cpickle_file = open(model_file_path, 'rb')
            infinite_adaptor_grammar = pickle.load(cpickle_file)
            print("successfully load model from %s" % (model_file_path))
            cpickle_file.close()
        except ValueError:
            print("warning: unsuccessfully load model from %s due to value error..." % (model_file_path))
            continue
        except EOFError:
            print("warning: unsuccessfully load model from %s due to EOF error..." % (model_file_path))
            continue

        non_terminal = nltk.grammar.Nonterminal(non_terminal_symbol)
        # assert non_terminal in infinite_adaptor_grammar._adapted_non_terminals, (
        #     non_terminal, infinite_adaptor_grammar._adapted_non_terminals)

        inference_parameter = (refer_docs, non_terminal, model_directory, model_file)
        infinite_adaptor_grammar.inference(train_docs, inference_parameter, number_of_samples, number_of_processes)

        '''
        #from launch_train import shuffle_lists
        #shuffle_lists(train_docs, refer_docs)
        if number_of_processes==0:
            infinite_adaptor_grammar.inference(train_docs, refer_docs, non_terminal, model_directory, model_file, number_of_samples)
        else:
            from hybrid_process import inference_process
            inference_process(infinite_adaptor_grammar, train_docs, refer_docs, non_terminal, model_directory, number_of_samples, number_of_processes)
        '''


if __name__ == '__main__':
    main()
