# coding: utf-8

import os
from tensor2tensor.data_generators import problem
from tensor2tensor.data_generators import text_problems
from tensor2tensor.utils import registry


@registry.register_problem
class Jddc(text_problems.Text2TextProblem):
    """JDDC多轮对话"""

    @property
    def approx_vocab_size(self):
        return 2 ** 17  # ~131k

    @property
    def is_generate_per_split(self):
        # generate_data will shard the data
        # into TRAIN and EVAL for us.
        return False

    @property
    def dataset_splits(self):
        """Splits of data to produce and number
        of output shards for each."""
        # 10% evaluation data
        return [{
            "split": problem.DatasetSplit.TRAIN,
            "shards": 799,
        }, {
            "split": problem.DatasetSplit.EVAL,
            "shards": 1,
        }]

    def generate_samples(self, data_dir, tmp_dir=None, dataset_split=None):
        del tmp_dir
        del dataset_split

        file_qa = os.path.join(data_dir, "qa_pairs.tsv")

        with open(file_qa, 'r') as f:
            qa_pairs = f.readlines()

        for row in qa_pairs:
            q, a = row.strip("\n").split("\t")
            # yield {
            #     "inputs": q.replace(" ", ""),
            #     "targets": a.replace(" ", ""),
            # }
            yield {
                "inputs": q,
                "targets": a,
            }

845c7c2d1ee97c89d5a13a4c1c34951a5a2986e5
<mask token>


@registry.register_problem
class Jddc(text_problems.Text2TextProblem):
    <mask token>
    <mask token>

    @property
    def is_generate_per_split(self):
        return False

    @property
    def dataset_splits(self):
        """Splits of data to produce and number
        of output shards for each."""
        return [{'split': problem.DatasetSplit.TRAIN, 'shards': 799}, {
            'split': problem.DatasetSplit.EVAL, 'shards': 1}]
    <mask token>

<mask token>


@registry.register_problem
class Jddc(text_problems.Text2TextProblem):
    <mask token>

    @property
    def approx_vocab_size(self):
        return 2 ** 17

    @property
    def is_generate_per_split(self):
        return False

    @property
    def dataset_splits(self):
        """Splits of data to produce and number
        of output shards for each."""
        return [{'split': problem.DatasetSplit.TRAIN, 'shards': 799}, {
            'split': problem.DatasetSplit.EVAL, 'shards': 1}]

    def generate_samples(self, data_dir, tmp_dir=None, dataset_split=None):
        del tmp_dir
        del dataset_split
        file_qa = os.path.join(data_dir, 'qa_pairs.tsv')
        with open(file_qa, 'r') as f:
            qa_pairs = f.readlines()
        for row in qa_pairs:
            q, a = row.strip('\n').split('\t')
            yield {'inputs': q, 'targets': a}

<mask token>


@registry.register_problem
class Jddc(text_problems.Text2TextProblem):
    """JDDC多轮对话"""

    @property
    def approx_vocab_size(self):
        return 2 ** 17

    @property
    def is_generate_per_split(self):
        return False

    @property
    def dataset_splits(self):
        """Splits of data to produce and number
        of output shards for each."""
        return [{'split': problem.DatasetSplit.TRAIN, 'shards': 799}, {
            'split': problem.DatasetSplit.EVAL, 'shards': 1}]

    def generate_samples(self, data_dir, tmp_dir=None, dataset_split=None):
        del tmp_dir
        del dataset_split
        file_qa = os.path.join(data_dir, 'qa_pairs.tsv')
        with open(file_qa, 'r') as f:
            qa_pairs = f.readlines()
        for row in qa_pairs:
            q, a = row.strip('\n').split('\t')
            yield {'inputs': q, 'targets': a}

import os
from tensor2tensor.data_generators import problem
from tensor2tensor.data_generators import text_problems
from tensor2tensor.utils import registry


@registry.register_problem
class Jddc(text_problems.Text2TextProblem):
    """JDDC多轮对话"""

    @property
    def approx_vocab_size(self):
        return 2 ** 17

    @property
    def is_generate_per_split(self):
        return False

    @property
    def dataset_splits(self):
        """Splits of data to produce and number
        of output shards for each."""
        return [{'split': problem.DatasetSplit.TRAIN, 'shards': 799}, {
            'split': problem.DatasetSplit.EVAL, 'shards': 1}]

    def generate_samples(self, data_dir, tmp_dir=None, dataset_split=None):
        del tmp_dir
        del dataset_split
        file_qa = os.path.join(data_dir, 'qa_pairs.tsv')
        with open(file_qa, 'r') as f:
            qa_pairs = f.readlines()
        for row in qa_pairs:
            q, a = row.strip('\n').split('\t')
            yield {'inputs': q, 'targets': a}

# coding: utf-8

import os
from tensor2tensor.data_generators import problem
from tensor2tensor.data_generators import text_problems
from tensor2tensor.utils import registry


@registry.register_problem
class Jddc(text_problems.Text2TextProblem):
    """JDDC多轮对话"""

    @property
    def approx_vocab_size(self):
        return 2 ** 17  # ~131k

    @property
    def is_generate_per_split(self):
        # generate_data will shard the data
        # into TRAIN and EVAL for us.
        return False

    @property
    def dataset_splits(self):
        """Splits of data to produce and number
        of output shards for each."""
        # 10% evaluation data
        return [{
            "split": problem.DatasetSplit.TRAIN,
            "shards": 799,
        }, {
            "split": problem.DatasetSplit.EVAL,
            "shards": 1,
        }]

    def generate_samples(self, data_dir, tmp_dir=None, dataset_split=None):
        del tmp_dir
        del dataset_split

        file_qa = os.path.join(data_dir, "qa_pairs.tsv")

        with open(file_qa, 'r') as f:
            qa_pairs = f.readlines()

        for row in qa_pairs:
            q, a = row.strip("\n").split("\t")
            # yield {
            #     "inputs": q.replace(" ", ""),
            #     "targets": a.replace(" ", ""),
            # }
            yield {
                "inputs": q,
                "targets": a,
            }
