# coding: utf-8

import unittest

from dnlp.aux2 import Tokenizer


class MyTestCase(unittest.TestCase):
    def test_something(self):
        self.assertEqual(True, False)

    def test_tokenize(self):
        self.assertTrue(True)

        print(Tokenizer.token('官方太小气，连个膜也不给，wifi经常断89^-^。'))

    def test_c_tokenize(self):
        self.assertTrue(True)

        import re

        EXTRACT_TOKEN_RE = r'([a-z0-9]+)'

        txt = '太窝火了，WIFI连不上'.lower()

        for x in re.split(EXTRACT_TOKEN_RE, txt):
            print('->', x)


if __name__ == '__main__':
    unittest.main()

e4216f91be6b5b711ace0c492c321b3c820d2dd2
<mask token>


class MyTestCase(unittest.TestCase):

    def test_something(self):
        self.assertEqual(True, False)

    def test_tokenize(self):
        self.assertTrue(True)
        print(Tokenizer.token('官方太小气，连个膜也不给，wifi经常断89^-^。'))
    <mask token>


<mask token>

<mask token>


class MyTestCase(unittest.TestCase):

    def test_something(self):
        self.assertEqual(True, False)

    def test_tokenize(self):
        self.assertTrue(True)
        print(Tokenizer.token('官方太小气，连个膜也不给，wifi经常断89^-^。'))

    def test_c_tokenize(self):
        self.assertTrue(True)
        import re
        EXTRACT_TOKEN_RE = '([a-z0-9]+)'
        txt = '太窝火了，WIFI连不上'.lower()
        for x in re.split(EXTRACT_TOKEN_RE, txt):
            print('->', x)


<mask token>

<mask token>


class MyTestCase(unittest.TestCase):

    def test_something(self):
        self.assertEqual(True, False)

    def test_tokenize(self):
        self.assertTrue(True)
        print(Tokenizer.token('官方太小气，连个膜也不给，wifi经常断89^-^。'))

    def test_c_tokenize(self):
        self.assertTrue(True)
        import re
        EXTRACT_TOKEN_RE = '([a-z0-9]+)'
        txt = '太窝火了，WIFI连不上'.lower()
        for x in re.split(EXTRACT_TOKEN_RE, txt):
            print('->', x)


if __name__ == '__main__':
    unittest.main()

import unittest
from dnlp.aux2 import Tokenizer


class MyTestCase(unittest.TestCase):

    def test_something(self):
        self.assertEqual(True, False)

    def test_tokenize(self):
        self.assertTrue(True)
        print(Tokenizer.token('官方太小气，连个膜也不给，wifi经常断89^-^。'))

    def test_c_tokenize(self):
        self.assertTrue(True)
        import re
        EXTRACT_TOKEN_RE = '([a-z0-9]+)'
        txt = '太窝火了，WIFI连不上'.lower()
        for x in re.split(EXTRACT_TOKEN_RE, txt):
            print('->', x)


if __name__ == '__main__':
    unittest.main()

# coding: utf-8

import unittest

from dnlp.aux2 import Tokenizer


class MyTestCase(unittest.TestCase):
    def test_something(self):
        self.assertEqual(True, False)

    def test_tokenize(self):
        self.assertTrue(True)

        print(Tokenizer.token('官方太小气，连个膜也不给，wifi经常断89^-^。'))

    def test_c_tokenize(self):
        self.assertTrue(True)

        import re

        EXTRACT_TOKEN_RE = r'([a-z0-9]+)'

        txt = '太窝火了，WIFI连不上'.lower()

        for x in re.split(EXTRACT_TOKEN_RE, txt):
            print('->', x)


if __name__ == '__main__':
    unittest.main()
