#
# Copyright (c) 2015 nexB Inc. and others. All rights reserved.
# http://nexb.com and https://github.com/nexB/scancode-toolkit/
# The ScanCode software is licensed under the Apache License version 2.0.
# Data generated with ScanCode require an acknowledgment.
# ScanCode is a trademark of nexB Inc.
#
# You may not use this software except in compliance with the License.
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#
# When you publish or redistribute any data created with ScanCode or any ScanCode
# derivative work, you must accompany this data with the following acknowledgment:
#
#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
#  ScanCode should be considered or used as legal advice. Consult an Attorney
#  for any legal advice.
#  ScanCode is a free software code scanning tool from nexB Inc. and others.
#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.

from __future__ import absolute_import, print_function

import os
import codecs

from commoncode.testcase import FileBasedTesting

from textcode import analysis
from textcode.analysis import Token
from textcode.analysis import text_lines

from licensedcode import index


TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')


class TestIndexBasedDetection(FileBasedTesting):
    test_data_dir = TEST_DATA_DIR

    def test_merge_positions(self):
        pos1 = Token(start=58, end=58, start_line=8, start_char=52, end_line=8, end_char=59)
        pos2 = Token(start=63, end=64, start_line=23, start_char=12, end_line=53, end_char=23)
        expected = Token(start=58, end=64, start_line=8, start_char=52, end_line=53, end_char=23)
        tst = index.merge_positions([pos2, pos1])
        assert expected == tst

    def test_merge_aligned_positions(self):
        pos11 = Token(start=58, end=58, start_line=8, start_char=52, end_line=8, end_char=59)
        pos12 = Token(start=63, end=64, start_line=23, start_char=12, end_line=53, end_char=23)

        pos21 = Token(start=12, end=24, start_line=12, start_char=52, end_line=12, end_char=59)
        pos22 = Token(start=15, end=35, start_line=45, start_char=12, end_line=54, end_char=36)

        expected = (
            Token(start=58, end=64, start_line=8, start_char=52, end_line=53, end_char=23),
            Token(start=12, end=35, start_line=12, start_char=52, end_line=54, end_char=36)
        )

        tst = index.merge_aligned_positions([(pos11, pos21,), (pos12, pos22,)])
        assert expected == tst

    def get_test_docs(self, base, subset=None):
        base = self.get_test_loc(base, copy=True)
        for docid in os.listdir(base):
            if (subset and docid in subset) or not subset:
                yield docid, text_lines(location=os.path.join(base, docid))

    def test_Index_index_several_unigrams(self):
        test_docs = self.get_test_docs('index/bsd', ['bsd-new', 'bsd-no-mod'])
        idx = index.Index(ngram_len=1)
        for docid, doc in test_docs:
            idx.index_one(docid, doc)
        unigrams_index = idx.indexes[1]

        assert 213 == idx.get_tokens_count('bsd-new')
        assert 234 == idx.get_tokens_count('bsd-no-mod')
        assert 138 == len(unigrams_index)

        pos = Token(start=61, end=61, start_line=8, start_char=52, end_line=8, end_char=59, value=u'minimum')
        expected_posting = ('bsd-no-mod', [pos],)
        assert expected_posting == unigrams_index['minimum'].items()[0]

    def test_get_tokens_count(self):
        base = self.get_test_loc('index/tokens_count', copy=True)
        docids = os.listdir(base)
        idx = index.Index(ngram_len=3)
        for docid in docids:
            doc = text_lines(location=os.path.join(base, docid))
            template = docid.startswith('tmpl')
            idx.index_one(docid, doc, template=template)
        indexes = [
            (idx.indexes[1], set(['all',
                                  'redistribution',
                                  'for',
                                  'is'
                                 ]),),
            (idx.indexes[2], set(['is allowed',
                                  'all and',
                                  'redistribution is',
                                  'allowed for',
                                 ]),),
            (idx.indexes[3], set(['for all and',
                                  'and any thing',
                                  'is allowed for',
                                  'all and any',
                                  'redistribution is allowed',
                                  'allowed for all',
                                 ]),)
        ]

        for idxi, expected_keys in indexes:
            assert expected_keys == set(idxi.keys())

        expected = {
            'plain1': 1,
            'plain2': 2,
            'plain3': 3,
            'plain4': 4,
            'plain5': 5,
            'tmpl10': 10,
            'tmpl2': 2,
            'tmpl3': 3,
            'tmpl4': 4,
            'tmpl5': 5,
            'tmpl5_2': 5,
            'tmpl6': 6,
            'tmpl7': 7,
            'tmpl8': 8,
            'tmpl9': 9,
        }

        result = {docid: idx.get_tokens_count(docid) for docid in docids}
        assert expected == result

    def test_Index_index_one_unigrams(self):
        test_docs = self.get_test_docs('index/bsd', ['bsd-new', 'bsd-no-mod'])
        idx = index.Index(ngram_len=1)
        for docid, doc in test_docs:
            idx.index_one(docid, doc)
        unigrams_index = idx.indexes[1]

        assert 213 == idx.get_tokens_count('bsd-new')
        assert 234 == idx.get_tokens_count('bsd-no-mod')
        assert 138 == len(unigrams_index)

        pos = Token(start=61, end=61, start_line=8, start_char=52, end_line=8, end_char=59, value=u'minimum')
        expected_posting = ('bsd-no-mod', [pos],)
        assert expected_posting == unigrams_index['minimum'].items()[0]

    def test_Index_index_one_trigrams_no_templates(self):
        test_docs = self.get_test_docs('index/bsd', ['bsd-new', 'bsd-no-mod'])
        idx = index.Index(ngram_len=3)

        for docid, doc in test_docs:
            idx.index_one(docid, doc)

        indexes = [(idx.indexes[1], 0,),
                   (idx.indexes[2], 0,),
                   (idx.indexes[3], 280,)]
        for idxi, expected_len in indexes:
            assert expected_len == len(idxi)

        assert 213 == idx.get_tokens_count('bsd-new')
        assert 234 == idx.get_tokens_count('bsd-no-mod')

    def test_Index_index_one_trigrams_with_templates(self):
        test_docs = self.get_test_docs('index/bsd_templates2')
        idx = index.Index(ngram_len=3)
        for docid, doc in test_docs:
            idx.index_one(docid, doc, template=True)
        indexes = [
            (idx.indexes[1], 2,),
            (idx.indexes[2], 5,),
            (idx.indexes[3], 267,)
        ]

        for idxi, expected_len in indexes:
            assert expected_len == len(idxi)

        assert 211 == idx.get_tokens_count('bsd-new')
        assert 232 == idx.get_tokens_count('bsd-no-mod')

    def get_test_index(self, docs, ngram_len=3, template=False):
        idx = index.Index(ngram_len)
        for docid, doc in docs:
            idx.index_one(docid, doc, template)
        return idx

    def test_Index_exact_match_simple(self):
        test_docs = self.get_test_docs('index/bsd')
        idx = self.get_test_index(test_docs, ngram_len=1)
        test_query_doc = self.get_test_loc('index/querysimple')
        expected = {
            'bsd-new':
                [(Token(start=0, start_line=0, start_char=0, end_line=6, end_char=753, end=212),
                  Token(start=0, start_line=4, start_char=0, end_line=12, end_char=607, end=212))
                ],
            'bsd-no-mod':
                [(Token(start=0, start_line=0, start_char=0, end_line=0, end_char=49, end=7),
                  Token(start=0, start_line=4, start_char=0, end_line=4, end_char=49, end=7))
                ],
            'bsd-original':
                [(Token(start=0, start_line=0, start_char=0, end_line=0, end_char=9, end=0),
                  Token(start=29, start_line=6, start_char=59, end_line=6, end_char=68, end=29)
                 ),
                 (Token(start=0, start_line=0, start_char=0, end_line=0, end_char=9, end=0),
                  Token(start=47, start_line=7, start_char=62, end_line=7, end_char=71, end=47)
                 ),
                 (Token(start=0, start_line=0, start_char=0, end_line=0, end_char=9, end=0),
                  Token(start=103, start_line=10, start_char=33, end_line=10, end_char=42, end=103)
                 ),
                 (Token(start=0, start_line=0, start_char=0, end_line=0, end_char=9, end=0),
                  Token(start=137, start_line=12, start_char=117, end_line=12, end_char=126, end=137)
                 )
                ],
            'bsd-original-uc':
                [(Token(start=0, start_line=0, start_char=0, end_line=0, end_char=9, end=0),
                  Token(start=29, start_line=6, start_char=59, end_line=6, end_char=68, end=29)),
                 (Token(start=0, start_line=0, start_char=0, end_line=0, end_char=9, end=0),
                  Token(start=47, start_line=7, start_char=62, end_line=7, end_char=71, end=47)),
                 (Token(start=0, start_line=0, start_char=0, end_line=0, end_char=9, end=0),
                  Token(start=103, start_line=10, start_char=33, end_line=10, end_char=42, end=103)),
                 (Token(start=0, start_line=0, start_char=0, end_line=0, end_char=9, end=0),
                  Token(start=137, start_line=12, start_char=117, end_line=12, end_char=126, end=137))
                ],
            'bsd-simplified':
                [(Token(start=0, start_line=0, start_char=3, end_line=7, end_char=73, end=67),
                  Token(start=0, start_line=4, start_char=0, end_line=7, end_char=207, end=67))
                 ]
        }

        matches = idx.match(text_lines(test_query_doc))
        for k, val in matches.items():
            assert expected[k] == val

    def test_Index_exact_match_unigrams_perfect(self):
        test_docs = self.get_test_docs('index/bsd')
        idx = self.get_test_index(test_docs, ngram_len=1, template=False)
        test_query_doc = self.get_test_loc('index/queryperfect')

        expected = {
            'bsd-new': [
                (Token(start=0, start_line=0, start_char=0, end_line=6, end_char=753, end=212),
                 Token(start=0, start_line=5, start_char=0, end_line=11, end_char=753, end=212))
            ]
        }

        matches = idx.match(text_lines(test_query_doc))

        assert {} != matches
        for k, val in matches.items():
            assert expected[k] == val

        with codecs.open(test_query_doc, encoding='utf-8') as td:
            actual = td.read().splitlines(True)
            expected = u''.join(actual[5:-2])[:-2]
            query_match_pos = matches['bsd-new'][0][-1]
            tst = analysis.doc_subset(text_lines(location=test_query_doc), query_match_pos)
            tst = u''.join(tst)
            assert expected == tst

    def test_Index_exact_match_ngrams_perfect_minimalist(self):
        index_doc = [u'name is joker, name is joker']
        #                 0  1     2     3  4     5
        idx = index.Index(ngram_len=3)
        idx.index_one('tst', text_lines(index_doc), template=False)

        query_doc = [u'Hi my name is joker, name is joker yes.']
        # match         0  1   |2  3     4     5  6     7|  8
        expected = {
            'tst': [
                (Token(start=0, start_line=0, start_char=0, end_line=0, end_char=28, end=5),
                 Token(start=2, start_line=0, start_char=6, end_line=0, end_char=34, end=7))
            ]
        }
        matches = idx.match(query_doc)

        assert {} != matches
        for k, val in matches.items():
#              assert [] == val
            assert expected[k] == val

    def test_Index_exact_match_ngrams_perfect_single_index_doc_in_index_minimal(self):
        test_docs = self.get_test_docs('index/mini')
        idx = self.get_test_index(test_docs, ngram_len=3, template=False)
        test_query_doc = self.get_test_loc('index/queryperfect-mini')

        expected = {
            'bsd-new': [
                (Token(start=0, start_line=0, start_char=0, end_line=0, end_char=94, end=13),
                 Token(start=1, start_line=2, start_char=0, end_line=2, end_char=94, end=14))
            ]
        }
        matches = idx.match(text_lines(test_query_doc))

        assert {} != matches
        for k, val in matches.items():
            assert expected[k] == val

    def test_Index_exact_match_ngrams_templates_perfect_minimalist(self):
        index_doc = [u'name is joker, {{}} name is joker']
        idx = index.Index(ngram_len=3)
        idx.index_one('tst', text_lines(index_doc), template=True)

        query_doc = [u'Hi my name is joker the joker name is joker yes.']
        #              012345678901234567890123456789012345678901234567
        #                        11111111112222222222333333333344444444
        expected = {
            'tst': [
                (Token(start=0, start_line=0, start_char=0, end_line=0, end_char=33, end=5),
                 Token(start=2, start_line=0, start_char=6, end_line=0, end_char=43, end=9))
            ]
        }

        matches = idx.match(text_lines(query_doc))

        assert {} != matches
        for k, val in matches.items():
            assert expected[k] == val

    def test_Index_exact_match_ngrams_template_perfect_multi_index_doc_in_index(self):
        test_docs = self.get_test_docs('index/bsd_templates')
        idx = self.get_test_index(test_docs, ngram_len=3, template=True)
        test_query_doc = self.get_test_loc('index/queryperfect_single_template')

        expected = {
            'bsd-new':[
                 (Token(start=0, start_line=0, start_char=0, end_line=6, end_char=753, end=210),
                  Token(start=4, start_line=5, start_char=0, end_line=11, end_char=753, end=216))
            ]
        }
        matches = idx.match(text_lines(test_query_doc))
        assert {} != matches
        for k, val in matches.items():
            assert expected[k] == val

    def test_Index_exact_match_return_one_match_with_correct_offsets(self):
        index_doc = [u'A one. A two. A three.']
        idx = index.Index(ngram_len=4)
        idx.index_one('tst', text_lines(index_doc), template=False)
        query_doc = [u'some junk. A one. A two. A three.']
        #                         1111111111222222222233
        #              012345678901234567890123456789012

        matches = idx.match(query_doc)
        match = matches['tst']
        assert 1 == len(match)
        index_pos, query_pos = match[0]
        assert 11 == query_pos.start_char
        assert 32 == query_pos.end_char
        assert 0 == index_pos.start_char
        assert 21 == index_pos.end_char

    def test_Index_exact_match_to_indexed_template_with_short_tokens_around_gaps(self):
        # was failing when a gapped token (from a template) starts at a
        # beginning of an index doc and at a position less than ngram length

        # setup
        idx = index.Index(ngram_len=4)
        index_doc = text_lines(self.get_test_loc('index/templates/idx.txt'))
        idx.index_one('idx', text_lines(index_doc), template=True)

        # test index
        quad_grams_index = idx._get_index_for_len(4)
        assert 205 == len(quad_grams_index)
        assert u'software without prior written' in quad_grams_index

        # test match
        query_doc = text_lines(self.get_test_loc('index/templates/query.txt'))
        matches = idx.match(query_doc)
        assert 1 == len(matches)

        # we expect a single match to the idx doc
        matched_query_doc_position = matches['idx'][0][1]
        expected = Token(start=0, start_line=0, start_char=0, end_line=39, end_char=34, end=276)
        assert expected == matched_query_doc_position
