from itertools import chain

from utils import generate_ngrams

def distinct(lines, N):
    ngrams = [generate_ngrams(line, N) for line in lines]
    ngrams_set = set(chain.from_iterable(ngrams))

    return float(len(ngrams_set)) / sum(len(ngram) for ngram in ngrams)

def read_lines(filename):
    fr = open(filename, 'r')
    lines = fr.readlines()
    fr.close()
    return lines

if __name__ == "__main__":
    #lines = ["hello world", "this new", "hello world"]
    lines = read_lines('test.txt')
    print(distinct(lines, 2))


2e4dd0098ce47106ccde317e149b91f525f197f3
<mask token>


def read_lines(filename):
    fr = open(filename, 'r')
    lines = fr.readlines()
    fr.close()
    return lines


<mask token>

<mask token>


def distinct(lines, N):
    ngrams = [generate_ngrams(line, N) for line in lines]
    ngrams_set = set(chain.from_iterable(ngrams))
    return float(len(ngrams_set)) / sum(len(ngram) for ngram in ngrams)


def read_lines(filename):
    fr = open(filename, 'r')
    lines = fr.readlines()
    fr.close()
    return lines


<mask token>

<mask token>


def distinct(lines, N):
    ngrams = [generate_ngrams(line, N) for line in lines]
    ngrams_set = set(chain.from_iterable(ngrams))
    return float(len(ngrams_set)) / sum(len(ngram) for ngram in ngrams)


def read_lines(filename):
    fr = open(filename, 'r')
    lines = fr.readlines()
    fr.close()
    return lines


if __name__ == '__main__':
    lines = read_lines('test.txt')
    print(distinct(lines, 2))

from itertools import chain
from utils import generate_ngrams


def distinct(lines, N):
    ngrams = [generate_ngrams(line, N) for line in lines]
    ngrams_set = set(chain.from_iterable(ngrams))
    return float(len(ngrams_set)) / sum(len(ngram) for ngram in ngrams)


def read_lines(filename):
    fr = open(filename, 'r')
    lines = fr.readlines()
    fr.close()
    return lines


if __name__ == '__main__':
    lines = read_lines('test.txt')
    print(distinct(lines, 2))

from itertools import chain

from utils import generate_ngrams

def distinct(lines, N):
    ngrams = [generate_ngrams(line, N) for line in lines]
    ngrams_set = set(chain.from_iterable(ngrams))

    return float(len(ngrams_set)) / sum(len(ngram) for ngram in ngrams)

def read_lines(filename):
    fr = open(filename, 'r')
    lines = fr.readlines()
    fr.close()
    return lines

if __name__ == "__main__":
    #lines = ["hello world", "this new", "hello world"]
    lines = read_lines('test.txt')
    print(distinct(lines, 2))

