from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk

from timeit import default_timer as timer

es = Elasticsearch("http://127.0.0.1:9200/")
es.indices.create(index='nomes', ignore=400) #ignore=400 --> ignora se já existir

start = timer()


fi = open('data.txt')
txt = fi.read()
fi.close()

pessoa = []
first = True

for reg in txt.split("\n"):
    registo = reg.split('#')
    nome = registo[0].strip()
    if nome != '':
        pessoa.append({'nome': nome, 'abertura': registo[1], 'balcao': registo[2]})


threads = []

start = timer()
cnt = 0
data_list = []

def gendata(t_pessoa):
    for reg in t_pessoa:
        yield {
            "_index": "nomes",
            "_type": "document",
            "conta": reg,
        }

def worker(data_list):
    bulk(es, gendata(data_list))

print('inicio')
for i in pessoa:
    data_list.append(i)
    if cnt % 1000 == 0:
        print(cnt)
        bulk(es, gendata(data_list))
        data_list = []

    cnt += 1

es.indices.refresh()
end = timer()

end = timer()
print('tempo: ' + str(end - start)) # Time in seconds, e.g. 5.38091952400282

print(end - start) # Time in seconds, e.g. 5.38091952400282

5e1ac3652a92c0b1f52aff5012c9227bbb7db39a
<mask token>


def gendata(t_pessoa):
    for reg in t_pessoa:
        yield {'_index': 'nomes', '_type': 'document', 'conta': reg}


def worker(data_list):
    bulk(es, gendata(data_list))


<mask token>

<mask token>
es.indices.create(index='nomes', ignore=400)
<mask token>
fi.close()
<mask token>
for reg in txt.split('\n'):
    registo = reg.split('#')
    nome = registo[0].strip()
    if nome != '':
        pessoa.append({'nome': nome, 'abertura': registo[1], 'balcao':
            registo[2]})
<mask token>


def gendata(t_pessoa):
    for reg in t_pessoa:
        yield {'_index': 'nomes', '_type': 'document', 'conta': reg}


def worker(data_list):
    bulk(es, gendata(data_list))


print('inicio')
for i in pessoa:
    data_list.append(i)
    if cnt % 1000 == 0:
        print(cnt)
        bulk(es, gendata(data_list))
        data_list = []
    cnt += 1
es.indices.refresh()
<mask token>
print('tempo: ' + str(end - start))
print(end - start)

<mask token>
es = Elasticsearch('http://127.0.0.1:9200/')
es.indices.create(index='nomes', ignore=400)
start = timer()
fi = open('data.txt')
txt = fi.read()
fi.close()
pessoa = []
first = True
for reg in txt.split('\n'):
    registo = reg.split('#')
    nome = registo[0].strip()
    if nome != '':
        pessoa.append({'nome': nome, 'abertura': registo[1], 'balcao':
            registo[2]})
threads = []
start = timer()
cnt = 0
data_list = []


def gendata(t_pessoa):
    for reg in t_pessoa:
        yield {'_index': 'nomes', '_type': 'document', 'conta': reg}


def worker(data_list):
    bulk(es, gendata(data_list))


print('inicio')
for i in pessoa:
    data_list.append(i)
    if cnt % 1000 == 0:
        print(cnt)
        bulk(es, gendata(data_list))
        data_list = []
    cnt += 1
es.indices.refresh()
end = timer()
end = timer()
print('tempo: ' + str(end - start))
print(end - start)

from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
from timeit import default_timer as timer
es = Elasticsearch('http://127.0.0.1:9200/')
es.indices.create(index='nomes', ignore=400)
start = timer()
fi = open('data.txt')
txt = fi.read()
fi.close()
pessoa = []
first = True
for reg in txt.split('\n'):
    registo = reg.split('#')
    nome = registo[0].strip()
    if nome != '':
        pessoa.append({'nome': nome, 'abertura': registo[1], 'balcao':
            registo[2]})
threads = []
start = timer()
cnt = 0
data_list = []


def gendata(t_pessoa):
    for reg in t_pessoa:
        yield {'_index': 'nomes', '_type': 'document', 'conta': reg}


def worker(data_list):
    bulk(es, gendata(data_list))


print('inicio')
for i in pessoa:
    data_list.append(i)
    if cnt % 1000 == 0:
        print(cnt)
        bulk(es, gendata(data_list))
        data_list = []
    cnt += 1
es.indices.refresh()
end = timer()
end = timer()
print('tempo: ' + str(end - start))
print(end - start)

from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk

from timeit import default_timer as timer

es = Elasticsearch("http://127.0.0.1:9200/")
es.indices.create(index='nomes', ignore=400) #ignore=400 --> ignora se já existir

start = timer()


fi = open('data.txt')
txt = fi.read()
fi.close()

pessoa = []
first = True

for reg in txt.split("\n"):
    registo = reg.split('#')
    nome = registo[0].strip()
    if nome != '':
        pessoa.append({'nome': nome, 'abertura': registo[1], 'balcao': registo[2]})


threads = []

start = timer()
cnt = 0
data_list = []

def gendata(t_pessoa):
    for reg in t_pessoa:
        yield {
            "_index": "nomes",
            "_type": "document",
            "conta": reg,
        }

def worker(data_list):
    bulk(es, gendata(data_list))

print('inicio')
for i in pessoa:
    data_list.append(i)
    if cnt % 1000 == 0:
        print(cnt)
        bulk(es, gendata(data_list))
        data_list = []

    cnt += 1

es.indices.refresh()
end = timer()

end = timer()
print('tempo: ' + str(end - start)) # Time in seconds, e.g. 5.38091952400282

print(end - start) # Time in seconds, e.g. 5.38091952400282
