from pymongo import MongoClient


def retrieve_one_tweet_per_time():
    client = MongoClient(port=27017)
    db = client.european_elections
    collection = db.tweets
    cursor = collection.find({}).batch_size(10)
    remove_duplicates(cursor, db, collection)


def remove_duplicates(cursor, db, collection):
    id_list = []

    for document in cursor:

        try:

            if document["id"] not in id_list:

                id_list.append(document["id"])

            else:

                collection.delete_one(document)

        except:

            pass


retrieve_one_tweet_per_time()

ccf80878656f421030416b205b1e3d352c0b8a30
<mask token>


def remove_duplicates(cursor, db, collection):
    id_list = []
    for document in cursor:
        try:
            if document['id'] not in id_list:
                id_list.append(document['id'])
            else:
                collection.delete_one(document)
        except:
            pass


<mask token>

<mask token>


def retrieve_one_tweet_per_time():
    client = MongoClient(port=27017)
    db = client.european_elections
    collection = db.tweets
    cursor = collection.find({}).batch_size(10)
    remove_duplicates(cursor, db, collection)


def remove_duplicates(cursor, db, collection):
    id_list = []
    for document in cursor:
        try:
            if document['id'] not in id_list:
                id_list.append(document['id'])
            else:
                collection.delete_one(document)
        except:
            pass


<mask token>

<mask token>


def retrieve_one_tweet_per_time():
    client = MongoClient(port=27017)
    db = client.european_elections
    collection = db.tweets
    cursor = collection.find({}).batch_size(10)
    remove_duplicates(cursor, db, collection)


def remove_duplicates(cursor, db, collection):
    id_list = []
    for document in cursor:
        try:
            if document['id'] not in id_list:
                id_list.append(document['id'])
            else:
                collection.delete_one(document)
        except:
            pass


retrieve_one_tweet_per_time()

from pymongo import MongoClient


def retrieve_one_tweet_per_time():
    client = MongoClient(port=27017)
    db = client.european_elections
    collection = db.tweets
    cursor = collection.find({}).batch_size(10)
    remove_duplicates(cursor, db, collection)


def remove_duplicates(cursor, db, collection):
    id_list = []
    for document in cursor:
        try:
            if document['id'] not in id_list:
                id_list.append(document['id'])
            else:
                collection.delete_one(document)
        except:
            pass


retrieve_one_tweet_per_time()

from pymongo import MongoClient


def retrieve_one_tweet_per_time():
    client = MongoClient(port=27017)
    db = client.european_elections
    collection = db.tweets
    cursor = collection.find({}).batch_size(10)
    remove_duplicates(cursor, db, collection)


def remove_duplicates(cursor, db, collection):
    id_list = []

    for document in cursor:

        try:

            if document["id"] not in id_list:

                id_list.append(document["id"])

            else:

                collection.delete_one(document)

        except:

            pass


retrieve_one_tweet_per_time()
