#! /usr/bin/env python

import sys

current_word = None
current_count = 0
word = None
word_counts = {}
sum = 0.0

#input from STDIN
for line in sys.stdin:
    line = line.strip()
    word, count = line.split('\t', 1)

    try:
        count = int(count)
    except ValueError:
        continue

    # works because hadoop sorts map output
    # same word
    if current_word == word:
       current_count += count
       sum += count

    # new word, save word count and start new
    else:
        if current_word:
            word_counts[current_word] = current_count
        current_count = count
        current_word = word
        sum += count

# went through all mapped items
for word, count in word_counts.iteritems():
    print('{}\tcount: {}\tavg use per word: {}'.format(word, count, 
                                                       round(float(count/sum), 4)))
    

422c3c9387b8a5fbfe3e30cad42c21b2ddb4f1a5
<mask token>

<mask token>
for line in sys.stdin:
    line = line.strip()
    word, count = line.split('\t', 1)
    try:
        count = int(count)
    except ValueError:
        continue
    if current_word == word:
        current_count += count
        sum += count
    else:
        if current_word:
            word_counts[current_word] = current_count
        current_count = count
        current_word = word
        sum += count
for word, count in word_counts.iteritems():
    print('{}\tcount: {}\tavg use per word: {}'.format(word, count, round(
        float(count / sum), 4)))

<mask token>
current_word = None
current_count = 0
word = None
word_counts = {}
sum = 0.0
for line in sys.stdin:
    line = line.strip()
    word, count = line.split('\t', 1)
    try:
        count = int(count)
    except ValueError:
        continue
    if current_word == word:
        current_count += count
        sum += count
    else:
        if current_word:
            word_counts[current_word] = current_count
        current_count = count
        current_word = word
        sum += count
for word, count in word_counts.iteritems():
    print('{}\tcount: {}\tavg use per word: {}'.format(word, count, round(
        float(count / sum), 4)))

import sys
current_word = None
current_count = 0
word = None
word_counts = {}
sum = 0.0
for line in sys.stdin:
    line = line.strip()
    word, count = line.split('\t', 1)
    try:
        count = int(count)
    except ValueError:
        continue
    if current_word == word:
        current_count += count
        sum += count
    else:
        if current_word:
            word_counts[current_word] = current_count
        current_count = count
        current_word = word
        sum += count
for word, count in word_counts.iteritems():
    print('{}\tcount: {}\tavg use per word: {}'.format(word, count, round(
        float(count / sum), 4)))

#! /usr/bin/env python

import sys

current_word = None
current_count = 0
word = None
word_counts = {}
sum = 0.0

#input from STDIN
for line in sys.stdin:
    line = line.strip()
    word, count = line.split('\t', 1)

    try:
        count = int(count)
    except ValueError:
        continue

    # works because hadoop sorts map output
    # same word
    if current_word == word:
       current_count += count
       sum += count

    # new word, save word count and start new
    else:
        if current_word:
            word_counts[current_word] = current_count
        current_count = count
        current_word = word
        sum += count

# went through all mapped items
for word, count in word_counts.iteritems():
    print('{}\tcount: {}\tavg use per word: {}'.format(word, count, 
                                                       round(float(count/sum), 4)))
    
