import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time

data_jeremy = pd.read_csv('scraper/data.csv')

links = list(set(list(data_jeremy['link'])))
users = list(set(list(data_jeremy['Facebook user id'])))

print "Links"
print links[:5]
print "Users"
print users[:5]

matrix = pd.DataFrame(np.zeros((len(links), len(users))))
matrix.index = links
matrix.columns = users

print "matrix_size:",
print matrix.shape

for i in data_jeremy.iterrows():
	matrix.loc[i[1][0], i[1][2]] = 1

start = time.time()
count = 0
for i in matrix:
	count += 1
	if count % len(matrix)/10 == 0:
		print 'Count = %r/%r' %(count,len(matrix))
	if sum(matrix[i]) == 1:
		matrix.drop(i, 1, inplace=True)
print 'Time for drop = %r' %(time.time() - start)
        
def PCA(matrix):
    A = matrix
    M = (A-np.mean(A.T,axis=1)).T # subtract the mean (along columns)
    [latent,coeff] = np.linalg.eig(np.cov(M)) # attention:not always sorted
    score = np.dot(coeff.T,M) # projection of the data in the new space
    return coeff,score,latent

matrix = matrix.T
coef, score, eigenvalues = PCA(matrix)

principal_components = coef[:,:2]
print "Principal Components"
print principal_components
compressed = np.dot(matrix, principal_components)
plt.scatter(compressed[:,0], compressed[:,1])
plt.show()
eba469a21eab80cf8760da188baaa00bca2a30ba
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time

data_jeremy = pd.read_csv('scraper/data.csv')

links = list(set(list(data_jeremy['link'])))
users = list(set(list(data_jeremy['Facebook user id'])))

print "Links"
print links[:5]
print "Users"
print users[:5]

matrix = pd.DataFrame(np.zeros((len(links), len(users))))
matrix.index = links
matrix.columns = users

print "matrix_size:",
print matrix.shape

for i in data_jeremy.iterrows():
	matrix.loc[i[1][0], i[1][2]] = 1

start = time.time()
count = 0
for i in matrix:
	count += 1
	if count % len(matrix)/10 == 0:
		print 'Count = %r/%r' %(count,len(matrix))
	if sum(matrix[i]) == 1:
		matrix.drop(i, 1, inplace=True)
print 'Time for drop = %r' %(time.time() - start)
        
def PCA(matrix):
    A = matrix
    M = (A-np.mean(A.T,axis=1)).T # subtract the mean (along columns)
    [latent,coeff] = np.linalg.eig(np.cov(M)) # attention:not always sorted
    score = np.dot(coeff.T,M) # projection of the data in the new space
    return coeff,score,latent

matrix = matrix.T
coef, score, eigenvalues = PCA(matrix)

principal_components = coef[:,:2]
print "Principal Components"
print principal_components
compressed = np.dot(matrix, principal_components)
plt.scatter(compressed[:,0], compressed[:,1])
plt.show()