python实现协同过滤算法
实现基于MapReduce协同过滤，需要三个阶段，如下所示
第一个MapReduce：通过ui矩阵得到归一化后的ui矩阵
map阶段：以i为key进行分区排序，相同的key的hash编码放到同一个partition中，
#!/usr/local/bin/python
import sys
for line in sys.stdin:
u, i, s = line.strip().split(',')
print "%s\t%s\t%s" % (i, u, s)
reduce阶段：利用同一个i被所用用户打过分的score,对其进行归一化操作
#!/usr/local/bin/python
import sys
import math
cur_item = None
user_score_list = []
for line in sys.stdin:
item, user, score = line.strip().split('\t')
if cur_item == None:
cur_item = item
if item != cur_item:
sum = 0.0
for tuple in user_score_list:
(u, s) = tuple
sum += pow(s,2)
sum = math.sqrt(sum)
for tuple in user_score_list:
(u, s) = tuple
print "%s\t%s\t%\t" % (u,cur_item,float(s/sum))
user_score_list = []
cur_item = item
user_score_list.append((user,float(score))
for tuple in user_score_list:
(u, s) = tuple
sum += pow(s, 2)
sum = math.sqrt(sum)
for tuple in user_score_list:
(u, s) = tuple
print "%s\t%s\t%s" % (u, cur_item, float(s / sum))
第二个MapReduce：
map阶段：为了得到ii 矩阵必须以u为key，得到(u,i,s)
#!/usr/local/bin/python
import sys
for line in sys.stdin:
u, i, s = line.strip().split('\t')
print "%s\t%s\t%s" % (u, i, s)
reduce阶段：对同一个用户，计算所有打过分的item之间归一化后的分数的乘积，得到 ii 矩阵
#!/usr/local/bin/python
import sys
cur_user = None
item_score_list = []
for line in sys.stdin:
user, item, score = line.strip().split(\t')
if cur_user == None:
cur_user = user
if cur_user != user:
for i in range(0,len(item_score_list) -1):
for j in range(i +1,len(item_score_list)):
item_a, score_a = item_score_list[i]
item_b, score_b = item_score_list[j]
print "%s\t%s\t%s" % (item_a, item_b, score_a * score_b)
print "%s\t%s\t%s" % (item_b, item_a, score_a * score_b)
item_score_list = []
cur_user = user
item_user_score.append((item, float(score)))
for i in range(0, len(item_score_list) - 1):
for j in range(i + 1, len(item_score_list)):
item_a, score_a = item_score_list[i]
item_b, score_b = item_score_list[j]
print "%s\t%s\t%s" % (item_a, item_b, score_a * score_b)
print "%s\t%s\t%s" % (item_b, item_a, score_a * score_b)
第三个MapReduce：
map阶段：以item_a_item_b key，调用map函数
#!/usr/local/bin/python
import sys
for line in sys.stdin:
item_a, item_b, s = line.strip().split('\t')
print "%s\t%s" % (item_a +"_" + item_b, s)
reduce阶段： 对相同的key进行聚合，对value值score进行求和,就得到item与item之间的相似度
#!/usr/local/bin/python
import sys
cur_ii_pair = None
score = 0.0
for line in sys.stdin:
ii_pair, s = line.strip().split('\t')
if cur_ii_pair == None:
cur_ii_pair = ii_pair
if cur_ii_pair != ii_pair:
item_a, item_b = cur_ii_pair.split('_')
print "%s\t%s\t%s" % (item_a, item_b, sum)
cur_ii_pair = ii_pair
score = 0.0
score += float(s)
item_a, item_b = cur_ii_pair.split('_')
print "%s\t%s\t%s" % (item_a, item_b, sum)
