[GitHub]Taming Big Data with Apache Spark and Python

1. from pyspark import SparkConf, SparkContext
   
2. 
   
3. conf = SparkConf().setMaster("local").setAppName("SpendByCustomerSorted")
   
4. sc = SparkContext(conf = conf)
   
5. 
   
6. def extractCustomerPricePairs(line):
   
7.     fields = line.split(',')
   
8.     return (int(fields[0]), float(fields[2]))
   
9. 
   
10. input = sc.textFile("file:///sparkcourse/customer-orders.csv")
    
11. mappedInput = input.map(extractCustomerPricePairs)
    
12. totalByCustomer = mappedInput.reduceByKey(lambda x, y: x + y)
    
13. 
    
14. #Changed for Python 3 compatibility:
    
15. #flipped = totalByCustomer.map(lambda (x,y):(y,x))
    
16. flipped = totalByCustomer.map(lambda x: (x[1], x[0]))
    
17. 
    
18. totalByCustomerSorted = flipped.sortByKey()
    
19. 
    
20. results = totalByCustomerSorted.collect();
    
21. for result in results:
    
22.     print(result)

复制代码

ok
ok
ok