Differences
This shows you the differences between two versions of the page.
Both sides previous revision Previous revision | Next revision Both sides next revision | ||
spark:spark-introduction [2014/10/03 15:02] straka |
spark:spark-introduction [2014/10/06 11:18] straka |
||
---|---|---|---|
Line 27: | Line 27: | ||
words = wiki.flatMap(lambda line: line.split()) | words = wiki.flatMap(lambda line: line.split()) | ||
counts = words.map(lambda word: (word, 1)).reduceByKey(lambda c1,c2: c1+c2) | counts = words.map(lambda word: (word, 1)).reduceByKey(lambda c1,c2: c1+c2) | ||
- | sorted = counts.sortBy(lambda (word, | + | sorted = counts.sortBy(lambda (word, |
sorted.saveAsTextFile(' | sorted.saveAsTextFile(' | ||
| | ||
Line 35: | Line 35: | ||
| | ||
| | ||
- | | + | |
- | | + | |
The output of ' | The output of ' | ||
Line 43: | Line 43: | ||
val words = wiki.flatMap(line => line.split(" | val words = wiki.flatMap(line => line.split(" | ||
val counts = words.map(word => (word, | val counts = words.map(word => (word, | ||
- | val sorted = counts.sortBy({case (word, count) => count}, false) | + | val sorted = counts.sortBy({case (word, count) => count}, |
sorted.saveAsTextFile(' | sorted.saveAsTextFile(' | ||
| | ||
Line 50: | Line 50: | ||
| | ||
| | ||
- | | + | |
- | | + | |