Both sides previous revision
Previous revision
Next revision
|
Previous revision
Next revision
Both sides next revision
|
courses:mapreduce-tutorial:step-31 [2012/02/06 13:10] straka |
courses:mapreduce-tutorial:step-31 [2012/02/06 13:24] straka |
This example reads the keys of ''/net/projects/hadoop/examples/inputs/numbers-small'', computes the sum of all the keys and print it: | This example reads the keys of ''/net/projects/hadoop/examples/inputs/numbers-small'', computes the sum of all the keys and print it: |
<code java Sum.java> | <code java Sum.java> |
| import java.io.IOException; |
| |
| import org.apache.hadoop.conf.*; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.io.*; |
import org.apache.hadoop.mapreduce.*; | import org.apache.hadoop.mapreduce.*; |
import org.apache.hadoop.mapreduce.lib.allreduce.*; | import org.apache.hadoop.mapreduce.lib.allreduce.*; |
less step-31-out/part-* | less step-31-out/part-* |
| |
To run on a cluster with //C// machines using //C// mappers: | To run on a cluster using specified number of machines: |
rm -rf step-31-out; /net/projects/hadoop/bin/hadoop Sum.jar -c C `/net/projects/hadoop/bin/compute-splitsize /net/projects/hadoop/examples/inputs/numbers-small C` /net/projects/hadoop/examples/inputs/numbers-small step-31-out | rm -rf step-31-out; M=#of_machines; INPUT=/net/projects/hadoop/examples/inputs/numbers-small; /net/projects/hadoop/bin/hadoop Sum.jar -c $M `/net/projects/hadoop/bin/compute-splitsize $INPUT $M` $INPUT step-31-out |
less step-31-out/part-* | less step-31-out/part-* |
| |
# $EDITOR Statistics.java | # $EDITOR Statistics.java |
make -f /net/projects/hadoop/java/Makefile Statistics.jar | make -f /net/projects/hadoop/java/Makefile Statistics.jar |
rm -rf step-31-out; /net/projects/hadoop/bin/hadoop Statistics.jar -c C `/net/projects/hadoop/bin/compute-splitsize /net/projects/hadoop/examples/inputs/numbers-small C` /net/projects/hadoop/examples/inputs/numbers-small step-31-out | rm -rf step-31-out; M=#of_machines; INPUT=/net/projects/hadoop/examples/inputs/numbers-small; /net/projects/hadoop/bin/hadoop Statistics.jar -c $M `/net/projects/hadoop/bin/compute-splitsize $INPUT $M` $INPUT step-31-out |
less step-31-out/part-* | less step-31-out/part-* |
| |
# $EDITOR Median.java | # $EDITOR Median.java |
make -f /net/projects/hadoop/java/Makefile Median.jar | make -f /net/projects/hadoop/java/Makefile Median.jar |
rm -rf step-31-out; /net/projects/hadoop/bin/hadoop Median.jar -c C `/net/projects/hadoop/bin/compute-splitsize /net/projects/hadoop/examples/inputs/numbers-small C` /net/projects/hadoop/examples/inputs/numbers-small step-31-out | rm -rf step-31-out; M=#of_machines; INPUT=/net/projects/hadoop/examples/inputs/numbers-small; /net/projects/hadoop/bin/hadoop Median.jar -c $M `/net/projects/hadoop/bin/compute-splitsize $INPUT $M` $INPUT step-31-out |
less step-31-out/part-* | less step-31-out/part-* |
| |