import java.io.IOException; import org.apache.commons.logging.*; import org.apache.hadoop.conf.*; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.*; import org.apache.hadoop.mapreduce.*; import org.apache.hadoop.mapreduce.lib.allreduce.*; import org.apache.hadoop.mapreduce.lib.input.*; import org.apache.hadoop.mapreduce.lib.output.*; import org.apache.hadoop.util.*; public class Statistics extends Configured implements Tool { public static class TheMapper extends AllReduceMapper{ int[] keys = new int[64]; int keys_num = 0; public void map(Text key, Text value, Context context) throws IOException, InterruptedException { if (keys_num == keys.length) { int[] new_keys = new int[2*keys_num]; System.arraycopy(keys, 0, new_keys, 0, keys_num); keys = new_keys; } keys[keys_num++] = Integer.parseInt(key.toString()); } public void cooperate(Context context, boolean writeResults) throws IOException, InterruptedException { // INSERT CODE HERE } } // Job configuration public int run(String[] args) throws Exception { if (args.length < 2) { System.err.printf("Usage: %s.jar in-path out-path", this.getClass().getName()); return 1; } Job job = new Job(getConf(), this.getClass().getName()); job.setJarByClass(this.getClass()); job.setMapperClass(TheMapper.class); AllReduce.init(job); job.setOutputKeyClass(DoubleWritable.class); job.setOutputValueClass(NullWritable.class); job.setInputFormatClass(KeyValueTextInputFormat.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); return job.waitForCompletion(true) ? 0 : 1; } // Main method public static void main(String[] args) throws Exception { int res = ToolRunner.run(new Statistics(), args); System.exit(res); } }