Differences
This shows you the differences between two versions of the page.
Both sides previous revision Previous revision Next revision | Previous revision Next revision Both sides next revision | ||
courses:mapreduce-tutorial:step-29 [2012/01/29 17:26] straka |
courses:mapreduce-tutorial:step-29 [2012/02/05 19:03] straka |
||
---|---|---|---|
Line 1: | Line 1: | ||
- | ====== MapReduce Tutorial : Custom | + | ====== MapReduce Tutorial : Custom |
- | Every custom format reading keys of type '' | + | ====== Custom sorting comparator ====== |
- | ===== WholeFileInputFormat ===== | + | The keys are sorted before processed by a reducer, using a |
+ | [[http:// | ||
- | We start by creating '' | + | <code java> |
+ | public static class IntPair implements WritableComparable< | ||
+ | private int first = 0; | ||
+ | private int second = 0; | ||
- | The main functionality lays in '' | + | public void set(int left, int right) { first = left; second = right; } |
+ | public int getFirst() { return first; } | ||
+ | public int getSecond() { return second; } | ||
- | <code java> | + | |
- | public | + | |
- | // Helper class, which does the actual work -- reads the (path, content) input pair. | + | |
- | public static class WholeFileRecordReader extends RecordReader< | + | } |
- | | + | |
- | | + | |
- | | + | |
- | | + | } |
- | | + | |
- | | + | |
- | | + | |
- | | + | |
- | file = split.getPath(); | + | else return second < o.second ? -1 : second |
- | | + | } |
- | key = null; | + | } |
- | value = null; | + | </ |
- | | + | |
- | FileSystem fs = file.getFileSystem(context.getConfiguration()); | + | If we would like in a Hadoop job to sort the '' |
- | in = fs.open(split.getPath()); | + | |
- | CompressionCodecFactory compressionCodecs = new CompressionCodecFactory(context.getConfiguration()); | + | <code java> |
- | | + | public static class IntPair implements WritableComparable< |
- | | + | |
- | in = new DataInputStream(codec.createInputStream(in)); | + | public static class FirstOnlyComparator implements RawComparator< |
+ | public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { | ||
+ | | ||
+ | | ||
+ | return first1 < first2 ? -1 : first1 == first2 ? 0 : 1; | ||
} | } | ||
+ | public int compare(IntPair x, IntPair y) { | ||
+ | return x.getFirst() < y.getFirst() ? -1 : x.getFirst() == y.getFirst() ? 0 : 1; | ||
+ | } | ||
+ | } | ||
+ | } | ||
- | public boolean nextKeyValue() throws IOException { | + | ... |
- | if (value_read) return false; | + | |
- | byte[] data = new byte[length]; | + | job.setSortComparatorClass(IntPair.FirstOnlyComparator.class); |
- | in.readFully(data); | + | </ |
+ | Notice we used helper function '' | ||
- | key = new Text(file.toString()); | + | ====== Grouping comparator ====== |
- | value = new BytesWritable(data); | + | |
- | value_read | + | |
- | return true; | + | In a reduce, it is guaranteed that keys are processed in ascending order. Sometimes it would be useful if the //values associated with one key// could also be processed in ascending order. |
- | } | + | |
- | public Text getCurrentKey() { return | + | That is possible only to some degree. The (key, value) pairs are compared //using the key only//. After the (key, value) |
- | public BytesWritable getCurrentValue() { return | + | |
- | public float getProgress() { return value_read ? 0 : 1; } | + | |
- | public synchronized void close() throws IOException { if (in != null) { in.close(); in = null; } } | + | |
- | } | + | |
- | // Use the helper class as a RecordReader in out file format. | + | As an example, consider that the input consists of ('' |
- | | + | |
- | | + | - These pairs are sorted by the '' |
- | } | + | - The custom grouping comparator is used, which groups the '' |
+ | <code java> | ||
+ | </ | ||
- | // Do not allow splitting. | + | ---- |
- | protected boolean isSplittable(JobContext context, Path filename) { | + | |
- | return false; | + | |
- | } | + | |
- | } | + | |
- | </code> | + | < |
+ | <table style=" | ||
+ | < | ||
+ | <td style=" | ||
+ | <td style=" | ||
+ | <td style=" | ||
+ | </ | ||
+ | </ | ||
+ | </html> | ||