Differences
This shows you the differences between two versions of the page.
Both sides previous revision Previous revision | Next revision Both sides next revision | ||
courses:mapreduce-tutorial:step-29 [2012/01/29 17:26] straka |
courses:mapreduce-tutorial:step-29 [2012/01/29 17:40] straka |
||
---|---|---|---|
Line 2: | Line 2: | ||
Every custom format reading keys of type '' | Every custom format reading keys of type '' | ||
+ | |||
+ | ===== FileAsPathInputFormat ===== | ||
+ | |||
+ | We start by creating '' | ||
+ | |||
+ | <code java> | ||
+ | public static class FileAsPathInputFormat extends FileInputFormat< | ||
+ | public static class FileAsPathRecordReader extends RecordReader< | ||
+ | private Path file; | ||
+ | long start, length; | ||
+ | private Text key, value; | ||
+ | | ||
+ | public void initialize(InputSplit genericSplit, | ||
+ | FileSplit split = (FileSplit) genericSplit; | ||
+ | file = split.getPath(); | ||
+ | start = split.getStart(); | ||
+ | length = split.getLength(); | ||
+ | key = null; | ||
+ | value = null; | ||
+ | } | ||
+ | public boolean nextKeyValue() throws IOException { | ||
+ | if (key != null) return false; | ||
+ | | ||
+ | key = new Text(file.toString()); | ||
+ | value = new Text(String.format(" | ||
+ | | ||
+ | return true; | ||
+ | } | ||
+ | | ||
+ | public Text getCurrentKey() { return key; } | ||
+ | public Text getCurrentValue() { return value; } | ||
+ | public float getProgress() { return (key == null) ? 0 : 1; } | ||
+ | public synchronized void close() throws IOException {} | ||
+ | } | ||
+ | | ||
+ | public RecordReader< | ||
+ | return new FileAsPathRecordReader(); | ||
+ | } | ||
+ | | ||
+ | protected boolean isSplittable(JobContext context, Path filename) { | ||
+ | CompressionCodec codec = new CompressionCodecFactory(context.getConfiguration()).getCodec(filename); | ||
+ | return codec == null; | ||
+ | } | ||
+ | } | ||
+ | </ | ||
===== WholeFileInputFormat ===== | ===== WholeFileInputFormat ===== |