Changes between Version 1 and Version 2 of waue/2009/0715

Timestamp:

Jul 15, 2009, 3:59:30 PM (16 years ago)

Author:

waue

Comment:

--

waue/2009/0715

@@ -1 +1 @@
 {{{
-                public void map(LongWritable key, Text value,
-                                OutputCollector<Text, IntWritable> output, Reporter reporter)
-                                throws IOException {
-                        String line = (caseSensitive) ? value.toString() : value.toString()
-                                        .toLowerCase();
+public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
+public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
 
-                        for (String pattern : patternsToSkip) {
-                                line = line.replaceAll(pattern, "");
-                        }
+while (tokenizer.hasMoreTokens()) {
+                private final static IntWritable one = new IntWritable(1);
+                private Text word = new Text();
+                output.collect(word, one);
 
-                        StringTokenizer tokenizer = new StringTokenizer(line);
-                        while (tokenizer.hasMoreTokens()) {
-                                word.set(tokenizer.nextToken());
-                                output.collect(word, one);
-                                reporter.incrCounter(Counters.INPUT_WORDS, 1);
-                        }
+}}}
 
-                        if ((++numRecords % 100) == 0) {
-                                reporter.setStatus("Finished processing " + numRecords
-                                                + " records " + "from the input file: " + inputFile);
-                        }
-                }
-        }
+{{{
+public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
+public void reduce(Text key, Iterator<IntWritable> values,OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
+int sum = 0;
+while (values.hasNext()) {
+        sum += values.next().get();
+}
+output.collect(key, new IntWritable(sum));
 
-        public static class Reduce extends MapReduceBase implements
-                        Reducer<Text, IntWritable, Text, IntWritable> {
-                public void reduce(Text key, Iterator<IntWritable> values,
-                                OutputCollector<Text, IntWritable> output, Reporter reporter)
-                                throws IOException {
-                        int sum = 0;
-                        while (values.hasNext()) {
-                                sum += values.next().get();
-                        }
-                        output.collect(key, new IntWritable(sum));
-                }
-        }
 }}}