Changes between Version 12 and Version 13 of Streaming

Timestamp:

Sep 13, 2009, 8:07:57 PM (16 years ago)

Author:

waue

Comment:

--

Streaming

@@ -47 +47 @@
  = 用php實做mapReduce =
  * [http://www.hadoop.tw/2008/09/php-hadoop.html 用 "單機" 跟 "PHP" 開發 Hadoop 程式] from Hadoop Taiwan User Group
+{{{
+$ cd /opt/hadoop/
+$ sudo apt-get install php5-cli
+}}}
 
+{{{
+$ gedit mapper.php
+}}}
+
+{{{
+#!php
+#!/usr/bin/php
+<?php
+
+$word2count = array();
+
+// 標準輸入為 STDIN (standard input)
+while (($line = fgets(STDIN)) !== false) {
+   // 移除小寫與空白
+   $line = strtolower(trim($line));
+   // 將行拆解成各個字於words 陣列中
+   $words = preg_split('/\W/', $line, 0, PREG_SPLIT_NO_EMPTY);
+   // 將字+1
+   foreach ($words as $word) {
+       $word2count[$word] += 1;
+   }
+}
+
+// 將結果寫到 STDOUT (standard output)
+foreach ($word2count as $word => $count) {
+   // 印出 [字 , "tab符號" ,  "數字" , "結束字元"]
+   echo $word, chr(9), $count, PHP_EOL;
+}
+?>
+}}}
+
+{{{
+$ gedit reducer.php
+}}}
+
+{{{
+#!php
+#!/usr/bin/php
+<?php
+
+$word2count = array();
+
+// 輸入為 STDIN
+while (($line = fgets(STDIN)) !== false) {
+    // 移除多餘的空白
+    $line = trim($line);
+    // 每一行的格式為 (單字 "tab" 數字) ，紀錄到($word, $count)
+    list($word, $count) = explode(chr(9), $line);
+    // 轉換格式string -> int
+    $count = intval($count);
+    // 加總
+    if ($count > 0) $word2count[$word] += $count;
+}
+
+// 此行不必要，但可讓output排列更完整
+ksort($word2count);
+
+// 將結果寫到 STDOUT (standard output)
+foreach ($word2count as $word => $count) {
+    echo $word, chr(9), $count, PHP_EOL;
+}
+
+?> 
+}}}
+
+{{{
+$ chmod 755 *.php
+$ echo "i love hadoop, hadoop love u" | ./mapper.php | ./reducer.php
+$ bin/hadoop jar hadoop/contrib/hadoop-*-streaming.jar -mapper mapper.php -reducer reducer.php -input lab4_input -output stream_out2
+}}}
+
+ * 檢查結果
+
+{{{
+$ haddop dfs -cat stream_out2/part-00000 | more
+}}}
 
  = Python 實做 =