Changes between Version 4 and Version 5 of YMU110509/Lab9

Timestamp:

Jun 20, 2011, 2:38:05 PM (14 years ago)

Author:

jazz

Comment:

--

YMU110509/Lab9

@@ -17 +17 @@
 ~$ gunzip test_long.fa.gz
 ~$ for ((i=1;i<100;i++)); do hadoop fs -put test_long.fa sample-$i.fa; done
-~$ for ((i=1;i<20;i++)); do echo /user/h998/sample-$i.fa; done > sample-01.txt
-~$ for ((i=20;i<40;i++)); do echo /user/h998/sample-$i.fa; done > sample-02.txt
-~$ for ((i=40;i<60;i++)); do echo /user/h998/sample-$i.fa; done > sample-03.txt
-~$ for ((i=60;i<80;i++)); do echo /user/h998/sample-$i.fa; done > sample-04.txt
-~$ for ((i=80;i<100;i++)); do echo /user/h998/sample-$i.fa; done > sample-05.txt
+~$ for ((i=1;i<20;i++)); do echo /user/$(whoami)/sample-$i.fa; done > sample-01.txt
+~$ for ((i=20;i<40;i++)); do echo /user/$(whoami)/sample-$i.fa; done > sample-02.txt
+~$ for ((i=40;i<60;i++)); do echo /user/$(whoami)/sample-$i.fa; done > sample-03.txt
+~$ for ((i=60;i<80;i++)); do echo /user/$(whoami)/sample-$i.fa; done > sample-04.txt
+~$ for ((i=80;i<100;i++)); do echo /user/$(whoami)/sample-$i.fa; done > sample-05.txt
 ~$ hadoop fs -mkdir lab9_input
 ~$ hadoop fs -put sample-0* lab9_input
+}}}
+ * 檢查輸入檔案
+{{{
+~$ hadoop fs -ls
+~$ hadoop fs -ls lab9_input
 }}}
 
@@ -31 +36 @@
 {{{
 #!sh
+#!/bin/bash
 
+id="h998"
+mkdir -p /tmp/$id
+host=`hostname`
+pwd=`pwd`
+uid=`whoami`
+
+while read line; do 
+  input=$line
+  filename=`basename $input`
+  echo "$uid@$host:$pwd> hadoop fs -get $input /tmp/$id/$filename"
+  echo "$uid@$host:$pwd> velveth output-$filename 17 -fasta -short /tmp/$id/$filename"
+  echo "$uid@$host:$pwd> hadoop fs -put output-$filename ."
+done
+rm -rf /tmp/$id
+}}}
+ * 接著，讓我們在本地端先驗證一下 testmapper.sh 的運作
+{{{
+~$ head -n 10 sample-01.txt > sample-00.txt
+~$ cat > testmapper.sh << EOF
+#!/bin/bash
+
+id="h998"
+mkdir -p /tmp/\$id
+host=\`hostname\`
+pwd=\`pwd\`
+uid=\`whoami\`
+
+while read line; do 
+  input=\$line
+  filename=\`basename \$input\`
+  echo "\$uid@$host:\$pwd> hadoop fs -get \$input /tmp/\$id/\$filename"
+  echo "\$uid@$host:\$pwd> velveth output-\$filename 17 -fasta -short /tmp/\$id/\$filename"
+  echo "\$uid@$host:\$pwd> hadoop fs -put output-\$filename ."
+done
+rm -rf /tmp/\$id
+EOF
+~$ chmod a+x testmapper.sh
+~$ cat sample-00.txt | ./testmapper.sh 
+h998@hadoop:/home/h998> hadoop fs -get /user/h998/sample-1.fa /tmp/h998/sample-1.fa
+h998@hadoop:/home/h998> velveth output-sample-1.fa 17 -fasta -short /tmp/h998/sample-1.fa
+h998@hadoop:/home/h998> hadoop fs -put output-sample-1.fa .
+h998@hadoop:/home/h998> hadoop fs -get /user/h998/sample-2.fa /tmp/h998/sample-2.fa
+}}}
+ * 讓我們用 Hadoop Streaming 的方式來執行 testmapper.sh
+{{{
+~$ hadoop jar hadoop-streaming.jar -input lab9_input -output lab9_out1 -mapper testmapper.sh -file testmapper.sh 
+}}}
+ * 觀察 lab9_out1 的結果，看與本機執行有何不同呢？
+{{{
+~$ hadoop fs -cat /user/$(whoami)/lab9_out1/part-00000 | head
+hadoop@hadoop104:/var/lib/hadoop/cache/hadoop/mapred/local/taskTracker/jobcache/job_201106041247_1820/attempt_201106041247_1820_m_000002_0/work> hadoop fs -get /user/h998/sample-60.fa /tmp/h998/sample-60.fa  
+hadoop@hadoop104:/var/lib/hadoop/cache/hadoop/mapred/local/taskTracker/jobcache/job_201106041247_1820/attempt_201106041247_1820_m_000002_0/work> hadoop fs -get /user/h998/sample-61.fa /tmp/h998/sample-61.fa  
+hadoop@hadoop104:/var/lib/hadoop/cache/hadoop/mapred/local/taskTracker/jobcache/job_201106041247_1820/attempt_201106041247_1820_m_000002_0/work> hadoop fs -get /user/h998/sample-62.fa /tmp/h998/sample-62.fa  
 }}}