Changes between Initial Version and Version 1 of III130615/Lab10


Ignore:
Timestamp:
Jun 12, 2013, 5:26:37 PM (11 years ago)
Author:
jazz
Comment:

--

Legend:

Unmodified
Added
Removed
Modified
  • III130615/Lab10

    v1 v1  
     1[[PageOutline]]
     2
     3◢ <[wiki:III130615/Lab9 實作九]> | <[wiki:III130615 回課程大綱]> ▲ | <[wiki:III130615/Lab11 實作十一]> ◣
     4
     5{{{
     6#!text
     7請連線至 hadoop.nchc.org.tw 作練習
     8}}}
     9 * ssh://hadoop.nchc.org.tw
     10 * https://hadoop.nchc.org.tw
     11
     12= 實作十 Lab10 =
     13
     14{{{
     15#!html
     16<div style="text-align: center;"><big style="font-weight: bold;"><big>HDFS、MapReduce 與 Hadoop Streaming 觀念驗證<br/>Running Hadoop Streaming with HDFS and MapReduce</big></big></div>
     17}}}
     18
     19== 準備輸入資料集 Input Dataset ==
     20
     21 * 首先準備輸入,包含兩部份:(1) 供 Velvet 計算的 *.fa 檔案,這裡為了方便示範起見,採用 test_long.fa 當範本,並複製 99 份不同檔名,當輸入 (2) 供 Mapper 運算的輸入檔案(內含 HDFS 的檔名路徑)
     22{{{
     23~$ cp /usr/share/doc/velvet-example/examples/data/test_long.fa.gz .
     24~$ gunzip test_long.fa.gz
     25~$ for ((i=1;i<100;i++)); do hadoop fs -put test_long.fa input-$i.fa; done
     26~$ for ((i=1;i<20;i++)); do echo /user/$(whoami)/input-$i.fa; done > split-01.txt
     27~$ for ((i=20;i<40;i++)); do echo /user/$(whoami)/input-$i.fa; done > split-02.txt
     28~$ for ((i=40;i<60;i++)); do echo /user/$(whoami)/input-$i.fa; done > split-03.txt
     29~$ for ((i=60;i<80;i++)); do echo /user/$(whoami)/input-$i.fa; done > split-04.txt
     30~$ for ((i=80;i<100;i++)); do echo /user/$(whoami)/input-$i.fa; done > split-05.txt
     31~$ hadoop fs -mkdir lab10_input
     32~$ hadoop fs -put split-0* lab10_input
     33}}}
     34 * 檢查輸入檔案
     35{{{
     36~$ hadoop fs -ls
     37~$ hadoop fs -ls lab10_input
     38}}}
     39
     40== 觀察 Hadoop Streaming 執行身份與工作目錄 ==
     41
     42 * 撰寫 testmapper.sh
     43{{{
     44#!sh
     45#!/bin/bash
     46
     47id="h998"
     48mkdir -p /tmp/$id
     49host=`hostname`
     50pwd=`pwd`
     51uid=`whoami`
     52
     53while read line; do
     54  input=$line
     55  filename=`basename $input`
     56  echo "$uid@$host:$pwd> hadoop fs -get $input /tmp/$id/$filename"
     57  echo "$uid@$host:$pwd> velveth output-$filename 17 -fasta -short /tmp/$id/$filename"
     58  echo "$uid@$host:$pwd> hadoop fs -put output-$filename ."
     59done
     60rm -rf /tmp/$id
     61}}}
     62 * 接著,讓我們在本地端先驗證一下 testmapper.sh 的運作
     63{{{
     64~$ head -n 2 split-01.txt > split-00.txt
     65~$ cat > testmapper.sh << EOF
     66#!/bin/bash
     67
     68id="`whoami`"
     69mkdir -p /tmp/\$id
     70host=\`hostname\`
     71pwd=\`pwd\`
     72uid=\`whoami\`
     73
     74while read line; do
     75  input=\$line
     76  filename=\`basename \$input\`
     77  echo "\$uid@\$host:\$pwd> hadoop fs -get \$input /tmp/\$id/\$filename"
     78  echo "\$uid@\$host:\$pwd> velveth output-\$filename 17 -fasta -short /tmp/\$id/\$filename"
     79  echo "\$uid@\$host:\$pwd> hadoop fs -put output-\$filename ."
     80done
     81rm -rf /tmp/\$id
     82EOF
     83~$ chmod a+x testmapper.sh
     84~$ cat sample-00.txt | ./testmapper.sh
     85h998@hadoop:/home/h998> hadoop fs -get /user/h998/input-1.fa /tmp/h998/input-1.fa
     86h998@hadoop:/home/h998> velveth output-input-1.fa 17 -fasta -short /tmp/h998/input-1.fa
     87h998@hadoop:/home/h998> hadoop fs -put output-input-1.fa .
     88h998@hadoop:/home/h998> hadoop fs -get /user/h998/input-2.fa /tmp/h998/input-2.fa
     89}}}
     90 * 讓我們用 Hadoop Streaming 的方式來執行 testmapper.sh
     91{{{
     92~$ hadoop jar hadoop-streaming.jar -input lab10_input -output lab10_out1 -mapper testmapper.sh -file testmapper.sh
     93}}}
     94 * 觀察 lab10_out1 的結果,看與本機執行有何不同呢?
     95{{{
     96~$ hadoop fs -cat /user/$(whoami)/lab10_out1/part-00000 | head
     97hadoop@hadoop104:/var/lib/hadoop/cache/hadoop/mapred/local/taskTracker/jobcache/job_201106041247_1820/attempt_201106041247_1820_m_000002_0/work> hadoop fs -get /user/h998/input-60.fa /tmp/h998/input-60.fa   
     98hadoop@hadoop104:/var/lib/hadoop/cache/hadoop/mapred/local/taskTracker/jobcache/job_201106041247_1820/attempt_201106041247_1820_m_000002_0/work> hadoop fs -get /user/h998/input-61.fa /tmp/h998/input-61.fa   
     99hadoop@hadoop104:/var/lib/hadoop/cache/hadoop/mapred/local/taskTracker/jobcache/job_201106041247_1820/attempt_201106041247_1820_m_000002_0/work> hadoop fs -get /user/h998/input-62.fa /tmp/h998/input-62.fa   
     100}}}
     101
     102== 實作透過 Hadoop Streaming 執行 99 組 velvet 運算 ==
     103
     104 * 撰寫 mapper.sh
     105{{{
     106#!sh
     107#!/bin/bash
     108
     109id="h998"
     110mkdir -p /tmp/$id
     111host=`hostname`
     112pwd=`pwd`
     113uid=`whoami`
     114
     115while read line; do
     116  input=$line
     117  filename=`basename $input`
     118  echo "$uid@$host> hadoop fs -get $input /tmp/$id/$filename"
     119  hadoop fs -get $input /tmp/$id/$filename
     120  echo "$uid@$host> velveth output-$filename 17 -fasta -short /tmp/$id/$filename"
     121  velveth output-$filename 17 -fasta -short /tmp/$id/$filename
     122  echo "$uid@$host> hadoop fs -put output-$filename /user/$id/."
     123  hadoop fs -put output-$filename /user/$id/.
     124  hadoop fs -chown $id /user/$id/output-$filename
     125done
     126rm -rf /tmp/$id
     127}}}
     128 * 於本機測試 mapper.sh
     129{{{
     130~$ cat > mapper.sh << EOF
     131#!/bin/bash
     132
     133id="`whoami`"
     134mkdir -p /tmp/\$id
     135host=\`hostname\`
     136pwd=\`pwd\`
     137uid=\`whoami\`
     138
     139while read line; do
     140  input=\$line
     141  filename=\`basename \$input\`
     142  echo "\$uid@\$host> hadoop fs -get \$input /tmp/\$id/\$filename"
     143  hadoop fs -get \$input /tmp/\$id/\$filename
     144  echo "\$uid@\$host> velveth output-\$filename 17 -fasta -short /tmp/\$id/\$filename"
     145  velveth output-\$filename 17 -fasta -short /tmp/\$id/\$filename
     146  echo "\$uid@\$host> hadoop fs -put output-\$filename /user/\$id/."
     147  hadoop fs -put output-\$filename /user/\$id/.
     148  hadoop fs -chown \$id /user/\$id/output-\$filename
     149done
     150rm -rf /tmp/\$id
     151EOF
     152~$ chmod a+x mapper.sh
     153~$ cat sample-00.txt | ./mapper.sh
     154~$ hadoop fs -rmr output-*
     155~$ rm -rf output-sample-*
     156}}}
     157 * 接著用 hadoop streaming 來執行
     158{{{
     159~$ hadoop jar hadoop-streaming.jar -input lab10_input -output lab10_out2 -mapper mapper.sh -file mapper.sh
     160}}}