source: nutchez-0.1/bin/nutchez-func.sh @ 217

#!/bin/bash

# Author: WeiYu Chen <waue _at_ nchc org tw>

# License: GPL

# Description: Eazily use for Nutch

# .





#DIALOG='dialog --backtitle "     NutchEz Setup Menu           -- powered by NCHC "'

DIALOG=dialog



# set 1 to display more for debug, 

VERB=0



init_nutchez () {

  if ! [ -e ~/.nutchez ] ;then

    # copy from /etc/nutch

    mkdir ~/.nutchez

    cp -rf /etc/nutch/conf ~/.nutchez/

    mkdir ~/.nutchez/log

    cp -rf /etc/nutch/sav ~/.nutchez/

    chown -R $LOGNAME:$LOGNAME ~/.nutchez

  fi

  

}



echo_dialog_v () {

  if [[ $VERB -eq 1 ]]; then

    $DIALOG --msgbox "$1" 16 51

  fi

}



test_file () {

  if ! test -e "$1" ; then

    echo_dialog_v "test_file: \n can not find $1"

    echo "" >  $1

  else

     echo_dialog_v "test_file: \n Touch  $1 ! \n Its content is \n `cat $1`"

  fi

}



check_if_root() {

   if [[ ! "$UID" -eq "0" ]]; then

     echo_dialog_v "Hi [$LOGNAME] !! "

     echo_dialog_v "You need to run this script \"`basename $0`\" as root."

     exit 1

   fi

}



promote_tempfile () {

  echo_dialog_v "7. chang tmp as txt"

  rm -f ~/.nutchez/sav/n.*.txt

  mv /tmp/n.urls.tmp ~/.nutchez/sav/n.urls.txt

  mv /tmp/n.robot.tmp ~/.nutchez/sav/n.robot.txt

  mv /tmp/n.crawler.tmp ~/.nutchez/sav/n.crawler.txt

  mv /tmp/n.tomcat.tmp ~/.nutchez/sav/n.tomcat.txt

  rm -f /tmp/n.*.tmp

}



clean_tempfile () {

  echo_dialog_v "7. delete tmp"

  rm -f /tmp/n.*.tmp

}



setup_nutchez () {    

  if ! [ -e ~/.nutchez/urls ] ; then

    # make url list dir

    mkdir ~/.nutchez/urls

  fi



  if  [ -e ~/.nutchez/urls/urls.txt ] ; then

    rm ~/.nutchez/urls/urls.txt

  fi



  cp ~/.nutchez/sav/n.urls.txt ~/.nutchez/urls/urls.txt

  if ! [ -z $NOCONTINUE ]; then

    if [[ $NOCONTINUE -eq 1 ]]; then

      echo_dialog_v " delete the ~/.nutchez/search/*"

      DATE=`date +%Y%m%d%H%M%S`

      mv ~/.nutchez/search ~/.nutchez/search-$DATE

      $DIALOG --msgbox "上次搜尋的結果改放到 ~/.nutchez/search-$DATE " 0 0

  #    rm -f /tmp/search

  #    rm -rf ~/.nutchez/search/*

    fi

  fi



  if [ -e ~/.nutchez/conf/nutch-site.xml ] ; then

    # set nutch-site.xml

    sed -i -e "4s/<value>[a-zA-Z0-9]*</<value>$ROBOT</" ~/.nutchez/conf/nutch-site.xml

  fi

}



install_tomcat (){

  if ! [ -e ~/.nutchez/tomcat ] ;then

    # isntall tomcat to home

    cp -rf /opt/nutchez/tomcat ~/.nutchez/

    chown -R $LOGNAME:$LOGNAME ~/.nutchez/tomcat/

    # make search dir

    if ! [ -e ~/.nutchez/search ] ;then

      mkdir ~/.nutchez/search

    fi



   # setup search dir

    cd ~

    HOMEDIR=`pwd`/.nutchez/search

    sed -i -e "8s|<value>[a-zA-Z0-9/]*<|<value>$HOMEDIR<|" ~/.nutchez/tomcat/webapps/ROOT/WEB-INF/classes/nutch-site.xml



  fi

  # change explorer port

  sed -i -e "67s/<Connector port=\"[0-9]*\"/<Connector port=\"$PORT\"/" ~/.nutchez/tomcat/conf/server.xml

}





show_urls (){

  # show urls : ok =0 ,cancel = 1

  echo_dialog_v "2. show_urls !"

  test_file  ~/.nutchez/sav/n.urls.txt

  echo_dialog_v "2.1 test_file ~/.nutchez/sav return : $?"

  # dialog begin

  dialog --title "請輸入你要抓取的網址（一行一個網址）" --editbox ~/.nutchez/sav/n.urls.txt 16 51 2>/tmp/n.urls.tmp

  RET=$?

  echo_dialog_v "2.1 cat url: `cat /tmp/n.urls.tmp`"

  return $RET

}



setup_robot () {

  test_file ~/.nutchez/sav/n.robot.txt

  echo_dialog_v "3. setup_robot"

  # dialog

  dialog --title "設定機器人名稱" --nocancel --inputbox " 這個爬網機器人，你要將他取名為：\n\n (ps: 這個設定只是禮貌性宣告，並不會對結果造成影響) \n" 16 55 "`cat ~/.nutchez/sav/n.robot.txt`" 2>/tmp/n.robot.tmp

  echo_dialog_v "3.1 cat robot : `cat /tmp/n.robot.tmp`"

}



setup_crawler () {

  echo_dialog_v "4. setup_crawler"

  test_file ~/.nutchez/sav/n.crawler.txt

  dialog --title "設定抓取深度" --nocancel --inputbox " 對於每個網址，你需要NutchEz爬多深呢？\n\n (ps: 初次體驗建議將深度設為1來感受需要多久)  \n " 16 51 "`cat ~/.nutchez/sav/n.crawler.txt`" 2>/tmp/n.crawler.tmp

  echo_dialog_v "4.1 cat robot : `cat /tmp/n.robot.tmp`"

}



setup_tomcat () {

  echo_dialog_v "5. setup_tomcat"

  test_file ~/.nutchez/sav/n.tomcat.txt

  dialog --title "設定網頁伺服器" --nocancel --inputbox " 你希望NutchEz將網頁伺服器開在哪個port \n\n (ps: 請選擇一個沒用到的port以免造成衝突 \n 也請盡量不要設成80以免造成你誤以為是apache的混淆) \n " 16 51 "`cat ~/.nutchez/sav/n.tomcat.txt`" 2>/tmp/n.tomcat.tmp

  echo_dialog_v "5.1 cat tomcat : `cat /tmp/n.tomcat.tmp`"

}



continue_previous () {

  echo_dialog_v "6. setup_tomcat"

$DIALOG --title "清除上次搜尋" --clear \

        --yesno "你是否要清除上一次爬網所得的結果，\n否則將加入到URL列裡增加搜尋負擔 \n\n ps: 選no的話，會跑相當相當久，\n 請慎重考慮之\n" 16 51



case $? in

  0)

    NOCONTINUE=1;;

  1)

    NOCONTINUE=0;;

  255)

    echo "ESC pressed.";;

esac

echo_dialog_v " 6continue = $CONTINUE"

}



final_confirm () {

  echo_dialog_v "7. final_confirm : start =0 , back =1 "

  tempfile=/tmp/n.finalcheck.tmp



  echo " \n 1. 你所選擇要爬取的網址為 : \n " > $tempfile

  cat /tmp/n.urls.tmp >> $tempfile

  echo " \n\n 2. 對於這個爬網機器人，你取名為 : \n" >> $tempfile

  cat /tmp/n.robot.tmp >> $tempfile

  echo " \n\n 3. 爬網的深度，你設定為 : \n " >> $tempfile

  cat /tmp/n.crawler.tmp >> $tempfile

  echo " \n\n 4. NutchEz將會把你的搜尋結果呈現在這個Port : \n " >> $tempfile

  cat /tmp/n.tomcat.tmp >> $tempfile

  if [[ $NOCONTINUE -eq 0 ]];then

    echo " \n\n 5. 是否要清除上一次的收尋結果 : \n " >> $tempfile

    echo_dialog_v " 7continue = $CONTINUE"

    echo "NO" >> $tempfile

  elif [[ $NOCONTINUE -eq 1 ]];then

    echo " \n\n 5. 是否要清除上一次的收尋結果繼續搜尋 : \n " >> $tempfile

    echo_dialog_v " 7continue = $CONTINUE"

    echo "YES" >> $tempfile

  else

    echo_dialog_v " 無資料可匯入 "

  fi

  MSG=`cat $tempfile`

  echo_dialog_v "7.1 final message :\n $MSG"

  #read READ

  $DIALOG --title "請檢查你的選擇 ! \n\n 若所有的設定都是正確的，你可以按 \"ok\",\n 若你按了 \"reset\" 則會重頭開始設定, \n 若你選擇 \"exit\" 則會跳出NutchEz的設定選單 \n ps: reset 與 exit都不會把資料記成預設值，請放心使用 " --clear \

        --extra-button --extra-label "reset" --ok-label "ok" --cancel-label "exit" \

        --yesno "$MSG" 26 51

  RET=$?

  echo_dialog_v "final return = $RET"  

  return $RET

}



# define paramaters



set_nutchez_p () {

  ROBOT=`cat ~/.nutchez/sav/n.robot.txt`

  URLS=`cat ~/.nutchez/sav/n.urls.txt`

  DEPTH=`cat ~/.nutchez/sav/n.crawler.txt`

  PORT=`cat ~/.nutchez/sav/n.tomcat.txt`

}

start_crawl () {



  echo_dialog_v "7. start_crawl"

  setup_nutchez

  install_tomcat

  echo_dialog_v "/opt/nutchez/nutch/bin/nutch crawl ~/.nutchez/urls -dir ~/.nutchez/search -depth $DEPTH"

  echo_dialog_v "nutch conf dir = $NUTCH_CONF_DIR"

  /opt/nutchez/nutch/bin/nutch crawl ~/.nutchez/urls -dir ~/.nutchez/search -depth $DEPTH -topN 5000 -threads 1000

}



start_tomcat () {

  echo_dialog_v "8. start_tomcat "

  echo_dialog_v "/opt/nutchez/nutch/tomcat/bin/startup.sh"

  #if [ -e /tmp/search ];then

  #  rm -rf /tmp/search

  #fi

  #ln -sf ~/.nutchez/search/ /tmp/



  pid_tc=$(ps axw -eo pid,command |\

  grep "catalina" | grep "java" |\

  grep "start" | awk '{print $1}')

  if [ -z "$pid_tc" ]; then

      echo_dialog_v "no another tomcat is running"

  else

      echo_dialog_v "tomcat had been started and the pid is $pid_tc"

      echo_dialog_v "stop it first"

      kill -9 $pid_tc

      if [ -z $? ];then

    echo_dialog_v " tomcat ($pid_tc) is  killed ..."

      else

    echo_dialog_v "kill error ..." 

      fi

  fi

  echo "Starting Tomcat ...."

  ~/.nutchez/tomcat/bin/startup.sh

  sleep 3

}



show_report () {

  echo_dialog_v "9. show_report "

  FIREFOX=`which firefox`

  RET=$?

  if [[ $RET -eq 0 ]];then 

    $FIREFOX -D 0.0 http://localhost:$PORT

    RET=$?

  fi

  if ! [[ $RET -eq 0 ]];then

    $DIALOG --msgbox "恭喜你已經完成了! \n 你可以用瀏覽器瀏覽: \n  http://host_ip:$PORT" 0 0 

  fi

}