draft/getUbot.R

In leoluyi/CRMaddress: Crawling address in Taiwan

getUbot <- function(){
  final_data=list()#最後的資料表格，先設定為空白
  district_data = data.frame()
  brand_nm = "聯邦銀行"
  
  #---匯整區域代碼表開始---#
  
  url="http://web.ubot.com.tw/map/default.asp"
  res=GET(url)
  con=content(res,"text",encoding="utf-8")
  par=htmlParse(con,encoding="utf-8")
  
  header_code=xpathSApply(par,"//div[@class='upperBar']/select/option",xmlGetAttr,'value')
  header_name=xpathSApply(par,"//div[@class='upperBar']/select/option",xmlValue)
  
  #大地區的選項,共有六組
  big_area=list()
  for(i in 1:6){
    big_area$code[i]=header_code[i]
    big_area$name[i]=header_name[i]
  }
  
  for (big_area_no in 1:length(big_area$code)){ 
    
    area=big_area$code[big_area_no]
    area_nm = big_area$name[big_area_no]
    area_key=paste("C1=1&","C2=",area,sep="")
    
    url_key=paste(url,"?",area_key,sep="")
    ser_res=GET(url_key)
    ser_con=content(ser_res,"text",encoding="utf-8")
    ser_par=htmlParse(ser_con,encoding="utf-8")
    district_code=xpathSApply(ser_par,"//div[@class='upperBar']/select/option",xmlGetAttr,'value')
    district_name=xpathSApply(ser_par,"//div[@class='upperBar']/select/option",xmlValue)
    
    dis=list()
    for(i in 7:length(district_name)){
      dis$code[i-6]=district_code[i]
      dis$name[i-6]=district_name[i]
      dis_cnt=length(dis$code)
    }
    temp = data.frame("area_1_no"=area,"area_1_nm"=area_nm,"area_sub_url"=area_key,
                      "area_2_no" = dis$code,"area_2_no" = dis$name) 
    district_data = rbind( district_data,temp ) 
  }
  
  #---匯整區域代碼表結束---#
  
  
  #---分行資料開始---#
  for( serch_code in 1 : nrow( district_data )){
    
    #   for(serch_code in 1:dis_cnt){
    
    code_1 = district_data$area_2_no[serch_code]
    area_key = district_data$area_sub_url[serch_code]  
    key_1=paste(area_key,"&","ZipCode=",code_1,sep="")
    url_1=paste(url,"?",key_1,sep="")
    res_1=GET(url_1)
    con_1=content(res_1,"text",encoding="utf-8")
    par_1=htmlParse(res_1,encoding="utf-8")
    
    
    #開始分析資料
    name_1=xpathSApply(par_1,"//div[@class='firstChap']",xmlValue)  #data_name
    detail_1=xpathSApply(par_1,"//td",xmlValue)
    src_1=xpathSApply(par_1,"//iframe",xmlGetAttr,'src')
    
    store_nm=as.list(name_1) #分行名字,並改以list物件方式記錄
    
    #設定node，抓取服務該項的html資料，再做parsing
    service_1=getNodeSet(par_1,"//div/div[@class='serviceIcon']")
    
    #確認總計該zip_code中有幾家分行
    service_1_cnt=length(service_1) 
    service=list()
    ATMb=list()
    deposit=list()
    bankbookChecker=list()
    
    #------------------------------------------------------------------------確認各分行開始
    #從該區的第一家分行開始確認其特別服務的狀況
    for(bank_no in 1:service_1_cnt){
      temp_1=xmlChildren(service_1[[bank_no]])
      temp_1_cnt=length(temp_1)
      
      ATMb[[bank_no]]="N"
      deposit[[bank_no]]="N"
      bankbookChecker[[bank_no]]="N"
      
      #確認此區域內的各分行的三種服務提供狀況，此時ATMb會有多筆，對應多家分行狀況，
      #但我們預設為N，若有則該項會變成Y
      for(t_1 in 1:temp_1_cnt){
        service[t_1]=xmlToList(temp_1[t_1]$div)
        if(service[t_1]=="ATMb"){ATMb[[bank_no]]="Y"}
        if(service[t_1]=="deposit"){deposit[[bank_no]]="Y"}
        if(service[t_1]=="bankbookChecker"){bankbookChecker[[bank_no]]="Y"}
      }
    }#特別服務確認到此為此
    
    #確認各分行的資訊細節，在這裡我們用出現地址時，來區分分行，實為逼于得已
    detail_1=getNodeSet(par_1,"//table[@class='column2']/tr/td")
    detail_str=c()
    total=length(detail_1)#總筆數
    for( t_2 in 1:total){
      detail_temp=xmlChildren(detail_1[[t_2]])
      if(length(detail_temp)==0){detail_str[t_2]="NA"}else{  
        detail_temp=xmlToList(detail_temp$text)
        detail_str[t_2]=detail_temp
      }
    }
    
    #開始將各家分行的細節資料加入
    t_3=1 #明細資料筆數的第一筆開始
    no=0  #該區的分行的第一家開始
    tel=list()
    fax=list()
    address=list()
    time=list()
    memo=list()
    check=list()
    repeat
    {
      if(detail_str[t_3]=="地　　址："){
        no=no+1 
        address[[no]]=detail_str[t_3+1]
        
        addr_temp = gsub("\t","",address[[no]])
        addr_temp = gsub("\r","",addr_temp)      
        addr_temp = gsub("\n","",addr_temp)
        addr_temp = gsub(" ","",addr_temp)      
        address[[no]] = addr_temp 
        
      }
      if(detail_str[t_3]=="電　　話："){tel[[no]]=detail_str[t_3+1]}
      if(detail_str[t_3]=="傳　　真："){fax[[no]]=detail_str[t_3+1]}
      if(detail_str[t_3]=="營業時間："){time[[no]]=detail_str[t_3+1]}  
      if(detail_str[t_3]=="備　　註："){memo[[no]]=detail_str[t_3+1]}
      if(detail_str[t_3]=="Swift Code："){check[[no]]=detail_str[t_3+1]}
      t_3=t_3+1
      if(t_3>total) break
    }#確認各分行的明細過程結束
    
    
    #抓取各家分行的位置經緯度
    src=list()
    gisx=list()
    gisy=list()
    link=list()
    for(t_4 in 1:service_1_cnt){
      xy=paste("http://web.ubot.com.tw/map/",cbind(src_1[t_4]),sep="")
      xy_res=GET(xy)
      xy_con=content(xy_res,"text",encoding="utf-8")
      matches=gregexpr('[0-9]{2,3}(\\.)[0-9]{4,7}',xy_con)
      src=regmatches(xy_con,matches)
      src_temp=strsplit(src[[1]],",",fixed=T)
      t_5=1
      t_6=length(src_temp)
      repeat
      {
        src_temp_str=strsplit(src_temp[[t_5]],".",fixed=T)
        if(nchar(src_temp_str[[1]][1])==2){gisy[[t_4]]=src_temp[[t_5]]}
        if(nchar(src_temp_str[[1]][1])==3){gisx[[t_4]]=src_temp[[t_5]]} 
        t_5=t_5+1
        if(t_5>t_6) break
      }#確認各分行的明細過程結束
      link[[t_4]]=xy
    }#確認經緯度過程結束
    
    #------------------------------------------------------------------------確認各分行結束
    
    #將資料組成表格
    
    result=list("store_nm"=cbind(store_nm),"ATM"=cbind(ATMb),"deposit"=cbind(deposit),"bankbookChecker"=cbind(bankbookChecker),
                "addr"=cbind(address),"tel"=cbind(tel),"fax"=cbind(fax),"time"=cbind(time),#"memo"=cbind(memo),
                "src"=cbind(link),"swift_code"=cbind(check),"gisx"=cbind(gisx),"gisy"=cbind(gisy))
    result=do.call(cbind,result)
    result=cbind(brand_nm,result)
    final_data=rbind(final_data,result)
    
  }
  #---分行資料結束---#
  final_data = apply(final_data,2,unlist)
  as.data.frame(final_data, stringsAsFactors=FALSE)
}