Web scraping with R

library(rvest)

url <- "http://www.zillow.com/homes/for_sale/VT/58_rid/any_days/45.537137,-69.062805,42.187829,-75.846863_rect/7_zm/"
# I manually checked the source html to find the correct node to read
more.urls <- read_html(url) %>%
  html_nodes(".zsg-pagination a") %>%
  html_attr("href") 
# See how high the pagination goes
more.urls <- more.urls[!is.na(more.urls)]
more.urls <- strsplit(more.urls, '/')
more.urls <- lapply(more.urls, function(x) x[length(x)])
more.urls <- as.numeric(gsub('[_p]','', more.urls))
more.urls <- max(more.urls)
# Now paste together a url for each page
urls <- c(url, paste0(url,2:more.urls,'_p/'))

urls

# The scraper function
getZillow <- function(urls) {
    # return a list of html <article></article> tags, one for each URL
    lapply(urls, function(u) {
        # call out the at-bat url in case there is an error
        cat(u, '\n')
        houses <- read_html(u) %>%
          html_nodes("article")
        houses
    })
}

zdata2 <- getZillow(urls)

as.character(zdata[[1]][1])

# Here's how I put it all together
zdata3 <- parseZillow(zdata2)
zdata4 <- do.call(rbind.data.frame, zdata3)
# Let numbers be numeric rather than factor
for(i in c('lat', 'long', 'price', 'pricepersqft', 'beds','baths','sqft', 'acres', 'yr_built')) zdata4[i] <- as.numeric(as.character(zdata4[,i]))
write.csv(zdata4, 'data/RS_2016May01.csv', row.names=FALSE)


# Here are the parsers
parseZillow <- function(zhouse.l, debugFun=function(...) NULL) {
    len.h <-1:length(zhouse.l)
    lapply(1:length(zhouse.l), function(x) {
        debugFun(x, '\n')
        houses <- zhouse.l[[x]]
        propdata <- tryCatch(houses %>% html_node(".property-data"), error=function(e) NULL)
        if(!is.null(propdata)) {
            parsePropData(houses, debugFun)
        } else {
            parsePhotoCard(houses, debugFun)
        }
    })
}

parsePropData <- function(houses, debugFun) {
    debugFun(x, '\n')
    z_id <- houses %>% html_attr("id")

    debugFun('    address\n')
    address <- houses %>%
      html_node(".property-address a") %>%
      html_attr("href") %>%
      strsplit("/") %>%
      pluck(3, character(1))
      
    debugFun('    geo\n')  
    geo <- houses %>%
      html_nodes(".property-address meta") %>%
      html_attr("content") 
    geo <- as.numeric(geo)
    geo <- data.frame(lat=geo[seq(1,length(geo), by=2)], long=geo[seq(2,length(geo), by=2)])

    debugFun('    price\n')  
    price <- houses %>%
      html_node(".price-large") %>%
      html_text()

    price <- gsub('[^0-9]', '', price)

    params <- houses %>%
      html_node(".property-data") %>%
      html_text() %>%
      strsplit(", ") 
      
    params.l <- lapply(params, function(x) {
        if(length(x)) {
            x <- gsub('Studio', '0.5 bds', x)
            
            debugFun('    beds\n')  
            beds <- grepl('bds', x)
            if(beds) {
                beds <- x %>%
                strsplit('bds')
                beds <- beds[[1]][1]
              } 
            
            debugFun('    baths\n')  
            baths <- grepl('ba', x)
            if(baths) {
                baths <- x %>%
                strsplit('ba')
                baths <- strsplit(baths[[1]], ' ')[[1]]
                baths <- baths[length(baths)]
            } 
            
            debugFun('    house_area\n')  
            house_area <- grepl('sqft [^a-z]+ ', x)
            if(house_area) {
                house_area <- x %>%
                strsplit('sqft [^a-z]+ ')
                house_area <- strsplit(house_area[[1]], ' ')[[1]]
                house_area <- house_area[length(house_area)]
            } 

            debugFun('    lot_ac\n')  
            lot_ac <- grepl('ac lot', x)
            if(lot_ac) {
                lot_ac <- x %>%
                strsplit('ac lot')
                lot_ac <- strsplit(lot_ac[[1]], ' ')[[1]]
                lot_ac <- lot_ac[length(lot_ac)]
            }
            
            lot_sqft <- grepl('sqft lot', x)
            if(lot_sqft) {
                lot_ac <- x %>%
                strsplit('sqft lot')
                lot_ac <- gsub('[^0-9.]', '', lot_ac) %>%
                as.numeric() 
                lot_ac <- lot_ac/43560
            }
            
            debugFun('    year_built\n')  
            year_built <- grepl('Built', x)
            if(year_built) {
                year_built <- x %>%
                strsplit('Built')
                year_built <- year_built[[1]][2]
            } 
            
            out <- c(beds=beds, baths=baths, sqft=house_area, acres=lot_ac, yr_built=year_built)
            out.vals <- gsub('[^0-9.]', '', out)
            out.df <- data.frame(t(as.numeric(out.vals)))
            names(out.df) <- names(out)
            out.df
        } else data.frame(beds=NA, baths=NA, sqft=NA, acres=NA, yr_built=NA)
    })

    params.df <- do.call(rbind.data.frame, params.l)

    data.frame(z_id=z_id, address=address, geo, price=price, params.df)
}


parsePhotoCard <- function(houses, debugFun) {
    z_id <- houses %>% html_attr("id")

    debugFun('    address\n')
    address <- houses %>%
      html_node(".zsg-photo-card-spec a") %>%
      html_attr("href") %>%
      strsplit("/") %>%
      pluck(3, character(1))
      
    debugFun('    geo\n')  
    
    lat <- houses %>% html_attr("data-latitude")
    lon <- houses %>% html_attr("data-longitude")
#    geo <- houses %>%
#      html_nodes(".property-address meta") %>%
#      html_attr("content") 

    geo <- data.frame(lat=as.numeric(lat)*10e-7, long=as.numeric(lon)*10e-7)

    debugFun('    price\n')  
    price <- houses %>%
      html_nodes(".zsg-photo-card-status")%>%
      html_text()
      
    price <- price!='Auction'
    for(i in 1:length(price)) {
        if(price[i]) {
            price[i] <- houses[i] %>%
              html_nodes(".zsg-photo-card-spec") %>% 
              html_nodes(".zsg-photo-card-price") %>%
              html_text()
        }
    }

    price <- gsub('[^0-9]', '', price)

    params <- houses %>%
      html_nodes(".zsg-photo-card-spec") %>% 
      html_nodes(".zsg-photo-card-info") %>%
      html_text() %>%
      strsplit(", ")  
      
    params.l <- lapply(params, function(x) {
        debugFun(x, '\n')
        if(length(x)) {
            x <- gsub('Studio', '0.5 bds', x)
            x <- gsub('--', 'NA', x)
            debugFun('    beds\n')  
            beds <- grepl('bds', x)
            if(beds) {
                beds <- x %>%
                strsplit('bds')
                beds <- beds[[1]][1]
              } 
            
            debugFun('    baths\n')  
            baths <- grepl('ba', x)
            if(baths) {
                baths <- x %>%
                strsplit('ba')
                baths <- strsplit(baths[[1]], ' ')[[1]]
                baths <- baths[length(baths)]
            } 
            
            debugFun('    house_area\n')  
            house_area <- grepl('sqft', x)
            if(house_area && grepl('k sqft', x)) {
                house_area <- x %>%
                strsplit('k sqft')
                house_area <- strsplit(house_area[[1]], ' ')[[1]]
                house_area <- house_area[length(house_area)]
                house_area <- gsub('[^0-9]', '', house_area)
                house_area <- as.numeric(house_area) * 100
            } else {
                house_area <- x %>%
                strsplit('sqft')
                house_area <- strsplit(house_area[[1]], ' ')[[1]]
                house_area <- house_area[length(house_area)]
            }

            debugFun('    lot_ac\n')  
            lot_ac <- grepl('ac lot', x)
            if(lot_ac) {
                lot_ac <- x %>%
                strsplit('ac lot')
                lot_ac <- strsplit(lot_ac[[1]], ' ')[[1]]
                lot_ac <- lot_ac[length(lot_ac)]
            }
            
            lot_sqft <- grepl('sqft lot', x)
            if(lot_sqft) {
                lot_ac <- x %>%
                strsplit('sqft lot')
                lot_ac <- gsub('[^0-9.]', '', lot_ac) %>%
                as.numeric() 
                lot_ac <- lot_ac/43560
            }
            
            debugFun('    year_built\n')  
            year_built <- grepl('Built', x)
            if(year_built) {
                year_built <- x %>%
                strsplit('Built')
                year_built <- year_built[[1]][2]
            } 
            
            out <- c(beds=beds, baths=baths, sqft=house_area, acres=lot_ac, yr_built=year_built)
            out.vals <- gsub('[^0-9.]', '', out)
            out.df <- data.frame(t(as.numeric(out.vals)))
            names(out.df) <- names(out)
            out.df
        } else data.frame(beds=NA, baths=NA, sqft=NA, acres=NA, yr_built=NA)
    })

    params.df <- do.call(rbind.data.frame, params.l)

    data.frame(z_id=z_id, address=address, geo, price=price, params.df)
}