czech_theatre_institute.Rmd

---
title: "R Notebook"
output: html_notebook
---

```{r}
library(RSelenium)
library(RCurl)
library(XML)
library(rvest)
library(tidyverse)
library(stringr)
library(svMisc)
library(splitstackshape)
library(stringdist)
library(RJDBC)
library(purrr)
library(sqldf)
library(binman)

#empty table
table_data <- data.frame(rekord = as.character(), stringsAsFactors=FALSE)
#browser
#list_versions("chromedriver")
rD <- rsDriver(port=4444L,browser="chrome", chromever="78.0.3904.11")
remDr <- rD$client
url <- "https://vis.idu.cz/Productions.aspx"
remDr$navigate(url)
webElem <- remDr$findElement(using = 'css selector', "#ctl00_pageContent_tabPage_ucSearch_pnlSearch_lupPerson1_txtValues_I")
webElem$clickElement()
Sys.sleep(3)
sbox <- remDr$findElement(using = 'css selector', "#ctl00_pageContent_tabPage_ucSearch_pnlSearch_lupPerson1_popup_gridLookup_DXFREditorcol1_I")
sbox$sendKeysToElement(list("Topol, Josef"))
Sys.sleep(5)
webElem2 <- remDr$findElement(using = 'css selector', "table.tabTight table.c.tabTight.tabContent:nth-child(3) tr.cContent:nth-child(1) td.cContent div.content div.dxtcLite.tabPageControl.dxtcSys.dxtc-flex.dxtc-top.dxtc-pc:nth-child(2) div.dxtc-content div.dxpnlControl:nth-child(2) table.search table.insearch div.dxpcLite.dxpclW:nth-child(16) div.dxpc-mainDiv.dxpc-shadow div.dxpc-contentWrapper div.dxpc-content table.dxgvControl.dxgv:nth-child(2) td:nth-child(1) table.dxgvTable:nth-child(1) tbody:nth-child(1) tr.dxgvDataRow.dxgvLVR.dxgvFocusedRow:nth-child(3) > td.dxgv.dx-al:nth-child(2)")
webElem2$clickElement()
webElem3 <- remDr$findElement(using = "xpath", "//div[@id='ctl00_pageContent_tabPage_ucSearch_pnlSearch_btnSearch_CD']//span[@class='dx-vam'][contains(text(),'Search')]")
webElem3$clickElement()
webElem4 <- remDr$findElement(using = 'css selector', "table.tabTight table.c.tabTight.tabContent:nth-child(3) tr.cContent:nth-child(1) td.cContent div.content div.dxtcLite.tabPageControl.dxtcSys.dxtc-flex.dxtc-top.dxtc-pc:nth-child(1) div.dxtc-content div.dxpnlControl:nth-child(2) div.dxpnlControl table.dxgvControl.roundCorner.dxgv:nth-child(6) div.dxgvPagerBottomPanel td.rightmedskip.wmax:nth-child(5) tr:nth-child(2) td.center div:nth-child(2) a.dxp-button.dxp-bi.inlineTable > img.dxWeb_pAll")
webElem4$clickElement()

zzz <- remDr$findElement(using = 'css selector', "#ctl00_pageContent_tabPage_ucSearch_pnlSearch_resultList_pnlList_gvResultList_DXMainTable")
test <- zzz$
test <- as.data.frame(zzz)
zzz <- unlist(strsplit(zzz[[1]], "\n"))



tabele <- readHTMLTable(getURL(zzz$getCurrentUrl()), stringsAsFactors = FALSE)
length(tabele)

x <- 0:151
for (i in x) {
  progress(match(i,x), max.value = length(x)) 
  css1 <- "#ctl00_pageContent_tabPage_ucSearch_pnlSearch_resultList_pnlList_gvResultList_cell"
  css2 <- x[match(i,x)]
  css3 <- "_3_HyperLink1_"
  css <- paste(css1,css2,css3,css2,sep = "")
  record <- remDr$findElement(using = 'css selector', css)
  record$clickElement()
  webpage <- data.frame(rekord = as.character(record$getCurrentUrl()))
  table_data <- rbind(table_data,webpage)
  remDr$goBack()
}
write.csv2(table_data, "C:/Users/Cezary/Desktop/topol_josef_links.csv", row.names = F, na = '', fileEncoding = 'Windows-1250')



sbox <- remdir$findElement(using = 'name',"q")
sbox$sendKeysToElement(list("laptops",key="enter"))

#rD$server$stop()
#rm(rD)
#gc()
#iteracje

x <- seq(4,7500, by=2)

for (i in x) {
  progress(match(i,x), max.value = length(x))  
  remDr$navigate(url)
  webElem1 <- remDr$findElement("css selector", "#sbox")
  webElem1$sendKeysToElement(list('*', key = 'enter'))
  
  selector_left <- "div:nth-child(2) div:nth-child(2) div:nth-child(2) div:nth-child(2) > a:nth-child("
  selector_mid <- i
  selector_right <- ")"
  selector <- paste(selector_left,selector_mid,selector_right, sep = "")
  
  webElem2 <- remDr$findElement(using = 'css selector', selector)
  data_person <- webElem2$getElementText()
  data_person <- as.character(data_person)
  webElem2$clickElement()
  tryCatch({
  role <- remDr$findElement(using = 'css selector', 'h3')
  role <- role$getElementText()
  role <- as.character(role)
  }, error=function(e){
                      role <<- "Brak danych (CR)"
  })
  data_person2 <- remDr$findElement(using = 'css selector', 'h2')
  data_person2 <- data_person2$getElementText()
  data_person2 <- as.character(data_person2)
  description <- remDr$findElement(using = 'css selector', '.indent')
  description <- description$getElementText()
  description <- as.character(description)
  
  iteration <- cbind(data_person,role,data_person2,description)
  
  table_data <- rbind(table_data,iteration)
}
write.csv2(table_data, "C:/Users/Cezary/Desktop/czech_literary_prizes_webscraping_file.csv", row.names = F, na = '', fileEncoding = 'Windows-1250')
table_data <- read.csv2("C:/Users/Cezary/Desktop/czech_literary_prizes_1.csv", encoding = "Windows-1250", header = TRUE, stringsAsFactors = FALSE)
table_data <- table_data %>%
  filter(role == "ocenění:") %>%
  select(-role)

#mutations

table_data <- mutate(table_data,
                     person_year = ifelse(grepl("\\d{4}", table_data$data_person2), as.character(str_replace_all(table_data$data_person2, "(.*)(, )(\\d.*)", "\\3")
),NA))

table_data <- mutate(table_data,
                     single_prize = str_replace_all(table_data$description, "(\\n)(\\d{4} [A-ZAÁÀÂÃÄĂĀÅĄÆEÉÈÊËĚĒĖĘIÍÌÎÏĪĮIOÓÒÔÕÖŐØŒUÚÙÛÜŰŪůYÝCĆČçDĎĐĐGĢĞKĶLŁĻNŃÑŇŅŊRŘSŚŠŞSßTŤŦÞŢ8ZŻŹŽ])", "|\\2"))

table_data <- cSplit(table_data, "single_prize", sep = "|", direction = "long")

table_data$single_prize <- str_replace_all(table_data$single_prize,"\\n", "|")

table_data <- mutate(table_data,
                     prize_year = str_replace_all(table_data$single_prize, "(^\\d{4})(.*)", "\\1"),
                     book_title__reason = ifelse(grepl("\\|",table_data$single_prize),as.character(str_replace_all(table_data$single_prize, "(^\\d{4} )(.*)(\\|)(.*)", "\\4")),NA),
                     rest = ifelse(grepl("\\|",table_data$single_prize),as.character(str_replace_all(table_data$single_prize, "(^\\d{4} )(.*)(\\|)(.*)", "\\2")),as.character(str_replace_all(table_data$single_prize, "(^\\d{4} )(.*)($)", "\\2"))))


table_data <- mutate(table_data,
                     prize_name = ifelse(grepl("\\,",table_data$rest), as.character(str_replace_all(table_data$rest, "(.*?)(, )(.*)", "\\1")),as.character(table_data$rest)),
                     prize_category__place = ifelse(grepl("\\,",table_data$rest), as.character(str_replace_all(table_data$rest, "(.*?)(, )(.*)", "\\3")),NA))
table_data <- table_data %>%
  select(data_person,person_year,prize_name,prize_year,prize_category__place,book_title__reason)
colnames(table_data) <- c("person","person_year","prize_name","prize_year","prize_category__place","book_title__reason")
                       
write.csv2(table_data, "C:/Users/Cezary/Desktop/czech_literary_prizes.csv", row.names = F, na = '', fileEncoding = 'Windows-1250')

literary_prizes <- read.csv2("C:/Users/Cezary/Desktop/czech_literary_prizes.csv", encoding = "Windows-1250", header = TRUE, stringsAsFactors = FALSE) 
```

```{r api dla Czechów}

library(httr)
library(jsonlite)
library(lubridate)
library(viafr)
library(dplyr)
library(purrr)
library(stringr)
library(tidyverse)
library(XML)
library(methods)
library(svMisc)
options(stringsAsFactors = FALSE)

url  <- "http://www.viaf.org"
path <- "eurlex/directory_code"

tt_GET <- function(path, ...){
  if (is.null(path))
    stop("Nothing to parse")
  uri <- "http://www.viaf.org/"
  # call api
  req <- httr::GET(uri, path = path, ...)
  # check for http status
  tt_check(req)
  req
}
tt_check <- function(req) {
  if (req$status_code < 400)
    return(invisible())
  stop("HTTP failure: ", req$status_code, "\n", call. = FALSE)
}

tt_parse_js <- function(req) {
  text <- httr::content(req, as = "text")
  if (identical(text, ""))
    stop("Not output to parse", call. = FALSE)
  jsonlite::fromJSON(text)
}

viaf_get <- function(viaf = NULL) {
  if(is.null(viaf))
    stop("No viaf provided")
  tt_GET(path = paste0("viaf/", viaf), httr::add_headers(Accept = "application/json+links")) %>%
    tt_parse_js()
}

viaf_suggest <- function(query = NULL){
  if(is.null(query))
    stop ("No query provided")
  tt_GET(path = "viaf/AutoSuggest", query = list(query = query)) %>%
  tt_parse_js()
}

ludzie <- table_data %>%
  select(person) %>%
  unique()
ludzie <- data.frame(person = ludzie$person[2001:3253])

x <- 1:length(ludzie$person)
test3 <- data.frame(term = as.character(),displayForm = as.character(), nametype = as.character(), viafid = as.integer(), stringsAsFactors = FALSE)

for (i in x) {
  svMisc::progress(match(i,x), max.value = length(x))

  test <- viaf_suggest(ludzie$person[i])
  tryCatch({
  test2 <- test$result %>%
    select(term,displayForm,nametype,viafid)
  }, error=function(e){
                   test2 <<- data.frame(term = ludzie$person[i],displayForm = NA, nametype = NA, viafid =NA, stringsAsFactors = FALSE)
})
  test3 <- rbind(test3,test2)
  
}

pierwsza_lista <- test3

test3 <- rbind(pierwsza_lista,test3)
test3 <- test3 %>%
  unique() %>%
  filter(grepl("personal|corporate",nametype))

test3 <- ddply(test3, .(viafid, nametype), summarize, term = paste(term, collapse="|"))

#ZROBIĆ pętle, iteracje, najpierw testowo 100 Viafów, a potem całość (może podwójne for dla linki1,2,3 i dla iteracji wewnątrz)

test4 <- test3 %>%
  select(viafid) %>%
  filter(!is.na(viafid)) %>%
  unique()

tabela_viaf <- data.frame(Plec = character(), VIAF = character(), Zawod = character(), Linki = character(), Tworczosc = character(), Jezyk = character(), Narodowosc = character(), data_urodzenia = character(), data_smierci = character(), stringsAsFactors=FALSE)
start_time <- Sys.time()
for (i in x) {

tryCatch({
poczatek_url <- "http://viaf.org/viaf/"
srodek_url <- test4$viafid[i]
koniec_url <- "/viaf.xml"
url <- paste(poczatek_url, srodek_url, koniec_url, sep = "")
xml <- xmlParse(url)

tryCatch({
a <- setNames(xmlToDataFrame(nodes = getNodeSet(xml, "//ns1:VIAFCluster/ns1:fixed/ns1:gender")),"Plec")
a2 <- paste(unlist(a), collapse = "~")
}, error=function(e){
                    a2 <<- "No data here"
})

tryCatch({
  d <- setNames(xmlToDataFrame(nodes = getNodeSet(xml, "//ns1:VIAFCluster/ns1:viafID")),"VIAF")
d2 <- paste(unlist(d), collapse = "~")
}, error=function(e){
                    d2 <<- "No data here"
})
tryCatch({
  e <- setNames(xmlToDataFrame(nodes = getNodeSet(xml, "//ns1:VIAFCluster/ns1:occupation/ns1:data/ns1:text")),"Zawod")
e2 <- paste(unlist(e), collapse = "~")
}, error=function(e){
                    e2 <<- "No data here"
})
tryCatch({
  f <- setNames(xmlToDataFrame(nodes = getNodeSet(xml, "//ns1:VIAFCluster/ns1:xLinks/ns1:xLink")),"Linki")[1]
f2 <- paste(unlist(f), collapse = "~")
}, error=function(e){
                    f2 <<- "No data here"
})
tryCatch({
  g <- setNames(xmlToDataFrame(nodes = getNodeSet(xml, "//ns1:VIAFCluster/ns1:titles/ns1:work/ns1:title")),"Tworczosc")
g2 <- paste(unlist(g), collapse = "~")
}, error=function(e){
                    g2 <<- "No data here"
})
tryCatch({
  h <- setNames(xmlToDataFrame(nodes = getNodeSet(xml, "//ns1:VIAFCluster/ns1:languageOfEntity/ns1:data/ns1:text")),"Jezyk")
h2 <- paste(unlist(h), collapse = "~")
}, error=function(e){
                    h2 <<- "No data here"
})
tryCatch({
  j <- setNames(xmlToDataFrame(nodes = getNodeSet(xml, "//ns1:VIAFCluster/ns1:nationalityOfEntity/ns1:data/ns1:text")),"Narodowosc")
j2 <- paste(unlist(j), collapse = "~")
}, error=function(e){
                    j2 <<- "No data here"
})
tryCatch({
  s <- setNames(xmlToDataFrame(nodes = getNodeSet(xml, "//ns1:VIAFCluster/ns1:birthDate")),"data_urodzenia")
s2 <- paste(unlist(s), collapse = "~")
}, error=function(e){
                    s2 <<- "No data here"
})
tryCatch({
  w <- setNames(xmlToDataFrame(nodes = getNodeSet(xml, "//ns1:VIAFCluster/ns1:deathDate")),"data_smierci")
w2 <- paste(unlist(w), collapse = "~")
}, error=function(e){
                    w2 <<- "No data here"
})
total_i <- data.frame(Plec = a2, VIAF = d2, Zawod = e2, Linki = f2, Tworczosc = g2, Jezyk = h2, Narodowosc = j2, data_urodzenia = s2, data_smierci = w2)

tabela_viaf <- rbind(tabela_viaf,total_i)
}, error=function(e){NULL})

  svMisc::progress(i, max.value = length(test4$viafid))#w zależnosci od paczki zmienia się nazwa pliku(3/4)
  
}
colnames(tabela_viaf) <- c("Sex", "VIAF", "Occupation", "Links", "Works", "Language", "Nationality", "Date_of_birth", "Date_of_death")
end_time <- Sys.time()
tabela_viaf <- merge(test3, tabela_viaf, by.x = "viafid", by.y = "VIAF", all = TRUE)
tabela_viaf <- tabela_viaf %>%
  select(term,viafid,nametype,Date_of_birth,Date_of_death,Sex,Occupation,Links,Works,Language,Nationality)

```