diff --git a/TR1/HMDHFDplus/R/HFDutils.R b/TR1/HMDHFDplus/R/HFDutils.R index 4c24580..c601957 100644 --- a/TR1/HMDHFDplus/R/HFDutils.R +++ b/TR1/HMDHFDplus/R/HFDutils.R @@ -329,7 +329,7 @@ getHFCcountries <- function(names = FALSE){ Codes <- html_table( html_element( - read_html("http://www.fertilitydata.org/cgi-bin/country_codes.php"), + read_html("https://www.fertilitydata.org/Codes/CountryCodes/country_codes.php"), "table"), header = TRUE) diff --git a/TR1/HMDHFDplus/R/readHFD.R b/TR1/HMDHFDplus/R/readHFD.R index 12c6987..0c759fe 100755 --- a/TR1/HMDHFDplus/R/readHFD.R +++ b/TR1/HMDHFDplus/R/readHFD.R @@ -206,70 +206,70 @@ https://www.humanfertility.org/Account/UserAgreement")) # readHFCweb() ############################################################################ -##' @title readHFCweb get HFC data straight from the web into R! -##' -##' @description This concatenates the necessary url and calls \code{read.csv()} with all the necessary defaults to avoid annoying surprises#. \code{Age} is given as an integer, along with an \code{AgeInterval}. The default behavior is to change the \code{AgeInterval} column to #character and produce a logical indicator column, \code{OpenInterval}. \code{Year} is also given as the starting year of a \code{YearInterval}, rather than the original \code{Year1} and \code{Year2} columns. The column \code{Country} is renamed \code{CNTRY}. All #other original columns are maintained. Output is invisibly returned, so you must assign it to take a look. -##' -##' @param CNTRY character string of the HCD short code. Only one! Run \code{getHFCcountries(FALSE)} to see what the options are. -##' @param item character string of the data product code, which is the base file name, but excluding the country code and file extension \code{.txt}. For instance, \code{"ASFRstand_TOT"}, \code{"ASFRstand_BO"}, \code{"TFRMAB_TOT"}, \code{"TFRMAB_BO"}. Only one item! -##' @param fixup logical. Default \code{TRUE}. Should column classes be coerced to those more similar to HFD, HMD? -##' @param ... optional arguments passed to \code{read.csv()}. Not required. -##' -##' @export -##' -##' @importFrom httr HEAD -##' @importFrom utils read.csv -##' -##' @examples -##' \dontrun{ -##' DF <- readHFCweb("CZE","TFRMAB_TOT") -##' head(DF) -##' DF <- readHFCweb("CZE","ASFRstand_BO") -##' head(DF) -##' -##' # get ASFRstand_BO for all countries where available: -##' Countries <- getHFCcountries() -##' # takes a minute to run -##' -##' urls <- paste0("http://www.fertilitydata.org/data/", -##' Countries,"/", Countries, "_", "ASFRstand_BO", ".txt") -##' -##' HaveBO <- RCurl::url.exists(urls) -##' # we grab data for these countries: -##' (Countries <- Countries[HaveBO]) -##' -##' # Also takes 1-15 min depending on internet connection and machine -##' # read in one at a time and stick together into long data.frame -##' allBO <- do.call(rbind, -##' # this is the iteration of reading in -##' lapply(Countries, function(CNTRY){ -##' readHFCweb(CNTRY, item = "ASFRstand_BO") -##' })) # closes off the meta-rbind thingy -##' dim(allBO) # [1] 133237 31 -##' unique(allBO$CNTRY) -##' -##' } -##' -## readHFCweb <- function(CNTRY, item, fixup = TRUE, ...){ -## # concatenate url: -## fileurl <- paste0("https://www.fertilitydata.org/data/", CNTRY, "/", CNTRY, "_", item, ".txt") -## -## # read in with needed arguments: -## if (httr::HEAD(fileurl)$all_headers[[1]]$status == 200){ -## con <- url(fileurl) -## DF <- read.csv(url(fileurl), stringsAsFactors = FALSE, na.strings = ".", strip.white = TRUE, ...) -## # close(con) -## # optionally use standard columns: -## if (fixup){ -## DF <- HFCparse(DF) -## } -## return(invisible(DF)) -## } else { -## cat("Either the CNTRY code or data item are not available\nCheck names.\nNULL returned\n") -## NULL -## } -## -## } +#' @title readHFCweb get HFC data straight from the web into R! +#' +#' @description This concatenates the necessary url and calls \code{read.csv()} with all the necessary defaults to avoid annoying surprises#. \code{Age} is given as an integer, along with an \code{AgeInterval}. The default behavior is to change the \code{AgeInterval} column to #character and produce a logical indicator column, \code{OpenInterval}. \code{Year} is also given as the starting year of a \code{YearInterval}, rather than the original \code{Year1} and \code{Year2} columns. The column \code{Country} is renamed \code{CNTRY}. All #other original columns are maintained. Output is invisibly returned, so you must assign it to take a look. +#' +#' @param CNTRY character string of the HCD short code. Only one! Run \code{getHFCcountries(FALSE)} to see what the options are. +#' @param item character string of the data product code, which is the base file name, but excluding the country code and file extension \code{.txt}. For instance, \code{"ASFRstand_TOT"}, \code{"ASFRstand_BO"}, \code{"TFRMAB_TOT"}, \code{"TFRMAB_BO"}. Only one item! +#' @param fixup logical. Default \code{TRUE}. Should column classes be coerced to those more similar to HFD, HMD? +#' @param ... optional arguments passed to \code{read.csv()}. Not required. +#' +#' @export +#' +#' @importFrom httr HEAD +#' @importFrom utils read.csv +#' +#' @examples +#' \dontrun{ +#' DF <- readHFCweb("CZE","TFRMAB_TOT") +#' head(DF) +#' DF <- readHFCweb("CZE","ASFRstand_BO") +#' head(DF) +#' +#' # get ASFRstand_BO for all countries where available: +#' Countries <- getHFCcountries() +#' # takes a minute to run +#' +#' urls <- paste0("http://www.fertilitydata.org/data/", +#' Countries,"/", Countries, "_", "ASFRstand_BO", ".txt") +#' +#' HaveBO <- RCurl::url.exists(urls) +#' # we grab data for these countries: +#' (Countries <- Countries[HaveBO]) +#' +#' # Also takes 1-15 min depending on internet connection and machine +#' # read in one at a time and stick together into long data.frame +#' allBO <- do.call(rbind, +#' # this is the iteration of reading in +#' lapply(Countries, function(CNTRY){ +#' readHFCweb(CNTRY, item = "ASFRstand_BO") +#' })) # closes off the meta-rbind thingy +#' dim(allBO) # [1] 133237 31 +#' unique(allBO$CNTRY) +#' +#' } +#' +readHFCweb <- function(CNTRY, item, fixup = TRUE, ...){ + # concatenate url: + fileurl <- paste0("https://www.fertilitydata.org/File/GetFile/Country/", CNTRY, "/", CNTRY, "_", item, ".txt") + + # read in with needed arguments: + if (httr::GET(fileurl)$all_headers[[1]]$status == 200){ + con <- url(fileurl) + DF <- read.csv(url(fileurl), stringsAsFactors = FALSE, na.strings = ".", strip.white = TRUE) + # close(con) + # optionally use standard columns: + if (fixup){ + DF <- HFCparse(DF) + } + return(invisible(DF)) + } else { + cat("Either the CNTRY code or data item are not available\nCheck names.\nNULL returned\n") + NULL + } + +} # end \ No newline at end of file