From 003a1cbee783c76939b2840b59b33dc2246be029 Mon Sep 17 00:00:00 2001 From: Lee Shibley Date: Sat, 16 Nov 2019 14:24:04 -0600 Subject: [PATCH 1/6] (WIP) calling dark sky API for weather data --- bikegroup/darkskycalls.R | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 bikegroup/darkskycalls.R diff --git a/bikegroup/darkskycalls.R b/bikegroup/darkskycalls.R new file mode 100644 index 0000000..5bb6fbc --- /dev/null +++ b/bikegroup/darkskycalls.R @@ -0,0 +1,85 @@ +######################### +### Call Dark Sky API ### +######################### + +# ---- +## Hypothesis: adverse weather conditions play a significant role in frequency and severity of traffic accidents. +## To test: add weather data from DarkSky API +## How: API call for each time (date & time of day) and place (latitude & longitude) observed in the data + +# install libraries ---- +install.packages("httr") +#Require the package so you can use it +require("httr") +library(httr) + +install.packages("jsonlite") +#Require the package so you can use it +require("jsonlite") +library(jsonlite) + +# create function: API to dataframe ---- +apiData <- function(call){ + as.data.frame( + fromJSON( + content( + GET(call), "text") + ) + ) +} + +# create function: API call writer ---- +apiCallWriter <- function(lat, lon, time, key = "9cbfb8d2e1502c2e06cac69d30b1c984"){ + call = paste0("https://api.darksky.net/forecast/", + key, "/", + lat, ",", + lon, ",", + time, "?", + "exclude=currently,flags") + return(call) +} + +# create function: unix time ---- +unixTime <- function(date, time, tz = "America/Chicago", format = "%m/%d/%Y %H:%M"){ + humanReadableTime = paste(date, time) # space in between + epochTime = as.numeric(as.POSIXct(humanReadableTime, format = format, tz = tz)) + return(epochTime) +} + +# ---- +# crash <- readRDS("data-idot/COMBINED CRASH DATA.Rds") %>% +# select(lat, lon, CrashHour, CrashMonth) +crash2 = crash %>% + select(lat, lon, CrashHour, CrashMonth, CrashDate, CrashDay) %>% + mutate(CrashYear = str_sub(CrashDate, -4, -1), + CrashUnixTime = unixTime(CrashDate, CrashHour)) + +masterWeather = setNames(data.table(matrix(nrow = 0, ncol = 21)), c("lat", "lon", "CrashUnixTime", + ## DarkSky hourly columns + "hourly.summary", + "hourly.icon", + "hourly.data.time", + "hourly.data.summary", + "hourly.data.icon", + "hourly.data.precipIntensity", + "hourly.data.precipProbability", + "hourly.data.temperature", + "hourly.data.apparentTemperature", + "hourly.data.dewPoint", + "hourly.data.humidity", + "hourly.data.pressure", + "hourly.data.windSpeed", + "hourly.data.windGust", + "hourly.data.windBearing", + "hourly.data.cloudCover", + "hourly.data.uvIndex", + "hourly.data.visibility")) +i = 1 ## test out with first row +call = apiCallWriter(crash2$lat[i], crash2$lon[i], crash2$CrashUnixTime[i]) +test = apiData(call) + +# sample darksky call ---- +call = apiCallWriter(lat = 41.90337, lon = -87.68947, time = 1576612000) +test = apiData(call) +## whatever the day of the epoch time within the location's time zone, API call returns that entire day + From 62e0b6b4851060e7ee1181404737417761c32575 Mon Sep 17 00:00:00 2001 From: Lee Shibley Date: Sat, 16 Nov 2019 15:15:24 -0600 Subject: [PATCH 2/6] call DarkSky API to add weather features to crash data --- bikegroup/darkskycalls.R | 122 +++++++++++++++++++++++++++------------ 1 file changed, 85 insertions(+), 37 deletions(-) diff --git a/bikegroup/darkskycalls.R b/bikegroup/darkskycalls.R index 5bb6fbc..7acf7f4 100644 --- a/bikegroup/darkskycalls.R +++ b/bikegroup/darkskycalls.R @@ -1,6 +1,6 @@ -######################### -### Call Dark Sky API ### -######################### +########################### +#### Call Dark Sky API #### +########################### # ---- ## Hypothesis: adverse weather conditions play a significant role in frequency and severity of traffic accidents. @@ -29,7 +29,8 @@ apiData <- function(call){ } # create function: API call writer ---- -apiCallWriter <- function(lat, lon, time, key = "9cbfb8d2e1502c2e06cac69d30b1c984"){ +apiCallWriter <- function(lat, lon, time, key = "1234567890abcdefg" ## change this fake key to your own, given from DarkSky.net + ){ call = paste0("https://api.darksky.net/forecast/", key, "/", lat, ",", @@ -40,7 +41,8 @@ apiCallWriter <- function(lat, lon, time, key = "9cbfb8d2e1502c2e06cac69d30b1c98 } # create function: unix time ---- -unixTime <- function(date, time, tz = "America/Chicago", format = "%m/%d/%Y %H:%M"){ +unixTime <- function(date, time, tz = "America/Chicago", format = "%m/%d/%Y %H:%M:%S"){ + time = format(strptime(time, "%I:%M %p"), format="%H:%M:%S") ## convert to 24hr format humanReadableTime = paste(date, time) # space in between epochTime = as.numeric(as.POSIXct(humanReadableTime, format = format, tz = tz)) return(epochTime) @@ -49,37 +51,83 @@ unixTime <- function(date, time, tz = "America/Chicago", format = "%m/%d/%Y %H:% # ---- # crash <- readRDS("data-idot/COMBINED CRASH DATA.Rds") %>% # select(lat, lon, CrashHour, CrashMonth) -crash2 = crash %>% - select(lat, lon, CrashHour, CrashMonth, CrashDate, CrashDay) %>% - mutate(CrashYear = str_sub(CrashDate, -4, -1), - CrashUnixTime = unixTime(CrashDate, CrashHour)) +darkskyinput = crash %>% + select(ICN, CrashID, lat, lon, TimeOfCrash, CrashDate) %>% + mutate(CrashUnixTime = unixTime(CrashDate, TimeOfCrash)) -masterWeather = setNames(data.table(matrix(nrow = 0, ncol = 21)), c("lat", "lon", "CrashUnixTime", - ## DarkSky hourly columns - "hourly.summary", - "hourly.icon", - "hourly.data.time", - "hourly.data.summary", - "hourly.data.icon", - "hourly.data.precipIntensity", - "hourly.data.precipProbability", - "hourly.data.temperature", - "hourly.data.apparentTemperature", - "hourly.data.dewPoint", - "hourly.data.humidity", - "hourly.data.pressure", - "hourly.data.windSpeed", - "hourly.data.windGust", - "hourly.data.windBearing", - "hourly.data.cloudCover", - "hourly.data.uvIndex", - "hourly.data.visibility")) -i = 1 ## test out with first row -call = apiCallWriter(crash2$lat[i], crash2$lon[i], crash2$CrashUnixTime[i]) -test = apiData(call) - -# sample darksky call ---- -call = apiCallWriter(lat = 41.90337, lon = -87.68947, time = 1576612000) -test = apiData(call) -## whatever the day of the epoch time within the location's time zone, API call returns that entire day +masterWeather = setNames(data.table(matrix(nrow = 0, + ncol = 24)), + c(## City of Chicago crash data columns + "ICN", + "CrashID", + "CrashDate", + "TimeOfCrash", + + ## Feature Engineering for DarkSky columns + "CrashUnixTime", + "MinutesFromInput", + + ## relevant DarkSky weather data columns + "hourly.summary", + "hourly.icon", + "hourly.data.time", + "hourly.data.summary", + "hourly.data.icon", + "hourly.data.precipIntensity", + "hourly.data.precipProbability", + "hourly.data.temperature", + "hourly.data.apparentTemperature", + "hourly.data.dewPoint", + "hourly.data.humidity", + "hourly.data.pressure", + "hourly.data.windSpeed", + "hourly.data.windGust", + "hourly.data.windBearing", + "hourly.data.cloudCover", + "hourly.data.uvIndex", + "hourly.data.visibility")) +for(i in c(1:nrow(darkskyinput))){ ## you only get 1,000 free calls per day, run on a small subset to test/develop + call = apiCallWriter(darkskyinput$lat[i], darkskyinput$lon[i], darkskyinput$CrashUnixTime[i]) + print(call) + sub = apiData(call) %>% + mutate(TimeFromInput = abs(hourly.data.time - darkskyinput$CrashUnixTime[i]), + CrashUnixTime = darkskyinput$CrashUnixTime[i], + TimeOfCrash = darkskyinput$TimeOfCrash[i], + CrashDate = darkskyinput$CrashDate[i], + ICN = darkskyinput$ICN[i], + CrashID = darkskyinput$CrashID[i]) %>% + ## whatever the day of the epoch time within the location's time zone, + ## DarkSky API call returns hourly weather data for the entire day. + ## here, choose only the weather data for hour nearest the crash + filter(TimeFromInput == min(TimeFromInput)) %>% + mutate(TimeFromInput = TimeFromInput / 60) %>% + rename(MinutesFromInput = TimeFromInput) %>% + select("ICN", + "CrashID", + "CrashDate", + "TimeOfCrash", + "CrashUnixTime", + "MinutesFromInput", + "hourly.summary", + "hourly.icon", + "hourly.data.time", + "hourly.data.summary", + "hourly.data.icon", + "hourly.data.precipIntensity", + "hourly.data.precipProbability", + "hourly.data.temperature", + "hourly.data.apparentTemperature", + "hourly.data.dewPoint", + "hourly.data.humidity", + "hourly.data.pressure", + "hourly.data.windSpeed", + "hourly.data.windGust", + "hourly.data.windBearing", + "hourly.data.cloudCover", + "hourly.data.uvIndex", + "hourly.data.visibility") + + masterWeather = rbind(masterWeather, sub) + } +View(masterWeather) \ No newline at end of file From e882b1240d04e675c3e2ebedc3aa666ef192af05 Mon Sep 17 00:00:00 2001 From: Lee Shibley Date: Sat, 16 Nov 2019 15:19:36 -0600 Subject: [PATCH 3/6] add newline to end of file --- bikegroup/darkskycalls.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bikegroup/darkskycalls.R b/bikegroup/darkskycalls.R index 7acf7f4..0bddca6 100644 --- a/bikegroup/darkskycalls.R +++ b/bikegroup/darkskycalls.R @@ -130,4 +130,5 @@ for(i in c(1:nrow(darkskyinput))){ ## you only get 1,000 free calls per day, run masterWeather = rbind(masterWeather, sub) } -View(masterWeather) \ No newline at end of file +View(masterWeather) + From 7857cec77975be09f8ed5c8eb2434f43504028b8 Mon Sep 17 00:00:00 2001 From: Lee Shibley Date: Tue, 10 Dec 2019 13:37:29 -0600 Subject: [PATCH 4/6] update comments --- bikegroup/darkskycalls.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bikegroup/darkskycalls.R b/bikegroup/darkskycalls.R index 0bddca6..ab92330 100644 --- a/bikegroup/darkskycalls.R +++ b/bikegroup/darkskycalls.R @@ -4,8 +4,8 @@ # ---- ## Hypothesis: adverse weather conditions play a significant role in frequency and severity of traffic accidents. -## To test: add weather data from DarkSky API -## How: API call for each time (date & time of day) and place (latitude & longitude) observed in the data +## To test: obtain weather data from DarkSky API +## How: API call for each time (date & time of day) and place (latitude & longitude) of a traffic accident observed in our data # install libraries ---- install.packages("httr") From 9a2e0fd1154f72e360e38543fb092357cbd48af2 Mon Sep 17 00:00:00 2001 From: Lee Shibley Date: Tue, 10 Dec 2019 13:38:31 -0600 Subject: [PATCH 5/6] update comments --- bikegroup/darkskycalls.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bikegroup/darkskycalls.R b/bikegroup/darkskycalls.R index ab92330..120bdce 100644 --- a/bikegroup/darkskycalls.R +++ b/bikegroup/darkskycalls.R @@ -5,7 +5,7 @@ # ---- ## Hypothesis: adverse weather conditions play a significant role in frequency and severity of traffic accidents. ## To test: obtain weather data from DarkSky API -## How: API call for each time (date & time of day) and place (latitude & longitude) of a traffic accident observed in our data +## How: API call for each time (date, time of day) and place (latitude & longitude) of the traffic accidents observed in our data # install libraries ---- install.packages("httr") From 609e08c20a6cd284c7c7d4b6547acbd079164108 Mon Sep 17 00:00:00 2001 From: Lee Shibley Date: Tue, 10 Dec 2019 13:42:31 -0600 Subject: [PATCH 6/6] update comments --- bikegroup/darkskycalls.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bikegroup/darkskycalls.R b/bikegroup/darkskycalls.R index 120bdce..e140f7d 100644 --- a/bikegroup/darkskycalls.R +++ b/bikegroup/darkskycalls.R @@ -4,8 +4,8 @@ # ---- ## Hypothesis: adverse weather conditions play a significant role in frequency and severity of traffic accidents. -## To test: obtain weather data from DarkSky API -## How: API call for each time (date, time of day) and place (latitude & longitude) of the traffic accidents observed in our data +## To test: augment our existing crash data with weather data from DarkSky API +## How: API call for the time (date, time of day) and place (latitude & longitude) of each traffic accident in our data # install libraries ---- install.packages("httr")