diff --git a/arbiter/arbiter.cpp b/arbiter/arbiter.cpp index fae5f3a..d23ccf1 100644 --- a/arbiter/arbiter.cpp +++ b/arbiter/arbiter.cpp @@ -102,6 +102,10 @@ Arbiter::Arbiter(const std::string s) { m_drivers[d->type()] = std::move(d); } + if (auto d = OneDrive::create(*m_pool, c.value("onedrive", json()).dump())) + { + m_drivers[d->type()] = std::move(d); + } #endif #endif diff --git a/arbiter/arbiter.hpp b/arbiter/arbiter.hpp index 20d73d1..a77e62d 100644 --- a/arbiter/arbiter.hpp +++ b/arbiter/arbiter.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/arbiter/drivers/CMakeLists.txt b/arbiter/drivers/CMakeLists.txt index 4599b56..3ec7062 100644 --- a/arbiter/drivers/CMakeLists.txt +++ b/arbiter/drivers/CMakeLists.txt @@ -8,6 +8,7 @@ set( "${BASE}/fs.cpp" "${BASE}/google.cpp" "${BASE}/s3.cpp" + "${BASE}/onedrive.cpp" ) set( @@ -17,6 +18,7 @@ set( "${BASE}/fs.hpp" "${BASE}/google.hpp" "${BASE}/s3.hpp" + "${BASE}/onedrive.hpp" "${BASE}/test.hpp" ) diff --git a/arbiter/drivers/google.hpp b/arbiter/drivers/google.hpp index cee4fb1..a861e39 100644 --- a/arbiter/drivers/google.hpp +++ b/arbiter/drivers/google.hpp @@ -78,5 +78,4 @@ class Google::Auth #ifdef ARBITER_CUSTOM_NAMESPACE } -#endif - +#endif \ No newline at end of file diff --git a/arbiter/drivers/onedrive.cpp b/arbiter/drivers/onedrive.cpp new file mode 100644 index 0000000..3714f7f --- /dev/null +++ b/arbiter/drivers/onedrive.cpp @@ -0,0 +1,266 @@ +#ifndef ARBITER_IS_AMALGAMATION +#include +#include +#include +#include +#include +#include +#include +#endif + +#include + +#ifdef ARBITER_CUSTOM_NAMESPACE +namespace ARBITER_CUSTOM_NAMESPACE +{ +#endif + +namespace arbiter +{ + +using namespace internal; + +namespace { + + +const std::string hostUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; + +std::string getBaseEndpoint(const std::string path) +{ + return hostUrl + path; +} + +std::string getBinaryEndpoint(const std::string path) +{ + return path + ":/content"; +} + +std::string getChildrenEndpoint(const std::string path) +{ + return path + ":/children"; +} + +std::string getRefreshUrl() +{ + return "https://login.microsoftonline.com/common/oauth2/v2.0/token"; +} + +std::vector buildBody(const http::Query& query) +{ + const std::string body(http::buildQueryString(query)); + return std::vector(body.begin(), body.end()); +} + +http::Query parseUrlQueryString(std::string url) +{ + return url.find("?") != std::string::npos ? http::parseQueryString(url) : http::Query({ }); +} + +}//unnamed + +namespace drivers +{ + +OneDrive::OneDrive(http::Pool& pool, std::unique_ptr auth) + : Https(pool) + , m_auth(std::move(auth)) +{ } + +std::unique_ptr OneDrive::create(http::Pool& pool, const std::string s) +{ + if (auto auth = Auth::create(s)) + { + return makeUnique(pool, std::move(auth)); + } + + return std::unique_ptr(); +} + +std::unique_ptr OneDrive::tryGetSize(const std::string path) const +{ + const std::string endpoint(getBaseEndpoint(path)); + http::Headers headers(m_auth->headers()); + headers["Content-Type"] = "application/x-www-form-urlencoded"; + drivers::Https https(m_pool); + + const auto res(https.internalGet(endpoint, headers)); + + if (!res.ok()) + { + std::cout << + "Failed get - " << res.code() << ": " << res.str() << std::endl; + return std::unique_ptr(); + } + + //parse the data into a json object and extract size key value + const auto obj = json::parse(res.data()); + if (obj.count("size")) + { + return makeUnique(obj.at("size").get()); + } + + return std::unique_ptr(); +} + +std::vector OneDrive::processList(std::string path, bool recursive) const +{ + const std::string endpoint(getBaseEndpoint(path)); + std::vector result; + + std::string pageUrl(getChildrenEndpoint(endpoint)); + + http::Headers headers(m_auth->headers()); + headers["Content-Type"] = "application/json"; + drivers::Https https(m_pool); + + //iterate through files and folders located in path parent + //limit to 200 items per list, @odata.nextLink will be provided as url for + //next set of items + do + { + const http::Query queries(parseUrlQueryString(pageUrl)); + + auto res(https.internalGet(getChildrenEndpoint(endpoint), headers, queries)); + + const json obj(json::parse(res.data())); + + if (!obj.contains("value") || !res.ok()) + { + std::cout << "Could not get OneDrive item " << path + << " with response code " << res.code() << ":" << res.str() << std::endl; + return std::vector(); + } + + //parse list + const json list(obj.at("value")); + for (auto& i: list.items()) + { + const auto data(i.value()); + const std::string fileName(data.at("name").get()); + const std::string filePath(path + "/" + fileName); + + result.push_back(filePath); + if (data.contains("folder") && recursive) + { + //restart process with new file head + const auto children(processList(filePath, recursive)); + + //add result of children processes to the parent + result.insert(result.end(), children.begin(), children.end()); + } + } + + //check for link to next set + if (obj.contains("@odata.nextLink")) + pageUrl = obj.at("@odata.nextLink"); + else + break; + + } while (true); + + return result; +} + +std::vector OneDrive::glob(std::string path, bool verbose) const +{ + path.pop_back(); + bool recursive(path.back() == '*'); + if (recursive) + path.pop_back(); + + if (path.back() == '/') + path.pop_back(); + + return processList(path, recursive); +} + +bool OneDrive::get( + const std::string path, + std::vector& data, + const http::Headers userHeaders, + const http::Query query) const +{ + const std::string endpoint(getBaseEndpoint(path)); + http::Headers headers(m_auth->headers()); + headers["Content-Type"] = "application/octet-stream"; + headers.insert(userHeaders.begin(), userHeaders.end()); + + drivers::Https https(m_pool); + const auto res(https.internalGet(getBinaryEndpoint(endpoint), headers)); + + if (!res.ok()) { + std::cout << + "Failed get - " << res.code() << ": " << res.str() << std::endl; + return false; + } + + data = res.data(); + return true; +} + +OneDrive::Auth::Auth(std::string s) { + const json config = json::parse(s); + m_token = config.at("access_token").get(); + m_refresh = config.at("refresh_token").get(); + m_redirect = config.at("redirect_uri").get(); + m_id = config.at("client_id").get(); + m_secret = config.at("client_secret").get(); +} + +std::unique_ptr OneDrive::Auth::create(const std::string s) +{ + return makeUnique(s); +} + +void OneDrive::Auth::refresh() +{ + //only refresh if we get within 2 minutes of the token ending + const auto now(Time().asUnix()); + if (m_expiration - now > 120) + return; + + http::Pool pool; + drivers::Https https(pool); + const http::Headers headers({ + { "Accept", "application/json" }, + { "Content-Type", "application/x-www-form-urlencoded" } + }); + + const auto encoded = buildBody({ + { "access_token", m_token }, + { "refresh_token", m_refresh }, + { "client_id", m_id }, + { "client_secret", m_secret }, + { "scope", "offline_access+files.read.all+user.read.all" }, + { "grant_type", "refresh_token" } + }); + + const auto res(https.internalPost(getRefreshUrl(), encoded, headers)); + const auto response(json::parse(res.str())); + if (res.code() != 200) + { + std::cout << res.code() << ": Failed to refresh token" << res.str() << std::endl; + throw new ArbiterError("Failed to refresh token. " + res.str()); + } + + //reset the token, refresh, and expiration time + m_token = response.at("access_token").get(); + m_refresh = response.at("refresh_token").get(); + m_expiration = now + response.at("expires_in").get(); +} + +http::Headers OneDrive::Auth::headers() { + std::lock_guard lock(m_mutex); + refresh(); + m_headers["Accept"] = "application/json"; + m_headers["Authorization"] = "Bearer " + getToken(); + return m_headers; +} + +}//drivers + +}//arbiter + +#ifdef ARBITER_CUSTOM_NAMESPACE +}; +#endif \ No newline at end of file diff --git a/arbiter/drivers/onedrive.hpp b/arbiter/drivers/onedrive.hpp new file mode 100644 index 0000000..c169387 --- /dev/null +++ b/arbiter/drivers/onedrive.hpp @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include +#include + +#ifndef ARBITER_IS_AMALGAMATION +#include +#endif + +#ifdef ARBITER_CUSTOM_NAMESPACE +namespace ARBITER_CUSTOM_NAMESPACE +{ +#endif + +namespace arbiter +{ + +namespace drivers +{ + +class OneDrive : public Https +{ + class Auth; + +public: + OneDrive(http::Pool& pool, std::unique_ptr auth); + + static std::unique_ptr create(http::Pool& pool, std::string j); + + virtual std::string type() const override { return "od"; }; + virtual std::unique_ptr tryGetSize( + std::string path) const override; +private: + virtual bool get( + std::string path, + std::vector& data, + http::Headers headers, + http::Query query) const override; + + virtual std::vector glob( + std::string path, + bool verbose) const override; + + std::vector processList(std::string path, bool recursive) const; + + std::unique_ptr m_auth; + +}; + +class OneDrive::Auth +{ +public: + Auth(std::string s); + static std::unique_ptr create(std::string s); + std::string getToken() { return m_token; }; + void refresh(); + http::Headers headers(); + +private: + //auth variables necessary for refreshing token + std::string m_refresh; + std::string m_redirect; + std::string m_id; + std::string m_secret; + std::string m_token; + int64_t m_expiration = 0; + + mutable http::Headers m_headers; + mutable std::mutex m_mutex; +}; + +} // namespace drivers +} // namespace arbiter + +#ifdef ARBITER_CUSTOM_NAMESPACE +} +#endif diff --git a/arbiter/util/curl.cpp b/arbiter/util/curl.cpp index 6bf4a83..0fe0205 100644 --- a/arbiter/util/curl.cpp +++ b/arbiter/util/curl.cpp @@ -235,7 +235,8 @@ void Curl::init( m_headers = nullptr; // Set path. - const std::string path(rawPath + buildQueryString(query)); + const std::string qs(buildQueryString(query)); + const std::string path(rawPath + (qs.size() ? "?" + qs : "")); curl_easy_setopt(m_curl, CURLOPT_URL, path.c_str()); // Needed for multithreaded Curl usage. diff --git a/arbiter/util/http.cpp b/arbiter/util/http.cpp index 38174f7..107ba4d 100644 --- a/arbiter/util/http.cpp +++ b/arbiter/util/http.cpp @@ -21,6 +21,28 @@ namespace ARBITER_CUSTOM_NAMESPACE namespace arbiter { + +namespace +{ + +// Slices a string between positions [begin, end), where end may be npos. +std::string slice( + const std::string& s, + const std::size_t begin, + const std::size_t end) +{ + assert(end >= begin); + return s.substr(begin, end == std::string::npos ? end : end - begin); +} + +std::size_t advance(const std::string& s, const std::size_t pos) +{ + if (pos == std::string::npos) return pos; + return pos + 1; +}; + +} + namespace http { @@ -58,11 +80,51 @@ std::string buildQueryString(const Query& query) std::string(), [](const std::string& out, const Query::value_type& keyVal) { - const char sep(out.empty() ? '?' : '&'); - return out + sep + keyVal.first + '=' + keyVal.second; + return ( + (out.empty() ? out : out + '&') + + keyVal.first + + (keyVal.second.size() ? ('=' + keyVal.second) : "") + ); }); } +Query parseQueryString(const std::string s) +{ + std::size_t cur = s.find_first_of("?"); + if (cur == std::string::npos) + cur = 0; + else + cur += 1; + + const std::size_t end = s.size(); + + http::Query result; + while (cur < end) + { + const std::size_t delimiterPos = s.find_first_of("&=", cur); + const std::string key = slice(s, cur, delimiterPos); + + if (delimiterPos == std::string::npos || s.at(delimiterPos) == '&') + { + // If we've reached the end of the string, or the next delimiter is + // an '&' character, then we have a valueless key. + result[key] = ""; + cur = advance(s, delimiterPos); + } + else + { + // Otherwise, extract the value. + cur = delimiterPos + 1; + const std::size_t ampersandPos = s.find_first_of("&", cur); + const std::string val = slice(s, cur, ampersandPos); + result[key] = val; + cur = advance(s, ampersandPos); + } + } + + return result; +} + Resource::Resource( Pool& pool, Curl& curl, @@ -200,4 +262,3 @@ void Pool::release(const std::size_t id) #ifdef ARBITER_CUSTOM_NAMESPACE } #endif - diff --git a/arbiter/util/http.hpp b/arbiter/util/http.hpp index 4c8ed33..497d72d 100644 --- a/arbiter/util/http.hpp +++ b/arbiter/util/http.hpp @@ -36,6 +36,10 @@ ARBITER_DLL std::string sanitize(std::string path, std::string exclusions = "/") */ ARBITER_DLL std::string buildQueryString(const http::Query& query); +/** Return Query from a given url path + */ +ARBITER_DLL Query parseQueryString(const std::string s); + /** @cond arbiter_internal */ class ARBITER_DLL Pool; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 114e784..e4a20ad 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -6,7 +6,6 @@ set_target_properties(arbiter-test COMPILE_DEFINITIONS ARBITER_DLL_IMPORT) - # We're overriding the test with a custom command for individual test output # and colors, which cmake doesn't like. set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS 1 CACHE INTERNAL "No dev warnings")