From ec0d4e5cb5840c738a4530099584ab44711af653 Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Fri, 24 Apr 2020 15:27:55 -0500 Subject: [PATCH 01/14] wip --- arbiter/arbiter.cpp | 1 + arbiter/drivers/CMakeLists.txt | 2 + arbiter/drivers/google.hpp | 3 +- arbiter/drivers/onedrive.cpp | 102 +++++++++++++++++++++++++++++++++ arbiter/drivers/onedrive.hpp | 69 ++++++++++++++++++++++ test/CMakeLists.txt | 7 ++- test/odTest.cpp | 30 ++++++++++ 7 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 arbiter/drivers/onedrive.cpp create mode 100644 arbiter/drivers/onedrive.hpp create mode 100644 test/odTest.cpp diff --git a/arbiter/arbiter.cpp b/arbiter/arbiter.cpp index fae5f3a..ed67349 100644 --- a/arbiter/arbiter.cpp +++ b/arbiter/arbiter.cpp @@ -35,6 +35,7 @@ namespace json config; std::string path("~/.arbiter/config.json"); + std::cout << "path is " << path << std::endl; if (auto p = env("ARBITER_CONFIG_FILE")) path = *p; else if (auto p = env("ARBITER_CONFIG_PATH")) path = *p; diff --git a/arbiter/drivers/CMakeLists.txt b/arbiter/drivers/CMakeLists.txt index 4599b56..3ec7062 100644 --- a/arbiter/drivers/CMakeLists.txt +++ b/arbiter/drivers/CMakeLists.txt @@ -8,6 +8,7 @@ set( "${BASE}/fs.cpp" "${BASE}/google.cpp" "${BASE}/s3.cpp" + "${BASE}/onedrive.cpp" ) set( @@ -17,6 +18,7 @@ set( "${BASE}/fs.hpp" "${BASE}/google.hpp" "${BASE}/s3.hpp" + "${BASE}/onedrive.hpp" "${BASE}/test.hpp" ) diff --git a/arbiter/drivers/google.hpp b/arbiter/drivers/google.hpp index cee4fb1..a861e39 100644 --- a/arbiter/drivers/google.hpp +++ b/arbiter/drivers/google.hpp @@ -78,5 +78,4 @@ class Google::Auth #ifdef ARBITER_CUSTOM_NAMESPACE } -#endif - +#endif \ No newline at end of file diff --git a/arbiter/drivers/onedrive.cpp b/arbiter/drivers/onedrive.cpp new file mode 100644 index 0000000..a8761ce --- /dev/null +++ b/arbiter/drivers/onedrive.cpp @@ -0,0 +1,102 @@ +#ifndef ARBITER_IS_AMALGAMATION +#include +#include +#include +#include +#include +#endif + +#include + +#ifdef ARBITER_CUSTOM_NAMESPACE +namespace ARBITER_CUSTOM_NAMESPACE +{ +#endif + +namespace arbiter +{ + +using namespace internal; + +namespace drivers +{ + +std::string baseOneDriveUrl = "graph.microsoft.com/v1.0/me/drive/root/"; + +OneDrive::OneDrive(http::Pool& pool, std::unique_ptr auth) + : Https(pool) + , m_auth(std::move(auth)) +{ } + +std::unique_ptr OneDrive::tryGetSize(const std::string path) const +{ + http::Headers headers(m_auth->headers()); + std::string constructed = baseOneDriveUrl + path; + + drivers::Https https(m_pool); + const auto res(https.internalHead(constructed, headers)); + + if (res.ok()) + { + if (res.headers().count("Content-Length")) + { + const auto& s(res.headers().at("Content-Length")); + return makeUnique(std::stoull(s)); + } + else if (res.headers().count("content-length")) + { + const auto& s(res.headers().at("content-length")); + return makeUnique(std::stoull(s)); + } + } + + return std::unique_ptr(); +} + +OneDrive::Auth::Auth(std::string s) + : m_token("Bearer " + json::parse(s).at("onedrive_token").get()) + { } + +http::Headers OneDrive::Auth::headers() const { + std::lock_guard lock(m_mutex); + return m_headers; +} + +bool OneDrive::get( + const std::string path, + std::vector& data, + const http::Headers userHeaders, + const http::Query query) const +{ + http::Headers headers(m_auth->headers()); + headers.insert(userHeaders.begin(), userHeaders.end()); + std::string constructed = baseOneDriveUrl + path + ":/content"; + + drivers::Https https(m_pool); + const auto res(https.internalGet(constructed, headers)); + + if (res.ok()) + { + data = res.data(); + return true; + } + else + { + std::cout << + "Failed get - " << res.code() << ": " << res.str() << std::endl; + return false; + } +} + +std::unique_ptr OneDrive::Auth::create(const std::string s) +{ + return makeUnique(s); +} + +}//drivers + +}//arbiter + +#ifdef ARBITER_CUSTOM_NAMESPACE +}; +#endif \ No newline at end of file diff --git a/arbiter/drivers/onedrive.hpp b/arbiter/drivers/onedrive.hpp new file mode 100644 index 0000000..991b62a --- /dev/null +++ b/arbiter/drivers/onedrive.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include +#include + +#ifndef ARBITER_IS_AMALGAMATION +#include +#endif + +#ifdef ARBITER_CUSTOM_NAMESPACE +namespace ARBITER_CUSTOM_NAMESPACE +{ +#endif + +namespace arbiter +{ + +namespace drivers +{ + +class OneDrive : public Https +{ + class Auth; + +public: + OneDrive(http::Pool& pool, std::unique_ptr auth); + + static std::unique_ptr create(http::Pool& pool, std::string j); + + //return type of driver + virtual std::string type() const override { return "od"; }; + //return size of the file + virtual std::unique_ptr tryGetSize( + std::string path) const override; + + + +private: + virtual bool get( + std::string path, + std::vector& data, + http::Headers headers, + http::Query query) const override; + + std::unique_ptr m_auth; + +}; + +class OneDrive::Auth +{ +public: + Auth(std::string s); + static std::unique_ptr create(std::string s); + http::Headers headers() const; + +private: + const std::string m_token; + mutable http::Headers m_headers; + mutable std::mutex m_mutex; +}; + +} // namespace drivers +} // namespace arbiter + +#ifdef ARBITER_CUSTOM_NAMESPACE +} +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 114e784..e84901a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -5,8 +5,13 @@ set_target_properties(arbiter-test PROPERTIES COMPILE_DEFINITIONS ARBITER_DLL_IMPORT) +add_executable(odtest odTest.cpp) +target_link_libraries(odtest PRIVATE arbiter) +set_target_properties(odtest + PROPERTIES + COMPILE_DEFINITIONS ARBITER_DLL_IMPORT) + - # We're overriding the test with a custom command for individual test output # and colors, which cmake doesn't like. set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS 1 CACHE INTERNAL "No dev warnings") diff --git a/test/odTest.cpp b/test/odTest.cpp new file mode 100644 index 0000000..c6cf166 --- /dev/null +++ b/test/odTest.cpp @@ -0,0 +1,30 @@ +#include +#include +#include +#include +#include +#include +#include + +#ifndef ARBITER_IS_AMALGAMATION +#include +#endif + +#include "config.hpp" + +#ifdef ARBITER_CUSTOM_NAMESPACE +namespace ARBITER_CUSTOM_NAMESPACE +{ +#endif +using namespace arbiter; + +int main() { + Arbiter a; + std::string path("od://Documents/autzen.las"); + a.get(path); + return 0; +} + +#ifdef ARBITER_CUSTOM_NAMESPACE +} +#endif \ No newline at end of file From ef7a19484fd7c8be2651a8c1ad5017c81549771b Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Wed, 29 Apr 2020 16:36:35 -0500 Subject: [PATCH 02/14] get and tryGetSize working --- arbiter/arbiter.cpp | 4 +++ arbiter/arbiter.hpp | 1 + arbiter/drivers/CMakeLists.txt | 1 + arbiter/drivers/onedrive.cpp | 62 +++++++++++++++++++++++++--------- arbiter/drivers/onedrive.hpp | 4 +-- test/odTest.cpp | 4 ++- 6 files changed, 56 insertions(+), 20 deletions(-) diff --git a/arbiter/arbiter.cpp b/arbiter/arbiter.cpp index ed67349..ad9aad9 100644 --- a/arbiter/arbiter.cpp +++ b/arbiter/arbiter.cpp @@ -103,6 +103,10 @@ Arbiter::Arbiter(const std::string s) { m_drivers[d->type()] = std::move(d); } + if (auto d = OneDrive::create(*m_pool, c.value("onedrive", json()).dump())) + { + m_drivers[d->type()] = std::move(d); + } #endif #endif diff --git a/arbiter/arbiter.hpp b/arbiter/arbiter.hpp index 20d73d1..a77e62d 100644 --- a/arbiter/arbiter.hpp +++ b/arbiter/arbiter.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/arbiter/drivers/CMakeLists.txt b/arbiter/drivers/CMakeLists.txt index 3ec7062..01665c2 100644 --- a/arbiter/drivers/CMakeLists.txt +++ b/arbiter/drivers/CMakeLists.txt @@ -21,6 +21,7 @@ set( "${BASE}/onedrive.hpp" "${BASE}/test.hpp" ) +message("SOURCES ${SOURCES}") install(FILES ${HEADERS} DESTINATION include/arbiter/${MODULE}) diff --git a/arbiter/drivers/onedrive.cpp b/arbiter/drivers/onedrive.cpp index a8761ce..56a0673 100644 --- a/arbiter/drivers/onedrive.cpp +++ b/arbiter/drivers/onedrive.cpp @@ -8,6 +8,20 @@ #include +#ifdef ARBITER_OPENSSL +#include +#include +#include +#include +#include + +// See: https://www.openssl.org/docs/manmaster/man3/OPENSSL_VERSION_NUMBER.html +# if OPENSSL_VERSION_NUMBER >= 0x010100000 +# define ARBITER_OPENSSL_ATLEAST_1_1 +# endif + +#endif + #ifdef ARBITER_CUSTOM_NAMESPACE namespace ARBITER_CUSTOM_NAMESPACE { @@ -21,47 +35,56 @@ using namespace internal; namespace drivers { -std::string baseOneDriveUrl = "graph.microsoft.com/v1.0/me/drive/root/"; +std::string baseOneDriveUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; OneDrive::OneDrive(http::Pool& pool, std::unique_ptr auth) : Https(pool) , m_auth(std::move(auth)) { } +std::unique_ptr OneDrive::create(http::Pool& pool, const std::string s) +{ + if (auto auth = Auth::create(s)) + { + return makeUnique(pool, std::move(auth)); + } + + return std::unique_ptr(); +} + std::unique_ptr OneDrive::tryGetSize(const std::string path) const { http::Headers headers(m_auth->headers()); + headers["Content-Type"] = "application/json"; + headers["Accept"] = "application/json"; std::string constructed = baseOneDriveUrl + path; drivers::Https https(m_pool); - const auto res(https.internalHead(constructed, headers)); + const auto res(https.internalGet(constructed, headers)); if (res.ok()) { - if (res.headers().count("Content-Length")) + //parse the data into a json object and extract size key value + const auto obj = json::parse(res.data()); + if (obj.find("size") != obj.end()) { - const auto& s(res.headers().at("Content-Length")); - return makeUnique(std::stoull(s)); - } - else if (res.headers().count("content-length")) - { - const auto& s(res.headers().at("content-length")); - return makeUnique(std::stoull(s)); + return makeUnique(obj.at("size").get()); } } + else + { + std::cout << + "Failed get - " << res.code() << ": " << res.str() << std::endl; + return std::unique_ptr(); + } return std::unique_ptr(); } OneDrive::Auth::Auth(std::string s) - : m_token("Bearer " + json::parse(s).at("onedrive_token").get()) + : m_token("bearer " + json::parse(s).at("onedrive_token").get()) { } -http::Headers OneDrive::Auth::headers() const { - std::lock_guard lock(m_mutex); - return m_headers; -} - bool OneDrive::get( const std::string path, std::vector& data, @@ -69,6 +92,7 @@ bool OneDrive::get( const http::Query query) const { http::Headers headers(m_auth->headers()); + headers["Content-Type"] = "application/octet-stream"; headers.insert(userHeaders.begin(), userHeaders.end()); std::string constructed = baseOneDriveUrl + path + ":/content"; @@ -93,6 +117,12 @@ std::unique_ptr OneDrive::Auth::create(const std::string s) return makeUnique(s); } +http::Headers OneDrive::Auth::headers() const { + std::lock_guard lock(m_mutex); + m_headers["Authorization"] = m_token; + return m_headers; +} + }//drivers }//arbiter diff --git a/arbiter/drivers/onedrive.hpp b/arbiter/drivers/onedrive.hpp index 991b62a..33d2e02 100644 --- a/arbiter/drivers/onedrive.hpp +++ b/arbiter/drivers/onedrive.hpp @@ -34,9 +34,6 @@ class OneDrive : public Https //return size of the file virtual std::unique_ptr tryGetSize( std::string path) const override; - - - private: virtual bool get( std::string path, @@ -44,6 +41,7 @@ class OneDrive : public Https http::Headers headers, http::Query query) const override; + std::unique_ptr m_auth; }; diff --git a/test/odTest.cpp b/test/odTest.cpp index c6cf166..6f5b1ab 100644 --- a/test/odTest.cpp +++ b/test/odTest.cpp @@ -21,7 +21,9 @@ using namespace arbiter; int main() { Arbiter a; std::string path("od://Documents/autzen.las"); - a.get(path); + auto data = a.get(path); + auto size = a.tryGetSize(path); + return 0; } From 1633160402f0cf126e57228ecb5b9fdad5db2f96 Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Tue, 26 May 2020 14:35:46 -0500 Subject: [PATCH 03/14] refresh and access tokens now working correctly --- arbiter/drivers/onedrive.cpp | 220 ++++++++++++++++++++++++++++++----- arbiter/drivers/onedrive.hpp | 41 ++++++- test/odTest.cpp | 32 ----- 3 files changed, 228 insertions(+), 65 deletions(-) delete mode 100644 test/odTest.cpp diff --git a/arbiter/drivers/onedrive.cpp b/arbiter/drivers/onedrive.cpp index 56a0673..dd86d95 100644 --- a/arbiter/drivers/onedrive.cpp +++ b/arbiter/drivers/onedrive.cpp @@ -2,8 +2,10 @@ #include #include #include -#include #include +#include +#include +#include #endif #include @@ -32,10 +34,53 @@ namespace arbiter using namespace internal; -namespace drivers +namespace { + +static std::string getQueries(const std::string url) { + json result; + + //find position of queries in url + std::string::size_type pos(url.find("?")); + if (pos == std::string::npos) + { + return result.dump(); + } + + std::string queries(url.substr(pos + 1)); + + do + { + //get positional values + //find next query + std::string::size_type nextQueryPos(queries.find_first_of("&")); + //if it doesn't exist, go to the end of the string + if (nextQueryPos == std::string::npos) + nextQueryPos = queries.size(); + const std::string::size_type separator(queries.find("=")); + + //create key-value pair for json + const std::string key(queries.substr(0, separator)); + const std::string value(queries.substr(separator + 1, nextQueryPos - (separator + 1))); + + result.push_back(json::object_t::value_type(key, value)); + + /*BREAK HERE*/ + //was this the last one? if yes, break. if no, resize queries and continue + if (nextQueryPos == queries.size()) + break; + + queries = queries.substr(nextQueryPos + 1); + + } while (true); + + return result.dump(); +} + +} -std::string baseOneDriveUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; +namespace drivers +{ OneDrive::OneDrive(http::Pool& pool, std::unique_ptr auth) : Https(pool) @@ -54,36 +99,101 @@ std::unique_ptr OneDrive::create(http::Pool& pool, const std::string s std::unique_ptr OneDrive::tryGetSize(const std::string path) const { + Resource resource(path); http::Headers headers(m_auth->headers()); - headers["Content-Type"] = "application/json"; + headers["Authorization"] = m_auth->getToken(); + headers["Content-Type"] = "application/x-www-form-urlencoded"; headers["Accept"] = "application/json"; - std::string constructed = baseOneDriveUrl + path; - drivers::Https https(m_pool); - const auto res(https.internalGet(constructed, headers)); - if (res.ok()) - { - //parse the data into a json object and extract size key value - const auto obj = json::parse(res.data()); - if (obj.find("size") != obj.end()) - { - return makeUnique(obj.at("size").get()); - } - } - else + const auto res(https.internalGet(resource.endpoint(), headers)); + + if (!res.ok()) { std::cout << "Failed get - " << res.code() << ": " << res.str() << std::endl; return std::unique_ptr(); } + //parse the data into a json object and extract size key value + const auto obj = json::parse(res.data()); + if (obj.find("size") != obj.end()) + { + return makeUnique(obj.at("size").get()); + } + return std::unique_ptr(); } -OneDrive::Auth::Auth(std::string s) - : m_token("bearer " + json::parse(s).at("onedrive_token").get()) - { } +std::vector OneDrive::processList(std::string path, bool recursive) const +{ + std::vector result; + + Resource resource(path); + std::string pageUrl(resource.childrenEndpoint()); + + http::Headers headers(m_auth->headers()); + headers["Content-Type"] = "application/json"; + drivers::Https https(m_pool); + + //iterate through files and folders located in path parent + //limit to 200 items per list, @odata.nextLink will be provided as url for + //next set of items + do + { + http::Query queries; + auto res(https.internalGet(resource.childrenEndpoint(), headers, queries)); + const json obj(json::parse(res.data())); + const json parsedQueries(json::parse(getQueries(pageUrl))); + for (auto& it: parsedQueries.items()) + queries[it.key()] = it.value(); + + if (!obj.contains("value") || !res.ok()) + { + std::cout << "Could not get OneDrive item " << resource.childrenEndpoint() + << " with response code " << res.code() << std::endl; + return std::vector(); + } + + const json list(obj.at("value")); + for (auto& i: list.items()) + { + const auto data(i.value()); + const std::string fileName(data.at("name").get()); + const std::string filePath(path + "/" + fileName); + + result.push_back(filePath); + if (data.contains("folder") && recursive) + { + //restart process with new file head + const auto children(processList(filePath, recursive)); + //TODO: look into more efficient way to do this + result.insert(result.end(), children.begin(), children.end()); + } + } + + if (obj.contains("@odata.nextLink")) + pageUrl = obj.at("@odata.nextLink"); + else + pageUrl = ""; + + } while (pageUrl.size() > 0); + + return result; +} + +std::vector OneDrive::glob(std::string path, bool verbose) const +{ + path.pop_back(); + bool recursive(path.back() == '*'); + if (recursive) + path.pop_back(); + + if (path.back() == '/') + path.pop_back(); + + return processList(path, recursive); +} bool OneDrive::get( const std::string path, @@ -91,35 +201,83 @@ bool OneDrive::get( const http::Headers userHeaders, const http::Query query) const { + Resource resource(path); http::Headers headers(m_auth->headers()); headers["Content-Type"] = "application/octet-stream"; headers.insert(userHeaders.begin(), userHeaders.end()); - std::string constructed = baseOneDriveUrl + path + ":/content"; drivers::Https https(m_pool); - const auto res(https.internalGet(constructed, headers)); + const auto res(https.internalGet(resource.binaryEndpoint(), headers)); - if (res.ok()) - { - data = res.data(); - return true; - } - else - { + if (!res.ok()) { std::cout << "Failed get - " << res.code() << ": " << res.str() << std::endl; return false; } + + data = res.data(); + return true; } +OneDrive::Auth::Auth(std::string s) + : m_token(json::parse(s).at("access_token").get()) + , m_refresh(json::parse(s).at("refresh_token").get()) + , m_redirect(json::parse(s).at("redirect_uri").get()) + , m_id(json::parse(s).at("client_id").get()) + , m_secret(json::parse(s).at("client_secret").get()) + , m_tenant(json::parse(s).at("tenant_id").get()) + , m_expiration(Time().asUnix()) + { } + std::unique_ptr OneDrive::Auth::create(const std::string s) { return makeUnique(s); } -http::Headers OneDrive::Auth::headers() const { +void OneDrive::Auth::refresh() +{ + const auto now(Time().asUnix()); + if (m_expiration - now > 120) + return; + + http::Pool pool; + drivers::Https https(pool); + http::Headers headers({ }); + + headers["Content-Type"] = "application/x-www-form-urlencoded"; + headers["Accept"] = "application/json"; + + std::string scope = "offline_access+files.readwrite.all+user.read+user.readwrite+user.readbasic.all+user.read.all+directory.read.all+directory.accessasuser.all"; + json bodyJson = { + { "access_token", m_token }, + { "refresh_token", m_refresh }, + { "client_id", m_id }, + { "client_secret", m_secret }, + { "scope", scope }, + { "grant_type", "refresh_token" } + }; + std::string body = ""; + + for (auto& i: bodyJson.items()) + body += i.key() + "=" + i.value().get() + "&"; + body.pop_back(); + std::vector content(body.begin(), body.end()); + + const auto res(https.internalPost(this->getRefreshUrl(), content, headers)); + const auto response(json::parse(res.str())); + std::cout << "response, " << res.str() << std::endl; + + m_token = response.at("access_token").get(); + m_refresh = response.at("refresh_token").get(); + + m_expiration = now + 3599; + +} + +http::Headers OneDrive::Auth::headers() { std::lock_guard lock(m_mutex); - m_headers["Authorization"] = m_token; + refresh(); + m_headers["Authorization"] = "Bearer " + getToken(); return m_headers; } diff --git a/arbiter/drivers/onedrive.hpp b/arbiter/drivers/onedrive.hpp index 33d2e02..540c400 100644 --- a/arbiter/drivers/onedrive.hpp +++ b/arbiter/drivers/onedrive.hpp @@ -23,6 +23,7 @@ namespace drivers class OneDrive : public Https { class Auth; + class Resource; public: OneDrive(http::Pool& pool, std::unique_ptr auth); @@ -41,6 +42,11 @@ class OneDrive : public Https http::Headers headers, http::Query query) const override; + virtual std::vector glob( + std::string path, + bool verbose) const override; + + std::vector processList(std::string path, bool recursive) const; std::unique_ptr m_auth; @@ -51,12 +57,43 @@ class OneDrive::Auth public: Auth(std::string s); static std::unique_ptr create(std::string s); - http::Headers headers() const; + std::string getToken() { return m_token; }; + // static std::string getToken(std::string s); + std::string getRefreshUrl() { return m_authUrl + "/common/oauth2/v2.0/token"; } + void refresh(); + http::Headers headers(); private: - const std::string m_token; + std::string m_refresh; + std::string m_redirect; + std::string m_id; + std::string m_secret; + std::string m_token; + std::string m_tenant; + int64_t m_expiration; + mutable http::Headers m_headers; mutable std::mutex m_mutex; + const std::string m_authUrl = "https://login.microsoftonline.com/"; +}; + +class OneDrive::Resource +{ +public: + Resource(std::string path) { + m_path = path; + m_baseUrl = hostUrl + path; + + } + const std::string endpoint() const { return std::string(m_baseUrl); } + const std::string binaryEndpoint() const { return std::string(m_baseUrl + ":/content"); } + const std::string childrenEndpoint() const { return std::string(m_baseUrl + ":/children"); } + +private: + std::string m_baseUrl; + std::string m_path; + const std::string hostUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; + }; } // namespace drivers diff --git a/test/odTest.cpp b/test/odTest.cpp deleted file mode 100644 index 6f5b1ab..0000000 --- a/test/odTest.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#ifndef ARBITER_IS_AMALGAMATION -#include -#endif - -#include "config.hpp" - -#ifdef ARBITER_CUSTOM_NAMESPACE -namespace ARBITER_CUSTOM_NAMESPACE -{ -#endif -using namespace arbiter; - -int main() { - Arbiter a; - std::string path("od://Documents/autzen.las"); - auto data = a.get(path); - auto size = a.tryGetSize(path); - - return 0; -} - -#ifdef ARBITER_CUSTOM_NAMESPACE -} -#endif \ No newline at end of file From 9ebdc2218221337843ca038b63f8210fbc0a50ed Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Thu, 28 May 2020 01:11:49 -0500 Subject: [PATCH 04/14] removing unnecessary logging and variables --- arbiter/drivers/onedrive.cpp | 44 +++++++++++++++++++----------------- arbiter/drivers/onedrive.hpp | 5 +--- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/arbiter/drivers/onedrive.cpp b/arbiter/drivers/onedrive.cpp index dd86d95..96d46de 100644 --- a/arbiter/drivers/onedrive.cpp +++ b/arbiter/drivers/onedrive.cpp @@ -101,9 +101,7 @@ std::unique_ptr OneDrive::tryGetSize(const std::string path) const { Resource resource(path); http::Headers headers(m_auth->headers()); - headers["Authorization"] = m_auth->getToken(); headers["Content-Type"] = "application/x-www-form-urlencoded"; - headers["Accept"] = "application/json"; drivers::Https https(m_pool); const auto res(https.internalGet(resource.endpoint(), headers)); @@ -145,6 +143,7 @@ std::vector OneDrive::processList(std::string path, bool recursive) auto res(https.internalGet(resource.childrenEndpoint(), headers, queries)); const json obj(json::parse(res.data())); const json parsedQueries(json::parse(getQueries(pageUrl))); + for (auto& it: parsedQueries.items()) queries[it.key()] = it.value(); @@ -155,6 +154,7 @@ std::vector OneDrive::processList(std::string path, bool recursive) return std::vector(); } + //parse list const json list(obj.at("value")); for (auto& i: list.items()) { @@ -167,17 +167,19 @@ std::vector OneDrive::processList(std::string path, bool recursive) { //restart process with new file head const auto children(processList(filePath, recursive)); - //TODO: look into more efficient way to do this + + //add result of children processes to the parent result.insert(result.end(), children.begin(), children.end()); } } + //check for link to next set if (obj.contains("@odata.nextLink")) pageUrl = obj.at("@odata.nextLink"); else - pageUrl = ""; + break; - } while (pageUrl.size() > 0); + } while (true); return result; } @@ -219,15 +221,15 @@ bool OneDrive::get( return true; } -OneDrive::Auth::Auth(std::string s) - : m_token(json::parse(s).at("access_token").get()) - , m_refresh(json::parse(s).at("refresh_token").get()) - , m_redirect(json::parse(s).at("redirect_uri").get()) - , m_id(json::parse(s).at("client_id").get()) - , m_secret(json::parse(s).at("client_secret").get()) - , m_tenant(json::parse(s).at("tenant_id").get()) - , m_expiration(Time().asUnix()) - { } +OneDrive::Auth::Auth(std::string s) { + const auto config = json::parse(s); + m_token = config.at("access_token").get(); + m_refresh = config.at("refresh_token").get(); + m_redirect = config.at("redirect_uri").get(); + m_id = config.at("client_id").get(); + m_secret = config.at("client_secret").get(); + m_expiration = Time().asUnix(); +} std::unique_ptr OneDrive::Auth::create(const std::string s) { @@ -236,6 +238,7 @@ std::unique_ptr OneDrive::Auth::create(const std::string s) void OneDrive::Auth::refresh() { + //only refresh if we get within 2 minutes of the token ending const auto now(Time().asUnix()); if (m_expiration - now > 120) return; @@ -244,20 +247,20 @@ void OneDrive::Auth::refresh() drivers::Https https(pool); http::Headers headers({ }); - headers["Content-Type"] = "application/x-www-form-urlencoded"; headers["Accept"] = "application/json"; + headers["Content-Type"] = "application/x-www-form-urlencoded"; - std::string scope = "offline_access+files.readwrite.all+user.read+user.readwrite+user.readbasic.all+user.read.all+directory.read.all+directory.accessasuser.all"; json bodyJson = { { "access_token", m_token }, { "refresh_token", m_refresh }, { "client_id", m_id }, { "client_secret", m_secret }, - { "scope", scope }, + { "scope", "offline_access+files.read.all+user.read.all" }, { "grant_type", "refresh_token" } }; - std::string body = ""; + //create url-encoded body + std::string body = ""; for (auto& i: bodyJson.items()) body += i.key() + "=" + i.value().get() + "&"; body.pop_back(); @@ -265,18 +268,17 @@ void OneDrive::Auth::refresh() const auto res(https.internalPost(this->getRefreshUrl(), content, headers)); const auto response(json::parse(res.str())); - std::cout << "response, " << res.str() << std::endl; + //reset the token, refresh, and expiration time m_token = response.at("access_token").get(); m_refresh = response.at("refresh_token").get(); - m_expiration = now + 3599; - } http::Headers OneDrive::Auth::headers() { std::lock_guard lock(m_mutex); refresh(); + m_headers["Accept"] = "application/json"; m_headers["Authorization"] = "Bearer " + getToken(); return m_headers; } diff --git a/arbiter/drivers/onedrive.hpp b/arbiter/drivers/onedrive.hpp index 540c400..7bb6000 100644 --- a/arbiter/drivers/onedrive.hpp +++ b/arbiter/drivers/onedrive.hpp @@ -30,9 +30,7 @@ class OneDrive : public Https static std::unique_ptr create(http::Pool& pool, std::string j); - //return type of driver virtual std::string type() const override { return "od"; }; - //return size of the file virtual std::unique_ptr tryGetSize( std::string path) const override; private: @@ -64,12 +62,12 @@ class OneDrive::Auth http::Headers headers(); private: + //auth variables necessary for refreshing token std::string m_refresh; std::string m_redirect; std::string m_id; std::string m_secret; std::string m_token; - std::string m_tenant; int64_t m_expiration; mutable http::Headers m_headers; @@ -83,7 +81,6 @@ class OneDrive::Resource Resource(std::string path) { m_path = path; m_baseUrl = hostUrl + path; - } const std::string endpoint() const { return std::string(m_baseUrl); } const std::string binaryEndpoint() const { return std::string(m_baseUrl + ":/content"); } From fa752caa3f275116d8a5ed9095f7600edd9f9a47 Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Thu, 28 May 2020 09:09:43 -0500 Subject: [PATCH 05/14] remvong log --- arbiter/arbiter.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/arbiter/arbiter.cpp b/arbiter/arbiter.cpp index ad9aad9..d23ccf1 100644 --- a/arbiter/arbiter.cpp +++ b/arbiter/arbiter.cpp @@ -35,7 +35,6 @@ namespace json config; std::string path("~/.arbiter/config.json"); - std::cout << "path is " << path << std::endl; if (auto p = env("ARBITER_CONFIG_FILE")) path = *p; else if (auto p = env("ARBITER_CONFIG_PATH")) path = *p; From e391999fc756653722e13fe3bb6317c0339316ed Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Thu, 28 May 2020 09:16:46 -0500 Subject: [PATCH 06/14] removing testing pieces from cmake file --- test/CMakeLists.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e84901a..e4a20ad 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -5,12 +5,6 @@ set_target_properties(arbiter-test PROPERTIES COMPILE_DEFINITIONS ARBITER_DLL_IMPORT) -add_executable(odtest odTest.cpp) -target_link_libraries(odtest PRIVATE arbiter) -set_target_properties(odtest - PROPERTIES - COMPILE_DEFINITIONS ARBITER_DLL_IMPORT) - # We're overriding the test with a custom command for individual test output # and colors, which cmake doesn't like. From 23910d0cffe80d78b27918898348442987e55993 Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Thu, 28 May 2020 09:18:13 -0500 Subject: [PATCH 07/14] removing unnecessary logs --- arbiter/drivers/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/arbiter/drivers/CMakeLists.txt b/arbiter/drivers/CMakeLists.txt index 01665c2..3ec7062 100644 --- a/arbiter/drivers/CMakeLists.txt +++ b/arbiter/drivers/CMakeLists.txt @@ -21,7 +21,6 @@ set( "${BASE}/onedrive.hpp" "${BASE}/test.hpp" ) -message("SOURCES ${SOURCES}") install(FILES ${HEADERS} DESTINATION include/arbiter/${MODULE}) From 08a4318e5d497bdeb34146904f5dda62c522c4fc Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Thu, 28 May 2020 12:15:44 -0500 Subject: [PATCH 08/14] removed resource class, added to unnamed namespace --- arbiter/drivers/onedrive.cpp | 81 ++++++++++++++++++++++++++---------- arbiter/drivers/onedrive.hpp | 24 +---------- 2 files changed, 59 insertions(+), 46 deletions(-) diff --git a/arbiter/drivers/onedrive.cpp b/arbiter/drivers/onedrive.cpp index 96d46de..5af2706 100644 --- a/arbiter/drivers/onedrive.cpp +++ b/arbiter/drivers/onedrive.cpp @@ -36,6 +36,42 @@ using namespace internal; namespace { + +const std::string hostUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; + +const std::string getBaseEndpoint(const std::string path) +{ + return std::string(hostUrl + path); +} + +const std::string getBinaryEndpoint(const std::string path) +{ + return std::string(path + ":/content"); +} + +const std::string getChildrenEndpoint(const std::string path) +{ + return std::string(path + ":/children"); +} + +std::string getRefreshUrl() +{ + return "https://login.microsoftonline.com/common/oauth2/v2.0/token"; +} + +std::vector buildBody(const http::Query& query) +{ + const std::string acc(std::accumulate( + query.begin(), + query.end(), + std::string(), + [](const std::string& out, const http::Query::value_type& keyVal) + { + return out + '&' + keyVal.first + '=' + keyVal.second; + })); + return std::vector(acc.begin(), acc.end()); +} + static std::string getQueries(const std::string url) { json result; @@ -99,12 +135,12 @@ std::unique_ptr OneDrive::create(http::Pool& pool, const std::string s std::unique_ptr OneDrive::tryGetSize(const std::string path) const { - Resource resource(path); + const std::string endpoint(getBaseEndpoint(path)); http::Headers headers(m_auth->headers()); headers["Content-Type"] = "application/x-www-form-urlencoded"; drivers::Https https(m_pool); - const auto res(https.internalGet(resource.endpoint(), headers)); + const auto res(https.internalGet(endpoint, headers)); if (!res.ok()) { @@ -125,10 +161,10 @@ std::unique_ptr OneDrive::tryGetSize(const std::string path) const std::vector OneDrive::processList(std::string path, bool recursive) const { + const std::string endpoint(getBaseEndpoint(path)); std::vector result; - Resource resource(path); - std::string pageUrl(resource.childrenEndpoint()); + std::string pageUrl(getChildrenEndpoint(endpoint)); http::Headers headers(m_auth->headers()); headers["Content-Type"] = "application/json"; @@ -140,7 +176,7 @@ std::vector OneDrive::processList(std::string path, bool recursive) do { http::Query queries; - auto res(https.internalGet(resource.childrenEndpoint(), headers, queries)); + auto res(https.internalGet(pageUrl, headers, queries)); const json obj(json::parse(res.data())); const json parsedQueries(json::parse(getQueries(pageUrl))); @@ -149,7 +185,7 @@ std::vector OneDrive::processList(std::string path, bool recursive) if (!obj.contains("value") || !res.ok()) { - std::cout << "Could not get OneDrive item " << resource.childrenEndpoint() + std::cout << "Could not get OneDrive item " << path << " with response code " << res.code() << std::endl; return std::vector(); } @@ -203,13 +239,13 @@ bool OneDrive::get( const http::Headers userHeaders, const http::Query query) const { - Resource resource(path); + const std::string endpoint(getBaseEndpoint(path)); http::Headers headers(m_auth->headers()); headers["Content-Type"] = "application/octet-stream"; headers.insert(userHeaders.begin(), userHeaders.end()); drivers::Https https(m_pool); - const auto res(https.internalGet(resource.binaryEndpoint(), headers)); + const auto res(https.internalGet(getBinaryEndpoint(endpoint), headers)); if (!res.ok()) { std::cout << @@ -222,13 +258,12 @@ bool OneDrive::get( } OneDrive::Auth::Auth(std::string s) { - const auto config = json::parse(s); + const json config = json::parse(s); m_token = config.at("access_token").get(); m_refresh = config.at("refresh_token").get(); m_redirect = config.at("redirect_uri").get(); m_id = config.at("client_id").get(); m_secret = config.at("client_secret").get(); - m_expiration = Time().asUnix(); } std::unique_ptr OneDrive::Auth::create(const std::string s) @@ -245,29 +280,29 @@ void OneDrive::Auth::refresh() http::Pool pool; drivers::Https https(pool); - http::Headers headers({ }); - - headers["Accept"] = "application/json"; - headers["Content-Type"] = "application/x-www-form-urlencoded"; + http::Headers headers({ + { "Accept", "application/json" }, + { "Content-Type", "application/x-www-form-urlencoded" } + }); - json bodyJson = { + http::Query body({ { "access_token", m_token }, { "refresh_token", m_refresh }, { "client_id", m_id }, { "client_secret", m_secret }, { "scope", "offline_access+files.read.all+user.read.all" }, { "grant_type", "refresh_token" } - }; + }); - //create url-encoded body - std::string body = ""; - for (auto& i: bodyJson.items()) - body += i.key() + "=" + i.value().get() + "&"; - body.pop_back(); - std::vector content(body.begin(), body.end()); + const auto encoded = buildBody(body); - const auto res(https.internalPost(this->getRefreshUrl(), content, headers)); + const auto res(https.internalPost(getRefreshUrl(), encoded, headers)); const auto response(json::parse(res.str())); + if (res.code() != 200) + { + std::cout << res.code() + ": Failed to refresh token" + res.str() << std::endl; + throw new ArbiterError(res.code() + ": Failed to refresh token" + res.str()); + } //reset the token, refresh, and expiration time m_token = response.at("access_token").get(); diff --git a/arbiter/drivers/onedrive.hpp b/arbiter/drivers/onedrive.hpp index 7bb6000..c169387 100644 --- a/arbiter/drivers/onedrive.hpp +++ b/arbiter/drivers/onedrive.hpp @@ -23,7 +23,6 @@ namespace drivers class OneDrive : public Https { class Auth; - class Resource; public: OneDrive(http::Pool& pool, std::unique_ptr auth); @@ -56,8 +55,6 @@ class OneDrive::Auth Auth(std::string s); static std::unique_ptr create(std::string s); std::string getToken() { return m_token; }; - // static std::string getToken(std::string s); - std::string getRefreshUrl() { return m_authUrl + "/common/oauth2/v2.0/token"; } void refresh(); http::Headers headers(); @@ -68,29 +65,10 @@ class OneDrive::Auth std::string m_id; std::string m_secret; std::string m_token; - int64_t m_expiration; + int64_t m_expiration = 0; mutable http::Headers m_headers; mutable std::mutex m_mutex; - const std::string m_authUrl = "https://login.microsoftonline.com/"; -}; - -class OneDrive::Resource -{ -public: - Resource(std::string path) { - m_path = path; - m_baseUrl = hostUrl + path; - } - const std::string endpoint() const { return std::string(m_baseUrl); } - const std::string binaryEndpoint() const { return std::string(m_baseUrl + ":/content"); } - const std::string childrenEndpoint() const { return std::string(m_baseUrl + ":/children"); } - -private: - std::string m_baseUrl; - std::string m_path; - const std::string hostUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; - }; } // namespace drivers From 5f9c03a305927dc89a68b03d620622f50d16cdba Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Thu, 28 May 2020 12:19:28 -0500 Subject: [PATCH 09/14] pull expiration time from json --- arbiter/drivers/onedrive.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbiter/drivers/onedrive.cpp b/arbiter/drivers/onedrive.cpp index 5af2706..f11edbd 100644 --- a/arbiter/drivers/onedrive.cpp +++ b/arbiter/drivers/onedrive.cpp @@ -307,7 +307,7 @@ void OneDrive::Auth::refresh() //reset the token, refresh, and expiration time m_token = response.at("access_token").get(); m_refresh = response.at("refresh_token").get(); - m_expiration = now + 3599; + m_expiration = now + response.at("expires_in").get(); } http::Headers OneDrive::Auth::headers() { From bb775633583ba88dfae75510742d1e6f660ac34f Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Thu, 28 May 2020 12:25:07 -0500 Subject: [PATCH 10/14] adding const where possible --- arbiter/drivers/onedrive.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arbiter/drivers/onedrive.cpp b/arbiter/drivers/onedrive.cpp index f11edbd..7760eb2 100644 --- a/arbiter/drivers/onedrive.cpp +++ b/arbiter/drivers/onedrive.cpp @@ -54,12 +54,12 @@ const std::string getChildrenEndpoint(const std::string path) return std::string(path + ":/children"); } -std::string getRefreshUrl() +const std::string getRefreshUrl() { return "https://login.microsoftonline.com/common/oauth2/v2.0/token"; } -std::vector buildBody(const http::Query& query) +const std::vector buildBody(const http::Query& query) { const std::string acc(std::accumulate( query.begin(), From 25e8585a36957fad5b1a8024ce86393906bff2b1 Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Thu, 28 May 2020 12:49:00 -0500 Subject: [PATCH 11/14] removing unnecessary consts --- arbiter/drivers/onedrive.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/arbiter/drivers/onedrive.cpp b/arbiter/drivers/onedrive.cpp index 7760eb2..7a04371 100644 --- a/arbiter/drivers/onedrive.cpp +++ b/arbiter/drivers/onedrive.cpp @@ -39,27 +39,27 @@ namespace { const std::string hostUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; -const std::string getBaseEndpoint(const std::string path) +std::string getBaseEndpoint(const std::string path) { return std::string(hostUrl + path); } -const std::string getBinaryEndpoint(const std::string path) +std::string getBinaryEndpoint(const std::string path) { return std::string(path + ":/content"); } -const std::string getChildrenEndpoint(const std::string path) +std::string getChildrenEndpoint(const std::string path) { return std::string(path + ":/children"); } -const std::string getRefreshUrl() +std::string getRefreshUrl() { return "https://login.microsoftonline.com/common/oauth2/v2.0/token"; } -const std::vector buildBody(const http::Query& query) +std::vector buildBody(const http::Query& query) { const std::string acc(std::accumulate( query.begin(), @@ -72,7 +72,7 @@ const std::vector buildBody(const http::Query& query) return std::vector(acc.begin(), acc.end()); } -static std::string getQueries(const std::string url) +std::string getQueries(const std::string url) { json result; @@ -280,12 +280,12 @@ void OneDrive::Auth::refresh() http::Pool pool; drivers::Https https(pool); - http::Headers headers({ + const http::Headers headers({ { "Accept", "application/json" }, { "Content-Type", "application/x-www-form-urlencoded" } }); - http::Query body({ + const auto encoded = buildBody({ { "access_token", m_token }, { "refresh_token", m_refresh }, { "client_id", m_id }, @@ -294,8 +294,6 @@ void OneDrive::Auth::refresh() { "grant_type", "refresh_token" } }); - const auto encoded = buildBody(body); - const auto res(https.internalPost(getRefreshUrl(), encoded, headers)); const auto response(json::parse(res.str())); if (res.code() != 200) From 1d38db2f69ecca5cc7f7db56aafa6c84be927ef8 Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Fri, 29 May 2020 11:49:27 -0500 Subject: [PATCH 12/14] handle empty queries, add query util to util file --- arbiter/drivers/onedrive.cpp | 93 +++++++----------------------------- arbiter/util/curl.cpp | 3 +- arbiter/util/http.cpp | 57 +++++++++++++++++++++- arbiter/util/http.hpp | 4 ++ 4 files changed, 79 insertions(+), 78 deletions(-) diff --git a/arbiter/drivers/onedrive.cpp b/arbiter/drivers/onedrive.cpp index 7a04371..693a3b9 100644 --- a/arbiter/drivers/onedrive.cpp +++ b/arbiter/drivers/onedrive.cpp @@ -10,20 +10,6 @@ #include -#ifdef ARBITER_OPENSSL -#include -#include -#include -#include -#include - -// See: https://www.openssl.org/docs/manmaster/man3/OPENSSL_VERSION_NUMBER.html -# if OPENSSL_VERSION_NUMBER >= 0x010100000 -# define ARBITER_OPENSSL_ATLEAST_1_1 -# endif - -#endif - #ifdef ARBITER_CUSTOM_NAMESPACE namespace ARBITER_CUSTOM_NAMESPACE { @@ -41,17 +27,17 @@ const std::string hostUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; std::string getBaseEndpoint(const std::string path) { - return std::string(hostUrl + path); + return hostUrl + path; } std::string getBinaryEndpoint(const std::string path) { - return std::string(path + ":/content"); + return path + ":/content"; } std::string getChildrenEndpoint(const std::string path) { - return std::string(path + ":/children"); + return path + ":/children"; } std::string getRefreshUrl() @@ -61,56 +47,8 @@ std::string getRefreshUrl() std::vector buildBody(const http::Query& query) { - const std::string acc(std::accumulate( - query.begin(), - query.end(), - std::string(), - [](const std::string& out, const http::Query::value_type& keyVal) - { - return out + '&' + keyVal.first + '=' + keyVal.second; - })); - return std::vector(acc.begin(), acc.end()); -} - -std::string getQueries(const std::string url) -{ - json result; - - //find position of queries in url - std::string::size_type pos(url.find("?")); - if (pos == std::string::npos) - { - return result.dump(); - } - - std::string queries(url.substr(pos + 1)); - - do - { - //get positional values - //find next query - std::string::size_type nextQueryPos(queries.find_first_of("&")); - //if it doesn't exist, go to the end of the string - if (nextQueryPos == std::string::npos) - nextQueryPos = queries.size(); - const std::string::size_type separator(queries.find("=")); - - //create key-value pair for json - const std::string key(queries.substr(0, separator)); - const std::string value(queries.substr(separator + 1, nextQueryPos - (separator + 1))); - - result.push_back(json::object_t::value_type(key, value)); - - /*BREAK HERE*/ - //was this the last one? if yes, break. if no, resize queries and continue - if (nextQueryPos == queries.size()) - break; - - queries = queries.substr(nextQueryPos + 1); - - } while (true); - - return result.dump(); + const std::string body(http::buildQueryString(query)); + return std::vector(body.begin(), body.end()); } } @@ -151,7 +89,7 @@ std::unique_ptr OneDrive::tryGetSize(const std::string path) const //parse the data into a json object and extract size key value const auto obj = json::parse(res.data()); - if (obj.find("size") != obj.end()) + if (obj.count("size")) { return makeUnique(obj.at("size").get()); } @@ -169,24 +107,21 @@ std::vector OneDrive::processList(std::string path, bool recursive) http::Headers headers(m_auth->headers()); headers["Content-Type"] = "application/json"; drivers::Https https(m_pool); + http::Query queries({ }); //iterate through files and folders located in path parent //limit to 200 items per list, @odata.nextLink will be provided as url for //next set of items do { - http::Query queries; - auto res(https.internalGet(pageUrl, headers, queries)); - const json obj(json::parse(res.data())); - const json parsedQueries(json::parse(getQueries(pageUrl))); + auto res(https.internalGet(getChildrenEndpoint(endpoint), headers, queries)); - for (auto& it: parsedQueries.items()) - queries[it.key()] = it.value(); + const json obj(json::parse(res.data())); if (!obj.contains("value") || !res.ok()) { std::cout << "Could not get OneDrive item " << path - << " with response code " << res.code() << std::endl; + << " with response code " << res.code() << ":" << res.str() << std::endl; return std::vector(); } @@ -211,7 +146,15 @@ std::vector OneDrive::processList(std::string path, bool recursive) //check for link to next set if (obj.contains("@odata.nextLink")) + { pageUrl = obj.at("@odata.nextLink"); + const http::Query parsedQueries(http::getQueries(pageUrl)); + + for (auto& it: parsedQueries) + { + queries[it.first] = it.second; + } + } else break; diff --git a/arbiter/util/curl.cpp b/arbiter/util/curl.cpp index 6bf4a83..0fe0205 100644 --- a/arbiter/util/curl.cpp +++ b/arbiter/util/curl.cpp @@ -235,7 +235,8 @@ void Curl::init( m_headers = nullptr; // Set path. - const std::string path(rawPath + buildQueryString(query)); + const std::string qs(buildQueryString(query)); + const std::string path(rawPath + (qs.size() ? "?" + qs : "")); curl_easy_setopt(m_curl, CURLOPT_URL, path.c_str()); // Needed for multithreaded Curl usage. diff --git a/arbiter/util/http.cpp b/arbiter/util/http.cpp index 38174f7..05fab05 100644 --- a/arbiter/util/http.cpp +++ b/arbiter/util/http.cpp @@ -58,11 +58,64 @@ std::string buildQueryString(const Query& query) std::string(), [](const std::string& out, const Query::value_type& keyVal) { - const char sep(out.empty() ? '?' : '&'); - return out + sep + keyVal.first + '=' + keyVal.second; + return ( + (out.empty() ? out : out + '&') + + keyVal.first + + (keyVal.second.size() ? ('=' + keyVal.second) : "") + ); }); } +Query getQueries(const std::string url) +{ + http::Query result; + + //find position of queries in url + std::string::size_type pos(url.find("?")); + if (pos == std::string::npos) + { + return result; + } + + const std::string queries(url.substr(pos + 1)); + ulong start = 0; + + do + { + //get positional values + //find next query + std::string::size_type nextQueryPos(queries.find_first_of("&", start)); + std::string::size_type valuePos(queries.find_first_of("=", start)); + std::string::size_type adjust(1); + //if it doesn't exist, go to the end of the string + + if (nextQueryPos == std::string::npos) + nextQueryPos = queries.size(); + if (valuePos == std::string::npos) + { + valuePos = queries.size(); + adjust = 0; + } + + std::string::size_type separator(std::min(valuePos, nextQueryPos)); + + //create key-value pair for Query + const std::string key(queries.substr(start, separator - start)); + const std::string value(queries.substr(separator + adjust, nextQueryPos - (separator + adjust))); + result[key] = value; + + /*BREAK HERE*/ + //was this the last one? if yes, break. if no, resize queries and continue + if (nextQueryPos >= queries.size()) + break; + + start = nextQueryPos + 1; + + } while (true); + + return result; +} + Resource::Resource( Pool& pool, Curl& curl, diff --git a/arbiter/util/http.hpp b/arbiter/util/http.hpp index 4c8ed33..23db2c8 100644 --- a/arbiter/util/http.hpp +++ b/arbiter/util/http.hpp @@ -36,6 +36,10 @@ ARBITER_DLL std::string sanitize(std::string path, std::string exclusions = "/") */ ARBITER_DLL std::string buildQueryString(const http::Query& query); +/** Return Query from a given url path + */ +ARBITER_DLL Query getQueries(const std::string url); + /** @cond arbiter_internal */ class ARBITER_DLL Pool; From d719c779dd4ef5074e7b320434bcc0680ded8f35 Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Fri, 29 May 2020 12:47:07 -0500 Subject: [PATCH 13/14] removed queries from list --- arbiter/drivers/onedrive.cpp | 16 ++++------------ arbiter/util/http.cpp | 10 +++++----- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/arbiter/drivers/onedrive.cpp b/arbiter/drivers/onedrive.cpp index 693a3b9..72a3e5a 100644 --- a/arbiter/drivers/onedrive.cpp +++ b/arbiter/drivers/onedrive.cpp @@ -51,7 +51,7 @@ std::vector buildBody(const http::Query& query) return std::vector(body.begin(), body.end()); } -} +}//unnamed namespace drivers { @@ -107,13 +107,13 @@ std::vector OneDrive::processList(std::string path, bool recursive) http::Headers headers(m_auth->headers()); headers["Content-Type"] = "application/json"; drivers::Https https(m_pool); - http::Query queries({ }); //iterate through files and folders located in path parent //limit to 200 items per list, @odata.nextLink will be provided as url for //next set of items do { + const http::Query queries(http::getQueries(pageUrl)); auto res(https.internalGet(getChildrenEndpoint(endpoint), headers, queries)); const json obj(json::parse(res.data())); @@ -146,15 +146,7 @@ std::vector OneDrive::processList(std::string path, bool recursive) //check for link to next set if (obj.contains("@odata.nextLink")) - { pageUrl = obj.at("@odata.nextLink"); - const http::Query parsedQueries(http::getQueries(pageUrl)); - - for (auto& it: parsedQueries) - { - queries[it.first] = it.second; - } - } else break; @@ -241,8 +233,8 @@ void OneDrive::Auth::refresh() const auto response(json::parse(res.str())); if (res.code() != 200) { - std::cout << res.code() + ": Failed to refresh token" + res.str() << std::endl; - throw new ArbiterError(res.code() + ": Failed to refresh token" + res.str()); + std::cout << res.code() << ": Failed to refresh token" << res.str() << std::endl; + throw new ArbiterError("Failed to refresh token. " + res.str()); } //reset the token, refresh, and expiration time diff --git a/arbiter/util/http.cpp b/arbiter/util/http.cpp index 05fab05..3bd061c 100644 --- a/arbiter/util/http.cpp +++ b/arbiter/util/http.cpp @@ -78,17 +78,17 @@ Query getQueries(const std::string url) } const std::string queries(url.substr(pos + 1)); - ulong start = 0; + std::size_t start = 0; do { //get positional values //find next query - std::string::size_type nextQueryPos(queries.find_first_of("&", start)); - std::string::size_type valuePos(queries.find_first_of("=", start)); - std::string::size_type adjust(1); - //if it doesn't exist, go to the end of the string + std::size_t nextQueryPos(queries.find_first_of("&", start)); + std::size_t valuePos(queries.find_first_of("=", start)); + std::size_t adjust(1); + //if it doesn't exist, go to the end of the string if (nextQueryPos == std::string::npos) nextQueryPos = queries.size(); if (valuePos == std::string::npos) From 1ca4f16dcc90916858982171cd40a456ea0e1fba Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Fri, 29 May 2020 14:29:16 -0500 Subject: [PATCH 14/14] more explicit query parser --- arbiter/drivers/onedrive.cpp | 8 +++- arbiter/util/http.cpp | 90 ++++++++++++++++++++---------------- arbiter/util/http.hpp | 2 +- 3 files changed, 57 insertions(+), 43 deletions(-) diff --git a/arbiter/drivers/onedrive.cpp b/arbiter/drivers/onedrive.cpp index 72a3e5a..3714f7f 100644 --- a/arbiter/drivers/onedrive.cpp +++ b/arbiter/drivers/onedrive.cpp @@ -51,6 +51,11 @@ std::vector buildBody(const http::Query& query) return std::vector(body.begin(), body.end()); } +http::Query parseUrlQueryString(std::string url) +{ + return url.find("?") != std::string::npos ? http::parseQueryString(url) : http::Query({ }); +} + }//unnamed namespace drivers @@ -113,7 +118,8 @@ std::vector OneDrive::processList(std::string path, bool recursive) //next set of items do { - const http::Query queries(http::getQueries(pageUrl)); + const http::Query queries(parseUrlQueryString(pageUrl)); + auto res(https.internalGet(getChildrenEndpoint(endpoint), headers, queries)); const json obj(json::parse(res.data())); diff --git a/arbiter/util/http.cpp b/arbiter/util/http.cpp index 3bd061c..107ba4d 100644 --- a/arbiter/util/http.cpp +++ b/arbiter/util/http.cpp @@ -21,6 +21,28 @@ namespace ARBITER_CUSTOM_NAMESPACE namespace arbiter { + +namespace +{ + +// Slices a string between positions [begin, end), where end may be npos. +std::string slice( + const std::string& s, + const std::size_t begin, + const std::size_t end) +{ + assert(end >= begin); + return s.substr(begin, end == std::string::npos ? end : end - begin); +} + +std::size_t advance(const std::string& s, const std::size_t pos) +{ + if (pos == std::string::npos) return pos; + return pos + 1; +}; + +} + namespace http { @@ -66,52 +88,39 @@ std::string buildQueryString(const Query& query) }); } -Query getQueries(const std::string url) +Query parseQueryString(const std::string s) { - http::Query result; - - //find position of queries in url - std::string::size_type pos(url.find("?")); - if (pos == std::string::npos) - { - return result; - } + std::size_t cur = s.find_first_of("?"); + if (cur == std::string::npos) + cur = 0; + else + cur += 1; - const std::string queries(url.substr(pos + 1)); - std::size_t start = 0; + const std::size_t end = s.size(); - do + http::Query result; + while (cur < end) { - //get positional values - //find next query - std::size_t nextQueryPos(queries.find_first_of("&", start)); - std::size_t valuePos(queries.find_first_of("=", start)); - std::size_t adjust(1); - - //if it doesn't exist, go to the end of the string - if (nextQueryPos == std::string::npos) - nextQueryPos = queries.size(); - if (valuePos == std::string::npos) + const std::size_t delimiterPos = s.find_first_of("&=", cur); + const std::string key = slice(s, cur, delimiterPos); + + if (delimiterPos == std::string::npos || s.at(delimiterPos) == '&') { - valuePos = queries.size(); - adjust = 0; + // If we've reached the end of the string, or the next delimiter is + // an '&' character, then we have a valueless key. + result[key] = ""; + cur = advance(s, delimiterPos); } - - std::string::size_type separator(std::min(valuePos, nextQueryPos)); - - //create key-value pair for Query - const std::string key(queries.substr(start, separator - start)); - const std::string value(queries.substr(separator + adjust, nextQueryPos - (separator + adjust))); - result[key] = value; - - /*BREAK HERE*/ - //was this the last one? if yes, break. if no, resize queries and continue - if (nextQueryPos >= queries.size()) - break; - - start = nextQueryPos + 1; - - } while (true); + else + { + // Otherwise, extract the value. + cur = delimiterPos + 1; + const std::size_t ampersandPos = s.find_first_of("&", cur); + const std::string val = slice(s, cur, ampersandPos); + result[key] = val; + cur = advance(s, ampersandPos); + } + } return result; } @@ -253,4 +262,3 @@ void Pool::release(const std::size_t id) #ifdef ARBITER_CUSTOM_NAMESPACE } #endif - diff --git a/arbiter/util/http.hpp b/arbiter/util/http.hpp index 23db2c8..497d72d 100644 --- a/arbiter/util/http.hpp +++ b/arbiter/util/http.hpp @@ -38,7 +38,7 @@ ARBITER_DLL std::string buildQueryString(const http::Query& query); /** Return Query from a given url path */ -ARBITER_DLL Query getQueries(const std::string url); +ARBITER_DLL Query parseQueryString(const std::string s); /** @cond arbiter_internal */