diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/HTTP/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | src/HTTP/HTTPMessageParser.h | 3 | ||||
| -rw-r--r-- | src/HTTP/UrlClient.cpp | 611 | ||||
| -rw-r--r-- | src/HTTP/UrlClient.h | 141 |
4 files changed, 756 insertions, 1 deletions
diff --git a/src/HTTP/CMakeLists.txt b/src/HTTP/CMakeLists.txt index 03cda2adc..3a2001e67 100644 --- a/src/HTTP/CMakeLists.txt +++ b/src/HTTP/CMakeLists.txt @@ -13,6 +13,7 @@ SET (SRCS NameValueParser.cpp SslHTTPServerConnection.cpp TransferEncodingParser.cpp + UrlClient.cpp UrlParser.cpp ) @@ -27,6 +28,7 @@ SET (HDRS NameValueParser.h SslHTTPServerConnection.h TransferEncodingParser.h + UrlClient.h UrlParser.h ) diff --git a/src/HTTP/HTTPMessageParser.h b/src/HTTP/HTTPMessageParser.h index f07de0492..307714243 100644 --- a/src/HTTP/HTTPMessageParser.h +++ b/src/HTTP/HTTPMessageParser.h @@ -31,7 +31,8 @@ public: /** Called when an error has occured while parsing. */ virtual void OnError(const AString & a_ErrorDescription) = 0; - /** Called when the first line (request / status) is fully parsed. */ + /** Called when the first line of the request or response is fully parsed. + Doesn't check the validity of the line, only extracts the first complete line. */ virtual void OnFirstLine(const AString & a_FirstLine) = 0; /** Called when a single header line is parsed. */ diff --git a/src/HTTP/UrlClient.cpp b/src/HTTP/UrlClient.cpp new file mode 100644 index 000000000..f9e642b22 --- /dev/null +++ b/src/HTTP/UrlClient.cpp @@ -0,0 +1,611 @@ + +// UrlClient.cpp + +// Implements the cUrlClient class for high-level URL interaction + +#include "Globals.h" +#include "UrlClient.h" +#include "UrlParser.h" +#include "HTTPMessageParser.h" + + + + + +// fwd: +class cSchemeHandler; +typedef SharedPtr<cSchemeHandler> cSchemeHandlerPtr; + + + + + +class cUrlClientRequest: + public cNetwork::cConnectCallbacks, + public cTCPLink::cCallbacks +{ + friend class cHttpSchemeHandler; + +public: + static std::pair<bool, AString> Request( + const AString & a_Method, + const AString & a_URL, + cUrlClient::cCallbacks & a_Callbacks, + AStringMap && a_Headers, + const AString & a_Body, + AStringMap && a_Options + ) + { + // Create a new instance of cUrlClientRequest, wrapped in a SharedPtr, so that it has a controlled lifetime. + // Cannot use std::make_shared, because the constructor is not public + SharedPtr<cUrlClientRequest> ptr (new cUrlClientRequest( + a_Method, a_URL, a_Callbacks, std::move(a_Headers), a_Body, std::move(a_Options) + )); + return ptr->DoRequest(ptr); + } + + + /** Calls the error callback with the specified message, if it exists, and terminates the request. */ + void CallErrorCallback(const AString & a_ErrorMessage) + { + // Call the error callback: + m_Callbacks.OnError(a_ErrorMessage); + + // Terminate the request's TCP link: + auto link = m_Link; + if (link != nullptr) + { + link->Close(); + } + m_Self.reset(); + } + + + cUrlClient::cCallbacks & GetCallbacks() { return m_Callbacks; } + + void RedirectTo(const AString & a_RedirectUrl); + + bool ShouldAllowRedirects() const; + + +protected: + + /** Method to be used for the request */ + AString m_Method; + + /** URL that will be requested. */ + AString m_Url; + + /** Individual components of the URL that will be requested. */ + AString m_UrlScheme, m_UrlUsername, m_UrlPassword, m_UrlHost, m_UrlPath, m_UrlQuery, m_UrlFragment; + UInt16 m_UrlPort; + + /** Callbacks that report progress and results of the request. */ + cUrlClient::cCallbacks & m_Callbacks; + + /** Extra headers to be sent with the request (besides the normal ones). */ + AStringMap m_Headers; + + /** Body to be sent with the request, if any. */ + AString m_Body; + + /** Extra options to be used for the request. */ + AStringMap m_Options; + + /** SharedPtr to self, so that this object can keep itself alive for as long as it needs, + and pass self as callbacks to cNetwork functions. */ + SharedPtr<cUrlClientRequest> m_Self; + + /** The handler that "talks" the protocol specified in m_UrlScheme, handles all the sending and receiving. */ + SharedPtr<cSchemeHandler> m_SchemeHandler; + + /** The link handling the request. */ + cTCPLinkPtr m_Link; + + /** The number of redirect attempts that will still be followed. + If the response specifies a redirect and this is nonzero, the redirect is followed. + If the response specifies a redirect and this is zero, a redirect loop is reported as an error. */ + int m_NumRemainingRedirects; + + + cUrlClientRequest( + const AString & a_Method, + const AString & a_Url, + cUrlClient::cCallbacks & a_Callbacks, + AStringMap && a_Headers, + const AString & a_Body, + AStringMap && a_Options + ): + m_Method(a_Method), + m_Url(a_Url), + m_Callbacks(a_Callbacks), + m_Headers(std::move(a_Headers)), + m_Body(a_Body), + m_Options(std::move(a_Options)) + { + m_NumRemainingRedirects = GetStringMapInteger(m_Options, "MaxRedirects", 30); + } + + + std::pair<bool, AString> DoRequest(SharedPtr<cUrlClientRequest> a_Self); + + + // cNetwork::cConnectCallbacks override: TCP link connected: + virtual void OnConnected(cTCPLink & a_Link) override; + + // cNetwork::cConnectCallbacks override: An error has occurred: + virtual void OnError(int a_ErrorCode, const AString & a_ErrorMsg) override + { + m_Callbacks.OnError(Printf("Network error %d (%s)", a_ErrorCode, a_ErrorMsg.c_str())); + m_Self.reset(); + } + + + // cTCPLink::cCallbacks override: TCP link created + virtual void OnLinkCreated(cTCPLinkPtr a_Link) override + { + m_Link = a_Link; + } + + + /** Called when there's data incoming from the remote peer. */ + virtual void OnReceivedData(const char * a_Data, size_t a_Length) override; + + + /** Called when the remote end closes the connection. + The link is still available for connection information query (IP / port). + Sending data on the link is not an error, but the data won't be delivered. */ + virtual void OnRemoteClosed(void) override; +}; + + + + + +/** Represents a base class for an object that "talks" a specified URL protocol, such as HTTP or FTP. +Also provides a static factory method for creating an instance based on the scheme. +A descendant of this class is created for each request and handles all of the request's aspects, +from right after connecting to the TCP link till the link is closed. +For an example of a specific handler, see the cHttpSchemeHandler class. */ +class cSchemeHandler abstract +{ +public: + cSchemeHandler(cUrlClientRequest & a_ParentRequest): + m_ParentRequest(a_ParentRequest) + { + } + + // Force a virtual destructor in all descendants + virtual ~cSchemeHandler() {} + + /** Creates and returns a new handler for the specified scheme. + a_ParentRequest is the request which is to be handled by the handler. */ + static cSchemeHandlerPtr Create(const AString & a_Scheme, cUrlClientRequest & a_ParentRequest); + + /** Called when the link gets established. */ + virtual void OnConnected(cTCPLink & a_Link) = 0; + + /** Called when there's data incoming from the remote peer. */ + virtual void OnReceivedData(const char * a_Data, size_t a_Length) = 0; + + /** Called when the remote end closes the connection. + The link is still available for connection information query (IP / port). + Sending data on the link is not an error, but the data won't be delivered. */ + virtual void OnRemoteClosed(void) = 0; + +protected: + cUrlClientRequest & m_ParentRequest; +}; + + + + + +/** cSchemeHandler descendant that handles HTTP (and HTTPS) requests. */ +class cHttpSchemeHandler: + public cSchemeHandler, + protected cHTTPMessageParser::cCallbacks +{ + typedef cSchemeHandler Super; + +public: + cHttpSchemeHandler(cUrlClientRequest & a_ParentRequest, bool a_IsTls): + Super(a_ParentRequest), + m_Parser(*this), + m_IsTls(a_IsTls), + m_IsRedirect(false) + { + } + + + virtual void OnConnected(cTCPLink & a_Link) override + { + m_Link = &a_Link; + if (m_IsTls) + { + // TODO: Start TLS + } + else + { + SendRequest(); + } + } + + void SendRequest() + { + // Send the request: + auto requestLine = m_ParentRequest.m_UrlPath; + if (requestLine.empty()) + { + requestLine = "/"; + } + if (!m_ParentRequest.m_UrlQuery.empty()) + { + requestLine.push_back('?'); + requestLine.append(m_ParentRequest.m_UrlQuery); + } + m_Link->Send(Printf("%s %s HTTP/1.1\r\n", m_ParentRequest.m_Method.c_str(), requestLine.c_str())); + m_Link->Send(Printf("Host: %s\r\n", m_ParentRequest.m_UrlHost.c_str())); + m_Link->Send(Printf("Content-Length: %u\r\n", static_cast<unsigned>(m_ParentRequest.m_Body.size()))); + for (auto itr = m_ParentRequest.m_Headers.cbegin(), end = m_ParentRequest.m_Headers.cend(); itr != end; ++itr) + { + m_Link->Send(Printf("%s: %s\r\n", itr->first.c_str(), itr->second.c_str())); + } // for itr - m_Headers[] + m_Link->Send("\r\n", 2); + m_Link->Send(m_ParentRequest.m_Body); + + // Notify the callbacks that the request has been sent: + m_ParentRequest.GetCallbacks().OnRequestSent(); + } + + + virtual void OnReceivedData(const char * a_Data, size_t a_Length) override + { + auto res = m_Parser.Parse(a_Data, a_Length); + if (res == AString::npos) + { + m_ParentRequest.CallErrorCallback("Failed to parse HTTP response"); + return; + } + } + + + virtual void OnRemoteClosed(void) override + { + m_Link = nullptr; + } + + + // cHTTPResponseParser::cCallbacks overrides: + virtual void OnError(const AString & a_ErrorDescription) override + { + m_ParentRequest.CallErrorCallback(a_ErrorDescription); + m_Link = nullptr; + } + + + virtual void OnFirstLine(const AString & a_FirstLine) override + { + // Find the first space, parse the result code between it and the second space: + auto idxFirstSpace = a_FirstLine.find(' '); + if (idxFirstSpace == AString::npos) + { + m_ParentRequest.CallErrorCallback(Printf("Failed to parse HTTP status line \"%s\", no space delimiter.", a_FirstLine.c_str())); + return; + } + auto idxSecondSpace = a_FirstLine.find(' ', idxFirstSpace + 1); + if (idxSecondSpace == AString::npos) + { + m_ParentRequest.CallErrorCallback(Printf("Failed to parse HTTP status line \"%s\", missing second space delimiter.", a_FirstLine.c_str())); + return; + } + int resultCode; + auto resultCodeStr = a_FirstLine.substr(idxFirstSpace + 1, idxSecondSpace - idxFirstSpace - 1); + if (!StringToInteger(resultCodeStr, resultCode)) + { + m_ParentRequest.CallErrorCallback(Printf("Failed to parse HTTP result code from response \"%s\"", resultCodeStr.c_str())); + return; + } + + // Check for redirects, follow if allowed by the options: + switch (resultCode) + { + case cUrlClient::HTTP_STATUS_MULTIPLE_CHOICES: + case cUrlClient::HTTP_STATUS_MOVED_PERMANENTLY: + case cUrlClient::HTTP_STATUS_FOUND: + case cUrlClient::HTTP_STATUS_SEE_OTHER: + case cUrlClient::HTTP_STATUS_TEMPORARY_REDIRECT: + { + m_IsRedirect = true; + return; + } + } + m_ParentRequest.GetCallbacks().OnStatusLine(a_FirstLine.substr(1, idxFirstSpace), resultCode, a_FirstLine.substr(idxSecondSpace + 1)); + } + + + virtual void OnHeaderLine(const AString & a_Key, const AString & a_Value) override + { + if (m_IsRedirect) + { + if (a_Key == "Location") + { + m_RedirectLocation = a_Value; + } + } + else + { + m_ParentRequest.GetCallbacks().OnHeader(a_Key, a_Value); + } + } + + + /** Called when all the headers have been parsed. */ + virtual void OnHeadersFinished(void) override + { + if (!m_IsRedirect) + { + m_ParentRequest.GetCallbacks().OnHeadersFinished(); + } + } + + + /** Called for each chunk of the incoming body data. */ + virtual void OnBodyData(const void * a_Data, size_t a_Size) override + { + if (!m_IsRedirect) + { + m_ParentRequest.GetCallbacks().OnBodyData(a_Data, a_Size); + } + } + + + /** Called when the entire body has been reported by OnBodyData(). */ + virtual void OnBodyFinished(void) override + { + if (m_IsRedirect) + { + if (m_RedirectLocation.empty()) + { + m_ParentRequest.CallErrorCallback("Invalid redirect, there's no location to redirect to"); + } + else + { + m_ParentRequest.RedirectTo(m_RedirectLocation); + } + } + else + { + m_ParentRequest.GetCallbacks().OnBodyFinished(); + } + } + +protected: + + /** The network link. */ + cTCPLink * m_Link; + + /** If true, the TLS should be started on the link before sending the request (used for https). */ + bool m_IsTls; + + /** Parser of the HTTP response message. */ + cHTTPMessageParser m_Parser; + + /** Set to true if the first line contains a redirecting HTTP status code and the options specify to follow redirects. + If true, and the parent request allows redirects, neither headers not the body contents are reported through the callbacks, + and after the entire request is parsed, the redirect is attempted. */ + bool m_IsRedirect; + + /** The Location where the request should be redirected. + Only used when m_IsRedirect is true. */ + AString m_RedirectLocation; +}; + + + + + +//////////////////////////////////////////////////////////////////////////////// +// cSchemeHandler: + +cSchemeHandlerPtr cSchemeHandler::Create(const AString & a_Scheme, cUrlClientRequest & a_ParentRequest) +{ + auto lowerScheme = StrToLower(a_Scheme); + if (lowerScheme == "http") + { + return std::make_shared<cHttpSchemeHandler>(a_ParentRequest, false); + } + else if (lowerScheme == "https") + { + return std::make_shared<cHttpSchemeHandler>(a_ParentRequest, true); + } + + return nullptr; +} + + + + + +//////////////////////////////////////////////////////////////////////////////// +// cUrlClientRequest: + +void cUrlClientRequest::RedirectTo(const AString & a_RedirectUrl) +{ + // Check that redirection is allowed: + m_Callbacks.OnRedirecting(a_RedirectUrl); + if (!ShouldAllowRedirects()) + { + CallErrorCallback(Printf("Redirect to \"%s\" not allowed", a_RedirectUrl.c_str())); + return; + } + + // Do the actual redirect: + m_Link->Close(); + m_Url = a_RedirectUrl; + m_NumRemainingRedirects = m_NumRemainingRedirects - 1; + auto res = DoRequest(m_Self); + if (!res.first) + { + m_Callbacks.OnError(Printf("Redirection failed: %s", res.second.c_str())); + return; + } +} + + + + + +bool cUrlClientRequest::ShouldAllowRedirects() const +{ + return (m_NumRemainingRedirects > 0); +} + + + + + +void cUrlClientRequest::OnConnected(cTCPLink & a_Link) +{ + m_Callbacks.OnConnected(a_Link); + m_SchemeHandler->OnConnected(a_Link); +} + + + + + +void cUrlClientRequest::OnReceivedData(const char * a_Data, size_t a_Length) +{ + auto handler = m_SchemeHandler; + if (handler != nullptr) + { + handler->OnReceivedData(a_Data, a_Length); + } +} + + + + + +void cUrlClientRequest::OnRemoteClosed() +{ + // Notify the callback: + auto handler = m_SchemeHandler; + if (handler != nullptr) + { + handler->OnRemoteClosed(); + } + + // Let ourselves be deleted + m_Self.reset(); +} + + + + + +std::pair<bool, AString> cUrlClientRequest::DoRequest(SharedPtr<cUrlClientRequest> a_Self) +{ + // We need a shared pointer to self, care must be taken not to pass any other ptr: + ASSERT(a_Self.get() == this); + + m_Self = a_Self; + + // Parse the URL: + auto res = cUrlParser::Parse(m_Url, m_UrlScheme, m_UrlUsername, m_UrlPassword, m_UrlHost, m_UrlPort, m_UrlPath, m_UrlQuery, m_UrlFragment); + if (!res.first) + { + return res; + } + + // Get a handler that will work with the specified scheme: + m_SchemeHandler = cSchemeHandler::Create(m_UrlScheme, *this); + if (m_SchemeHandler == nullptr) + { + return std::make_pair(false, Printf("Unknown Url scheme: %s", m_UrlScheme.c_str())); + } + + if (!cNetwork::Connect(m_UrlHost, m_UrlPort, m_Self, m_Self)) + { + return std::make_pair(false, "Network connection failed"); + } + return std::make_pair(true, AString()); +} + + + + + +//////////////////////////////////////////////////////////////////////////////// +// cUrlClient: + +std::pair<bool, AString> cUrlClient::Request( + const AString & a_Method, + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options +) +{ + return cUrlClientRequest::Request( + a_Method, a_URL, a_Callbacks, std::move(a_Headers), std::move(a_Body), std::move(a_Options) + ); +} + + + + + +std::pair<bool, AString> cUrlClient::Get( + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap a_Headers, + AString a_Body, + AStringMap a_Options +) +{ + return cUrlClientRequest::Request( + "GET", a_URL, a_Callbacks, std::move(a_Headers), std::move(a_Body), std::move(a_Options) + ); +} + + + + + +std::pair<bool, AString> cUrlClient::Post( + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options +) +{ + return cUrlClientRequest::Request( + "POST", a_URL, a_Callbacks, std::move(a_Headers), std::move(a_Body), std::move(a_Options) + ); +} + + + + + +std::pair<bool, AString> cUrlClient::Put( + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options +) +{ + return cUrlClientRequest::Request( + "PUT", a_URL, a_Callbacks, std::move(a_Headers), std::move(a_Body), std::move(a_Options) + ); +} + + + + + diff --git a/src/HTTP/UrlClient.h b/src/HTTP/UrlClient.h new file mode 100644 index 000000000..42086a4f1 --- /dev/null +++ b/src/HTTP/UrlClient.h @@ -0,0 +1,141 @@ + +// UrlClient.h + +// Declares the cUrlClient class for high-level URL interaction + +/* +Options that can be set via the Options parameter to the cUrlClient calls: +"MaxRedirects": The maximum number of allowed redirects before the client refuses a redirect with an error + +Behavior: +- If a redirect is received, and redirection is allowed, the redirection is reported via OnRedirecting() callback +and the request is restarted at the redirect URL, without reporting any of the redirect's headers nor body +- If a redirect is received and redirection is not allowed (maximum redirection attempts have been reached), +the OnRedirecting() callback is called with the redirect URL and then the request terminates with an OnError() callback, +without reporting the redirect's headers nor body. +*/ + + + + + +#pragma once + +#include "../OSSupport/Network.h" + + + + + +class cUrlClient +{ +public: + /** Callbacks that are used for progress and result reporting. */ + class cCallbacks + { + public: + /** Called when the TCP connection is established. */ + virtual void OnConnected(cTCPLink & a_Link) {}; + + /** Called for TLS connections, when the server certificate is received. + Return true to continue with the request, false to abort. + The default implementation does nothing and continues with the request. + TODO: The certificate parameter needs a representation! */ + virtual bool OnCertificateReceived() { return true; } + + /** Called after the entire request has been sent to the remote peer. */ + virtual void OnRequestSent() {}; + + /** Called after the first line of the response is parsed, unless the response is an allowed redirect. */ + virtual void OnStatusLine(const AString & a_HttpVersion, int a_StatusCode, const AString & a_Rest) {} + + /** Called when a single HTTP header is received and parsed, unless the response is an allowed redirect + Called once for each incoming header. */ + virtual void OnHeader(const AString & a_Key, const AString & a_Value) {}; + + /** Called when the HTTP headers have been fully parsed, unless the response is an allowed redirect. + There will be no more OnHeader() calls. */ + virtual void OnHeadersFinished() {}; + + /** Called when the next fragment of the response body is received, unless the response is an allowed redirect. + This can be called multiple times, as data arrives over the network. */ + virtual void OnBodyData(const void * a_Data, size_t a_Size) {}; + + /** Called after the response body has been fully reported by OnBody() calls, unless the response is an allowed redirect. + There will be no more OnBody() calls. */ + virtual void OnBodyFinished() {}; + + /** Called when an asynchronous error is encountered. */ + virtual void OnError(const AString & a_ErrorMsg) {}; + + /** Called when a redirect is to be followed. + This is called even if the redirecting is prohibited by the options; in such an event, this call will be + followed by OnError(). + If a response indicates a redirect (and the request allows redirecting), the regular callbacks + OnStatusLine(), OnHeader(), OnHeadersFinished(), OnBodyData() and OnBodyFinished() are not called + for such a response; instead, the redirect is silently attempted. */ + virtual void OnRedirecting(const AString & a_NewLocation) {}; + }; + + + /** Used for HTTP status codes. */ + enum eHTTPStatus + { + HTTP_STATUS_OK = 200, + HTTP_STATUS_MULTIPLE_CHOICES = 300, // MAY have a redirect using the "Location" header + HTTP_STATUS_MOVED_PERMANENTLY = 301, // redirect using the "Location" header + HTTP_STATUS_FOUND = 302, // redirect using the "Location" header + HTTP_STATUS_SEE_OTHER = 303, // redirect using the "Location" header + HTTP_STATUS_TEMPORARY_REDIRECT = 307, // redirect using the "Location" header + }; + + + /** Makes a network request to the specified URL, using the specified method (if applicable). + The response is reported via the a_ResponseCallback callback, in a single call. + The metadata about the response (HTTP headers) are reported via a_InfoCallback before the a_ResponseCallback call. + If there is an asynchronous error, it is reported in via the a_ErrorCallback. + If there is an immediate error (misformatted URL etc.), the function returns false and an error message. + a_Headers contains additional headers to use for the request. + a_Body specifies optional body to include with the request, if applicable. + a_Options contains various options for the request that govern the request behavior, but aren't sent to the server, + such as the proxy server, whether to follow redirects, and client certificate for TLS. */ + static std::pair<bool, AString> Request( + const AString & a_Method, + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options + ); + + /** Alias for Request("GET", ...) */ + static std::pair<bool, AString> Get( + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap a_Headers = AStringMap(), + AString a_Body = AString(), + AStringMap a_Options = AStringMap() + ); + + /** Alias for Request("POST", ...) */ + static std::pair<bool, AString> Post( + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options + ); + + /** Alias for Request("PUT", ...) */ + static std::pair<bool, AString> Put( + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options + ); +}; + + + + |