From 6c760ee348dfa61560660c214799b793ce17513b Mon Sep 17 00:00:00 2001 From: Mattes D Date: Mon, 22 Aug 2016 19:49:33 +0200 Subject: UrlClient: Basic HTTP implementation. --- src/HTTP/CMakeLists.txt | 2 + src/HTTP/HTTPMessageParser.h | 3 +- src/HTTP/UrlClient.cpp | 611 +++++++++++++++++++++++++++++++++++++++++++ src/HTTP/UrlClient.h | 141 ++++++++++ tests/HTTP/CMakeLists.txt | 26 +- tests/HTTP/UrlClientTest.cpp | 162 ++++++++++++ 6 files changed, 941 insertions(+), 4 deletions(-) create mode 100644 src/HTTP/UrlClient.cpp create mode 100644 src/HTTP/UrlClient.h create mode 100644 tests/HTTP/UrlClientTest.cpp diff --git a/src/HTTP/CMakeLists.txt b/src/HTTP/CMakeLists.txt index 03cda2adc..3a2001e67 100644 --- a/src/HTTP/CMakeLists.txt +++ b/src/HTTP/CMakeLists.txt @@ -13,6 +13,7 @@ SET (SRCS NameValueParser.cpp SslHTTPServerConnection.cpp TransferEncodingParser.cpp + UrlClient.cpp UrlParser.cpp ) @@ -27,6 +28,7 @@ SET (HDRS NameValueParser.h SslHTTPServerConnection.h TransferEncodingParser.h + UrlClient.h UrlParser.h ) diff --git a/src/HTTP/HTTPMessageParser.h b/src/HTTP/HTTPMessageParser.h index f07de0492..307714243 100644 --- a/src/HTTP/HTTPMessageParser.h +++ b/src/HTTP/HTTPMessageParser.h @@ -31,7 +31,8 @@ public: /** Called when an error has occured while parsing. */ virtual void OnError(const AString & a_ErrorDescription) = 0; - /** Called when the first line (request / status) is fully parsed. */ + /** Called when the first line of the request or response is fully parsed. + Doesn't check the validity of the line, only extracts the first complete line. */ virtual void OnFirstLine(const AString & a_FirstLine) = 0; /** Called when a single header line is parsed. */ diff --git a/src/HTTP/UrlClient.cpp b/src/HTTP/UrlClient.cpp new file mode 100644 index 000000000..f9e642b22 --- /dev/null +++ b/src/HTTP/UrlClient.cpp @@ -0,0 +1,611 @@ + +// UrlClient.cpp + +// Implements the cUrlClient class for high-level URL interaction + +#include "Globals.h" +#include "UrlClient.h" +#include "UrlParser.h" +#include "HTTPMessageParser.h" + + + + + +// fwd: +class cSchemeHandler; +typedef SharedPtr cSchemeHandlerPtr; + + + + + +class cUrlClientRequest: + public cNetwork::cConnectCallbacks, + public cTCPLink::cCallbacks +{ + friend class cHttpSchemeHandler; + +public: + static std::pair Request( + const AString & a_Method, + const AString & a_URL, + cUrlClient::cCallbacks & a_Callbacks, + AStringMap && a_Headers, + const AString & a_Body, + AStringMap && a_Options + ) + { + // Create a new instance of cUrlClientRequest, wrapped in a SharedPtr, so that it has a controlled lifetime. + // Cannot use std::make_shared, because the constructor is not public + SharedPtr ptr (new cUrlClientRequest( + a_Method, a_URL, a_Callbacks, std::move(a_Headers), a_Body, std::move(a_Options) + )); + return ptr->DoRequest(ptr); + } + + + /** Calls the error callback with the specified message, if it exists, and terminates the request. */ + void CallErrorCallback(const AString & a_ErrorMessage) + { + // Call the error callback: + m_Callbacks.OnError(a_ErrorMessage); + + // Terminate the request's TCP link: + auto link = m_Link; + if (link != nullptr) + { + link->Close(); + } + m_Self.reset(); + } + + + cUrlClient::cCallbacks & GetCallbacks() { return m_Callbacks; } + + void RedirectTo(const AString & a_RedirectUrl); + + bool ShouldAllowRedirects() const; + + +protected: + + /** Method to be used for the request */ + AString m_Method; + + /** URL that will be requested. */ + AString m_Url; + + /** Individual components of the URL that will be requested. */ + AString m_UrlScheme, m_UrlUsername, m_UrlPassword, m_UrlHost, m_UrlPath, m_UrlQuery, m_UrlFragment; + UInt16 m_UrlPort; + + /** Callbacks that report progress and results of the request. */ + cUrlClient::cCallbacks & m_Callbacks; + + /** Extra headers to be sent with the request (besides the normal ones). */ + AStringMap m_Headers; + + /** Body to be sent with the request, if any. */ + AString m_Body; + + /** Extra options to be used for the request. */ + AStringMap m_Options; + + /** SharedPtr to self, so that this object can keep itself alive for as long as it needs, + and pass self as callbacks to cNetwork functions. */ + SharedPtr m_Self; + + /** The handler that "talks" the protocol specified in m_UrlScheme, handles all the sending and receiving. */ + SharedPtr m_SchemeHandler; + + /** The link handling the request. */ + cTCPLinkPtr m_Link; + + /** The number of redirect attempts that will still be followed. + If the response specifies a redirect and this is nonzero, the redirect is followed. + If the response specifies a redirect and this is zero, a redirect loop is reported as an error. */ + int m_NumRemainingRedirects; + + + cUrlClientRequest( + const AString & a_Method, + const AString & a_Url, + cUrlClient::cCallbacks & a_Callbacks, + AStringMap && a_Headers, + const AString & a_Body, + AStringMap && a_Options + ): + m_Method(a_Method), + m_Url(a_Url), + m_Callbacks(a_Callbacks), + m_Headers(std::move(a_Headers)), + m_Body(a_Body), + m_Options(std::move(a_Options)) + { + m_NumRemainingRedirects = GetStringMapInteger(m_Options, "MaxRedirects", 30); + } + + + std::pair DoRequest(SharedPtr a_Self); + + + // cNetwork::cConnectCallbacks override: TCP link connected: + virtual void OnConnected(cTCPLink & a_Link) override; + + // cNetwork::cConnectCallbacks override: An error has occurred: + virtual void OnError(int a_ErrorCode, const AString & a_ErrorMsg) override + { + m_Callbacks.OnError(Printf("Network error %d (%s)", a_ErrorCode, a_ErrorMsg.c_str())); + m_Self.reset(); + } + + + // cTCPLink::cCallbacks override: TCP link created + virtual void OnLinkCreated(cTCPLinkPtr a_Link) override + { + m_Link = a_Link; + } + + + /** Called when there's data incoming from the remote peer. */ + virtual void OnReceivedData(const char * a_Data, size_t a_Length) override; + + + /** Called when the remote end closes the connection. + The link is still available for connection information query (IP / port). + Sending data on the link is not an error, but the data won't be delivered. */ + virtual void OnRemoteClosed(void) override; +}; + + + + + +/** Represents a base class for an object that "talks" a specified URL protocol, such as HTTP or FTP. +Also provides a static factory method for creating an instance based on the scheme. +A descendant of this class is created for each request and handles all of the request's aspects, +from right after connecting to the TCP link till the link is closed. +For an example of a specific handler, see the cHttpSchemeHandler class. */ +class cSchemeHandler abstract +{ +public: + cSchemeHandler(cUrlClientRequest & a_ParentRequest): + m_ParentRequest(a_ParentRequest) + { + } + + // Force a virtual destructor in all descendants + virtual ~cSchemeHandler() {} + + /** Creates and returns a new handler for the specified scheme. + a_ParentRequest is the request which is to be handled by the handler. */ + static cSchemeHandlerPtr Create(const AString & a_Scheme, cUrlClientRequest & a_ParentRequest); + + /** Called when the link gets established. */ + virtual void OnConnected(cTCPLink & a_Link) = 0; + + /** Called when there's data incoming from the remote peer. */ + virtual void OnReceivedData(const char * a_Data, size_t a_Length) = 0; + + /** Called when the remote end closes the connection. + The link is still available for connection information query (IP / port). + Sending data on the link is not an error, but the data won't be delivered. */ + virtual void OnRemoteClosed(void) = 0; + +protected: + cUrlClientRequest & m_ParentRequest; +}; + + + + + +/** cSchemeHandler descendant that handles HTTP (and HTTPS) requests. */ +class cHttpSchemeHandler: + public cSchemeHandler, + protected cHTTPMessageParser::cCallbacks +{ + typedef cSchemeHandler Super; + +public: + cHttpSchemeHandler(cUrlClientRequest & a_ParentRequest, bool a_IsTls): + Super(a_ParentRequest), + m_Parser(*this), + m_IsTls(a_IsTls), + m_IsRedirect(false) + { + } + + + virtual void OnConnected(cTCPLink & a_Link) override + { + m_Link = &a_Link; + if (m_IsTls) + { + // TODO: Start TLS + } + else + { + SendRequest(); + } + } + + void SendRequest() + { + // Send the request: + auto requestLine = m_ParentRequest.m_UrlPath; + if (requestLine.empty()) + { + requestLine = "/"; + } + if (!m_ParentRequest.m_UrlQuery.empty()) + { + requestLine.push_back('?'); + requestLine.append(m_ParentRequest.m_UrlQuery); + } + m_Link->Send(Printf("%s %s HTTP/1.1\r\n", m_ParentRequest.m_Method.c_str(), requestLine.c_str())); + m_Link->Send(Printf("Host: %s\r\n", m_ParentRequest.m_UrlHost.c_str())); + m_Link->Send(Printf("Content-Length: %u\r\n", static_cast(m_ParentRequest.m_Body.size()))); + for (auto itr = m_ParentRequest.m_Headers.cbegin(), end = m_ParentRequest.m_Headers.cend(); itr != end; ++itr) + { + m_Link->Send(Printf("%s: %s\r\n", itr->first.c_str(), itr->second.c_str())); + } // for itr - m_Headers[] + m_Link->Send("\r\n", 2); + m_Link->Send(m_ParentRequest.m_Body); + + // Notify the callbacks that the request has been sent: + m_ParentRequest.GetCallbacks().OnRequestSent(); + } + + + virtual void OnReceivedData(const char * a_Data, size_t a_Length) override + { + auto res = m_Parser.Parse(a_Data, a_Length); + if (res == AString::npos) + { + m_ParentRequest.CallErrorCallback("Failed to parse HTTP response"); + return; + } + } + + + virtual void OnRemoteClosed(void) override + { + m_Link = nullptr; + } + + + // cHTTPResponseParser::cCallbacks overrides: + virtual void OnError(const AString & a_ErrorDescription) override + { + m_ParentRequest.CallErrorCallback(a_ErrorDescription); + m_Link = nullptr; + } + + + virtual void OnFirstLine(const AString & a_FirstLine) override + { + // Find the first space, parse the result code between it and the second space: + auto idxFirstSpace = a_FirstLine.find(' '); + if (idxFirstSpace == AString::npos) + { + m_ParentRequest.CallErrorCallback(Printf("Failed to parse HTTP status line \"%s\", no space delimiter.", a_FirstLine.c_str())); + return; + } + auto idxSecondSpace = a_FirstLine.find(' ', idxFirstSpace + 1); + if (idxSecondSpace == AString::npos) + { + m_ParentRequest.CallErrorCallback(Printf("Failed to parse HTTP status line \"%s\", missing second space delimiter.", a_FirstLine.c_str())); + return; + } + int resultCode; + auto resultCodeStr = a_FirstLine.substr(idxFirstSpace + 1, idxSecondSpace - idxFirstSpace - 1); + if (!StringToInteger(resultCodeStr, resultCode)) + { + m_ParentRequest.CallErrorCallback(Printf("Failed to parse HTTP result code from response \"%s\"", resultCodeStr.c_str())); + return; + } + + // Check for redirects, follow if allowed by the options: + switch (resultCode) + { + case cUrlClient::HTTP_STATUS_MULTIPLE_CHOICES: + case cUrlClient::HTTP_STATUS_MOVED_PERMANENTLY: + case cUrlClient::HTTP_STATUS_FOUND: + case cUrlClient::HTTP_STATUS_SEE_OTHER: + case cUrlClient::HTTP_STATUS_TEMPORARY_REDIRECT: + { + m_IsRedirect = true; + return; + } + } + m_ParentRequest.GetCallbacks().OnStatusLine(a_FirstLine.substr(1, idxFirstSpace), resultCode, a_FirstLine.substr(idxSecondSpace + 1)); + } + + + virtual void OnHeaderLine(const AString & a_Key, const AString & a_Value) override + { + if (m_IsRedirect) + { + if (a_Key == "Location") + { + m_RedirectLocation = a_Value; + } + } + else + { + m_ParentRequest.GetCallbacks().OnHeader(a_Key, a_Value); + } + } + + + /** Called when all the headers have been parsed. */ + virtual void OnHeadersFinished(void) override + { + if (!m_IsRedirect) + { + m_ParentRequest.GetCallbacks().OnHeadersFinished(); + } + } + + + /** Called for each chunk of the incoming body data. */ + virtual void OnBodyData(const void * a_Data, size_t a_Size) override + { + if (!m_IsRedirect) + { + m_ParentRequest.GetCallbacks().OnBodyData(a_Data, a_Size); + } + } + + + /** Called when the entire body has been reported by OnBodyData(). */ + virtual void OnBodyFinished(void) override + { + if (m_IsRedirect) + { + if (m_RedirectLocation.empty()) + { + m_ParentRequest.CallErrorCallback("Invalid redirect, there's no location to redirect to"); + } + else + { + m_ParentRequest.RedirectTo(m_RedirectLocation); + } + } + else + { + m_ParentRequest.GetCallbacks().OnBodyFinished(); + } + } + +protected: + + /** The network link. */ + cTCPLink * m_Link; + + /** If true, the TLS should be started on the link before sending the request (used for https). */ + bool m_IsTls; + + /** Parser of the HTTP response message. */ + cHTTPMessageParser m_Parser; + + /** Set to true if the first line contains a redirecting HTTP status code and the options specify to follow redirects. + If true, and the parent request allows redirects, neither headers not the body contents are reported through the callbacks, + and after the entire request is parsed, the redirect is attempted. */ + bool m_IsRedirect; + + /** The Location where the request should be redirected. + Only used when m_IsRedirect is true. */ + AString m_RedirectLocation; +}; + + + + + +//////////////////////////////////////////////////////////////////////////////// +// cSchemeHandler: + +cSchemeHandlerPtr cSchemeHandler::Create(const AString & a_Scheme, cUrlClientRequest & a_ParentRequest) +{ + auto lowerScheme = StrToLower(a_Scheme); + if (lowerScheme == "http") + { + return std::make_shared(a_ParentRequest, false); + } + else if (lowerScheme == "https") + { + return std::make_shared(a_ParentRequest, true); + } + + return nullptr; +} + + + + + +//////////////////////////////////////////////////////////////////////////////// +// cUrlClientRequest: + +void cUrlClientRequest::RedirectTo(const AString & a_RedirectUrl) +{ + // Check that redirection is allowed: + m_Callbacks.OnRedirecting(a_RedirectUrl); + if (!ShouldAllowRedirects()) + { + CallErrorCallback(Printf("Redirect to \"%s\" not allowed", a_RedirectUrl.c_str())); + return; + } + + // Do the actual redirect: + m_Link->Close(); + m_Url = a_RedirectUrl; + m_NumRemainingRedirects = m_NumRemainingRedirects - 1; + auto res = DoRequest(m_Self); + if (!res.first) + { + m_Callbacks.OnError(Printf("Redirection failed: %s", res.second.c_str())); + return; + } +} + + + + + +bool cUrlClientRequest::ShouldAllowRedirects() const +{ + return (m_NumRemainingRedirects > 0); +} + + + + + +void cUrlClientRequest::OnConnected(cTCPLink & a_Link) +{ + m_Callbacks.OnConnected(a_Link); + m_SchemeHandler->OnConnected(a_Link); +} + + + + + +void cUrlClientRequest::OnReceivedData(const char * a_Data, size_t a_Length) +{ + auto handler = m_SchemeHandler; + if (handler != nullptr) + { + handler->OnReceivedData(a_Data, a_Length); + } +} + + + + + +void cUrlClientRequest::OnRemoteClosed() +{ + // Notify the callback: + auto handler = m_SchemeHandler; + if (handler != nullptr) + { + handler->OnRemoteClosed(); + } + + // Let ourselves be deleted + m_Self.reset(); +} + + + + + +std::pair cUrlClientRequest::DoRequest(SharedPtr a_Self) +{ + // We need a shared pointer to self, care must be taken not to pass any other ptr: + ASSERT(a_Self.get() == this); + + m_Self = a_Self; + + // Parse the URL: + auto res = cUrlParser::Parse(m_Url, m_UrlScheme, m_UrlUsername, m_UrlPassword, m_UrlHost, m_UrlPort, m_UrlPath, m_UrlQuery, m_UrlFragment); + if (!res.first) + { + return res; + } + + // Get a handler that will work with the specified scheme: + m_SchemeHandler = cSchemeHandler::Create(m_UrlScheme, *this); + if (m_SchemeHandler == nullptr) + { + return std::make_pair(false, Printf("Unknown Url scheme: %s", m_UrlScheme.c_str())); + } + + if (!cNetwork::Connect(m_UrlHost, m_UrlPort, m_Self, m_Self)) + { + return std::make_pair(false, "Network connection failed"); + } + return std::make_pair(true, AString()); +} + + + + + +//////////////////////////////////////////////////////////////////////////////// +// cUrlClient: + +std::pair cUrlClient::Request( + const AString & a_Method, + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options +) +{ + return cUrlClientRequest::Request( + a_Method, a_URL, a_Callbacks, std::move(a_Headers), std::move(a_Body), std::move(a_Options) + ); +} + + + + + +std::pair cUrlClient::Get( + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap a_Headers, + AString a_Body, + AStringMap a_Options +) +{ + return cUrlClientRequest::Request( + "GET", a_URL, a_Callbacks, std::move(a_Headers), std::move(a_Body), std::move(a_Options) + ); +} + + + + + +std::pair cUrlClient::Post( + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options +) +{ + return cUrlClientRequest::Request( + "POST", a_URL, a_Callbacks, std::move(a_Headers), std::move(a_Body), std::move(a_Options) + ); +} + + + + + +std::pair cUrlClient::Put( + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options +) +{ + return cUrlClientRequest::Request( + "PUT", a_URL, a_Callbacks, std::move(a_Headers), std::move(a_Body), std::move(a_Options) + ); +} + + + + + diff --git a/src/HTTP/UrlClient.h b/src/HTTP/UrlClient.h new file mode 100644 index 000000000..42086a4f1 --- /dev/null +++ b/src/HTTP/UrlClient.h @@ -0,0 +1,141 @@ + +// UrlClient.h + +// Declares the cUrlClient class for high-level URL interaction + +/* +Options that can be set via the Options parameter to the cUrlClient calls: +"MaxRedirects": The maximum number of allowed redirects before the client refuses a redirect with an error + +Behavior: +- If a redirect is received, and redirection is allowed, the redirection is reported via OnRedirecting() callback +and the request is restarted at the redirect URL, without reporting any of the redirect's headers nor body +- If a redirect is received and redirection is not allowed (maximum redirection attempts have been reached), +the OnRedirecting() callback is called with the redirect URL and then the request terminates with an OnError() callback, +without reporting the redirect's headers nor body. +*/ + + + + + +#pragma once + +#include "../OSSupport/Network.h" + + + + + +class cUrlClient +{ +public: + /** Callbacks that are used for progress and result reporting. */ + class cCallbacks + { + public: + /** Called when the TCP connection is established. */ + virtual void OnConnected(cTCPLink & a_Link) {}; + + /** Called for TLS connections, when the server certificate is received. + Return true to continue with the request, false to abort. + The default implementation does nothing and continues with the request. + TODO: The certificate parameter needs a representation! */ + virtual bool OnCertificateReceived() { return true; } + + /** Called after the entire request has been sent to the remote peer. */ + virtual void OnRequestSent() {}; + + /** Called after the first line of the response is parsed, unless the response is an allowed redirect. */ + virtual void OnStatusLine(const AString & a_HttpVersion, int a_StatusCode, const AString & a_Rest) {} + + /** Called when a single HTTP header is received and parsed, unless the response is an allowed redirect + Called once for each incoming header. */ + virtual void OnHeader(const AString & a_Key, const AString & a_Value) {}; + + /** Called when the HTTP headers have been fully parsed, unless the response is an allowed redirect. + There will be no more OnHeader() calls. */ + virtual void OnHeadersFinished() {}; + + /** Called when the next fragment of the response body is received, unless the response is an allowed redirect. + This can be called multiple times, as data arrives over the network. */ + virtual void OnBodyData(const void * a_Data, size_t a_Size) {}; + + /** Called after the response body has been fully reported by OnBody() calls, unless the response is an allowed redirect. + There will be no more OnBody() calls. */ + virtual void OnBodyFinished() {}; + + /** Called when an asynchronous error is encountered. */ + virtual void OnError(const AString & a_ErrorMsg) {}; + + /** Called when a redirect is to be followed. + This is called even if the redirecting is prohibited by the options; in such an event, this call will be + followed by OnError(). + If a response indicates a redirect (and the request allows redirecting), the regular callbacks + OnStatusLine(), OnHeader(), OnHeadersFinished(), OnBodyData() and OnBodyFinished() are not called + for such a response; instead, the redirect is silently attempted. */ + virtual void OnRedirecting(const AString & a_NewLocation) {}; + }; + + + /** Used for HTTP status codes. */ + enum eHTTPStatus + { + HTTP_STATUS_OK = 200, + HTTP_STATUS_MULTIPLE_CHOICES = 300, // MAY have a redirect using the "Location" header + HTTP_STATUS_MOVED_PERMANENTLY = 301, // redirect using the "Location" header + HTTP_STATUS_FOUND = 302, // redirect using the "Location" header + HTTP_STATUS_SEE_OTHER = 303, // redirect using the "Location" header + HTTP_STATUS_TEMPORARY_REDIRECT = 307, // redirect using the "Location" header + }; + + + /** Makes a network request to the specified URL, using the specified method (if applicable). + The response is reported via the a_ResponseCallback callback, in a single call. + The metadata about the response (HTTP headers) are reported via a_InfoCallback before the a_ResponseCallback call. + If there is an asynchronous error, it is reported in via the a_ErrorCallback. + If there is an immediate error (misformatted URL etc.), the function returns false and an error message. + a_Headers contains additional headers to use for the request. + a_Body specifies optional body to include with the request, if applicable. + a_Options contains various options for the request that govern the request behavior, but aren't sent to the server, + such as the proxy server, whether to follow redirects, and client certificate for TLS. */ + static std::pair Request( + const AString & a_Method, + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options + ); + + /** Alias for Request("GET", ...) */ + static std::pair Get( + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap a_Headers = AStringMap(), + AString a_Body = AString(), + AStringMap a_Options = AStringMap() + ); + + /** Alias for Request("POST", ...) */ + static std::pair Post( + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options + ); + + /** Alias for Request("PUT", ...) */ + static std::pair Put( + const AString & a_URL, + cCallbacks & a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options + ); +}; + + + + diff --git a/tests/HTTP/CMakeLists.txt b/tests/HTTP/CMakeLists.txt index ed5c9daaf..1e2eb356a 100644 --- a/tests/HTTP/CMakeLists.txt +++ b/tests/HTTP/CMakeLists.txt @@ -11,6 +11,8 @@ set (HTTP_SRCS ${CMAKE_SOURCE_DIR}/src/HTTP/HTTPMessage.cpp ${CMAKE_SOURCE_DIR}/src/HTTP/HTTPMessageParser.cpp ${CMAKE_SOURCE_DIR}/src/HTTP/TransferEncodingParser.cpp + ${CMAKE_SOURCE_DIR}/src/HTTP/UrlClient.cpp + ${CMAKE_SOURCE_DIR}/src/HTTP/UrlParser.cpp ${CMAKE_SOURCE_DIR}/src/StringUtils.cpp ) @@ -19,13 +21,20 @@ set (HTTP_HDRS ${CMAKE_SOURCE_DIR}/src/HTTP/HTTPMessage.h ${CMAKE_SOURCE_DIR}/src/HTTP/HTTPMessageParser.h ${CMAKE_SOURCE_DIR}/src/HTTP/TransferEncodingParser.h + ${CMAKE_SOURCE_DIR}/src/HTTP/UrlClient.h + ${CMAKE_SOURCE_DIR}/src/HTTP/UrlParser.h ${CMAKE_SOURCE_DIR}/src/StringUtils.h ) +set (SHARED_SRCS + ${CMAKE_SOURCE_DIR}/src/OSSupport/Event.cpp +) + add_library(HTTP ${HTTP_SRCS} ${HTTP_HDRS} ) +target_link_libraries(HTTP Network OSSupport) if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") add_flags_cxx("-Wno-error=conversion -Wno-error=old-style-cast") @@ -35,11 +44,21 @@ endif() -# Define individual tests: +# Define individual test executables: # HTTPMessageParser_file: Feed file contents into a cHTTPResponseParser and print the callbacks as they're called: add_executable(HTTPMessageParser_file-exe HTTPMessageParser_file.cpp) -target_link_libraries(HTTPMessageParser_file-exe HTTP) +target_link_libraries(HTTPMessageParser_file-exe HTTP Network OSSupport) + +# UrlClientTest: Tests the UrlClient class by requesting a few things off the internet: +add_executable(UrlClientTest-exe UrlClientTest.cpp) +target_link_libraries(UrlClientTest-exe HTTP) + + + + + +# Define individual tests: # Test parsing the response file in 2-byte chunks (should go from response line parsing through headers parsing to body parsing, each within a different step): add_test(NAME HTTPMessageParser_file-test1-2 COMMAND HTTPMessageParser_file-exe ${CMAKE_CURRENT_SOURCE_DIR}/HTTPResponse1.data 2) @@ -63,7 +82,8 @@ add_test(NAME HTTPMessageParser_file-test4-512 COMMAND HTTPMessageParser_file-ex # Put all the tests into a solution folder (MSVC): set_target_properties( HTTPMessageParser_file-exe - PROPERTIES FOLDER Tests + UrlClientTest-exe + PROPERTIES FOLDER Tests/HTTP ) set_target_properties( HTTP diff --git a/tests/HTTP/UrlClientTest.cpp b/tests/HTTP/UrlClientTest.cpp new file mode 100644 index 000000000..5f70855fb --- /dev/null +++ b/tests/HTTP/UrlClientTest.cpp @@ -0,0 +1,162 @@ + +#include "Globals.h" +#include "HTTP/UrlClient.h" +#include "OSSupport/NetworkSingleton.h" + + + + + +class cCallbacks: + public cUrlClient::cCallbacks +{ +public: + cCallbacks(cEvent & a_Event): + m_Event(a_Event) + { + } + + virtual void OnConnected(cTCPLink & a_Link) override + { + LOG("Link connected to %s:%u", a_Link.GetRemoteIP().c_str(), a_Link.GetRemotePort()); + } + + virtual bool OnCertificateReceived() override + { + LOG("Server certificate received"); + return true; + } + + virtual void OnRequestSent() override + { + LOG("Request has been sent"); + } + + virtual void OnHeader(const AString & a_Key, const AString & a_Value) override + { + LOG("HTTP Header: \"%s\" -> \"%s\"", a_Key.c_str(), a_Value.c_str()); + } + + virtual void OnHeadersFinished() override + { + LOG("HTTP headers finished."); + } + + virtual void OnBodyData(const void * a_Data, size_t a_Size) override + { + AString body(reinterpret_cast(a_Data), a_Size); + LOG("Body part:\n%s", body.c_str()); + } + + /** Called after the response body has been fully reported by OnBody() calls. + There will be no more OnBody() calls. */ + virtual void OnBodyFinished() override + { + LOG("Body finished."); + m_Event.Set(); + } + + virtual void OnRedirecting(const AString & a_RedirectUrl) override + { + LOG("Redirecting to \"%s\".", a_RedirectUrl.c_str()); + } + + virtual void OnError(const AString & a_ErrorMsg) override + { + LOG("Error: %s", a_ErrorMsg.c_str()); + m_Event.Set(); + } + +protected: + cEvent & m_Event; +}; + + + + + +int TestRequest1() +{ + LOG("Running test 1"); + cEvent evtFinished; + cCallbacks callbacks(evtFinished); + AStringMap options; + options["MaxRedirects"] = "0"; + auto res = cUrlClient::Get("http://github.com", callbacks, AStringMap(), AString(), options); + if (res.first) + { + evtFinished.Wait(); + } + else + { + LOG("Immediate error: %s", res.second.c_str()); + return 1; + } + return 0; +} + + + + + +int TestRequest2() +{ + LOG("Running test 2"); + cEvent evtFinished; + cCallbacks callbacks(evtFinished); + auto res = cUrlClient::Get("http://github.com", callbacks); + if (res.first) + { + evtFinished.Wait(); + } + else + { + LOG("Immediate error: %s", res.second.c_str()); + return 1; + } + return 0; +} + + + + + +int TestRequests() +{ + auto res = TestRequest1(); + if (res != 0) + { + return res; + } + res = TestRequest2(); + if (res != 0) + { + return res; + } + return 0; +} + + + + + +int main() +{ + LOGD("Test started"); + + LOGD("Initializing cNetwork..."); + cNetworkSingleton::Get().Initialise(); + + LOGD("Testing..."); + auto res = TestRequests(); + + LOGD("Terminating cNetwork..."); + cNetworkSingleton::Get().Terminate(); + LOGD("cUrlClient test finished"); + + return res; +} + + + + -- cgit v1.2.3 From 641cb063bc71acc7f29d25b12c8713a8beb2018c Mon Sep 17 00:00:00 2001 From: Mattes D Date: Mon, 22 Aug 2016 19:53:34 +0200 Subject: cTCPLink supports TLS out of the box. --- src/Bindings/LuaTCPLink.cpp | 270 ++++----------------------------------- src/Bindings/LuaTCPLink.h | 54 -------- src/HTTP/UrlClient.cpp | 74 ++++++++++- src/HTTP/UrlClient.h | 28 ++-- src/OSSupport/Network.h | 33 +++++ src/OSSupport/TCPLinkImpl.cpp | 291 +++++++++++++++++++++++++++++++++++++++++- src/OSSupport/TCPLinkImpl.h | 75 +++++++++++ tests/HTTP/CMakeLists.txt | 4 + tests/HTTP/UrlClientTest.cpp | 105 +++++++++++++-- tests/Network/CMakeLists.txt | 13 +- 10 files changed, 618 insertions(+), 329 deletions(-) diff --git a/src/Bindings/LuaTCPLink.cpp b/src/Bindings/LuaTCPLink.cpp index 466240a7d..905dfc5ac 100644 --- a/src/Bindings/LuaTCPLink.cpp +++ b/src/Bindings/LuaTCPLink.cpp @@ -6,6 +6,8 @@ #include "Globals.h" #include "LuaTCPLink.h" #include "LuaServerHandle.h" +#include "../PolarSSL++/X509Cert.h" +#include "../PolarSSL++/CryptoKey.h" @@ -48,13 +50,6 @@ cLuaTCPLink::~cLuaTCPLink() bool cLuaTCPLink::Send(const AString & a_Data) { - // If running in SSL mode, push the data into the SSL context instead: - if (m_SslContext != nullptr) - { - m_SslContext->Send(a_Data); - return true; - } - // Safely grab a copy of the link: auto link = m_Link; if (link == nullptr) @@ -144,12 +139,6 @@ void cLuaTCPLink::Shutdown(void) cTCPLinkPtr link = m_Link; if (link != nullptr) { - if (m_SslContext != nullptr) - { - m_SslContext->NotifyClose(); - m_SslContext->ResetSelf(); - m_SslContext.reset(); - } link->Shutdown(); } } @@ -164,12 +153,6 @@ void cLuaTCPLink::Close(void) cTCPLinkPtr link = m_Link; if (link != nullptr) { - if (m_SslContext != nullptr) - { - m_SslContext->NotifyClose(); - m_SslContext->ResetSelf(); - m_SslContext.reset(); - } link->Close(); } @@ -186,46 +169,31 @@ AString cLuaTCPLink::StartTLSClient( const AString & a_OwnPrivKeyPassword ) { - // Check preconditions: - if (m_SslContext != nullptr) - { - return "TLS is already active on this link"; - } - if ( - (a_OwnCertData.empty() && !a_OwnPrivKeyData.empty()) || - (!a_OwnCertData.empty() && a_OwnPrivKeyData.empty()) - ) - { - return "Either provide both the certificate and private key, or neither"; - } - - // Create the SSL context: - m_SslContext.reset(new cLinkSslContext(*this)); - m_SslContext->Initialize(true); - - // Create the peer cert, if required: - if (!a_OwnCertData.empty() && !a_OwnPrivKeyData.empty()) + auto link = m_Link; + if (link != nullptr) { - auto OwnCert = std::make_shared(); - int res = OwnCert->Parse(a_OwnCertData.data(), a_OwnCertData.size()); - if (res != 0) + cX509CertPtr ownCert; + if (!a_OwnCertData.empty()) { - m_SslContext.reset(); - return Printf("Cannot parse peer certificate: -0x%x", res); + ownCert = std::make_shared(); + auto res = ownCert->Parse(a_OwnCertData.data(), a_OwnCertData.size()); + if (res != 0) + { + return Printf("Cannot parse client certificate: -0x%x", res); + } } - auto OwnPrivKey = std::make_shared(); - res = OwnPrivKey->ParsePrivate(a_OwnPrivKeyData.data(), a_OwnPrivKeyData.size(), a_OwnPrivKeyPassword); - if (res != 0) + cCryptoKeyPtr ownPrivKey; + if (!a_OwnPrivKeyData.empty()) { - m_SslContext.reset(); - return Printf("Cannot parse peer private key: -0x%x", res); + ownPrivKey = std::make_shared(); + auto res = ownPrivKey->ParsePrivate(a_OwnPrivKeyData.data(), a_OwnPrivKeyData.size(), a_OwnPrivKeyPassword); + if (res != 0) + { + return Printf("Cannot parse client private key: -0x%x", res); + } } - m_SslContext->SetOwnCert(OwnCert, OwnPrivKey); + return link->StartTLSClient(ownCert, ownPrivKey); } - m_SslContext->SetSelf(cLinkSslContextWPtr(m_SslContext)); - - // Start the handshake: - m_SslContext->Handshake(); return ""; } @@ -240,43 +208,25 @@ AString cLuaTCPLink::StartTLSServer( const AString & a_StartTLSData ) { - // Check preconditions: - if (m_SslContext != nullptr) - { - return "TLS is already active on this link"; - } - if (a_OwnCertData.empty() || a_OwnPrivKeyData.empty()) + auto link = m_Link; + if (link != nullptr) { - return "Provide the server certificate and private key"; - } - - // Create the SSL context: - m_SslContext.reset(new cLinkSslContext(*this)); - m_SslContext->Initialize(false); - // Create the peer cert: auto OwnCert = std::make_shared(); int res = OwnCert->Parse(a_OwnCertData.data(), a_OwnCertData.size()); if (res != 0) { - m_SslContext.reset(); return Printf("Cannot parse server certificate: -0x%x", res); } auto OwnPrivKey = std::make_shared(); res = OwnPrivKey->ParsePrivate(a_OwnPrivKeyData.data(), a_OwnPrivKeyData.size(), a_OwnPrivKeyPassword); if (res != 0) { - m_SslContext.reset(); return Printf("Cannot parse server private key: -0x%x", res); } - m_SslContext->SetOwnCert(OwnCert, OwnPrivKey); - m_SslContext->SetSelf(cLinkSslContextWPtr(m_SslContext)); - // Push the initial data: - m_SslContext->StoreReceivedData(a_StartTLSData.data(), a_StartTLSData.size()); - - // Start the handshake: - m_SslContext->Handshake(); + return link->StartTLSServer(OwnCert, OwnPrivKey, a_StartTLSData); + } return ""; } @@ -308,9 +258,6 @@ void cLuaTCPLink::Terminated(void) m_Link.reset(); } } - - // If the SSL context still exists, free it: - m_SslContext.reset(); } @@ -362,14 +309,6 @@ void cLuaTCPLink::OnLinkCreated(cTCPLinkPtr a_Link) void cLuaTCPLink::OnReceivedData(const char * a_Data, size_t a_Length) { - // If we're running in SSL mode, put the data into the SSL decryptor: - auto sslContext = m_SslContext; - if (sslContext != nullptr) - { - sslContext->StoreReceivedData(a_Data, a_Length); - return; - } - // Call the callback: m_Callbacks->CallTableFn("OnReceivedData", this, AString(a_Data, a_Length)); } @@ -380,13 +319,6 @@ void cLuaTCPLink::OnReceivedData(const char * a_Data, size_t a_Length) void cLuaTCPLink::OnRemoteClosed(void) { - // If running in SSL mode and there's data left in the SSL contect, report it: - auto sslContext = m_SslContext; - if (sslContext != nullptr) - { - sslContext->FlushBuffers(); - } - // Call the callback: m_Callbacks->CallTableFn("OnRemoteClosed", this); @@ -398,155 +330,3 @@ void cLuaTCPLink::OnRemoteClosed(void) -//////////////////////////////////////////////////////////////////////////////// -// cLuaTCPLink::cLinkSslContext: - -cLuaTCPLink::cLinkSslContext::cLinkSslContext(cLuaTCPLink & a_Link): - m_Link(a_Link) -{ -} - - - - - -void cLuaTCPLink::cLinkSslContext::SetSelf(cLinkSslContextWPtr a_Self) -{ - m_Self = a_Self; -} - - - - - -void cLuaTCPLink::cLinkSslContext::ResetSelf(void) -{ - m_Self.reset(); -} - - - - - -void cLuaTCPLink::cLinkSslContext::StoreReceivedData(const char * a_Data, size_t a_NumBytes) -{ - // Hold self alive for the duration of this function - cLinkSslContextPtr Self(m_Self); - - m_EncryptedData.append(a_Data, a_NumBytes); - - // Try to finish a pending handshake: - TryFinishHandshaking(); - - // Flush any cleartext data that can be "received": - FlushBuffers(); -} - - - - - -void cLuaTCPLink::cLinkSslContext::FlushBuffers(void) -{ - // Hold self alive for the duration of this function - cLinkSslContextPtr Self(m_Self); - - // If the handshake didn't complete yet, bail out: - if (!HasHandshaken()) - { - return; - } - - char Buffer[1024]; - int NumBytes; - while ((NumBytes = ReadPlain(Buffer, sizeof(Buffer))) > 0) - { - m_Link.ReceivedCleartextData(Buffer, static_cast(NumBytes)); - if (m_Self.expired()) - { - // The callback closed the SSL context, bail out - return; - } - } -} - - - - - -void cLuaTCPLink::cLinkSslContext::TryFinishHandshaking(void) -{ - // Hold self alive for the duration of this function - cLinkSslContextPtr Self(m_Self); - - // If the handshake hasn't finished yet, retry: - if (!HasHandshaken()) - { - Handshake(); - } - - // If the handshake succeeded, write all the queued plaintext data: - if (HasHandshaken()) - { - WritePlain(m_CleartextData.data(), m_CleartextData.size()); - m_CleartextData.clear(); - } -} - - - - - -void cLuaTCPLink::cLinkSslContext::Send(const AString & a_Data) -{ - // Hold self alive for the duration of this function - cLinkSslContextPtr Self(m_Self); - - // If the handshake hasn't completed yet, queue the data: - if (!HasHandshaken()) - { - m_CleartextData.append(a_Data); - TryFinishHandshaking(); - return; - } - - // The connection is all set up, write the cleartext data into the SSL context: - WritePlain(a_Data.data(), a_Data.size()); - FlushBuffers(); -} - - - - - -int cLuaTCPLink::cLinkSslContext::ReceiveEncrypted(unsigned char * a_Buffer, size_t a_NumBytes) -{ - // Hold self alive for the duration of this function - cLinkSslContextPtr Self(m_Self); - - // If there's nothing queued in the buffer, report empty buffer: - if (m_EncryptedData.empty()) - { - return POLARSSL_ERR_NET_WANT_READ; - } - - // Copy as much data as possible to the provided buffer: - size_t BytesToCopy = std::min(a_NumBytes, m_EncryptedData.size()); - memcpy(a_Buffer, m_EncryptedData.data(), BytesToCopy); - m_EncryptedData.erase(0, BytesToCopy); - return static_cast(BytesToCopy); -} - - - - - -int cLuaTCPLink::cLinkSslContext::SendEncrypted(const unsigned char * a_Buffer, size_t a_NumBytes) -{ - m_Link.m_Link->Send(a_Buffer, a_NumBytes); - return static_cast(a_NumBytes); -} - - - - diff --git a/src/Bindings/LuaTCPLink.h b/src/Bindings/LuaTCPLink.h index b8c886ef8..f4ca67018 100644 --- a/src/Bindings/LuaTCPLink.h +++ b/src/Bindings/LuaTCPLink.h @@ -10,7 +10,6 @@ #pragma once #include "../OSSupport/Network.h" -#include "../PolarSSL++/SslContext.h" #include "LuaState.h" @@ -90,54 +89,6 @@ public: ); protected: - // fwd: - class cLinkSslContext; - typedef SharedPtr cLinkSslContextPtr; - typedef WeakPtr cLinkSslContextWPtr; - - /** Wrapper around cSslContext that is used when this link is being encrypted by SSL. */ - class cLinkSslContext : - public cSslContext - { - cLuaTCPLink & m_Link; - - /** Buffer for storing the incoming encrypted data until it is requested by the SSL decryptor. */ - AString m_EncryptedData; - - /** Buffer for storing the outgoing cleartext data until the link has finished handshaking. */ - AString m_CleartextData; - - /** Shared ownership of self, so that this object can keep itself alive for as long as it needs. */ - cLinkSslContextWPtr m_Self; - - public: - cLinkSslContext(cLuaTCPLink & a_Link); - - /** Shares ownership of self, so that this object can keep itself alive for as long as it needs. */ - void SetSelf(cLinkSslContextWPtr a_Self); - - /** Removes the self ownership so that we can detect the SSL closure. */ - void ResetSelf(void); - - /** Stores the specified block of data into the buffer of the data to be decrypted (incoming from remote). - Also flushes the SSL buffers by attempting to read any data through the SSL context. */ - void StoreReceivedData(const char * a_Data, size_t a_NumBytes); - - /** Tries to read any cleartext data available through the SSL, reports it in the link. */ - void FlushBuffers(void); - - /** Tries to finish handshaking the SSL. */ - void TryFinishHandshaking(void); - - /** Sends the specified cleartext data over the SSL to the remote peer. - If the handshake hasn't been completed yet, queues the data for sending when it completes. */ - void Send(const AString & a_Data); - - // cSslContext overrides: - virtual int ReceiveEncrypted(unsigned char * a_Buffer, size_t a_NumBytes) override; - virtual int SendEncrypted(const unsigned char * a_Buffer, size_t a_NumBytes) override; - }; - /** The Lua table that holds the callbacks to be invoked. */ cLuaState::cTableRefPtr m_Callbacks; @@ -149,11 +100,6 @@ protected: /** The server that is responsible for this link, if any. */ cLuaServerHandleWPtr m_Server; - /** The SSL context used for encryption, if this link uses SSL. - If valid, the link uses encryption through this context. */ - cLinkSslContextPtr m_SslContext; - - /** Common code called when the link is considered as terminated. Releases m_Link, m_Callbacks and this from m_Server, each when applicable. */ void Terminated(void); diff --git a/src/HTTP/UrlClient.cpp b/src/HTTP/UrlClient.cpp index f9e642b22..9346882f1 100644 --- a/src/HTTP/UrlClient.cpp +++ b/src/HTTP/UrlClient.cpp @@ -7,6 +7,8 @@ #include "UrlClient.h" #include "UrlParser.h" #include "HTTPMessageParser.h" +#include "../PolarSSL++/X509Cert.h" +#include "../PolarSSL++/CryptoKey.h" @@ -67,6 +69,38 @@ public: bool ShouldAllowRedirects() const; + cX509CertPtr GetOwnCert() const + { + auto itr = m_Options.find("OwnCert"); + if (itr == m_Options.end()) + { + return nullptr; + } + cX509CertPtr cert; + if (!cert->Parse(itr->second.data(), itr->second.size())) + { + LOGD("OwnCert failed to parse"); + return nullptr; + } + return cert; + } + + cCryptoKeyPtr GetOwnPrivKey() const + { + auto itr = m_Options.find("OwnPrivKey"); + if (itr == m_Options.end()) + { + return nullptr; + } + cCryptoKeyPtr key; + auto passItr = m_Options.find("OwnPrivKeyPassword"); + auto pass = (passItr == m_Options.end()) ? AString() : passItr->second; + if (!key->ParsePrivate(itr->second.data(), itr->second.size(), pass)) + { + return nullptr; + } + return key; + } protected: @@ -148,6 +182,9 @@ protected: } + // cTCPLink::cCallbacks override: TLS handshake completed on the link: + virtual void OnTlsHandshakeCompleted(void) override; + /** Called when there's data incoming from the remote peer. */ virtual void OnReceivedData(const char * a_Data, size_t a_Length) override; @@ -188,6 +225,9 @@ public: /** Called when there's data incoming from the remote peer. */ virtual void OnReceivedData(const char * a_Data, size_t a_Length) = 0; + /** Called when the TLS handshake has completed on the underlying link. */ + virtual void OnTlsHandshakeCompleted(void) = 0; + /** Called when the remote end closes the connection. The link is still available for connection information query (IP / port). Sending data on the link is not an error, but the data won't be delivered. */ @@ -223,7 +263,7 @@ public: m_Link = &a_Link; if (m_IsTls) { - // TODO: Start TLS + m_Link->StartTLSClient(m_ParentRequest.GetOwnCert(), m_ParentRequest.GetOwnPrivKey()); } else { @@ -231,9 +271,12 @@ public: } } + + /** Sends the HTTP request over the link. + Common code for both HTTP and HTTPS. */ void SendRequest() { - // Send the request: + // Compose the request line: auto requestLine = m_ParentRequest.m_UrlPath; if (requestLine.empty()) { @@ -245,6 +288,8 @@ public: requestLine.append(m_ParentRequest.m_UrlQuery); } m_Link->Send(Printf("%s %s HTTP/1.1\r\n", m_ParentRequest.m_Method.c_str(), requestLine.c_str())); + + // Send the headers: m_Link->Send(Printf("Host: %s\r\n", m_ParentRequest.m_UrlHost.c_str())); m_Link->Send(Printf("Content-Length: %u\r\n", static_cast(m_ParentRequest.m_Body.size()))); for (auto itr = m_ParentRequest.m_Headers.cbegin(), end = m_ParentRequest.m_Headers.cend(); itr != end; ++itr) @@ -252,6 +297,8 @@ public: m_Link->Send(Printf("%s: %s\r\n", itr->first.c_str(), itr->second.c_str())); } // for itr - m_Headers[] m_Link->Send("\r\n", 2); + + // Send the body: m_Link->Send(m_ParentRequest.m_Body); // Notify the callbacks that the request has been sent: @@ -270,6 +317,12 @@ public: } + virtual void OnTlsHandshakeCompleted(void) override + { + SendRequest(); + } + + virtual void OnRemoteClosed(void) override { m_Link = nullptr; @@ -385,12 +438,12 @@ protected: /** The network link. */ cTCPLink * m_Link; - /** If true, the TLS should be started on the link before sending the request (used for https). */ - bool m_IsTls; - /** Parser of the HTTP response message. */ cHTTPMessageParser m_Parser; + /** If true, the TLS should be started on the link before sending the request (used for https). */ + bool m_IsTls; + /** Set to true if the first line contains a redirecting HTTP status code and the options specify to follow redirects. If true, and the parent request allows redirects, neither headers not the body contents are reported through the callbacks, and after the entire request is parsed, the redirect is attempted. */ @@ -475,6 +528,17 @@ void cUrlClientRequest::OnConnected(cTCPLink & a_Link) +void cUrlClientRequest::OnTlsHandshakeCompleted(void) +{ + // Notify the scheme handler and the callbacks: + m_SchemeHandler->OnTlsHandshakeCompleted(); + m_Callbacks.OnTlsHandshakeCompleted(); +} + + + + + void cUrlClientRequest::OnReceivedData(const char * a_Data, size_t a_Length) { auto handler = m_SchemeHandler; diff --git a/src/HTTP/UrlClient.h b/src/HTTP/UrlClient.h index 42086a4f1..652cc76f7 100644 --- a/src/HTTP/UrlClient.h +++ b/src/HTTP/UrlClient.h @@ -5,7 +5,10 @@ /* Options that can be set via the Options parameter to the cUrlClient calls: -"MaxRedirects": The maximum number of allowed redirects before the client refuses a redirect with an error +"MaxRedirects": The maximum number of allowed redirects before the client refuses a redirect with an error +"OwnCert": The client certificate to use, if requested by the server. Any string that can be parsed by cX509Cert. +"OwnPrivKey": The private key appropriate for OwnCert. Any string that can be parsed by cCryptoKey. +"OwnPrivKeyPassword": The password for OwnPrivKey. If not present or empty, no password is assumed. Behavior: - If a redirect is received, and redirection is allowed, the redirection is reported via OnRedirecting() callback @@ -34,8 +37,11 @@ public: class cCallbacks { public: + // Force a virtual destructor in descendants: + virtual ~cCallbacks() {} + /** Called when the TCP connection is established. */ - virtual void OnConnected(cTCPLink & a_Link) {}; + virtual void OnConnected(cTCPLink & a_Link) {} /** Called for TLS connections, when the server certificate is received. Return true to continue with the request, false to abort. @@ -43,30 +49,34 @@ public: TODO: The certificate parameter needs a representation! */ virtual bool OnCertificateReceived() { return true; } + /** Called for TLS connections, when the TLS handshake has been completed. + An empty default implementation is provided so that clients don't need to reimplement it unless they are interested in the event. */ + virtual void OnTlsHandshakeCompleted() { } + /** Called after the entire request has been sent to the remote peer. */ - virtual void OnRequestSent() {}; + virtual void OnRequestSent() {} /** Called after the first line of the response is parsed, unless the response is an allowed redirect. */ virtual void OnStatusLine(const AString & a_HttpVersion, int a_StatusCode, const AString & a_Rest) {} /** Called when a single HTTP header is received and parsed, unless the response is an allowed redirect Called once for each incoming header. */ - virtual void OnHeader(const AString & a_Key, const AString & a_Value) {}; + virtual void OnHeader(const AString & a_Key, const AString & a_Value) {} /** Called when the HTTP headers have been fully parsed, unless the response is an allowed redirect. There will be no more OnHeader() calls. */ - virtual void OnHeadersFinished() {}; + virtual void OnHeadersFinished() {} /** Called when the next fragment of the response body is received, unless the response is an allowed redirect. This can be called multiple times, as data arrives over the network. */ - virtual void OnBodyData(const void * a_Data, size_t a_Size) {}; + virtual void OnBodyData(const void * a_Data, size_t a_Size) {} /** Called after the response body has been fully reported by OnBody() calls, unless the response is an allowed redirect. There will be no more OnBody() calls. */ - virtual void OnBodyFinished() {}; + virtual void OnBodyFinished() {} /** Called when an asynchronous error is encountered. */ - virtual void OnError(const AString & a_ErrorMsg) {}; + virtual void OnError(const AString & a_ErrorMsg) {} /** Called when a redirect is to be followed. This is called even if the redirecting is prohibited by the options; in such an event, this call will be @@ -74,7 +84,7 @@ public: If a response indicates a redirect (and the request allows redirecting), the regular callbacks OnStatusLine(), OnHeader(), OnHeadersFinished(), OnBodyData() and OnBodyFinished() are not called for such a response; instead, the redirect is silently attempted. */ - virtual void OnRedirecting(const AString & a_NewLocation) {}; + virtual void OnRedirecting(const AString & a_NewLocation) {} }; diff --git a/src/OSSupport/Network.h b/src/OSSupport/Network.h index 1162d7fc6..78c5e92f0 100644 --- a/src/OSSupport/Network.h +++ b/src/OSSupport/Network.h @@ -20,6 +20,11 @@ typedef std::vector cTCPLinkPtrs; class cServerHandle; typedef SharedPtr cServerHandlePtr; typedef std::vector cServerHandlePtrs; +class cCryptoKey; +typedef SharedPtr cCryptoKeyPtr; +class cX509Cert; +typedef SharedPtr cX509CertPtr; + @@ -49,6 +54,10 @@ public: Sending data on the link is not an error, but the data won't be delivered. */ virtual void OnRemoteClosed(void) = 0; + /** Called when the TLS handshake has been completed and communication can continue regularly. + Has an empty default implementation, so that link callback descendants don't need to specify TLS handlers when they don't use TLS at all. */ + virtual void OnTlsHandshakeCompleted(void) {} + /** Called when an error is detected on the connection. */ virtual void OnError(int a_ErrorCode, const AString & a_ErrorMsg) = 0; }; @@ -90,6 +99,30 @@ public: Sends the RST packet, queued outgoing and incoming data is lost. */ virtual void Close(void) = 0; + /** Starts a TLS handshake as a client connection. + If a client certificate should be used for the connection, set the certificate into a_OwnCertData and + its corresponding private key to a_OwnPrivKeyData. If both are empty, no client cert is presented. + a_OwnPrivKeyPassword is the password to be used for decoding PrivKey, empty if not passworded. + Returns empty string on success, non-empty error description on failure. */ + virtual AString StartTLSClient( + cX509CertPtr a_OwnCert, + cCryptoKeyPtr a_OwnPrivKey + ) = 0; + + /** Starts a TLS handshake as a server connection. + Set the server certificate into a_CertData and its corresponding private key to a_OwnPrivKeyData. + a_OwnPrivKeyPassword is the password to be used for decoding PrivKey, empty if not passworded. + a_StartTLSData is any data that should be pushed into the TLS before reading more data from the remote. + This is used mainly for protocols starting TLS in the middle of communication, when the TLS start command + can be received together with the TLS Client Hello message in one OnReceivedData() call, to re-queue the + Client Hello message into the TLS handshake buffer. + Returns empty string on success, non-empty error description on failure. */ + virtual AString StartTLSServer( + cX509CertPtr a_OwnCert, + cCryptoKeyPtr a_OwnPrivKey, + const AString & a_StartTLSData + ) = 0; + /** Returns the callbacks that are used. */ cCallbacksPtr GetCallbacks(void) const { return m_Callbacks; } diff --git a/src/OSSupport/TCPLinkImpl.cpp b/src/OSSupport/TCPLinkImpl.cpp index b15b6282f..df70f3f72 100644 --- a/src/OSSupport/TCPLinkImpl.cpp +++ b/src/OSSupport/TCPLinkImpl.cpp @@ -48,6 +48,9 @@ cTCPLinkImpl::cTCPLinkImpl(evutil_socket_t a_Socket, cTCPLink::cCallbacksPtr a_L cTCPLinkImpl::~cTCPLinkImpl() { + // If the TLS context still exists, free it: + m_TlsContext.reset(); + bufferevent_free(m_BufferEvent); } @@ -129,7 +132,16 @@ bool cTCPLinkImpl::Send(const void * a_Data, size_t a_Length) LOGD("%s: Cannot send data, the link is already shut down.", __FUNCTION__); return false; } - return (bufferevent_write(m_BufferEvent, a_Data, a_Length) == 0); + + // If running in TLS mode, push the data into the TLS context instead: + if (m_TlsContext != nullptr) + { + m_TlsContext->Send(a_Data, a_Length); + return true; + } + + // Send the data: + return SendRaw(a_Data, a_Length); } @@ -138,6 +150,14 @@ bool cTCPLinkImpl::Send(const void * a_Data, size_t a_Length) void cTCPLinkImpl::Shutdown(void) { + // If running in TLS mode, notify the TLS layer: + if (m_TlsContext != nullptr) + { + m_TlsContext->NotifyClose(); + m_TlsContext->ResetSelf(); + m_TlsContext.reset(); + } + // If there's no outgoing data, shutdown the socket directly: if (evbuffer_get_length(bufferevent_get_output(m_BufferEvent)) == 0) { @@ -155,6 +175,14 @@ void cTCPLinkImpl::Shutdown(void) void cTCPLinkImpl::Close(void) { + // If running in TLS mode, notify the TLS layer: + if (m_TlsContext != nullptr) + { + m_TlsContext->NotifyClose(); + m_TlsContext->ResetSelf(); + m_TlsContext.reset(); + } + // Disable all events on the socket, but keep it alive: bufferevent_disable(m_BufferEvent, EV_READ | EV_WRITE); if (m_Server == nullptr) @@ -173,18 +201,98 @@ void cTCPLinkImpl::Close(void) +AString cTCPLinkImpl::StartTLSClient( + cX509CertPtr a_OwnCert, + cCryptoKeyPtr a_OwnPrivKey +) +{ + // Check preconditions: + if (m_TlsContext != nullptr) + { + return "TLS is already active on this link"; + } + if ( + ((a_OwnCert == nullptr) && (a_OwnPrivKey != nullptr)) || + ((a_OwnCert != nullptr) && (a_OwnPrivKey != nullptr)) + ) + { + return "Either provide both the certificate and private key, or neither"; + } + + // Create the TLS context: + m_TlsContext.reset(new cLinkTlsContext(*this)); + m_TlsContext->Initialize(true); + if (a_OwnCert != nullptr) + { + m_TlsContext->SetOwnCert(a_OwnCert, a_OwnPrivKey); + } + m_TlsContext->SetSelf(cLinkTlsContextWPtr(m_TlsContext)); + + // Start the handshake: + m_TlsContext->Handshake(); + return ""; +} + + + + + +AString cTCPLinkImpl::StartTLSServer( + cX509CertPtr a_OwnCert, + cCryptoKeyPtr a_OwnPrivKey, + const AString & a_StartTLSData +) +{ + // Check preconditions: + if (m_TlsContext != nullptr) + { + return "TLS is already active on this link"; + } + if ((a_OwnCert == nullptr) || (a_OwnPrivKey == nullptr)) + { + return "Provide the server certificate and private key"; + } + + // Create the TLS context: + m_TlsContext.reset(new cLinkTlsContext(*this)); + m_TlsContext->Initialize(false); + m_TlsContext->SetOwnCert(a_OwnCert, a_OwnPrivKey); + m_TlsContext->SetSelf(cLinkTlsContextWPtr(m_TlsContext)); + + // Push the initial data: + m_TlsContext->StoreReceivedData(a_StartTLSData.data(), a_StartTLSData.size()); + + // Start the handshake: + m_TlsContext->Handshake(); + return ""; +} + + + + + void cTCPLinkImpl::ReadCallback(bufferevent * a_BufferEvent, void * a_Self) { ASSERT(a_Self != nullptr); cTCPLinkImpl * Self = static_cast(a_Self); + ASSERT(Self->m_BufferEvent == a_BufferEvent); ASSERT(Self->m_Callbacks != nullptr); // Read all the incoming data, in 1024-byte chunks: char data[1024]; size_t length; + auto tlsContext = Self->m_TlsContext; while ((length = bufferevent_read(a_BufferEvent, data, sizeof(data))) > 0) { - Self->m_Callbacks->OnReceivedData(data, length); + if (tlsContext != nullptr) + { + ASSERT(tlsContext->IsLink(Self)); + tlsContext->StoreReceivedData(data, length); + } + else + { + Self->ReceivedCleartextData(data, length); + } } } @@ -262,6 +370,13 @@ void cTCPLinkImpl::EventCallback(bufferevent * a_BufferEvent, short a_What, void // If the connection has been closed, call the link callback and remove the connection: if (a_What & BEV_EVENT_EOF) { + // If running in TLS mode and there's data left in the TLS contect, report it: + auto tlsContext = Self->m_TlsContext; + if (tlsContext != nullptr) + { + tlsContext->FlushBuffers(); + } + Self->m_Callbacks->OnRemoteClosed(); if (Self->m_Server != nullptr) { @@ -357,6 +472,178 @@ void cTCPLinkImpl::DoActualShutdown(void) +bool cTCPLinkImpl::SendRaw(const void * a_Data, size_t a_Length) +{ + return (bufferevent_write(m_BufferEvent, a_Data, a_Length) == 0); +} + + + + + +void cTCPLinkImpl::ReceivedCleartextData(const char * a_Data, size_t a_Length) +{ + ASSERT(m_Callbacks != nullptr); + m_Callbacks->OnReceivedData(a_Data, a_Length); +} + + + + + +//////////////////////////////////////////////////////////////////////////////// +// cTCPLinkImpl::cLinkTlsContext: + +cTCPLinkImpl::cLinkTlsContext::cLinkTlsContext(cTCPLinkImpl & a_Link): + m_Link(a_Link) +{ +} + + + + + +void cTCPLinkImpl::cLinkTlsContext::SetSelf(cLinkTlsContextWPtr a_Self) +{ + m_Self = a_Self; +} + + + + + +void cTCPLinkImpl::cLinkTlsContext::ResetSelf(void) +{ + m_Self.reset(); +} + + + + + +void cTCPLinkImpl::cLinkTlsContext::StoreReceivedData(const char * a_Data, size_t a_NumBytes) +{ + // Hold self alive for the duration of this function + cLinkTlsContextPtr Self(m_Self); + + m_EncryptedData.append(a_Data, a_NumBytes); + + // Try to finish a pending handshake: + TryFinishHandshaking(); + + // Flush any cleartext data that can be "received": + FlushBuffers(); +} + + + + + +void cTCPLinkImpl::cLinkTlsContext::FlushBuffers(void) +{ + // Hold self alive for the duration of this function + cLinkTlsContextPtr Self(m_Self); + + // If the handshake didn't complete yet, bail out: + if (!HasHandshaken()) + { + return; + } + + char Buffer[1024]; + int NumBytes; + while ((NumBytes = ReadPlain(Buffer, sizeof(Buffer))) > 0) + { + m_Link.ReceivedCleartextData(Buffer, static_cast(NumBytes)); + if (m_Self.expired()) + { + // The callback closed the SSL context, bail out + return; + } + } +} + + + + + +void cTCPLinkImpl::cLinkTlsContext::TryFinishHandshaking(void) +{ + // Hold self alive for the duration of this function + cLinkTlsContextPtr Self(m_Self); + + // If the handshake hasn't finished yet, retry: + if (!HasHandshaken()) + { + Handshake(); + // If the handshake succeeded, write all the queued plaintext data: + if (HasHandshaken()) + { + m_Link.GetCallbacks()->OnTlsHandshakeCompleted(); + WritePlain(m_CleartextData.data(), m_CleartextData.size()); + m_CleartextData.clear(); + } + } +} + + + + + +void cTCPLinkImpl::cLinkTlsContext::Send(const void * a_Data, size_t a_Length) +{ + // Hold self alive for the duration of this function + cLinkTlsContextPtr Self(m_Self); + + // If the handshake hasn't completed yet, queue the data: + if (!HasHandshaken()) + { + m_CleartextData.append(reinterpret_cast(a_Data), a_Length); + TryFinishHandshaking(); + return; + } + + // The connection is all set up, write the cleartext data into the SSL context: + WritePlain(a_Data, a_Length); + FlushBuffers(); +} + + + + + +int cTCPLinkImpl::cLinkTlsContext::ReceiveEncrypted(unsigned char * a_Buffer, size_t a_NumBytes) +{ + // Hold self alive for the duration of this function + cLinkTlsContextPtr Self(m_Self); + + // If there's nothing queued in the buffer, report empty buffer: + if (m_EncryptedData.empty()) + { + return POLARSSL_ERR_NET_WANT_READ; + } + + // Copy as much data as possible to the provided buffer: + size_t BytesToCopy = std::min(a_NumBytes, m_EncryptedData.size()); + memcpy(a_Buffer, m_EncryptedData.data(), BytesToCopy); + m_EncryptedData.erase(0, BytesToCopy); + return static_cast(BytesToCopy); +} + + + + + +int cTCPLinkImpl::cLinkTlsContext::SendEncrypted(const unsigned char * a_Buffer, size_t a_NumBytes) +{ + m_Link.SendRaw(a_Buffer, a_NumBytes); + return static_cast(a_NumBytes); +} + + + + + //////////////////////////////////////////////////////////////////////////////// // cNetwork API: diff --git a/src/OSSupport/TCPLinkImpl.h b/src/OSSupport/TCPLinkImpl.h index bea21aeff..b54c1a2cc 100644 --- a/src/OSSupport/TCPLinkImpl.h +++ b/src/OSSupport/TCPLinkImpl.h @@ -14,6 +14,7 @@ #include "Network.h" #include #include +#include "../PolarSSL++/SslContext.h" @@ -64,9 +65,73 @@ public: virtual UInt16 GetRemotePort(void) const override { return m_RemotePort; } virtual void Shutdown(void) override; virtual void Close(void) override; + virtual AString StartTLSClient( + cX509CertPtr a_OwnCert, + cCryptoKeyPtr a_OwnPrivKey + ) override; + virtual AString StartTLSServer( + cX509CertPtr a_OwnCert, + cCryptoKeyPtr a_OwnPrivKey, + const AString & a_StartTLSData + ) override; protected: + // fwd: + class cLinkTlsContext; + typedef SharedPtr cLinkTlsContextPtr; + typedef WeakPtr cLinkTlsContextWPtr; + + /** Wrapper around cSslContext that is used when this link is being encrypted by SSL. */ + class cLinkTlsContext : + public cSslContext + { + cTCPLinkImpl & m_Link; + + /** Buffer for storing the incoming encrypted data until it is requested by the SSL decryptor. */ + AString m_EncryptedData; + + /** Buffer for storing the outgoing cleartext data until the link has finished handshaking. */ + AString m_CleartextData; + + /** Shared ownership of self, so that this object can keep itself alive for as long as it needs. */ + cLinkTlsContextWPtr m_Self; + + public: + cLinkTlsContext(cTCPLinkImpl & a_Link); + + /** Shares ownership of self, so that this object can keep itself alive for as long as it needs. */ + void SetSelf(cLinkTlsContextWPtr a_Self); + + /** Removes the self ownership so that we can detect the SSL closure. */ + void ResetSelf(void); + + /** Stores the specified block of data into the buffer of the data to be decrypted (incoming from remote). + Also flushes the SSL buffers by attempting to read any data through the SSL context. */ + void StoreReceivedData(const char * a_Data, size_t a_NumBytes); + + /** Tries to read any cleartext data available through the SSL, reports it in the link. */ + void FlushBuffers(void); + + /** Tries to finish handshaking the SSL. */ + void TryFinishHandshaking(void); + + /** Sends the specified cleartext data over the SSL to the remote peer. + If the handshake hasn't been completed yet, queues the data for sending when it completes. */ + void Send(const void * a_Data, size_t a_Length); + + // cSslContext overrides: + virtual int ReceiveEncrypted(unsigned char * a_Buffer, size_t a_NumBytes) override; + virtual int SendEncrypted(const unsigned char * a_Buffer, size_t a_NumBytes) override; + + /** Returns true if the context's associated TCP link is the same link as a_Link. */ + bool IsLink(cTCPLinkImpl * a_Link) + { + return (a_Link == &m_Link); + } + }; + + /** Callbacks to call when the connection is established. May be NULL if not used. Only used for outgoing connections (cNetwork::Connect()). */ cNetwork::cConnectCallbacksPtr m_ConnectCallbacks; @@ -99,6 +164,10 @@ protected: data is sent to the OS TCP stack, the socket gets shut down. */ bool m_ShouldShutdown; + /** The SSL context used for encryption, if this link uses SSL. + If valid, the link uses encryption through this context. */ + cLinkTlsContextPtr m_TlsContext; + /** Creates a new link to be queued to connect to a specified host:port. Used for outgoing connections created using cNetwork::Connect(). @@ -127,6 +196,12 @@ protected: /** Calls shutdown on the link and disables LibEvent writing. Called after all data from LibEvent buffers is sent to the OS TCP stack and shutdown() has been called before. */ void DoActualShutdown(void); + + /** Sends the data directly to the socket (without the optional TLS). */ + bool SendRaw(const void * a_Data, size_t a_Length); + + /** Called by the TLS when it has decoded a piece of incoming cleartext data from the socket. */ + void ReceivedCleartextData(const char * a_Data, size_t a_Length); }; diff --git a/tests/HTTP/CMakeLists.txt b/tests/HTTP/CMakeLists.txt index 1e2eb356a..cb2329adc 100644 --- a/tests/HTTP/CMakeLists.txt +++ b/tests/HTTP/CMakeLists.txt @@ -2,6 +2,7 @@ enable_testing() include_directories(${CMAKE_SOURCE_DIR}/src/) include_directories(${CMAKE_SOURCE_DIR}/lib/libevent/include) +include_directories(${CMAKE_SOURCE_DIR}/lib/polarssl/include) add_definitions(-DTEST_GLOBALS=1) @@ -75,6 +76,9 @@ add_test(NAME HTTPMessageParser_file-test3-2 COMMAND HTTPMessageParser_file-exe # Test parsing the request file in 512-byte chunks (should process everything in a single call): add_test(NAME HTTPMessageParser_file-test4-512 COMMAND HTTPMessageParser_file-exe ${CMAKE_CURRENT_SOURCE_DIR}/HTTPRequest1.data 512) +# Test the URLClient +add_test(NAME UrlClient-test COMMAND UrlClientTest-exe) + diff --git a/tests/HTTP/UrlClientTest.cpp b/tests/HTTP/UrlClientTest.cpp index 5f70855fb..97cdc6d6e 100644 --- a/tests/HTTP/UrlClientTest.cpp +++ b/tests/HTTP/UrlClientTest.cpp @@ -7,6 +7,7 @@ +/** Simple callbacks that dump the events to the console and signalize a cEvent when the request is finished. */ class cCallbacks: public cUrlClient::cCallbacks { @@ -14,53 +15,79 @@ public: cCallbacks(cEvent & a_Event): m_Event(a_Event) { + LOGD("Created a cCallbacks instance at %p", reinterpret_cast(this)); } + + ~cCallbacks() + { + LOGD("Deleting the cCallbacks instance at %p", reinterpret_cast(this)); + } + + virtual void OnConnected(cTCPLink & a_Link) override { LOG("Link connected to %s:%u", a_Link.GetRemoteIP().c_str(), a_Link.GetRemotePort()); } + virtual bool OnCertificateReceived() override { LOG("Server certificate received"); return true; } + + virtual void OnTlsHandshakeCompleted() override + { + LOG("TLS handshake has completed."); + } + + virtual void OnRequestSent() override { LOG("Request has been sent"); } + virtual void OnHeader(const AString & a_Key, const AString & a_Value) override { LOG("HTTP Header: \"%s\" -> \"%s\"", a_Key.c_str(), a_Value.c_str()); } + virtual void OnHeadersFinished() override { LOG("HTTP headers finished."); } + virtual void OnBodyData(const void * a_Data, size_t a_Size) override { - AString body(reinterpret_cast(a_Data), a_Size); - LOG("Body part:\n%s", body.c_str()); + #if 0 + // Output the whole received data blob: + AString body(reinterpret_cast(a_Data), a_Size); + LOG("Body part:\n%s", body.c_str()); + #else + // Output only the data size, to keep the log short: + LOG("Body part: %u bytes", static_cast(a_Size)); + #endif } - /** Called after the response body has been fully reported by OnBody() calls. - There will be no more OnBody() calls. */ + virtual void OnBodyFinished() override { LOG("Body finished."); m_Event.Set(); } + virtual void OnRedirecting(const AString & a_RedirectUrl) override { LOG("Redirecting to \"%s\".", a_RedirectUrl.c_str()); } + virtual void OnError(const AString & a_ErrorMsg) override { LOG("Error: %s", a_ErrorMsg.c_str()); @@ -75,7 +102,7 @@ protected: -int TestRequest1() +static int TestRequest1() { LOG("Running test 1"); cEvent evtFinished; @@ -99,7 +126,7 @@ int TestRequest1() -int TestRequest2() +static int TestRequest2() { LOG("Running test 2"); cEvent evtFinished; @@ -121,17 +148,69 @@ int TestRequest2() -int TestRequests() +static int TestRequest3() +{ + LOG("Running test 3"); + cEvent evtFinished; + cCallbacks callbacks(evtFinished); + AStringMap options; + options["MaxRedirects"] = "0"; + auto res = cUrlClient::Get("https://github.com", callbacks, AStringMap(), AString(), options); + if (res.first) + { + evtFinished.Wait(); + } + else + { + LOG("Immediate error: %s", res.second.c_str()); + return 1; + } + return 0; +} + + + + + +static int TestRequest4() { - auto res = TestRequest1(); - if (res != 0) + LOG("Running test 4"); + cEvent evtFinished; + cCallbacks callbacks(evtFinished); + auto res = cUrlClient::Get("https://github.com", callbacks); + if (res.first) { - return res; + evtFinished.Wait(); } - res = TestRequest2(); - if (res != 0) + else { - return res; + LOG("Immediate error: %s", res.second.c_str()); + return 1; + } + return 0; +} + + + + + +static int TestRequests() +{ + std::function tests[] = + { + &TestRequest1, + &TestRequest2, + &TestRequest3, + &TestRequest4, + }; + for (const auto & test: tests) + { + LOG("%s", AString(60, '-').c_str()); + auto res = test(); + if (res != 0) + { + return res; + } } return 0; } diff --git a/tests/Network/CMakeLists.txt b/tests/Network/CMakeLists.txt index c02a53456..c6c76385e 100644 --- a/tests/Network/CMakeLists.txt +++ b/tests/Network/CMakeLists.txt @@ -2,6 +2,7 @@ enable_testing() include_directories(${CMAKE_SOURCE_DIR}/src/) include_directories(${CMAKE_SOURCE_DIR}/lib/libevent/include) +include_directories(${CMAKE_SOURCE_DIR}/lib/polarssl/include) add_definitions(-DTEST_GLOBALS=1) @@ -15,6 +16,11 @@ set (Network_SRCS ${CMAKE_SOURCE_DIR}/src/OSSupport/NetworkSingleton.cpp ${CMAKE_SOURCE_DIR}/src/OSSupport/ServerHandleImpl.cpp ${CMAKE_SOURCE_DIR}/src/OSSupport/TCPLinkImpl.cpp + ${CMAKE_SOURCE_DIR}/src/PolarSSL++/CtrDrbgContext.cpp + ${CMAKE_SOURCE_DIR}/src/PolarSSL++/CryptoKey.cpp + ${CMAKE_SOURCE_DIR}/src/PolarSSL++/EntropyContext.cpp + ${CMAKE_SOURCE_DIR}/src/PolarSSL++/SslContext.cpp + ${CMAKE_SOURCE_DIR}/src/PolarSSL++/X509Cert.cpp ${CMAKE_SOURCE_DIR}/src/StringUtils.cpp ) @@ -27,6 +33,11 @@ set (Network_HDRS ${CMAKE_SOURCE_DIR}/src/OSSupport/NetworkSingleton.h ${CMAKE_SOURCE_DIR}/src/OSSupport/ServerHandleImpl.h ${CMAKE_SOURCE_DIR}/src/OSSupport/TCPLinkImpl.h + ${CMAKE_SOURCE_DIR}/src/PolarSSL++/CtrDrbgContext.h + ${CMAKE_SOURCE_DIR}/src/PolarSSL++/CryptoKey.h + ${CMAKE_SOURCE_DIR}/src/PolarSSL++/EntropyContext.h + ${CMAKE_SOURCE_DIR}/src/PolarSSL++/SslContext.h + ${CMAKE_SOURCE_DIR}/src/PolarSSL++/X509Cert.h ${CMAKE_SOURCE_DIR}/src/StringUtils.h ) @@ -35,7 +46,7 @@ add_library(Network ${Network_HDRS} ) -target_link_libraries(Network event_core event_extra) +target_link_libraries(Network event_core event_extra mbedtls) if (MSVC) target_link_libraries(Network ws2_32.lib) endif() -- cgit v1.2.3 From 74918ce805de260371ad1be0604ee7369f5f809b Mon Sep 17 00:00:00 2001 From: Mattes D Date: Tue, 16 Aug 2016 11:55:49 +0200 Subject: cUrlClient: Refactored callbacks to use UniquePtr. --- src/HTTP/UrlClient.cpp | 40 ++++++++++++++++++++-------------------- src/HTTP/UrlClient.h | 9 +++++---- tests/HTTP/UrlClientTest.cpp | 16 ++++++++-------- 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/src/HTTP/UrlClient.cpp b/src/HTTP/UrlClient.cpp index 9346882f1..29d5e28d7 100644 --- a/src/HTTP/UrlClient.cpp +++ b/src/HTTP/UrlClient.cpp @@ -32,7 +32,7 @@ public: static std::pair Request( const AString & a_Method, const AString & a_URL, - cUrlClient::cCallbacks & a_Callbacks, + cUrlClient::cCallbacksPtr && a_Callbacks, AStringMap && a_Headers, const AString & a_Body, AStringMap && a_Options @@ -41,7 +41,7 @@ public: // Create a new instance of cUrlClientRequest, wrapped in a SharedPtr, so that it has a controlled lifetime. // Cannot use std::make_shared, because the constructor is not public SharedPtr ptr (new cUrlClientRequest( - a_Method, a_URL, a_Callbacks, std::move(a_Headers), a_Body, std::move(a_Options) + a_Method, a_URL, std::move(a_Callbacks), std::move(a_Headers), a_Body, std::move(a_Options) )); return ptr->DoRequest(ptr); } @@ -51,7 +51,7 @@ public: void CallErrorCallback(const AString & a_ErrorMessage) { // Call the error callback: - m_Callbacks.OnError(a_ErrorMessage); + m_Callbacks->OnError(a_ErrorMessage); // Terminate the request's TCP link: auto link = m_Link; @@ -63,7 +63,7 @@ public: } - cUrlClient::cCallbacks & GetCallbacks() { return m_Callbacks; } + cUrlClient::cCallbacks & GetCallbacks() { return *m_Callbacks; } void RedirectTo(const AString & a_RedirectUrl); @@ -115,7 +115,7 @@ protected: UInt16 m_UrlPort; /** Callbacks that report progress and results of the request. */ - cUrlClient::cCallbacks & m_Callbacks; + cUrlClient::cCallbacksPtr m_Callbacks; /** Extra headers to be sent with the request (besides the normal ones). */ AStringMap m_Headers; @@ -145,14 +145,14 @@ protected: cUrlClientRequest( const AString & a_Method, const AString & a_Url, - cUrlClient::cCallbacks & a_Callbacks, + cUrlClient::cCallbacksPtr && a_Callbacks, AStringMap && a_Headers, const AString & a_Body, AStringMap && a_Options ): m_Method(a_Method), m_Url(a_Url), - m_Callbacks(a_Callbacks), + m_Callbacks(std::move(a_Callbacks)), m_Headers(std::move(a_Headers)), m_Body(a_Body), m_Options(std::move(a_Options)) @@ -170,7 +170,7 @@ protected: // cNetwork::cConnectCallbacks override: An error has occurred: virtual void OnError(int a_ErrorCode, const AString & a_ErrorMsg) override { - m_Callbacks.OnError(Printf("Network error %d (%s)", a_ErrorCode, a_ErrorMsg.c_str())); + m_Callbacks->OnError(Printf("Network error %d (%s)", a_ErrorCode, a_ErrorMsg.c_str())); m_Self.reset(); } @@ -486,7 +486,7 @@ cSchemeHandlerPtr cSchemeHandler::Create(const AString & a_Scheme, cUrlClientReq void cUrlClientRequest::RedirectTo(const AString & a_RedirectUrl) { // Check that redirection is allowed: - m_Callbacks.OnRedirecting(a_RedirectUrl); + m_Callbacks->OnRedirecting(a_RedirectUrl); if (!ShouldAllowRedirects()) { CallErrorCallback(Printf("Redirect to \"%s\" not allowed", a_RedirectUrl.c_str())); @@ -500,7 +500,7 @@ void cUrlClientRequest::RedirectTo(const AString & a_RedirectUrl) auto res = DoRequest(m_Self); if (!res.first) { - m_Callbacks.OnError(Printf("Redirection failed: %s", res.second.c_str())); + m_Callbacks->OnError(Printf("Redirection failed: %s", res.second.c_str())); return; } } @@ -520,7 +520,7 @@ bool cUrlClientRequest::ShouldAllowRedirects() const void cUrlClientRequest::OnConnected(cTCPLink & a_Link) { - m_Callbacks.OnConnected(a_Link); + m_Callbacks->OnConnected(a_Link); m_SchemeHandler->OnConnected(a_Link); } @@ -532,7 +532,7 @@ void cUrlClientRequest::OnTlsHandshakeCompleted(void) { // Notify the scheme handler and the callbacks: m_SchemeHandler->OnTlsHandshakeCompleted(); - m_Callbacks.OnTlsHandshakeCompleted(); + m_Callbacks->OnTlsHandshakeCompleted(); } @@ -607,14 +607,14 @@ std::pair cUrlClientRequest::DoRequest(SharedPtr cUrlClient::Request( const AString & a_Method, const AString & a_URL, - cCallbacks & a_Callbacks, + cCallbacksPtr && a_Callbacks, AStringMap && a_Headers, AString && a_Body, AStringMap && a_Options ) { return cUrlClientRequest::Request( - a_Method, a_URL, a_Callbacks, std::move(a_Headers), std::move(a_Body), std::move(a_Options) + a_Method, a_URL, std::move(a_Callbacks), std::move(a_Headers), std::move(a_Body), std::move(a_Options) ); } @@ -624,14 +624,14 @@ std::pair cUrlClient::Request( std::pair cUrlClient::Get( const AString & a_URL, - cCallbacks & a_Callbacks, + cCallbacksPtr && a_Callbacks, AStringMap a_Headers, AString a_Body, AStringMap a_Options ) { return cUrlClientRequest::Request( - "GET", a_URL, a_Callbacks, std::move(a_Headers), std::move(a_Body), std::move(a_Options) + "GET", a_URL, std::move(a_Callbacks), std::move(a_Headers), std::move(a_Body), std::move(a_Options) ); } @@ -641,14 +641,14 @@ std::pair cUrlClient::Get( std::pair cUrlClient::Post( const AString & a_URL, - cCallbacks & a_Callbacks, + cCallbacksPtr && a_Callbacks, AStringMap && a_Headers, AString && a_Body, AStringMap && a_Options ) { return cUrlClientRequest::Request( - "POST", a_URL, a_Callbacks, std::move(a_Headers), std::move(a_Body), std::move(a_Options) + "POST", a_URL, std::move(a_Callbacks), std::move(a_Headers), std::move(a_Body), std::move(a_Options) ); } @@ -658,14 +658,14 @@ std::pair cUrlClient::Post( std::pair cUrlClient::Put( const AString & a_URL, - cCallbacks & a_Callbacks, + cCallbacksPtr && a_Callbacks, AStringMap && a_Headers, AString && a_Body, AStringMap && a_Options ) { return cUrlClientRequest::Request( - "PUT", a_URL, a_Callbacks, std::move(a_Headers), std::move(a_Body), std::move(a_Options) + "PUT", a_URL, std::move(a_Callbacks), std::move(a_Headers), std::move(a_Body), std::move(a_Options) ); } diff --git a/src/HTTP/UrlClient.h b/src/HTTP/UrlClient.h index 652cc76f7..ac772235e 100644 --- a/src/HTTP/UrlClient.h +++ b/src/HTTP/UrlClient.h @@ -86,6 +86,7 @@ public: for such a response; instead, the redirect is silently attempted. */ virtual void OnRedirecting(const AString & a_NewLocation) {} }; + typedef UniquePtr cCallbacksPtr; /** Used for HTTP status codes. */ @@ -112,7 +113,7 @@ public: static std::pair Request( const AString & a_Method, const AString & a_URL, - cCallbacks & a_Callbacks, + cCallbacksPtr && a_Callbacks, AStringMap && a_Headers, AString && a_Body, AStringMap && a_Options @@ -121,7 +122,7 @@ public: /** Alias for Request("GET", ...) */ static std::pair Get( const AString & a_URL, - cCallbacks & a_Callbacks, + cCallbacksPtr && a_Callbacks, AStringMap a_Headers = AStringMap(), AString a_Body = AString(), AStringMap a_Options = AStringMap() @@ -130,7 +131,7 @@ public: /** Alias for Request("POST", ...) */ static std::pair Post( const AString & a_URL, - cCallbacks & a_Callbacks, + cCallbacksPtr && a_Callbacks, AStringMap && a_Headers, AString && a_Body, AStringMap && a_Options @@ -139,7 +140,7 @@ public: /** Alias for Request("PUT", ...) */ static std::pair Put( const AString & a_URL, - cCallbacks & a_Callbacks, + cCallbacksPtr && a_Callbacks, AStringMap && a_Headers, AString && a_Body, AStringMap && a_Options diff --git a/tests/HTTP/UrlClientTest.cpp b/tests/HTTP/UrlClientTest.cpp index 97cdc6d6e..d8412fddf 100644 --- a/tests/HTTP/UrlClientTest.cpp +++ b/tests/HTTP/UrlClientTest.cpp @@ -106,10 +106,10 @@ static int TestRequest1() { LOG("Running test 1"); cEvent evtFinished; - cCallbacks callbacks(evtFinished); + auto callbacks = cpp14::make_unique(evtFinished); AStringMap options; options["MaxRedirects"] = "0"; - auto res = cUrlClient::Get("http://github.com", callbacks, AStringMap(), AString(), options); + auto res = cUrlClient::Get("http://github.com", std::move(callbacks), AStringMap(), AString(), options); if (res.first) { evtFinished.Wait(); @@ -130,8 +130,8 @@ static int TestRequest2() { LOG("Running test 2"); cEvent evtFinished; - cCallbacks callbacks(evtFinished); - auto res = cUrlClient::Get("http://github.com", callbacks); + auto callbacks = cpp14::make_unique(evtFinished); + auto res = cUrlClient::Get("http://github.com", std::move(callbacks)); if (res.first) { evtFinished.Wait(); @@ -152,10 +152,10 @@ static int TestRequest3() { LOG("Running test 3"); cEvent evtFinished; - cCallbacks callbacks(evtFinished); + auto callbacks = cpp14::make_unique(evtFinished); AStringMap options; options["MaxRedirects"] = "0"; - auto res = cUrlClient::Get("https://github.com", callbacks, AStringMap(), AString(), options); + auto res = cUrlClient::Get("https://github.com", std::move(callbacks), AStringMap(), AString(), options); if (res.first) { evtFinished.Wait(); @@ -176,8 +176,8 @@ static int TestRequest4() { LOG("Running test 4"); cEvent evtFinished; - cCallbacks callbacks(evtFinished); - auto res = cUrlClient::Get("https://github.com", callbacks); + auto callbacks = cpp14::make_unique(evtFinished); + auto res = cUrlClient::Get("https://github.com", std::move(callbacks)); if (res.first) { evtFinished.Wait(); -- cgit v1.2.3 From 5ca371bb9a65cc322eb8327d81709985daefe173 Mon Sep 17 00:00:00 2001 From: Mattes D Date: Tue, 23 Aug 2016 13:20:43 +0200 Subject: cUrlClient: Exported to Lua API. --- Server/Plugins/APIDump/Classes/Network.lua | 174 ++++++++++++++- Server/Plugins/Debuggers/Debuggers.lua | 81 ++++++- Server/Plugins/Debuggers/Info.lua | 12 ++ src/Bindings/LuaState.cpp | 11 + src/Bindings/LuaState.h | 24 +++ src/Bindings/ManualBindings_Network.cpp | 334 +++++++++++++++++++++++++++++ src/OSSupport/TCPLinkImpl.cpp | 5 + 7 files changed, 639 insertions(+), 2 deletions(-) diff --git a/Server/Plugins/APIDump/Classes/Network.lua b/Server/Plugins/APIDump/Classes/Network.lua index 74f0a6488..e517b8022 100644 --- a/Server/Plugins/APIDump/Classes/Network.lua +++ b/Server/Plugins/APIDump/Classes/Network.lua @@ -1,7 +1,7 @@ -- Network.lua --- Defines the documentation for the cNetwork-related classes +-- Defines the documentation for the cNetwork-related classes and cUrlClient @@ -365,6 +365,178 @@ g_Server = nil Send = { Params = "RawData, RemoteHost, RemotePort", Return = "bool", Notes = "Sends the specified raw data (string) to the specified remote host. The RemoteHost can be either a hostname or an IP address; if it is a hostname, the endpoint will queue a DNS lookup first, if it is an IP address, the send operation is executed immediately. Returns true if there was no immediate error, false on any failure. Note that the return value needn't represent whether the packet was actually sent, only if it was successfully queued." }, }, }, -- cUDPEndpoint + + + cUrlClient = + { + Desc = + [[ + Implements high-level asynchronous access to URLs, such as downloading webpages over HTTP(S).

+

+ Note that unlike other languages' URL access libraries, this class implements asynchronous requests. + This means that the functions only start a request and return immediately. The request is then + fulfilled in the background, while the server continues to run. The response is delivered back to the + plugin using callbacks. This allows the plugin to start requests and not block the server until the + response is received.

+

+ The functions that make network requests are all static and have a dual interface. Either you can use + a single callback function, which gets called once the entire response is received or an error is + encountered. Or you can use a table of callback functions, each function being called whenever the + specific event happens during the request and response lifetime. See the Simple Callback and Callback + Table chapters later on this page for details and examples.

+

+ All the request function also support optional parameters for further customization of the request - + the Headers parameter specifies additional HTTP headers that are to be sent (as a dictionary-table of + key -> value), the RequestBody parameter specifying the optional body of the request (used mainly for + POST and PUT requests), and an Options parameter specifying additional options specific to the protocol + used. + ]], + + AdditionalInfo = + { + { + Header = "Simple Callback", + Contents = + [[ + When you don't need fine control for receiving the requests and are interested only in the result, + you can use the simple callback approach. Pass a single function as the Callback parameter, the + function will get called when the response is fully processed, either with the body of the response, + or with an error message: +

+cUrlClient:Get(url,
+	function (a_Body, a_Data)
+		if (a_Body) then
+			-- Response received correctly, a_Body contains the entire response body,
+			-- a_Data is a dictionary-table of the response's HTTP headers
+		else
+			-- There was an error, a_Data is the error message string
+		end
+	end
+)
+
+ ]], + }, + { + Header = "Callback Table", + Contents = + [[ + To provide complete control over the request and response handling, Cuberite allows plugins to pass + a table of callbacks as the Callback parameter. Then the respective functions are called for their + respective events during the lifetime of the request and response. This way it is possible to + process huge downloads that wouldn't fit into memory otherwise, or display detailed progress.

+ Each callback function receives a "self" as its first parameter, this allows the functions to + access the Callback table and any of its other members, allowing the use of Lua object idiom for + the table. See this forum post for an + example. +

+ The following callback functions are used by Cuberite. Any callback that is not assigned is + silently ignored. The table may also contain other functions and other values, those are silently + ignored.

+ + + + + + + + + + + + + +
CallbackParamsNotes
OnConnectedself, {{cTCPLink}}, RemoteIP, RemotePortCalled when the connection to the remote host is established. Note that current implementation doesn't provide the {{cTCPLink}} parameter and passes nil instead.
OnCertificateReceivedselfCalled for HTTPS URLs when the server's certificate is received. If the callback returns anything else than true, the connection is aborted. Note that the current implementation doesn't provide the certificate because there is no representation for the cert in Lua.
OnTlsHandshakeCompletedselfCalled for HTTPS URLs when the TLS is established on the connection.
OnRequestSentselfCalled after the entire request is sent to the server.
OnStatusLineself, HttpVersion, StatusCode, RestThe initial line of the response has been parsed. HttpVersion is typically "HTTP/1.1", StatusCode is the numerical HTTP status code reported by the server (200 being OK), Rest is the rest of the line, usually a short message in case of an error.
OnHeaderself, Name, ValueA new HTTP response header line has been received.
OnHeadersFinishedself, AllHeadersAll HTTP response headers have been parsed. AllHeaders is a dictionary-table containing all the headers received.
OnBodyDataself, DataA piece of the response body has been received. This callback is called repeatedly until the entire body is reported through its Data parameter.
OnBodyFinishedselfThe entire response body has been reported by OnBodyData(), the response has finished.
OnErrorself, ErrorMsgCalled whenever an error is detected. After this call, no other callback will get called.
OnRedirectingself, NewUrlCalled if the server returned a valid redirection HTTP status code and a Location header, and redirection is allowed by the Options.
+

+ The following example is adapted from the Debuggers plugin's "download" command, it downloads the + contents of an URL into a file. +

+function HandleConsoleDownload(a_Split)  -- Console command handler
+	-- Read the params from the command:
+	local url = a_Split[2]
+	local fnam = a_Split[3]
+	if (not(url) or not(fnam)) then
+		return true, "Missing parameters. Usage: download  "
+	end
+	
+	-- Define the cUrlClient callbacks
+	local callbacks =
+	{
+		OnStatusLine = function (self, a_HttpVersion, a_Status, a_Rest)
+			-- Only open the output file if the server reports a success:
+			if (a_Status ~= 200) then
+				LOG("Cannot download " .. url .. ", HTTP error code " .. a_Status)
+				return
+			end
+			local f, err = io.open(fnam, "wb")
+			if not(f) then
+				LOG("Cannot download " .. url .. ", error opening the file " .. fnam .. ": " .. (err or ""))
+				return
+			end
+			self.m_File = f
+		end,
+
+		OnBodyData = function (self, a_Data)
+			-- If the file has been opened, write the data:
+			if (self.m_File) then
+				self.m_File:write(a_Data)
+			end
+		end,
+		
+		OnBodyFinished = function (self)
+			-- If the file has been opened, close it and report success
+			if (self.m_File) then
+				self.m_File:close()
+				LOG("File " .. fnam .. " has been downloaded.")
+			end
+		end,
+	}
+	
+	-- Start the URL download:
+	local isSuccess, msg = cUrlClient:Get(url, callbacks)
+	if not(isSuccess) then
+		LOG("Cannot start an URL download: " .. (msg or ""))
+		return true
+	end
+	return true
+end
+
+]], + }, + { + Header = "Options", + Contents = + [[ + The requests support the following options, specified in the optional Options table parameter: + + + + + + +
Option nameDescription
MaxRedirectsMaximum number of HTTP redirects that the cUrlClient will follow. If the server still reports a redirect after reaching this many redirects, the cUrlClient reports an error. May be specified as either a number or a string parsable into a number. Default: 30.
OwnCertThe client certificate to use, if requested by the server. A string containing a PEM- or DER-encoded cert is expected.
OwnPrivKeyThe private key appropriate for OwnCert. A string containing a PEM- or DER-encoded private key is expected.
OwnPrivKeyPasswordThe password for OwnPrivKey. If not present or empty, no password is assumed.
+

+ Redirection: +

    +
  • If a redirect is received, and redirection is allowed by MaxRedirects, the redirection is + reported via OnRedirecting() callback and the request is restarted at the redirect URL, without + reporting any of the redirect's headers nor body.
    • +
  • If a redirect is received and redirection is not allowed (maximum redirection attempts have + been reached), the OnRedirecting() callback is called with the redirect URL and then the request + terminates with an OnError() callback, without reporting the redirect's headers nor body.
    • +
+ ]], + }, + }, + + Functions = + { + Delete = { Params = "URL, Callbacks, [Headers], [RequestBody], [Options]", Return = "bool, [ErrMsg]", IsStatic = true, Notes = "Starts a HTTP DELETE request. Alias for Request(\"DELETE\", ...). Returns true on succes, false and error message on immediate failure (unparsable URL etc.)."}, + Get = { Params = "URL, Callbacks, [Headers], [RequestBody], [Options]", Return = "bool, [ErrMsg]", IsStatic = true, Notes = "Starts a HTTP GET request. Alias for Request(\"GET\", ...). Returns true on succes, false and error message on immediate failure (unparsable URL etc.)."}, + Post = { Params = "URL, Callbacks, [Headers], [RequestBody], [Options]", Return = "bool, [ErrMsg]", IsStatic = true, Notes = "Starts a HTTP POST request. Alias for Request(\"POST\", ...). Returns true on succes, false and error message on immediate failure (unparsable URL etc.)."}, + Put = { Params = "URL, Callbacks, [Headers], [RequestBody], [Options]", Return = "bool, [ErrMsg]", IsStatic = true, Notes = "Starts a HTTP PUT request. Alias for Request(\"PUT\", ...). Returns true on succes, false and error message on immediate failure (unparsable URL etc.)."}, + Request = { Params = "Method, URL, Callbacks, [Headers], [RequestBody], [Options]", Return = "bool, [ErrMsg]", IsStatic = true, Notes = "Starts a request with the specified Method. Returns true on succes, false and error message on immediate failure (unparsable URL etc.)."}, + }, + }, -- cUrlClient } diff --git a/Server/Plugins/Debuggers/Debuggers.lua b/Server/Plugins/Debuggers/Debuggers.lua index da375cdff..f405d95ae 100644 --- a/Server/Plugins/Debuggers/Debuggers.lua +++ b/Server/Plugins/Debuggers/Debuggers.lua @@ -61,7 +61,7 @@ function Initialize(a_Plugin) -- TestUUIDFromName() -- TestRankMgr() TestFileExt() - TestFileLastMod() + -- TestFileLastMod() TestPluginInterface() local LastSelfMod = cFile:GetLastModificationTime(a_Plugin:GetLocalFolder() .. "/Debuggers.lua") @@ -2135,6 +2135,35 @@ end +function HandleConsoleTestUrlClient(a_Split, a_EntireCmd) + local url = a_Split[2] or "https://github.com" + local isSuccess, msg = cUrlClient:Get(url, + function (a_Body, a_SecondParam) + if not(a_Body) then + -- An error has occurred, a_SecondParam is the error message + LOG("Error while retrieving URL \"" .. url .. "\": " .. (a_SecondParam or "")) + return + end + -- Body received, a_SecondParam is the HTTP headers dictionary-table + assert(type(a_Body) == "string") + assert(type(a_SecondParam) == "table") + LOG("URL body received, length is " .. string.len(a_Body) .. " bytes and there are these headers:") + for k, v in pairs(a_SecondParam) do + LOG(" \"" .. k .. "\": \"" .. v .. "\"") + end + LOG("(headers list finished)") + end + ) + if not(isSuccess) then + LOG("cUrlClient request failed: " .. (msg or "")) + end + return true +end + + + + + function HandleConsoleTestUrlParser(a_Split, a_EntireCmd) LOG("Testing cUrlParser...") local UrlsToTest = @@ -2262,6 +2291,56 @@ end +function HandleConsoleDownload(a_Split) + -- Check params: + local url = a_Split[2] + local fnam = a_Split[3] + if (not(url) or not(fnam)) then + return true, "Missing parameters. Usage: download " + end + + local callbacks = + { + OnStatusLine = function (self, a_HttpVersion, a_Status, a_Rest) + if (a_Status ~= 200) then + LOG("Cannot download " .. url .. ", HTTP error code " .. a_Status) + return + end + + local f, err = io.open(fnam, "wb") + if not(f) then + LOG("Cannot download " .. url .. ", error opening the file " .. fnam .. ": " .. (err or "")) + return + end + self.m_File = f + end, + + OnBodyData = function (self, a_Data) + if (self.m_File) then + self.m_File:write(a_Data) + end + end, + + OnBodyFinished = function (self) + if (self.m_File) then + self.m_File:close() + LOG("File " .. fnam .. " has been downloaded.") + end + end, + } + + local isSuccess, msg = cUrlClient:Get(url, callbacks) + if not(isSuccess) then + LOG("Cannot start an URL download: " .. (msg or "")) + return true + end + return true +end + + + + + function HandleBlkCmd(a_Split, a_Player) -- Gets info about the block the player is looking at. local World = a_Player:GetWorld(); diff --git a/Server/Plugins/Debuggers/Info.lua b/Server/Plugins/Debuggers/Info.lua index 8fd04d0ae..6c7eabbf2 100644 --- a/Server/Plugins/Debuggers/Info.lua +++ b/Server/Plugins/Debuggers/Info.lua @@ -224,6 +224,12 @@ g_PluginInfo = HelpString = "Performs cBoundingBox API tests", }, + ["download"] = + { + Handler = HandleConsoleDownload, + HelpString = "Downloads a file from a specified URL", + }, + ["hash"] = { Handler = HandleConsoleHash, @@ -278,6 +284,12 @@ g_PluginInfo = HelpString = "Tests the cLineBlockTracer", }, + ["testurlclient"] = + { + Handler = HandleConsoleTestUrlClient, + HelpString = "Tests the cUrlClient", + }, + ["testurlparser"] = { Handler = HandleConsoleTestUrlParser, diff --git a/src/Bindings/LuaState.cpp b/src/Bindings/LuaState.cpp index ec6bdb48a..81893c946 100644 --- a/src/Bindings/LuaState.cpp +++ b/src/Bindings/LuaState.cpp @@ -2139,6 +2139,17 @@ cLuaState * cLuaState::QueryCanonLuaState(void) +void cLuaState::LogApiCallParamFailure(const char * a_FnName, const char * a_ParamNames) +{ + LOGWARNING("%s: Cannot read params: %s, bailing out.", a_FnName, a_ParamNames); + LogStackTrace(); + LogStackValues("Values on the stack"); +} + + + + + int cLuaState::ReportFnCallErrors(lua_State * a_LuaState) { LOGWARNING("LUA: %s", lua_tostring(a_LuaState, -1)); diff --git a/src/Bindings/LuaState.h b/src/Bindings/LuaState.h index cb68b9a98..32d346a19 100644 --- a/src/Bindings/LuaState.h +++ b/src/Bindings/LuaState.h @@ -301,6 +301,26 @@ public: return cLuaState(m_Ref.GetLuaState()).CallTableFn(m_Ref, a_FnName, std::forward(args)...); } + /** Calls the Lua function stored under the specified name in the referenced table, if still available. + A "self" parameter is injected in front of all the given parameters and is set to the callback table. + Returns true if callback has been called. + Returns false if the Lua state isn't valid anymore, or the function doesn't exist. */ + template + bool CallTableFnWithSelf(const char * a_FnName, Args &&... args) + { + auto cs = m_CS; + if (cs == nullptr) + { + return false; + } + cCSLock Lock(*cs); + if (!m_Ref.IsValid()) + { + return false; + } + return cLuaState(m_Ref.GetLuaState()).CallTableFn(m_Ref, a_FnName, m_Ref, std::forward(args)...); + } + /** Set the contained reference to the table in the specified Lua state's stack position. If another table has been previously contained, it is unreferenced first. Returns true on success, false on failure (not a table at the specified stack pos). */ @@ -741,6 +761,10 @@ public: Returns nullptr if the canon Lua state cannot be queried. */ cLuaState * QueryCanonLuaState(void); + /** Outputs to log a warning about API call being unable to read its parameters from the stack, + logs the stack trace and stack values. */ + void LogApiCallParamFailure(const char * a_FnName, const char * a_ParamNames); + protected: cCriticalSection m_CS; diff --git a/src/Bindings/ManualBindings_Network.cpp b/src/Bindings/ManualBindings_Network.cpp index c8565d23d..f59860938 100644 --- a/src/Bindings/ManualBindings_Network.cpp +++ b/src/Bindings/ManualBindings_Network.cpp @@ -2,6 +2,7 @@ // ManualBindings_Network.cpp // Implements the cNetwork-related API bindings for Lua +// Also implements the cUrlClient bindings #include "Globals.h" #include "LuaTCPLink.h" @@ -12,6 +13,7 @@ #include "LuaNameLookup.h" #include "LuaServerHandle.h" #include "LuaUDPEndpoint.h" +#include "../HTTP/UrlClient.h" @@ -903,6 +905,329 @@ static int tolua_cUDPEndpoint_Send(lua_State * L) +/** Used when the cUrlClient Lua request wants all the callbacks. +Maps each callback onto a Lua function callback in the callback table. */ +class cFullUrlClientCallbacks: + public cUrlClient::cCallbacks +{ +public: + /** Creates a new instance bound to the specified table of callbacks. */ + cFullUrlClientCallbacks(cLuaState::cTableRefPtr && a_Callbacks): + m_Callbacks(std::move(a_Callbacks)) + { + } + + + // cUrlClient::cCallbacks overrides: + virtual void OnConnected(cTCPLink & a_Link) override + { + // TODO: Cannot push a cTCPLink to Lua, need to translate via cLuaTCPLink + m_Callbacks->CallTableFnWithSelf("OnConnected", cLuaState::Nil, a_Link.GetRemoteIP(), a_Link.GetRemotePort()); + } + + + virtual bool OnCertificateReceived() override + { + // TODO: The received cert needs proper type specification from the underlying cUrlClient framework and in the Lua engine as well + bool res = true; + m_Callbacks->CallTableFnWithSelf("OnCertificateReceived", cLuaState::Return, res); + return res; + } + + + virtual void OnTlsHandshakeCompleted() override + { + m_Callbacks->CallTableFnWithSelf("OnTlsHandshakeCompleted"); + } + + + virtual void OnRequestSent() override + { + m_Callbacks->CallTableFnWithSelf("OnRequestSent"); + } + + + virtual void OnStatusLine(const AString & a_HttpVersion, int a_StatusCode, const AString & a_Rest) override + { + m_Callbacks->CallTableFnWithSelf("OnStatusLine", a_HttpVersion, a_StatusCode, a_Rest); + } + + + virtual void OnHeader(const AString & a_Key, const AString & a_Value) override + { + m_Callbacks->CallTableFnWithSelf("OnHeader", a_Key, a_Value); + m_Headers[a_Key] = a_Value; + } + + + virtual void OnHeadersFinished() override + { + m_Callbacks->CallTableFnWithSelf("OnHeadersFinished", m_Headers); + } + + + virtual void OnBodyData(const void * a_Data, size_t a_Size) override + { + m_Callbacks->CallTableFnWithSelf("OnBodyData", AString(reinterpret_cast(a_Data), a_Size)); + } + + + virtual void OnBodyFinished() override + { + m_Callbacks->CallTableFnWithSelf("OnBodyFinished"); + } + + + virtual void OnError(const AString & a_ErrorMsg) override + { + m_Callbacks->CallTableFnWithSelf("OnError", a_ErrorMsg); + } + + + virtual void OnRedirecting(const AString & a_NewLocation) override + { + m_Callbacks->CallTableFnWithSelf("OnRedirecting", a_NewLocation); + } + + +protected: + /** The Lua table containing the callbacks. */ + cLuaState::cTableRefPtr m_Callbacks; + + /** Accumulator for all the headers to be reported in the OnHeadersFinished() callback. */ + AStringMap m_Headers; +}; + + + + + +/** Used when the cUrlClient Lua request has just a single callback. +The callback is used to report the entire body at once, together with the HTTP headers, or to report an error: +callback("BodyContents", {headers}) +callback(nil, "ErrorMessage") +Accumulates the body contents into a single string until the body is finished. +Accumulates all HTTP headers into an AStringMap. */ +class cSimpleUrlClientCallbacks: + public cUrlClient::cCallbacks +{ +public: + /** Creates a new instance that uses the specified callback to report when request finishes. */ + cSimpleUrlClientCallbacks(cLuaState::cCallbackPtr && a_Callback): + m_Callback(std::move(a_Callback)) + { + } + + + virtual void OnHeader(const AString & a_Key, const AString & a_Value) override + { + m_Headers[a_Key] = a_Value; + } + + + virtual void OnBodyData(const void * a_Data, size_t a_Size) override + { + m_ResponseBody.append(reinterpret_cast(a_Data), a_Size); + } + + + virtual void OnBodyFinished() override + { + m_Callback->Call(m_ResponseBody, m_Headers); + } + + + virtual void OnError(const AString & a_ErrorMsg) override + { + m_Callback->Call(cLuaState::Nil, a_ErrorMsg); + } + + +protected: + + /** The callback to call when the request finishes. */ + cLuaState::cCallbackPtr m_Callback; + + /** The accumulator for the partial body data, so that OnBodyFinished() can send the entire thing at once. */ + AString m_ResponseBody; + + /** Accumulator for all the headers to be reported in the combined callback. */ + AStringMap m_Headers; +}; + + + + + +/** Common code shared among the cUrlClient request methods. +a_Method is the method name to be used in the request. +a_UrlStackIdx is the position on the Lua stack of the Url parameter. */ +static int tolua_cUrlClient_Request_Common(lua_State * a_LuaState, const AString & a_Method, int a_UrlStackIdx) +{ + // Check params: + cLuaState L(a_LuaState); + if (!L.CheckParamString(a_UrlStackIdx)) + { + return 0; + } + + // Read params: + AString url, requestBody; + AStringMap headers, options; + cLuaState::cTableRefPtr callbacks; + cLuaState::cCallbackPtr onCompleteBodyCallback; + if (!L.GetStackValues(a_UrlStackIdx, url)) + { + L.LogApiCallParamFailure("cUrlClient:Request()", Printf("URL (%d)", a_UrlStackIdx).c_str()); + L.Push(false); + L.Push("Invalid params"); + return 2; + } + cUrlClient::cCallbacksPtr urlClientCallbacks; + if (lua_istable(L, a_UrlStackIdx + 1)) + { + if (!L.GetStackValue(a_UrlStackIdx + 1, callbacks)) + { + L.LogApiCallParamFailure("cUrlClient:Request()", Printf("CallbacksTable (%d)", a_UrlStackIdx + 1).c_str()); + L.Push(false); + L.Push("Invalid Callbacks param"); + return 2; + } + urlClientCallbacks = cpp14::make_unique(std::move(callbacks)); + } + else if (lua_isfunction(L, a_UrlStackIdx + 1)) + { + if (!L.GetStackValue(a_UrlStackIdx + 1, onCompleteBodyCallback)) + { + L.LogApiCallParamFailure("cUrlClient:Request()", Printf("CallbacksFn (%d)", a_UrlStackIdx + 1).c_str()); + L.Push(false); + L.Push("Invalid OnCompleteBodyCallback param"); + return 2; + } + urlClientCallbacks = cpp14::make_unique(std::move(onCompleteBodyCallback)); + } + else + { + L.LogApiCallParamFailure("cUrlClient:Request()", Printf("Callbacks (%d)", a_UrlStackIdx + 1).c_str()); + L.Push(false); + L.Push("Invalid OnCompleteBodyCallback param"); + return 2; + } + if (!L.GetStackValues(a_UrlStackIdx + 2, cLuaState::cOptionalParam(headers), cLuaState::cOptionalParam(requestBody), cLuaState::cOptionalParam(options))) + { + L.LogApiCallParamFailure("cUrlClient:Request()", Printf("Header, Body or Options (%d, %d, %d)", a_UrlStackIdx + 2, a_UrlStackIdx + 3, a_UrlStackIdx + 4).c_str()); + L.Push(false); + L.Push("Invalid params"); + return 2; + } + + // Make the request: + auto res = cUrlClient::Request(a_Method, url, std::move(urlClientCallbacks), std::move(headers), std::move(requestBody), std::move(options)); + if (!res.first) + { + L.Push(false); + L.Push(res.second); + return 2; + } + L.Push(true); + return 1; +} + + + + + +/** Implements cUrlClient:Get() using cUrlClient::Request(). */ +static int tolua_cUrlClient_Delete(lua_State * a_LuaState) +{ + /* Function signatures: + cUrlClient:Delete(URL, {CallbacksFnTable}, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + cUrlClient:Delete(URL, OnCompleteBodyCallback, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + */ + + return tolua_cUrlClient_Request_Common(a_LuaState, "DELETE", 2); +} + + + + + +/** Implements cUrlClient:Get() using cUrlClient::Request(). */ +static int tolua_cUrlClient_Get(lua_State * a_LuaState) +{ + /* Function signatures: + cUrlClient:Get(URL, {CallbacksFnTable}, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + cUrlClient:Get(URL, OnCompleteBodyCallback, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + */ + + return tolua_cUrlClient_Request_Common(a_LuaState, "GET", 2); +} + + + + + +/** Implements cUrlClient:Post() using cUrlClient::Request(). */ +static int tolua_cUrlClient_Post(lua_State * a_LuaState) +{ + /* Function signatures: + cUrlClient:Post(URL, {CallbacksFnTable}, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + cUrlClient:Post(URL, OnCompleteBodyCallback, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + */ + + return tolua_cUrlClient_Request_Common(a_LuaState, "POST", 2); +} + + + + + +/** Implements cUrlClient:Put() using cUrlClient::Request(). */ +static int tolua_cUrlClient_Put(lua_State * a_LuaState) +{ + /* Function signatures: + cUrlClient:Put(URL, {CallbacksFnTable}, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + cUrlClient:Put(URL, OnCompleteBodyCallback, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + */ + + return tolua_cUrlClient_Request_Common(a_LuaState, "PUT", 2); +} + + + + + +/** Binds cUrlClient::Request(). */ +static int tolua_cUrlClient_Request(lua_State * a_LuaState) +{ + /* Function signatures: + cUrlClient:Request(Method, URL, {CallbacksFnTable}, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + cUrlClient:Request(Method, URL, OnCompleteBodyCallback, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + */ + + // Check that the Method param is a string: + cLuaState L(a_LuaState); + if (!L.CheckParamString(2)) + { + return 0; + } + + // Redirect the rest to the common code: + AString method; + if (!L.GetStackValue(2, method)) + { + L.LogApiCallParamFailure("cUrlClient:Request", "Method (2)"); + L.Push(cLuaState::Nil); + L.Push("Invalid params"); + return 2; + } + return tolua_cUrlClient_Request_Common(a_LuaState, method, 3); +} + + + + + //////////////////////////////////////////////////////////////////////////////// // Register the bindings: @@ -913,10 +1238,12 @@ void cManualBindings::BindNetwork(lua_State * tolua_S) tolua_usertype(tolua_S, "cServerHandle"); tolua_usertype(tolua_S, "cTCPLink"); tolua_usertype(tolua_S, "cUDPEndpoint"); + tolua_usertype(tolua_S, "cUrlClient"); tolua_cclass(tolua_S, "cNetwork", "cNetwork", "", nullptr); tolua_cclass(tolua_S, "cServerHandle", "cServerHandle", "", tolua_collect_cServerHandle); tolua_cclass(tolua_S, "cTCPLink", "cTCPLink", "", nullptr); tolua_cclass(tolua_S, "cUDPEndpoint", "cUDPEndpoint", "", tolua_collect_cUDPEndpoint); + tolua_cclass(tolua_S, "cUrlClient", "cUrlClient", "", nullptr); // Fill in the functions (alpha-sorted): tolua_beginmodule(tolua_S, "cNetwork"); @@ -953,6 +1280,13 @@ void cManualBindings::BindNetwork(lua_State * tolua_S) tolua_function(tolua_S, "Send", tolua_cUDPEndpoint_Send); tolua_endmodule(tolua_S); + tolua_beginmodule(tolua_S, "cUrlClient"); + tolua_function(tolua_S, "Delete", tolua_cUrlClient_Delete); + tolua_function(tolua_S, "Get", tolua_cUrlClient_Get); + tolua_function(tolua_S, "Post", tolua_cUrlClient_Post); + tolua_function(tolua_S, "Put", tolua_cUrlClient_Put); + tolua_function(tolua_S, "Request", tolua_cUrlClient_Request); + tolua_endmodule(tolua_S); } diff --git a/src/OSSupport/TCPLinkImpl.cpp b/src/OSSupport/TCPLinkImpl.cpp index df70f3f72..d55dc9da1 100644 --- a/src/OSSupport/TCPLinkImpl.cpp +++ b/src/OSSupport/TCPLinkImpl.cpp @@ -322,6 +322,11 @@ void cTCPLinkImpl::EventCallback(bufferevent * a_BufferEvent, short a_What, void { ASSERT(a_Self != nullptr); cTCPLinkImplPtr Self = static_cast(a_Self)->m_Self; + if (Self == nullptr) + { + // The link has already been freed + return; + } // If an error is reported, call the error callback: if (a_What & BEV_EVENT_ERROR) -- cgit v1.2.3