From f664f89d343518bd7fe0900012f3cc87322bf1ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Sun, 28 Apr 2019 19:25:56 -0300 Subject: [PATCH] HTTP Proxy Tunneling support. --HG-- branch : dev-proxy --- include/eepp/network/http.hpp | 80 ++++- include/eepp/network/ssl/sslsocket.hpp | 15 +- src/eepp/network/http.cpp | 295 ++++++++++++++---- .../ssl/backend/mbedtls/mbedtlssocket.cpp | 4 +- src/eepp/network/ssl/sslsocket.cpp | 29 +- src/examples/http_request/http_request.cpp | 2 +- 6 files changed, 338 insertions(+), 87 deletions(-) diff --git a/include/eepp/network/http.hpp b/include/eepp/network/http.hpp index 9bf2f74b8..fa81aacbb 100644 --- a/include/eepp/network/http.hpp +++ b/include/eepp/network/http.hpp @@ -45,6 +45,9 @@ class EE_API Http : NonCopyable { /** @return Method from a method name string. */ static Method methodFromString( std::string methodString ); + /** @return The method string from a method */ + static std::string methodToString( const Method& method ); + /** @brief Default constructor ** This constructor creates a GET request, with the root ** URI ("/") and an empty body. @@ -135,15 +138,6 @@ class EE_API Http : NonCopyable { /** Set the maximun number of redirects allowed if follow redirect is enabled. */ void setMaxRedirects( unsigned int maxRedirects ); - /** Sets the request proxy */ - void setProxy( const URI& uri ); - - /** @return The request proxy */ - const URI& getProxy() const; - - /** @return Is a proxy is need to be used */ - bool isProxied() const; - /** Definition of the current progress callback * @param http The http client * @param request The http request @@ -164,7 +158,8 @@ class EE_API Http : NonCopyable { /** @return True if the current request was cancelled */ const bool& isCancelled() const; - private: + + private: friend class Http; /** @brief Prepare the final request to send to the server @@ -173,6 +168,9 @@ class EE_API Http : NonCopyable { ** @return String containing the request, ready to be sent */ std::string prepare(const Http& http) const; + /** Prepares a http tunnel request */ + std::string prepareTunnel(const Http& http); + // Types typedef std::map FieldTable; @@ -317,8 +315,10 @@ class EE_API Http : NonCopyable { ** than the standard one, or use an unknown protocol. ** @param host Web server to connect to ** @param port Port to use for connection - ** @param useSSL force the SSL usage ( if compiled with the support of it ). If the host starts with https:// it will use it by default. */ - Http(const std::string& host, unsigned short port = 0, bool useSSL = false); + ** @param useSSL force the SSL usage ( if compiled with the support of it ). If the host starts with https:// it will use it by default. + ** @param proxy Set an http proxy for the host connection + */ + Http(const std::string& host, unsigned short port = 0, bool useSSL = false, URI proxy = URI()); ~Http(); @@ -332,8 +332,10 @@ class EE_API Http : NonCopyable { ** than the standard one, or use an unknown protocol. ** @param host Web server to connect to ** @param port Port to use for connection - ** @param useSSL force the SSL usage ( if compiled with the support of it ). If the host starts with https:// it will use it by default. */ - void setHost(const std::string& host, unsigned short port = 0, bool useSSL = false); + ** @param useSSL force the SSL usage ( if compiled with the support of it ). If the host starts with https:// it will use it by default. + ** @param proxy Set an http proxy for the host connection + */ + void setHost(const std::string& host, unsigned short port = 0, bool useSSL = false, URI proxy = URI()); /** @brief Send a HTTP request and return the server's response. ** You must have a valid host before sending a request (see setHost). @@ -409,6 +411,15 @@ class EE_API Http : NonCopyable { /** @return The URI from the schema + hostname + port */ URI getURI() const; + + /** Sets the request proxy */ + void setProxy( const URI& uri ); + + /** @return The request proxy */ + const URI& getProxy() const; + + /** @return Is a proxy is need to be used */ + bool isProxied() const; private: class AsyncRequest : public Thread { public: @@ -433,8 +444,45 @@ class EE_API Http : NonCopyable { IOStream * mStream; }; + class HttpConnection { + public: + HttpConnection(); + + HttpConnection( TcpSocket * socket ); + + ~HttpConnection(); + + void setSocket( TcpSocket * socket ); + + TcpSocket * getSocket() const; + + void disconnect(); + + const bool& isConnected() const; + + void setConnected( const bool& connected ); + + const bool& isTunneled() const; + + void setTunneled( const bool& tunneled ); + + const bool& isSSL() const; + + void setSSL( const bool& ssl ); + + const bool& isKeepAlive() const; + + void setKeepAlive( const bool& isKeepAlive ); + protected: + TcpSocket * mSocket; + bool mIsConnected; + bool mIsTunneled; + bool mIsSSL; + bool mIsKeepAlive; + }; + friend class AsyncRequest; - ThreadLocalPtr mConnection; ///< Connection to the host + ThreadLocalPtr mConnection; ///< Connection to the host IpAddress mHost; ///< Web host address std::string mHostName; ///< Web host name unsigned short mPort; ///< Port used for connection with host @@ -443,8 +491,6 @@ class EE_API Http : NonCopyable { bool mIsSSL; URI mProxy; - Http(const std::string& host, unsigned short port, bool useSSL, URI proxy); - void removeOldThreads(); Request prepareFields(const Http::Request& request); diff --git a/include/eepp/network/ssl/sslsocket.hpp b/include/eepp/network/ssl/sslsocket.hpp index 1161d65eb..0f1829a52 100644 --- a/include/eepp/network/ssl/sslsocket.hpp +++ b/include/eepp/network/ssl/sslsocket.hpp @@ -36,15 +36,22 @@ class EE_API SSLSocket : public TcpSocket { Status receive(Packet& packet); + Status sslConnect(const IpAddress& remoteAddress, unsigned short remotePort, Time timeout = Time::Zero); + + void sslDisconnect(); + + Status tcpConnect(const IpAddress& remoteAddress, unsigned short remotePort, Time timeout = Time::Zero); + + void tcpDisconnect(); + + Status tcpReceive(void* data, std::size_t size, std::size_t& received); + + Status tcpSend(const void* data, std::size_t size, std::size_t& sent); protected: friend class SSLSocketImpl; friend class OpenSSLSocket; friend class MbedTLSSocket; - Status tcp_receive(void* data, std::size_t size, std::size_t& received); - - Status tcp_send(const void* data, std::size_t size, std::size_t& sent); - SSLSocketImpl * mImpl; std::string mHostName; bool mValidateCertificate; diff --git a/src/eepp/network/http.cpp b/src/eepp/network/http.cpp index 40d5277fd..abe746e36 100644 --- a/src/eepp/network/http.cpp +++ b/src/eepp/network/http.cpp @@ -71,6 +71,20 @@ Http::Request::Method Http::Request::methodFromString( std::string methodString } } +std::string Http::Request::methodToString(const Http::Request::Method& method) { + switch (method) { + default : + case Get: return "GET"; + case Head: return "HEAD"; + case Post: return "POST"; + case Put: return "PUT"; + case Delete: return "DELETE"; + case Options: return "OPTIONS"; + case Patch: return "PATCH"; + case Connect: return "CONNECT"; + } +} + Http::Request::Request(const std::string& uri, Method method, const std::string& body, bool validateCertificate, bool validateHostname , bool followRedirect) : mValidateCertificate( validateCertificate ), mValidateHostname( validateHostname ), @@ -146,18 +160,6 @@ void Http::Request::setMaxRedirects(unsigned int maxRedirects) { mMaxRedirections = maxRedirects; } -void Http::Request::setProxy(const URI& uri) { - mProxy = uri; -} - -const URI& Http::Request::getProxy() const { - return mProxy; -} - -bool Http::Request::isProxied() const { - return !mProxy.empty(); -} - void Http::Request::setProgressCallback(const Http::Request::ProgressCallback& progressCallback) { mProgressCallback = progressCallback; } @@ -174,25 +176,36 @@ const bool &Http::Request::isCancelled() const { return mCancel; } +std::string Http::Request::prepareTunnel(const Http& http) { + std::ostringstream out; + + setMethod( Connect ); + + std::string method = methodToString( mMethod ); + + out << method << " " << http.getHostName() << ":" << http.getPort() << " "; + out << "HTTP/" << mMajorVersion << "." << mMinorVersion << "\r\n"; + + setField( "Host", String::format( "%s:%d", http.getHostName().c_str(), http.getPort() ) ); + setField( "Proxy-Connection", "Keep-Alive" ); + setField( "User-Agent", "eepp-network" ); + + for (FieldTable::const_iterator i = mFields.begin(); i != mFields.end(); ++i) + out << i->first << ": " << i->second << "\r\n"; + + out << "\r\n"; + + return out.str(); +} + std::string Http::Request::prepare(const Http& http) const { std::ostringstream out; // Convert the method to its string representation - std::string method; - switch (mMethod) { - default : - case Get: method = "GET"; break; - case Head: method = "HEAD"; break; - case Post: method = "POST"; break; - case Put: method = "PUT"; break; - case Delete: method = "DELETE"; break; - case Options: method = "OPTIONS"; break; - case Patch: method = "PATCH"; break; - case Connect: method = "CONNECT"; break; - } + std::string method = methodToString( mMethod ); // Write the first line containing the request type - if ( mProxy.empty() ) { + if ( http.getProxy().empty() ) { out << method << " " << mUri << " "; } else { URI uri = http.getURI(); @@ -411,20 +424,14 @@ Http::Http() : { } -Http::Http(const std::string& host, unsigned short port, bool useSSL) : - mConnection( NULL ), - mIsSSL( false ) -{ - setHost(host, port, useSSL); -} - Http::Http(const std::string & host, unsigned short port, bool useSSL, URI proxy) : mConnection( NULL ), mHostName(host), mPort(port), + mIsSSL( useSSL ), mProxy(proxy) { - setHost(host, port, useSSL); + setHost(host, port, useSSL, proxy); } Http::~Http() { @@ -440,12 +447,14 @@ Http::~Http() { } // Then we destroy the last open connection - TcpSocket * tcp = mConnection; + HttpConnection * connection = mConnection; - eeSAFE_DELETE( tcp ); + eeSAFE_DELETE( connection ); } -void Http::setHost(const std::string& host, unsigned short port, bool useSSL) { +void Http::setHost(const std::string& host, unsigned short port, bool useSSL, URI proxy) { + mProxy = proxy; + bool sameHost( host == mHostName && port == mPort && useSSL == mIsSSL ); // Check the protocol @@ -489,8 +498,8 @@ void Http::setHost(const std::string& host, unsigned short port, bool useSSL) { // and there's an open connection to the host, we close // the old connection to prepare a new one. if ( !sameHost && NULL != mConnection ) { - TcpSocket * tcp = mConnection; - eeSAFE_DELETE( tcp ); + HttpConnection * connection = mConnection; + eeSAFE_DELETE( connection ); mConnection = NULL; } } @@ -503,20 +512,32 @@ Http::Response Http::sendRequest(const Http::Request& request, Time timeout) { } Http::Response Http::downloadRequest(const Http::Request& request, IOStream& writeTo, Time timeout) { - if ( mProxy.empty() && request.isProxied() ) { - Http http( mHostName, mPort, mIsSSL, request.getProxy() ); - return http.downloadRequest( request, writeTo, timeout ); - } - if ( 0 == mHost.toInteger() ) { return Response(); } if ( NULL == mConnection ) { - TcpSocket * Conn = ( mProxy.empty() ? mIsSSL : ( SSLSocket::isSupported() && mProxy.getScheme() == "https" ) ) ? - SSLSocket::New( mHostName, request.getValidateCertificate(), request.getValidateHostname() ) : - TcpSocket::New(); - mConnection = Conn; + HttpConnection * connection = eeNew( HttpConnection, () ); + TcpSocket * socket = NULL; + + // If the http client is proxied and the end host use SSL + // We need to create an HTTP Tunnel against the proxy server + if ( isProxied() && mIsSSL && SSLSocket::isSupported() ) { + socket = SSLSocket::New( mHostName, request.getValidateCertificate(), request.getValidateHostname() ); + + connection->setSSL( true ); + } else { + bool isSSL = !isProxied() ? mIsSSL : ( SSLSocket::isSupported() && mProxy.getScheme() == "https" ); + + socket = isSSL ? SSLSocket::New( mHostName, request.getValidateCertificate(), request.getValidateHostname() ) : + TcpSocket::New(); + + connection->setSSL( isSSL ); + } + + connection->setSocket( socket ); + + mConnection = connection; } // First make sure that the request is valid -- add missing mandatory fields @@ -525,14 +546,77 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri // Prepare the response Response received; + // If not connected, try to connect to the server + if ( !mConnection->isConnected() ) { + // We need to create an HTTP Tunnel? + if ( isProxied() && mIsSSL && SSLSocket::isSupported() ) { + SSLSocket * sslSocket = reinterpret_cast( mConnection->getSocket() ); + + // For an HTTP Tunnel first we need to connect to the proxy server ( without TLS ) + if (sslSocket->tcpConnect( mHost, mProxy.getPort(), timeout ) != Socket::Done) { + return std::move(received); + } else { + mConnection->setConnected(true); + } + } else { + if (mConnection->getSocket()->connect(mHost, mProxy.empty() ? mPort : mProxy.getPort(), timeout) != Socket::Done) { + return std::move(received); + } else { + mConnection->setConnected(true); + } + } + } + // Connect the socket to the host - if (mConnection->connect(mHost, mProxy.empty() ? mPort : mProxy.getPort(), timeout) == Socket::Done) { + if (mConnection->isConnected()) { + // Create a HTTP Tunnel for SSL connections if not ready + if ( isProxied() && mIsSSL && !mConnection->isTunneled() ) { + // Create the HTTP Tunnel request + Request tunnelRequest; + std::string tunnelStr = tunnelRequest.prepareTunnel(*this); + + SSLSocket * sslSocket = reinterpret_cast( mConnection->getSocket() ); + std::size_t sent; + + // Send the request + if (sslSocket->tcpSend(tunnelStr.c_str(), tunnelStr.size(), sent) == Socket::Done) { + const std::size_t bufferSize = 1024; + char buffer[bufferSize+1]; + std::size_t readed = 0; + + // Get the proxy server response + if (sslSocket->tcpReceive(buffer, bufferSize, readed) == Socket::Done) { + // Parse the HTTP Tunnel request response + Response tunnelResponse; + std::string header; + header.append( buffer, readed ); + tunnelResponse.parse(header); + + if ( tunnelResponse.getStatus() == Response::Ok ) { + // Stablish the SSL connection if the response is positive + if (sslSocket->sslConnect( mHost, mProxy.getPort(), timeout ) != Socket::Done) { + return std::move(received); + } + } else { + return std::move(tunnelResponse); + } + } else { + return std::move(received); + } + + mConnection->setTunneled(true); + mConnection->setKeepAlive(true); + } + } + // Convert the request to string and send it through the connected socket std::string requestStr = toSend.prepare(*this); if (!requestStr.empty()) { + Socket::Status status; + // Send it through the socket - if (mConnection->send(requestStr.c_str(), requestStr.size()) == Socket::Done) { + if (mConnection->getSocket()->send(requestStr.c_str(), requestStr.size()) == Socket::Done) { // Wait for the server's response bool isnheader = false; std::size_t currentTotalBytes = 0; @@ -547,7 +631,7 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri bool newFileBuffer = false; bool chunked = false; - while (!request.isCancelled() && mConnection->receive(buffer, bufferSize, readed) == Socket::Done) { + while (!request.isCancelled() && ( status = mConnection->getSocket()->receive(buffer, bufferSize, readed) ) == Socket::Done) { if ( !isnheader ) { // calculate combined length of unprocessed data and new data len += readed; @@ -621,6 +705,11 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri fileBuffer.clear(); } + if ( received.getField("connection") == "closed" ) { + mConnection->setConnected(false); + mConnection->setTunneled(false); + } + // If a redirection is requested, and requests follows redirections, // send a new request to the redirection location. if ( ( received.getStatus() == Response::MovedPermanently || received.getStatus() == Response::MovedTemporarily ) && @@ -635,7 +724,8 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri newRequest.setUri( uri.getPathEtc() ); // Close the connection - mConnection->disconnect(); + if ( !mConnection->isKeepAlive() ) + mConnection->disconnect(); request.mRedirectionCount++; @@ -703,11 +793,20 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri } } } + + if ( status == Socket::Status::Disconnected ) { + mConnection->setConnected(false); + mConnection->setTunneled(false); + } + } else { + mConnection->setConnected(false); + mConnection->setTunneled(false); } } // Close the connection - mConnection->disconnect(); + if ( !mConnection->isKeepAlive() ) + mConnection->disconnect(); } return std::move(received); @@ -769,8 +868,8 @@ void Http::AsyncRequest::run() { } // The Async Request destroys the socket used to create the request - TcpSocket * tcp = mHttp->mConnection; - eeSAFE_DELETE( tcp ); + HttpConnection * connection = mHttp->mConnection; + eeSAFE_DELETE( connection ); mHttp->mConnection = NULL; mRunning = false; @@ -827,12 +926,28 @@ Http::Request Http::prepareFields(const Http::Request& request) { if (!mProxy.empty()) { toSend.setField("Accept", "*/*"); - toSend.setField("Proxy-connection", "close"); + if ( mIsSSL ) { + toSend.setField("Proxy-connection", "keep-alive"); + } else { + toSend.setField("Proxy-connection", "close"); + } } return std::move(toSend); } +void Http::setProxy(const URI& uri) { + setHost( mHostName, mPort, mIsSSL, uri ); +} + +const URI& Http::getProxy() const { + return mProxy; +} + +bool Http::isProxied() const { + return !mProxy.empty(); +} + void Http::sendAsyncRequest( AsyncResponseCallback cb, const Http::Request& request, Time timeout ) { AsyncRequest * thread = eeNew( AsyncRequest, ( this, cb, request, timeout ) ); @@ -872,11 +987,11 @@ void Http::downloadAsyncRequest(Http::AsyncResponseCallback cb, const Http::Requ mThreads.push_back( thread ); } -const IpAddress &Http::getHost() const { +const IpAddress& Http::getHost() const { return mHost; } -const std::string &Http::getHostName() const { +const std::string& Http::getHostName() const { return mHostName; } @@ -892,4 +1007,70 @@ URI Http::getURI() const { return URI( String::format( "%s://%s:%d", mIsSSL ? "https" : "http", mHostName.c_str(), mPort ) ); } +Http::HttpConnection::HttpConnection() : + mSocket(NULL), + mIsConnected(false), + mIsTunneled(false), + mIsSSL(false), + mIsKeepAlive(false) +{} + +Http::HttpConnection::HttpConnection(TcpSocket * socket) : + mSocket( socket ), + mIsConnected( false ), + mIsTunneled( false ), + mIsSSL( false ) +{} + +Http::HttpConnection::~HttpConnection() { + eeSAFE_DELETE(mSocket); +} + +void Http::HttpConnection::setSocket(TcpSocket * socket) { + mSocket = socket; +} + +TcpSocket *Http::HttpConnection::getSocket() const { + return mSocket; +} + +void Http::HttpConnection::disconnect() { + if ( NULL != mSocket ) + mSocket->disconnect(); + + mIsConnected = false; +} + +const bool &Http::HttpConnection::isConnected() const { + return mIsConnected; +} + +void Http::HttpConnection::setConnected(const bool & connected) { + mIsConnected = connected; +} + +const bool &Http::HttpConnection::isTunneled() const { + return mIsTunneled; +} + +void Http::HttpConnection::setTunneled(const bool & tunneled) { + mIsTunneled = tunneled; +} + +const bool &Http::HttpConnection::isSSL() const { + return mIsSSL; +} + +void Http::HttpConnection::setSSL(const bool & ssl) { + mIsSSL = ssl; +} + +const bool &Http::HttpConnection::isKeepAlive() const { + return mIsKeepAlive; +} + +void Http::HttpConnection::setKeepAlive(const bool & isKeepAlive) { + mIsKeepAlive = isKeepAlive; +} + }} diff --git a/src/eepp/network/ssl/backend/mbedtls/mbedtlssocket.cpp b/src/eepp/network/ssl/backend/mbedtls/mbedtlssocket.cpp index cf341e472..2f32240a0 100644 --- a/src/eepp/network/ssl/backend/mbedtls/mbedtlssocket.cpp +++ b/src/eepp/network/ssl/backend/mbedtls/mbedtlssocket.cpp @@ -72,7 +72,7 @@ int MbedTLSSocket::bio_send(void *ctx, const unsigned char *buf, size_t len) { MbedTLSSocket *sp = (MbedTLSSocket *)ctx; size_t sent; - Socket::Status err = sp->mSSLSocket->tcp_send((const void*)buf, len, sent); + Socket::Status err = sp->mSSLSocket->tcpSend((const void*)buf, len, sent); if (err != Socket::Done) { return MBEDTLS_ERR_SSL_INTERNAL_ERROR; @@ -90,7 +90,7 @@ int MbedTLSSocket::bio_recv(void *ctx, unsigned char *buf, size_t len) { MbedTLSSocket *sp = (MbedTLSSocket *)ctx; size_t got; - Socket::Status err = sp->mSSLSocket->tcp_receive(buf, len, got); + Socket::Status err = sp->mSSLSocket->tcpReceive(buf, len, got); if (err != Socket::Done) { return MBEDTLS_ERR_SSL_INTERNAL_ERROR; diff --git a/src/eepp/network/ssl/sslsocket.cpp b/src/eepp/network/ssl/sslsocket.cpp index fff6a3fb2..5967b8f39 100644 --- a/src/eepp/network/ssl/sslsocket.cpp +++ b/src/eepp/network/ssl/sslsocket.cpp @@ -134,16 +134,17 @@ SSLSocket::~SSLSocket() { Socket::Status SSLSocket::connect( const IpAddress& remoteAddress, unsigned short remotePort, Time timeout ) { Status status = Socket::Disconnected; - if ( ( status = TcpSocket::connect( remoteAddress, remotePort, timeout ) ) == Socket::Done ) { - status = mImpl->connect( remoteAddress, remotePort, timeout ); + if ( ( status = tcpConnect( remoteAddress, remotePort, timeout ) ) == Socket::Done ) { + status = sslConnect( remoteAddress, remotePort, timeout ); } return status; } void SSLSocket::disconnect() { - mImpl->disconnect(); - TcpSocket::disconnect(); + sslDisconnect(); + + tcpDisconnect(); } Socket::Status SSLSocket::send(const void* data, std::size_t size) { @@ -162,11 +163,27 @@ Socket::Status SSLSocket::receive(Packet& packet) { return TcpSocket::receive( packet ); } -Socket::Status SSLSocket::tcp_receive(void * data, std::size_t size, std::size_t & received) { +Socket::Status SSLSocket::sslConnect(const IpAddress & remoteAddress, unsigned short remotePort, Time timeout) { + return mImpl->connect( remoteAddress, remotePort, timeout ); +} + +void SSLSocket::sslDisconnect() { + mImpl->disconnect(); +} + +Socket::Status SSLSocket::tcpConnect(const IpAddress & remoteAddress, unsigned short remotePort, Time timeout) { + return TcpSocket::connect( remoteAddress, remotePort, timeout ); +} + +void SSLSocket::tcpDisconnect() { + TcpSocket::disconnect(); +} + +Socket::Status SSLSocket::tcpReceive(void * data, std::size_t size, std::size_t & received) { return TcpSocket::receive( data, size, received ); } -Socket::Status SSLSocket::tcp_send(const void * data, std::size_t size, std::size_t & sent) { +Socket::Status SSLSocket::tcpSend(const void * data, std::size_t size, std::size_t & sent) { return TcpSocket::send( data, size, sent ); } diff --git a/src/examples/http_request/http_request.cpp b/src/examples/http_request/http_request.cpp index 6b5c5fe25..2dec6548e 100644 --- a/src/examples/http_request/http_request.cpp +++ b/src/examples/http_request/http_request.cpp @@ -133,7 +133,7 @@ EE_MAIN_FUNC int main (int argc, char * argv []) { // Set the proxy for the request if ( proxy ) { - request.setProxy( URI( proxy.Get() ) ); + http.setProxy( URI( proxy.Get() ) ); } if ( !output ) {