diff --git a/include/eepp/network/http.hpp b/include/eepp/network/http.hpp index 659ef69bb..3bcf0d061 100644 --- a/include/eepp/network/http.hpp +++ b/include/eepp/network/http.hpp @@ -127,6 +127,12 @@ class EE_API Http : NonCopyable { /** Enables/Disables follow redirects */ void setFollowRedirect( bool follow ); + /** @return The maximun number of redirects allowd if follow redirect is enabled. */ + const unsigned int& getMaxRedirects() const; + + /** Set the maximun number of redirects allowed if follow redirect is enabled. */ + void setMaxRedirects( unsigned int maxRedirects ); + /** Definition of the current progress callback * @param http The http client * @param request The http request @@ -171,6 +177,7 @@ class EE_API Http : NonCopyable { bool mFollowRedirect; ///< Follows redirect response codes mutable bool mCancel; ///< Cancel state of current request ProgressCallback mProgressCallback; ///< Progress callback + unsigned int mMaxRedirections; ///< Maximun number of redirections allowed mutable unsigned int mRedirectionCount; ///< Number of redirections followed by the request }; diff --git a/include/eepp/network/uri.hpp b/include/eepp/network/uri.hpp index 9cce38f06..33f04fac1 100644 --- a/include/eepp/network/uri.hpp +++ b/include/eepp/network/uri.hpp @@ -213,6 +213,9 @@ class EE_API URI { /** Places the single path segments (delimited by slashes) into the given vector. */ void getPathSegments(std::vector& segments); + /** @return The last path segment if any */ + std::string getLastPathSegment(); + /** URI-encodes the given string by escaping reserved and non-ASCII * characters. The encoded string is appended to encodedStr. */ static void encode(const std::string& str, const std::string& reserved, std::string& encodedStr); diff --git a/include/eepp/system/filesystem.hpp b/include/eepp/system/filesystem.hpp index 36f799d1b..5da11ecf4 100644 --- a/include/eepp/system/filesystem.hpp +++ b/include/eepp/system/filesystem.hpp @@ -110,6 +110,13 @@ class EE_API FileSystem { /** @return Returns free disk space for a given path in bytes */ static Int64 getDiskFreeSpace(const std::string& path); + + /** Creates a file name available for the directory path. + * @example For file name "file-name-" will search the first available name + * in the file system starting from file-name-1, file-name-2, and so on. + * @return The file name found, otherwise empty string if error. + */ + static std::string fileGetNumberedFileNameFromPath( std::string directoryPath, const std::string& fileName, const std::string& separator = ".", const std::string& fileExtension = "" ); }; }} diff --git a/src/eepp/network/http.cpp b/src/eepp/network/http.cpp index 1b4e33820..7de52558b 100644 --- a/src/eepp/network/http.cpp +++ b/src/eepp/network/http.cpp @@ -13,6 +13,40 @@ using namespace EE::Network::SSL; namespace EE { namespace Network { +namespace { + +class IOFakeStreamString : public IOStream { + public: + ios_size read( char * data, ios_size size ) override { + return 0; + } + + ios_size write( const char * data, ios_size size ) override { + mStream.append( data, size ); + return std::move(size); + } + + ios_size seek( ios_size position ) override { + return 0; + } + + ios_size tell() override { + return getSize(); + } + + ios_size getSize() override { + return mStream.size(); + } + + bool isOpen() override { + return true; + } + + std::string mStream; +}; + +} + Http::Request::Method Http::Request::methodFromString( std::string methodString ) { String::toLowerInPlace(methodString); @@ -40,11 +74,12 @@ Http::Request::Request(const std::string& uri, Method method, const std::string& mValidateHostname( validateHostname ), mFollowRedirect( followRedirect ), mCancel( false ), + mMaxRedirections( 10 ), mRedirectionCount( 0 ) { setMethod(method); setUri(uri); - setHttpVersion(1, 0); + setHttpVersion(1, 1); setBody(body); } @@ -101,6 +136,14 @@ void Http::Request::setFollowRedirect(bool follow) { mFollowRedirect = follow; } +const unsigned int& Http::Request::getMaxRedirects() const { + return mMaxRedirections; +} + +void Http::Request::setMaxRedirects(unsigned int maxRedirects) { + mMaxRedirections = maxRedirects; +} + void Http::Request::setProgressCallback(const Http::Request::ProgressCallback& progressCallback) { mProgressCallback = progressCallback; } @@ -277,35 +320,7 @@ void Http::Response::parse(const std::string& data) { // Parse the other lines, which contain fields, one by one parseFields(in); - // Finally extract the body mBody.clear(); - - // Determine whether the transfer is chunked - if (String::toLower(getField("transfer-encoding")) != "chunked") { - // Not chunked - everything at once - std::copy(std::istreambuf_iterator(in), std::istreambuf_iterator(), std::back_inserter(mBody)); - } else { - // Chunked - have to read chunk by chunk - std::size_t length; - - // Read all chunks, identified by a chunk-size not being 0 - while (in >> std::hex >> length) { - // Drop the rest of the line (chunk-extension) - in.ignore(std::numeric_limits::max(), '\n'); - - // Copy the actual content data - std::istreambuf_iterator it(in); - std::istreambuf_iterator itEnd; - for (std::size_t i = 0; ((i < length) && (it != itEnd)); i++) - mBody.push_back(*it++); - } - - // Drop the rest of the line (chunk-extension) - in.ignore(std::numeric_limits::max(), '\n'); - - // Read all trailers (if present) - parseFields(in); - } } void Http::Response::parseFields(std::istream &in) { @@ -407,66 +422,10 @@ void Http::setHost(const std::string& host, unsigned short port, bool useSSL) { } Http::Response Http::sendRequest(const Http::Request& request, Time timeout) { - if ( 0 == mHost.toInteger() ) { - return Response(); - } - - if ( NULL == mConnection ) { - TcpSocket * Conn = mIsSSL ? SSLSocket::New( mHostName, request.getValidateCertificate(), request.getValidateHostname() ) : TcpSocket::New(); - mConnection = Conn; - } - - // First make sure that the request is valid -- add missing mandatory fields - Request toSend(prepareFields(request)); - - // Prepare the response - Response received; - - // Connect the socket to the host - if (mConnection->connect(mHost, mPort, timeout) == Socket::Done) { - // Convert the request to string and send it through the connected socket - std::string requestStr = toSend.prepare(); - - if (!requestStr.empty()) { - // Send it through the socket - if (mConnection->send(requestStr.c_str(), requestStr.size()) == Socket::Done) { - // Wait for the server's response - std::string receivedStr; - std::size_t size = 0; - char buffer[1024]; - - while (mConnection->receive(buffer, sizeof(buffer), size) == Socket::Done) { - receivedStr.append(buffer, buffer + size); - } - - // Build the Response object from the received data - received.parse(receivedStr); - } - } - - // Close the connection - mConnection->disconnect(); - } - - // If a redirection is requested, and requests follows redirections, - // send a new request to the redirection location. - if ( ( received.getStatus() == Response::MovedPermanently || received.getStatus() == Response::MovedTemporarily ) && - request.getFollowRedirect() ) { - - request.mRedirectionCount++; - - // Only continue redirecting if less than 10 redirections were done - if ( request.mRedirectionCount < 10 ) { - std::string location( received.getField("location") ); - URI uri( location ); - Http http( uri.getHost(), uri.getPort(), uri.getScheme() == "https" ? true : false ); - Http::Request newRequest( request ); - newRequest.setUri( uri.getPathEtc() ); - return http.sendRequest( request, timeout ); - } - } - - return std::move(received); + IOFakeStreamString stream; + Response response = downloadRequest( request, stream, timeout ); + response.mBody = std::move(stream.mStream); + return std::move(response); } Http::Response Http::downloadRequest(const Http::Request& request, IOStream& writeTo, Time timeout) { @@ -494,38 +453,41 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri // Send it through the socket if (mConnection->send(requestStr.c_str(), requestStr.size()) == Socket::Done) { // Wait for the server's response - int isnheader = 0; + bool isnheader = false; std::size_t currentTotalBytes = 0; std::size_t len = 0; char * eol; // end of line char * bol; // beginning of line - std::size_t size = 0; + std::size_t readed = 0; const std::size_t bufferSize = 1024; char buffer[bufferSize+1]; std::string header; + std::string fileBuffer; + bool newFileBuffer = false; + bool chunked = false; - while (!request.isCancelled() && mConnection->receive(buffer, bufferSize, size) == Socket::Done) { - if ( isnheader == 0 ) { + while (!request.isCancelled() && mConnection->receive(buffer, bufferSize, readed) == Socket::Done) { + if ( !isnheader ) { // calculate combined length of unprocessed data and new data - len += size; + len += readed; // NULL terminate buffer for string functions buffer[len] = '\0'; // checks if the header break happened to be the first line of the buffer - if ( !( strncmp( buffer, "\r\n", 2 ) ) ) { + if ( 0 == strncmp( buffer, "\r\n", 2 ) ) { if (len > 2) { currentTotalBytes += (len-2); - writeTo.write(buffer, (len-2)); + fileBuffer.append(buffer, buffer + (len-2)); } continue; } - if ( !( strncmp( buffer, "\n", 1 ) ) ) { + if ( 0 == strncmp( buffer, "\n", 1 ) ) { if ( len > 1 ) { currentTotalBytes += (len-1); - writeTo.write(buffer, (len-1)); + fileBuffer.append(buffer, buffer + (len-1)); } continue; @@ -534,14 +496,14 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri // process each line in buffer looking for header break bol = buffer; - while( ( eol = strchr( bol, '\n') ) != NULL ) { + while( !isnheader && ( eol = strchr( bol, '\n') ) != NULL ) { // update bol based upon the value of eol bol = eol + 1; // test if end of headers has been reached - if ( ( !( strncmp( bol, "\r\n", 2 ) ) ) || ( ! ( strncmp( bol, "\n", 1) ) ) ) { + if ( 0 == strncmp( bol, "\r\n", 2 ) || 0 == strncmp( bol, "\n", 1) ) { // note that end of headers has been reached - isnheader = 1; + isnheader = true; // update the value of bol to reflect the beginning of the line // immediately after the headers @@ -551,12 +513,12 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri bol += 1; // calculate the amount of data remaining in the buffer - len = size - ( bol - buffer ); + len = readed - ( bol - buffer ); // write remaining data to FILE stream if ( len > 0 ) { currentTotalBytes += len; - writeTo.write( bol, len ); + fileBuffer.append(bol, bol + len); } header.append( buffer, ( bol - buffer ) ); @@ -569,15 +531,22 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri // Build the Response object from the received data received.parse(header); + // Check if the response is chunked + chunked = received.getField("transfer-encoding") == "chunked"; + + // If is not chunked just save the file buffer and clear it + if ( !chunked && !fileBuffer.empty() ) { + writeTo.write( &fileBuffer[0], fileBuffer.size() ); + fileBuffer.clear(); + } + // If a redirection is requested, and requests follows redirections, // send a new request to the redirection location. if ( ( received.getStatus() == Response::MovedPermanently || received.getStatus() == Response::MovedTemporarily ) && request.getFollowRedirect() ) { - request.mRedirectionCount++; - // Only continue redirecting if less than 10 redirections were done - if ( request.mRedirectionCount < 10 ) { + if ( request.mRedirectionCount < request.getMaxRedirects() ) { std::string location( received.getField("location") ); URI uri( location ); Http http( uri.getHost(), uri.getPort(), uri.getScheme() == "https" ? true : false ); @@ -587,6 +556,8 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri // Close the connection mConnection->disconnect(); + request.mRedirectionCount++; + return http.downloadRequest( request, writeTo, timeout ); } } @@ -594,12 +565,48 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri } } - if ( isnheader == 0 ) { + if ( !isnheader ) { header.append( buffer, ( bol - buffer ) ); } } else { - currentTotalBytes += size; - writeTo.write( buffer, size ); + currentTotalBytes += readed; + + if ( chunked ) { + fileBuffer.append( buffer, buffer + readed ); + + if ( newFileBuffer ) { + if ( fileBuffer.substr( 0, 2 ) == "\r\n" ) { + fileBuffer = fileBuffer.substr( 2 ); + } + + newFileBuffer = false; + } + + std::string::size_type lenEnd = fileBuffer.find_first_of("\r\n"); + + if ( lenEnd != std::string::npos ) { + std::string::size_type firstCharPos = lenEnd + 2; + unsigned long length; + bool res = String::fromString( length, fileBuffer.substr(0, lenEnd), std::hex ); + + if ( res && length ) { + if ( fileBuffer.size() - firstCharPos >= length ) { + writeTo.write( &fileBuffer[firstCharPos], length ); + fileBuffer = fileBuffer.substr( firstCharPos + length ); + newFileBuffer = true; + + // Check if already have the \r\n of the next length in the buffer + if ( !fileBuffer.empty() && fileBuffer.substr( 0, 2 ) == "\r\n" ) { + // Remove it to be able to read the next length + fileBuffer = fileBuffer.substr( 2 ); + newFileBuffer = false; + } + } + } + } + } else { + writeTo.write( buffer, readed ); + } if ( request.getProgressCallback() ) { std::size_t length = 0; diff --git a/src/eepp/network/uri.cpp b/src/eepp/network/uri.cpp index f3c41c154..c1f734827 100644 --- a/src/eepp/network/uri.cpp +++ b/src/eepp/network/uri.cpp @@ -411,6 +411,17 @@ void URI::getPathSegments(std::vector& segments) { getPathSegments(mPath, segments); } +std::string URI::getLastPathSegment() { + std::vector segments; + getPathSegments( segments ); + + if ( !segments.empty() ) { + return segments[ segments.size() - 1 ]; + } + + return ""; +} + void URI::getPathSegments(const std::string& path, std::vector& segments) { std::string::const_iterator it = path.begin(); std::string::const_iterator end = path.end(); diff --git a/src/eepp/system/filesystem.cpp b/src/eepp/system/filesystem.cpp index 348207d74..142d873e0 100644 --- a/src/eepp/system/filesystem.cpp +++ b/src/eepp/system/filesystem.cpp @@ -580,4 +580,24 @@ Int64 FileSystem::getDiskFreeSpace(const std::string& path) { #endif } +std::string FileSystem::fileGetNumberedFileNameFromPath(std::string directoryPath, const std::string& fileName, const std::string& separator, const std::string& fileExtension) { + Uint32 fileNum = 1; + std::string fileNumName; + + if ( FileSystem::isDirectory( directoryPath ) ) { + dirPathAddSlashAtEnd( directoryPath ); + + while ( fileNum < 10000 ) { + fileNumName = String::format( std::string( "%s" + separator + "%d%s" ).c_str(), fileName.c_str(), fileNum, fileExtension.empty() ? "" : std::string( "." + fileExtension ).c_str() ); + + if ( !FileSystem::fileExists( directoryPath + fileNumName ) ) + return fileNumName; + + fileNum++; + } + } + + return ""; +} + }} diff --git a/src/examples/http_request/http_request.cpp b/src/examples/http_request/http_request.cpp index 9a0561a20..4ff57477b 100644 --- a/src/examples/http_request/http_request.cpp +++ b/src/examples/http_request/http_request.cpp @@ -1,6 +1,7 @@ #include #include +// Prints the response headers void printResponseHeaders( Http::Response& response ) { Http::Response::FieldTable headers = response.getHeaders(); @@ -18,6 +19,8 @@ EE_MAIN_FUNC int main (int argc, char * argv []) { args::ValueFlagList headers(parser, "header", "Pass custom header(s) to server", {'H', "header"}); args::Flag head(parser, "head", "Show document info", {'I',"head"}); args::Flag insecure(parser, "insecure", "Allow insecure server connections when using SSL", {'k',"insecure"}); + args::Flag location(parser, "location", "Follow redirects", {'L',"location"}); + args::ValueFlag maxRedirs(parser, "max-redirs", "Maximum number of redirects allowed", {"max-redirs"}); args::ValueFlag output(parser, "file", "Write to file instead of stdout", {'o', "output"}); args::Flag progress(parser, "progress", "Show current progress of a download", {'p',"progress"}); args::ValueFlag requestMethod(parser, "request", "Specify request command to use", {'X', "request"}); @@ -67,9 +70,10 @@ EE_MAIN_FUNC int main (int argc, char * argv []) { } }, asyncRequest, Seconds( 5 ) ); } else { - // If the user provided the URI, creates an instance of URI to parse it. + // If the user provided the URL, creates an instance of URI to parse it. URI uri( url.Get() ); + // If no scheme provided asume HTTP if ( uri.getScheme().empty() ) { uri = URI( "http://" + url.Get() ); } @@ -106,6 +110,23 @@ EE_MAIN_FUNC int main (int argc, char * argv []) { request.setBody( postData.Get() ); } + // If progress requested print a progress on screen + if ( progress ) { + request.setProgressCallback( []( const Http&, const Http::Request&, size_t totalBytes, size_t currentBytes ) { + std::cout << "\rDownloaded " << FileSystem::sizeToString( currentBytes ).c_str() << " of " << FileSystem::sizeToString( totalBytes ).c_str() << " "; + std::cout << std::flush; + return true; + }); + } + + // Set follow redirect + request.setFollowRedirect(location.Get()); + + // Set the maximun number of redirects + if ( maxRedirs ) { + request.setMaxRedirects(maxRedirs.Get()); + } + if ( !output ) { // Send the request Http::Response response = http.sendRequest(request); @@ -125,15 +146,30 @@ EE_MAIN_FUNC int main (int argc, char * argv []) { std::cout << "Error " << status << std::endl << response.getStatusDescription() << std::endl; } } else { - if ( progress ) { - request.setProgressCallback( []( const Http&, const Http::Request&, size_t totalBytes, size_t currentBytes ) { - std::cout << "\rDownloaded " << FileSystem::sizeToString( currentBytes ).c_str() << " of " << FileSystem::sizeToString( totalBytes ).c_str() << " "; - std::cout << std::flush; - return true; - }); + std::string path( output.Get() ); + + // If output path is a directory guess a file name + if ( FileSystem::isDirectory( path ) ) { + std::string lastPathSegment = uri.getLastPathSegment(); + + // If there's a path end segment + if ( !lastPathSegment.empty() ) { + FileSystem::dirPathAddSlashAtEnd( path ); + + // Save with the path end segment name + if ( !FileSystem::fileExists( path + lastPathSegment ) ) { + path += lastPathSegment; + } else { + path += FileSystem::fileGetNumberedFileNameFromPath( path, lastPathSegment ); + } + } else { + // Create a file name if no name found + path += FileSystem::fileGetNumberedFileNameFromPath( path, "eepp-network-file", "-" ); + } } - Http::Response response = http.downloadRequest(request, output.Get(), Seconds(5)); + // Download the request response into a file + Http::Response response = http.downloadRequest(request, path, Seconds(5)); if ( head ) printResponseHeaders(response);