diff --git a/include/eepp/network/http.hpp b/include/eepp/network/http.hpp index fa81aacbb..a65e02f5d 100644 --- a/include/eepp/network/http.hpp +++ b/include/eepp/network/http.hpp @@ -57,8 +57,9 @@ class EE_API Http : NonCopyable { ** @param validateCertificate Enables certificate validation for https request ** @param validateHostname Enables hostname validation for https request ** @param followRedirect Allow follor redirects to the request. + ** @param compressedResponse Set if the requested response should be compressed ( if available ) */ - Request(const std::string& uri = "/", Method method = Get, const std::string& body = "", bool validateCertificate = true, bool validateHostname = true, bool followRedirect = true); + Request(const std::string& uri = "/", Method method = Get, const std::string& body = "", bool validateCertificate = true, bool validateHostname = true, bool followRedirect = true, bool compressedResponse = false); /** @brief Set the value of a field ** The field is created if it doesn't exist. The name of @@ -159,6 +160,15 @@ class EE_API Http : NonCopyable { /** @return True if the current request was cancelled */ const bool& isCancelled() const; + /** @return If requests a compressed response */ + const bool& isCompressedResponse() const; + + /** Set to request a compressed response from the server + ** The returned response will be automatically decompressed + ** by the client. + */ + void setCompressedResponse(const bool& compressedResponse); + private: friend class Http; @@ -184,6 +194,7 @@ class EE_API Http : NonCopyable { bool mValidateCertificate; ///< Validates the SSL certificate in case of an HTTPS request bool mValidateHostname; ///< Validates the hostname in case of an HTTPS request bool mFollowRedirect; ///< Follows redirect response codes + bool mCompressedResponse; ///< Request comrpessed response mutable bool mCancel; ///< Cancel state of current request ProgressCallback mProgressCallback; ///< Progress callback unsigned int mMaxRedirections; ///< Maximun number of redirections allowed diff --git a/include/eepp/system.hpp b/include/eepp/system.hpp index 47d94a3db..bcffdedf4 100644 --- a/include/eepp/system.hpp +++ b/include/eepp/system.hpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,8 @@ #include #include #include +#include +#include #include #endif diff --git a/include/eepp/system/compression.hpp b/include/eepp/system/compression.hpp new file mode 100644 index 000000000..3f77e9eb7 --- /dev/null +++ b/include/eepp/system/compression.hpp @@ -0,0 +1,55 @@ +#ifndef EE_SYSTEM_COMPRESSION_HPP +#define EE_SYSTEM_COMPRESSION_HPP + +#include +#include + +namespace EE { namespace System { + +class Compression { + public: + enum Mode { + MODE_DEFLATE, + MODE_GZIP + }; + + enum Status { + OK = 0, + ERRNO = -1, + STREAM_ERROR = -2, + DATA_ERROR = -3, + MEM_ERROR = -4, + BUF_ERROR = -5, + VERSION_ERROR = -6 + }; + + struct ZlibConfig { + int level= -1; + }; + + struct GzipConfig { + int level = -1; + }; + + struct Config { + Config() {} + ZlibConfig zlib; + GzipConfig gzip; + }; + + static Status compress(Uint8* dst, Uint64 dstMaxSize, const Uint8* src, Uint64 srcSize, Mode mode = MODE_DEFLATE, const Config& config = Config()); + + static Status compress(IOStream& dst, IOStream& src, Mode mode = MODE_DEFLATE, const Config& config = Config()); + + static int getMaxCompressedBufferSize(Uint64 srcSize, Mode mode = MODE_DEFLATE, const Config& config = Config()); + + static Status decompress(Uint8* dst, Uint64 dstMaxSize, const Uint8* src, Uint64 srcSize, Mode mode = MODE_DEFLATE); + + static Status decompress(IOStream& dst, IOStream& src, Mode mode = MODE_DEFLATE); + + static std::size_t getModeDefaultChunkSize( const Mode& mode ); +}; + +}} + +#endif // EE_SYSTEM_COMPRESSION_HPP diff --git a/include/eepp/system/iostreaminflate.hpp b/include/eepp/system/iostreaminflate.hpp new file mode 100644 index 000000000..3ea1a1c45 --- /dev/null +++ b/include/eepp/system/iostreaminflate.hpp @@ -0,0 +1,48 @@ +#ifndef EE_SYSTEM_IOSTREAMINFLATE_HPP +#define EE_SYSTEM_IOSTREAMINFLATE_HPP + +#include +#include +#include + +namespace EE { namespace System { + +struct LocalStreamData; + +/** @brief Implementation of a inflating stream */ +class EE_API IOStreamInflate : public IOStream { + public: + static IOStreamInflate * New( IOStream& inOutStream, Compression::Mode mode ); + + /** @brief Use a stream as a input or output buffer + ** @param inOutStream Stream where the results will ve loaded or saved. + ** It must be used only for reading or writing, can't mix both calls. + ** @param mode Compression/Decompression method used + */ + IOStreamInflate( IOStream& inOutStream, Compression::Mode mode ); + + virtual ~IOStreamInflate(); + + ios_size read( char * data, ios_size size ); + + ios_size write( const char * data, ios_size size ); + + ios_size seek( ios_size position ); + + ios_size tell(); + + ios_size getSize(); + + bool isOpen(); + + const Compression::Mode& getMode() const; + protected: + IOStream& mStream; + Compression::Mode mMode; + SafeDataPointer mBuffer; + LocalStreamData * mLocalStream; +}; + +}} + +#endif // EE_SYSTEM_IOSTREAMINFLATE_HPP diff --git a/include/eepp/system/iostreamstring.hpp b/include/eepp/system/iostreamstring.hpp new file mode 100644 index 000000000..4f2b4ad84 --- /dev/null +++ b/include/eepp/system/iostreamstring.hpp @@ -0,0 +1,44 @@ +#ifndef EE_SYSTEM_IOSTREAMSTRING_HPP +#define EE_SYSTEM_IOSTREAMSTRING_HPP + +#include +#include + +namespace EE { namespace System { + +/** @brief Implementation of a memory stream file using an std::string as a container */ +class EE_API IOStreamString : public IOStream { + public: + IOStreamString(); + + ios_size read( char * data, ios_size size ); + + ios_size write( const char * data, ios_size size ); + + ios_size write( const std::string& string ); + + ios_size seek( ios_size position ); + + ios_size tell(); + + ios_size getSize(); + + bool isOpen(); + + void clear(); + + /** @return Pointer to the current position in the stream */ + const char * getPositionPointer(); + + /** @return The pointer to the beggining of the stream */ + const char * getStreamPointer() const; + + const std::string& getStream() const; + protected: + std::string mStream; + ios_size mPos; +}; + +}} + +#endif // EE_SYSTEM_IOSTREAMSTRING_HPP diff --git a/projects/linux/ee.files b/projects/linux/ee.files index 75889ec8b..daaec457c 100644 --- a/projects/linux/ee.files +++ b/projects/linux/ee.files @@ -241,6 +241,7 @@ ../../include/eepp/system/bitop.hpp ../../include/eepp/system/clock.hpp ../../include/eepp/system/color.hpp +../../include/eepp/system/compression.hpp ../../include/eepp/system/condition.hpp ../../include/eepp/system/container.hpp ../../include/eepp/system/directorypack.hpp @@ -249,8 +250,10 @@ ../../include/eepp/system/inifile.hpp ../../include/eepp/system/iostreamfile.hpp ../../include/eepp/system/iostream.hpp +../../include/eepp/system/iostreaminflate.hpp ../../include/eepp/system/iostreammemory.hpp ../../include/eepp/system/iostreampak.hpp +../../include/eepp/system/iostreamstring.hpp ../../include/eepp/system/iostreamzip.hpp ../../include/eepp/system/lock.hpp ../../include/eepp/system/log.hpp @@ -666,13 +669,16 @@ ../../src/eepp/system/base64.cpp ../../src/eepp/system/clock.cpp ../../src/eepp/system/color.cpp +../../src/eepp/system/compression.cpp ../../src/eepp/system/condition.cpp ../../src/eepp/system/directorypack.cpp ../../src/eepp/system/filesystem.cpp ../../src/eepp/system/inifile.cpp ../../src/eepp/system/iostreamfile.cpp +../../src/eepp/system/iostreaminflate.cpp ../../src/eepp/system/iostreammemory.cpp ../../src/eepp/system/iostreampak.cpp +../../src/eepp/system/iostreamstring.cpp ../../src/eepp/system/iostreamzip.cpp ../../src/eepp/system/lock.cpp ../../src/eepp/system/log.cpp diff --git a/src/eepp/network/http.cpp b/src/eepp/network/http.cpp index fc0a6c681..c4b63e65a 100644 --- a/src/eepp/network/http.cpp +++ b/src/eepp/network/http.cpp @@ -3,6 +3,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -15,62 +18,17 @@ namespace EE { namespace Network { #define PACKET_BUFFER_SIZE (16384) -namespace { - -class IOFakeStreamString : public IOStream { - public: - ios_size read( char * data, ios_size size ) override { - return 0; - } - - ios_size write( const char * data, ios_size size ) override { - mStream.append( data, size ); - return std::move(size); - } - - ios_size seek( ios_size position ) override { - return 0; - } - - ios_size tell() override { - return getSize(); - } - - ios_size getSize() override { - return mStream.size(); - } - - bool isOpen() override { - return true; - } - - std::string mStream; -}; - -} - Http::Request::Method Http::Request::methodFromString( std::string methodString ) { String::toLowerInPlace(methodString); - - if ( "get" == methodString ) { - return Method::Get; - } else if ( "head" == methodString ) { - return Method::Head; - } else if ( "post" == methodString ) { - return Method::Post; - } else if ( "put" == methodString ) { - return Method::Put; - } else if ( "delete" == methodString ) { - return Method::Delete; - } else if ( "options" == methodString ) { - return Method::Options; - } else if ( "patch" == methodString ) { - return Method::Patch; - } else if ( "connect" == methodString ) { - return Method::Connect; - } else { - return Method::Get; - } + if ( "get" == methodString ) return Method::Get; + else if ( "head" == methodString ) return Method::Head; + else if ( "post" == methodString ) return Method::Post; + else if ( "put" == methodString ) return Method::Put; + else if ( "delete" == methodString ) return Method::Delete; + else if ( "options" == methodString ) return Method::Options; + else if ( "patch" == methodString ) return Method::Patch; + else if ( "connect" == methodString ) return Method::Connect; + else return Method::Get; } std::string Http::Request::methodToString(const Http::Request::Method& method) { @@ -87,10 +45,11 @@ std::string Http::Request::methodToString(const Http::Request::Method& method) { } } -Http::Request::Request(const std::string& uri, Method method, const std::string& body, bool validateCertificate, bool validateHostname , bool followRedirect) : +Http::Request::Request(const std::string& uri, Method method, const std::string& body, bool validateCertificate, bool validateHostname , bool followRedirect, bool compressedResponse) : mValidateCertificate( validateCertificate ), mValidateHostname( validateHostname ), mFollowRedirect( followRedirect ), + mCompressedResponse( compressedResponse ), mCancel( false ), mMaxRedirections( 10 ), mRedirectionCount( 0 ) @@ -200,6 +159,14 @@ std::string Http::Request::prepareTunnel(const Http& http) { return out.str(); } +const bool& Http::Request::isCompressedResponse() const { + return mCompressedResponse; +} + +void Http::Request::setCompressedResponse(const bool& compressedResponse) { + mCompressedResponse = compressedResponse; +} + std::string Http::Request::prepare(const Http& http) const { std::ostringstream out; @@ -440,12 +407,12 @@ Http::~Http() { std::list::iterator itt; // First we wait to finish any request pending - for ( itt = mThreads.begin(); itt != mThreads.end(); ++itt ) { - (*itt)->wait(); + for ( auto&& itt : mThreads ) { + itt->wait(); } - for ( itt = mThreads.begin(); itt != mThreads.end(); ++itt ) { - eeDelete( *itt ); + for ( auto&& itt : mThreads ) { + eeDelete( itt ); } // Then we destroy the last open connection @@ -507,10 +474,10 @@ void Http::setHost(const std::string& host, unsigned short port, bool useSSL, UR } Http::Response Http::sendRequest(const Http::Request& request, Time timeout) { - IOFakeStreamString stream; + IOStreamString stream; Response response = downloadRequest( request, stream, timeout ); - response.mBody = std::move(stream.mStream); - return std::move(response); + response.mBody = stream.getStream(); + return response; } Http::Response Http::downloadRequest(const Http::Request& request, IOStream& writeTo, Time timeout) { @@ -556,13 +523,13 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri // For an HTTP Tunnel first we need to connect to the proxy server ( without TLS ) if (sslSocket->tcpConnect( mHost, mProxy.getPort(), timeout ) != Socket::Done) { - return std::move(received); + return received; } else { mConnection->setConnected(true); } } else { if (mConnection->getSocket()->connect(mHost, mProxy.empty() ? mPort : mProxy.getPort(), timeout) != Socket::Done) { - return std::move(received); + return received; } else { mConnection->setConnected(true); } @@ -596,13 +563,13 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri if ( tunnelResponse.getStatus() == Response::Ok ) { // Stablish the SSL connection if the response is positive if (sslSocket->sslConnect( mHost, mProxy.getPort(), timeout ) != Socket::Done) { - return std::move(received); + return received; } } else { - return std::move(tunnelResponse); + return tunnelResponse; } } else { - return std::move(received); + return received; } mConnection->setTunneled(true); @@ -619,19 +586,26 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri // Send it through the socket if (mConnection->getSocket()->send(requestStr.c_str(), requestStr.size()) == Socket::Done) { // Wait for the server's response - bool isnheader = false; std::size_t currentTotalBytes = 0; std::size_t len = 0; + std::size_t readed = 0; char * eol; // end of line char * bol; // beginning of line - std::size_t readed = 0; char buffer[PACKET_BUFFER_SIZE+1]; - std::string header; - std::string fileBuffer; - bool newFileBuffer = false; + std::string headerBuffer; + std::string chunkBuffer; + IOStreamString fileBuffer; + bool isnheader = false; bool chunked = false; + bool chunkNewBuffer = false; + bool chunkEnded = false; + bool compressed = false; + IOStreamInflate * inflateStream = NULL; + ios_size inflateChunkSize = 0; + std::size_t contentLength = 0; while (!request.isCancelled() && ( status = mConnection->getSocket()->receive(buffer, PACKET_BUFFER_SIZE, readed) ) == Socket::Done) { + // If we didn't receive the header yet, we will try to find the end of the header if ( !isnheader ) { // calculate combined length of unprocessed data and new data len += readed; @@ -643,7 +617,7 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri if ( 0 == strncmp( buffer, "\r\n", 2 ) ) { if (len > 2) { currentTotalBytes += (len-2); - fileBuffer.append(buffer, buffer + (len-2)); + chunkBuffer.append(buffer, buffer + (len-2)); } continue; @@ -652,7 +626,7 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri if ( 0 == strncmp( buffer, "\n", 1 ) ) { if ( len > 1 ) { currentTotalBytes += (len-1); - fileBuffer.append(buffer, buffer + (len-1)); + chunkBuffer.append(buffer, buffer + (len-1)); } continue; @@ -683,26 +657,40 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri // write remaining data to FILE stream if ( len > 0 ) { currentTotalBytes += len; - fileBuffer.append(bol, bol + len); + chunkBuffer.append(bol, bol + len); } - header.append( buffer, ( bol - buffer ) ); + headerBuffer.append( buffer, ( bol - buffer ) ); // reset length of left over data to zero and continue processing // non-header information len = 0; - if ( !header.empty() ) { + if ( !headerBuffer.empty() ) { // Build the Response object from the received data - received.parse(header); + received.parse(headerBuffer); + + headerBuffer.clear(); // Check if the response is chunked chunked = received.getField("transfer-encoding") == "chunked"; - // If is not chunked just save the file buffer and clear it - if ( !chunked && !fileBuffer.empty() ) { - writeTo.write( &fileBuffer[0], fileBuffer.size() ); - fileBuffer.clear(); + // Check if the content is compressed + std::string encoding( received.getField("content-encoding") ); + compressed = encoding == "gzip" || encoding == "deflate"; + + if ( compressed ) { + Compression::Mode compressionMode = "gzip" == encoding ? Compression::MODE_GZIP : Compression::MODE_DEFLATE; + + inflateChunkSize = Compression::getModeDefaultChunkSize( compressionMode ); + + inflateStream = IOStreamInflate::New( writeTo, compressionMode ); + } + + // Get the content length + if ( !received.getField("content-length").empty() ) { + if ( !String::fromString( contentLength, received.getField("content-length") ) ) + contentLength = 0; } if ( received.getField("connection") == "closed" ) { @@ -732,61 +720,108 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri return http.downloadRequest( request, writeTo, timeout ); } } + + // If is not chunked just save the file buffer and clear it + if ( !chunked && !chunkBuffer.empty() ) { + fileBuffer.write( &chunkBuffer[0], chunkBuffer.size() ); + chunkBuffer.clear(); + } } } } if ( !isnheader ) { - header.append( buffer, ( bol - buffer ) ); + headerBuffer.append( buffer, ( bol - buffer ) ); } } else { currentTotalBytes += readed; if ( chunked ) { - fileBuffer.append( buffer, buffer + readed ); + // If the chunk reading ended we just add the buffer received as a header + // Otherwise we process the buffer data as chunk + if ( !chunkEnded ) { + // Keep a chunk buffer until the end of chunk is found + chunkBuffer.append( buffer, buffer + readed ); - if ( newFileBuffer ) { - if ( fileBuffer.substr( 0, 2 ) == "\r\n" ) { - fileBuffer = fileBuffer.substr( 2 ); + // If the new chunk starts with \r\n and the last removed chunk + // did not contain the trailing \r\n, we remove it to detect + // correctly the next length data + if ( chunkNewBuffer ) { + if ( chunkBuffer.substr( 0, 2 ) == "\r\n" ) { + chunkBuffer = chunkBuffer.substr( 2 ); + } + + chunkNewBuffer = false; } - newFileBuffer = false; - } + // Check for the first \r\n to find the end of the length definition + std::string::size_type lenEnd = chunkBuffer.find_first_of("\r\n"); - std::string::size_type lenEnd = fileBuffer.find_first_of("\r\n"); + if ( lenEnd != std::string::npos ) { + std::string::size_type firstCharPos = lenEnd + 2; + unsigned long length; - if ( lenEnd != std::string::npos ) { - std::string::size_type firstCharPos = lenEnd + 2; - unsigned long length; - bool res = String::fromString( length, fileBuffer.substr(0, lenEnd), std::hex ); + // Get the length of the chunk + bool res = String::fromString( length, chunkBuffer.substr(0, lenEnd), std::hex ); - if ( res && length ) { - if ( fileBuffer.size() - firstCharPos >= length ) { - writeTo.write( &fileBuffer[firstCharPos], length ); - fileBuffer = fileBuffer.substr( firstCharPos + length ); - newFileBuffer = true; + // If the length is solved... + if ( res ) { + // And it's bigger than 0, means that there are more chunks + if ( length > 0 ) { + // Check if the chunk buffer size at least equals to the length reported + if ( chunkBuffer.size() - firstCharPos >= length ) { + // In that case write the chunk to the file buffer + fileBuffer.write( &chunkBuffer[firstCharPos], length ); - // Check if already have the \r\n of the next length in the buffer - if ( !fileBuffer.empty() && fileBuffer.substr( 0, 2 ) == "\r\n" ) { - // Remove it to be able to read the next length - fileBuffer = fileBuffer.substr( 2 ); - newFileBuffer = false; + // And keep the remaining not completed chunk + chunkBuffer = chunkBuffer.substr( firstCharPos + length ); + chunkNewBuffer = true; + + // Check if already have the \r\n of the next length in the buffer + if ( !chunkBuffer.empty() && chunkBuffer.substr( 0, 2 ) == "\r\n" ) { + // Remove it to be able to read the next length + chunkBuffer = chunkBuffer.substr( 2 ); + chunkNewBuffer = false; + } + } + } else { + // If the value is 0 means that the data ended + // But after this we can receive extra headers + chunkEnded = true; } } } + } else { + headerBuffer.append( buffer, buffer + readed ); } } else { - writeTo.write( buffer, readed ); + // If not chunked just write into the file buffer + fileBuffer.write( buffer, readed ); + } + + if ( compressed ) { + if ( fileBuffer.getSize() - inflateChunkSize >= 0 ) { + inflateStream->write( fileBuffer.getStreamPointer(), inflateChunkSize ); + + IOStreamString newFileBuffer; + + fileBuffer.seek(inflateChunkSize); + + std::size_t trailing = fileBuffer.getSize() - inflateChunkSize; + + if ( trailing > 0 ) + newFileBuffer.write( fileBuffer.getPositionPointer(), trailing ); + + fileBuffer = newFileBuffer; + } + } else { + fileBuffer.seek(0); + writeTo.write( fileBuffer.getPositionPointer(), fileBuffer.getSize() ); + fileBuffer.clear(); } if ( request.getProgressCallback() ) { - std::size_t length = 0; - - if ( !received.getField("content-length").empty() ) { - String::fromString( length, received.getField("content-length") ); - } - - if ( !request.getProgressCallback()( *this, request, length, currentTotalBytes ) ) { + if ( !request.getProgressCallback()( *this, request, contentLength, currentTotalBytes ) ) { request.mCancel = true; break; } @@ -794,10 +829,21 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri } } + if ( !headerBuffer.empty() ) { + std::istringstream in(headerBuffer); + received.parseFields(in); + } + + if ( compressed && fileBuffer.getSize() > 0 ) { + inflateStream->write( fileBuffer.getStreamPointer(), fileBuffer.getSize() ); + } + if ( status == Socket::Status::Disconnected ) { mConnection->setConnected(false); mConnection->setTunneled(false); } + + eeSAFE_DELETE( inflateStream ); } else { mConnection->setConnected(false); mConnection->setTunneled(false); @@ -809,11 +855,11 @@ Http::Response Http::downloadRequest(const Http::Request& request, IOStream& wri mConnection->disconnect(); } - return std::move(received); + return received; } Http::Response Http::downloadRequest(const Http::Request & request, std::string writePath, Time timeout) { - IOStreamFile file( writePath, "wb" ); + IOStreamFile file( writePath, "wb+" ); return downloadRequest( request, file, timeout ); } @@ -901,13 +947,11 @@ void Http::removeOldThreads() { Http::Request Http::prepareFields(const Http::Request& request) { Request toSend(request); - if (!toSend.hasField("User-Agent")) { + if (!toSend.hasField("User-Agent")) toSend.setField("User-Agent", "eepp-network"); - } - if (!toSend.hasField("Host")) { + if (!toSend.hasField("Host")) toSend.setField("Host", mHostName); - } if (!toSend.hasField("Content-Length")) { std::ostringstream out; @@ -915,13 +959,11 @@ Http::Request Http::prepareFields(const Http::Request& request) { toSend.setField("Content-Length", out.str()); } - if ((toSend.mMethod == Request::Post) && !toSend.hasField("Content-Type")) { + if ((toSend.mMethod == Request::Post) && !toSend.hasField("Content-Type")) toSend.setField("Content-Type", "application/x-www-form-urlencoded"); - } - if ((toSend.mMajorVersion * 10 + toSend.mMinorVersion >= 11) && !toSend.hasField("Connection")) { + if ((toSend.mMajorVersion * 10 + toSend.mMinorVersion >= 11) && !toSend.hasField("Connection")) toSend.setField("Connection", "close"); - } if (!mProxy.empty()) { toSend.setField("Accept", "*/*"); @@ -933,7 +975,10 @@ Http::Request Http::prepareFields(const Http::Request& request) { } } - return std::move(toSend); + if ( request.isCompressedResponse() ) + toSend.setField("Accept-Encoding", "gzip, deflate"); + + return toSend; } void Http::setProxy(const URI& uri) { diff --git a/src/eepp/system/compression.cpp b/src/eepp/system/compression.cpp new file mode 100644 index 000000000..17db69928 --- /dev/null +++ b/src/eepp/system/compression.cpp @@ -0,0 +1,167 @@ +#include +#include +#include +#include + +#include + +#define DEFLATE_CHUNK_SIZE (16384) + +namespace EE { namespace System { + +Compression::Status Compression::compress(Uint8* dst, Uint64 dstMaxSize, const Uint8* src, Uint64 srcSize, Mode mode, const Config& config) { + IOStreamMemory srcMem( (const char*)src, srcSize ); + IOStreamMemory dstMem( (char*)dst, dstMaxSize ); + return compress( dstMem, srcMem, mode, config ); +} + +Compression::Status Compression::compress(IOStream& dst, IOStream& src, Compression::Mode mode, const Config& config) { + switch (mode) { + case MODE_DEFLATE: + case MODE_GZIP: + { + int ret, flush; + unsigned have; + z_stream strm = {}; + char in[DEFLATE_CHUNK_SIZE]; + char out[DEFLATE_CHUNK_SIZE]; + int level = mode == MODE_DEFLATE ? config.zlib.level : config.gzip.level; + int windowBits = mode == Compression::MODE_DEFLATE ? MAX_WBITS : MAX_WBITS | 16; + + ret = deflateInit2(&strm, level, Z_DEFLATED, windowBits, 8, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) + return (Status)ret; + + src.seek(0); + + do { + strm.avail_in = src.read(in, DEFLATE_CHUNK_SIZE); + if ( strm.avail_in == 0 ) { + deflateEnd(&strm); + return Status::ERRNO; + } + + flush = src.tell() == src.getSize() ? Z_FINISH : Z_NO_FLUSH; + strm.next_in = (unsigned char*)in; + + do { + strm.avail_out = DEFLATE_CHUNK_SIZE; + strm.next_out = (unsigned char*)out; + + ret = deflate(&strm, flush); + + if ( ret == Z_STREAM_ERROR ) + return Status::STREAM_ERROR; + + have = DEFLATE_CHUNK_SIZE - strm.avail_out; + + if ( dst.write(out, have) != have ) { + deflateEnd(&strm); + + return Status::ERRNO; + } + } while (strm.avail_out == 0); + + if (strm.avail_in != 0) + return Status::DATA_ERROR; + } while (flush != Z_FINISH); + } + } + + return Status::OK; +} + +int Compression::getMaxCompressedBufferSize(Uint64 srcSize, Mode mode, const Config&) { + switch (mode) { + case MODE_DEFLATE: + case MODE_GZIP: + { + int windowBits = mode == MODE_DEFLATE ? MAX_WBITS : MAX_WBITS | 16; + + z_stream strm = {}; + int err = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, windowBits, 8, Z_DEFAULT_STRATEGY); + if (err != Z_OK) + return -1; + int aout = deflateBound(&strm, srcSize); + deflateEnd(&strm); + return aout; + } + } + + return -1; +} + +Compression::Status Compression::decompress(Uint8* dst, Uint64 dstMaxSize, const Uint8 * src, Uint64 srcSize, Mode mode) { + IOStreamMemory srcMem( (const char*)src, srcSize ); + IOStreamMemory dstMem( (char*)dst, dstMaxSize ); + return decompress( dstMem, srcMem, mode ); +} + +Compression::Status Compression::decompress(IOStream& dst, IOStream& src, Mode mode) { + switch (mode) { + case MODE_DEFLATE: + case MODE_GZIP: + { + SafeDataPointer buffer( DEFLATE_CHUNK_SIZE ); + SafeDataPointer bufferDst( DEFLATE_CHUNK_SIZE ); + + src.seek( 0 ); + + int windowBits = mode == Compression::MODE_DEFLATE ? MAX_WBITS : MAX_WBITS | 16; + + z_stream strm = {}; + strm.next_in = buffer.data; + + int err = inflateInit2(&strm, windowBits); + if (err != Z_OK) + return (Status)err; + + int zlibStatus; + int bytesRead; + unsigned int have; + Uint32 totalSize = src.getSize(); + Uint32 totalRead = 0; + + while ( totalRead < totalSize ) { + bytesRead = src.read( (char*)buffer.data, buffer.size ); + + strm.avail_in = bytesRead; + strm.next_in = buffer.data; + + do { + strm.avail_out = bufferDst.size; + strm.next_out = bufferDst.data; + zlibStatus = inflate(&strm, Z_NO_FLUSH); + + switch (zlibStatus) { + case Z_OK: + case Z_STREAM_END: + case Z_BUF_ERROR: + break; + default: + inflateEnd(&strm); + return (Status)zlibStatus; + } + + have = bufferDst.size- strm.avail_out; + + dst.write( (const char*)bufferDst.data, have ); + } while (strm.avail_out == 0); + + totalRead += bytesRead; + } + + inflateEnd(&strm); + + return Status::OK; + } + } + + return Status::ERRNO; +} + +std::size_t Compression::getModeDefaultChunkSize(const Mode&) { + return DEFLATE_CHUNK_SIZE; +} + +}} diff --git a/src/eepp/system/iostreaminflate.cpp b/src/eepp/system/iostreaminflate.cpp new file mode 100644 index 000000000..4ed40641c --- /dev/null +++ b/src/eepp/system/iostreaminflate.cpp @@ -0,0 +1,166 @@ +#include + +#include + +namespace EE { namespace System { + +struct LocalStreamData { + z_stream strm; + int state; +}; + +IOStreamInflate * IOStreamInflate::New(IOStream& inOutStream, Compression::Mode mode) { + return eeNew( IOStreamInflate, ( inOutStream, mode ) ); +} + +IOStreamInflate::IOStreamInflate(IOStream& inOutStream, Compression::Mode mode) : + mStream(inOutStream), + mMode(mode), + mBuffer(Compression::getModeDefaultChunkSize(mode)), + mLocalStream(eeNew(LocalStreamData,())) +{ + int windowBits = mode == Compression::MODE_DEFLATE ? MAX_WBITS : MAX_WBITS | 16; + + mLocalStream->strm = z_stream{}; + + mLocalStream->state = inflateInit2(&mLocalStream->strm, windowBits); +} + +IOStreamInflate::~IOStreamInflate() { + inflateEnd( &mLocalStream->strm ); + + eeSAFE_DELETE( mLocalStream ); +} + +ios_size IOStreamInflate::read(char * buffer, ios_size length) { + if ( mLocalStream->state != Z_OK || !mStream.isOpen() ) + return 0; + + z_stream& zstr = mLocalStream->strm; + + if (zstr.avail_in == 0) { + ios_size n = 0; + + if ( mStream.isOpen()) { + n = mStream.read((char*)mBuffer.data, mBuffer.size); + } + + zstr.next_in = (unsigned char*) mBuffer.data; + zstr.avail_in = n; + } + + zstr.next_out = (unsigned char*) buffer; + zstr.avail_out = length; + + for (;;) { + int rc = inflate(&zstr, Z_NO_FLUSH); + + if (rc == Z_DATA_ERROR) { + if (zstr.avail_in == 0) { + if (mStream.isOpen()) + rc = Z_OK; + else + rc = Z_STREAM_END; + } + } + + if (rc == Z_STREAM_END) { + return length - zstr.avail_out; + } + + if (rc != Z_OK) + return 0; + + if (zstr.avail_out == 0) + return static_cast(length); + + if (zstr.avail_in == 0) { + ios_size n = 0; + + if (mStream.isOpen()) { + n = mStream.read((char*)mBuffer.data, mBuffer.size); + } + + if (n > 0) { + zstr.next_in = (unsigned char*) mBuffer.data; + zstr.avail_in = n; + } else { + return length - zstr.avail_out; + } + } + } +} + +ios_size IOStreamInflate::write(const char * buffer, ios_size length) { + if ( mLocalStream->state != Z_OK || !mStream.isOpen() || length == 0 ) + return 0; + + z_stream& zstr = mLocalStream->strm; + + zstr.next_in = (unsigned char*) buffer; + zstr.avail_in = length; + zstr.next_out = mBuffer.data; + zstr.avail_out = mBuffer.size; + + for (;;) { + int rc = inflate(&zstr, Z_NO_FLUSH); + + if (rc == Z_STREAM_END) { + ios_size ret = mStream.write( (const char*)mBuffer.data, mBuffer.size - zstr.avail_out); + + if (ret == 0) + return 0; + + break; + } + + if (rc != Z_OK) + return 0; + + if (zstr.avail_out == 0) { + ios_size ret = mStream.write( (const char*)mBuffer.data, mBuffer.size); + + if (ret == 0) + return 0; + + zstr.next_out = (unsigned char*) mBuffer.data; + zstr.avail_out = mBuffer.size; + } + + if (zstr.avail_in == 0) { + ios_size ret = mStream.write( (const char*)mBuffer.data, mBuffer.size - zstr.avail_out); + + if (ret == 0) + return 0; + + zstr.next_out = (unsigned char*) mBuffer.data; + zstr.avail_out = mBuffer.size; + + break; + } + } + + return length; +} + +ios_size IOStreamInflate::seek(ios_size position) { + return mStream.seek( position ); +} + +ios_size IOStreamInflate::tell() { + return mStream.tell(); +} + +ios_size IOStreamInflate::getSize() { + return mStream.getSize(); +} + +bool IOStreamInflate::isOpen() { + return mStream.isOpen(); +} + +const Compression::Mode& IOStreamInflate::getMode() const { + return mMode; +} + +}} diff --git a/src/eepp/system/iostreamstring.cpp b/src/eepp/system/iostreamstring.cpp new file mode 100644 index 000000000..daa5cc12a --- /dev/null +++ b/src/eepp/system/iostreamstring.cpp @@ -0,0 +1,68 @@ +#include +#include + +namespace EE { namespace System { + +IOStreamString::IOStreamString() : + mPos(0) +{} + +ios_size IOStreamString::read(char * data, ios_size size) { + Int64 endPosition = mPos + size; + Int64 count = endPosition <= getSize() ? size : getSize() - mPos; + + if ( count > 0 ) { + memcpy( data, &mStream[mPos], static_cast( count ) ); + mPos += count; + } + + return count; +} + +ios_size IOStreamString::write(const char * data, ios_size size) { + mStream.insert( mPos, data, size ); + + mPos += size; + + return size; +} + +ios_size IOStreamString::write(const std::string& string) { + return write( string.c_str(), string.size() ); +} + +ios_size IOStreamString::seek(ios_size position) { + mPos = ( position < getSize() ) ? position : getSize(); + return mPos; +} + +ios_size IOStreamString::tell() { + return getSize(); +} + +ios_size IOStreamString::getSize() { + return mStream.size(); +} + +bool IOStreamString::isOpen() { + return true; +} + +void IOStreamString::clear() { + mStream.clear(); + mPos = 0; +} + +const char* IOStreamString::getPositionPointer() { + return &mStream[mPos]; +} + +const char* IOStreamString::getStreamPointer() const { + return mStream.c_str(); +} + +const std::string& IOStreamString::getStream() const { + return mStream; +} + +}} diff --git a/src/eepp/window/platform/x11/cursorx11.cpp b/src/eepp/window/platform/x11/cursorx11.cpp index 142cf2853..735aef5bf 100644 --- a/src/eepp/window/platform/x11/cursorx11.cpp +++ b/src/eepp/window/platform/x11/cursorx11.cpp @@ -38,7 +38,7 @@ CursorX11::CursorX11( const std::string& path, const Vector2i& hotspot, const st CursorX11::~CursorX11() { if ( None != mCursor ) - XFreeCursor( getPlatform()->GetDisplay(), mCursor ); + XFreeCursor( getPlatform()->getDisplay(), mCursor ); } void CursorX11::create() { @@ -65,11 +65,11 @@ void CursorX11::create() { image->xhot = mHotSpot.x; image->yhot = mHotSpot.y; - getPlatform()->Lock(); + getPlatform()->lock(); - mCursor = XcursorImageLoadCursor( getPlatform()->GetDisplay(), image ); + mCursor = XcursorImageLoadCursor( getPlatform()->getDisplay(), image ); - getPlatform()->Unlock(); + getPlatform()->unlock(); XcursorImageDestroy( image ); } diff --git a/src/eepp/window/platform/x11/x11impl.cpp b/src/eepp/window/platform/x11/x11impl.cpp index a4a5b8db3..1ad9a199b 100644 --- a/src/eepp/window/platform/x11/x11impl.cpp +++ b/src/eepp/window/platform/x11/x11impl.cpp @@ -40,18 +40,18 @@ X11Impl::~X11Impl() { } void X11Impl::minimizeWindow() { - Lock(); + lock(); XIconifyWindow( mDisplay, mX11Window, 0 ); XFlush( mDisplay ); - Unlock(); + unlock(); } void X11Impl::maximizeWindow() { // coded by Rafał Maj, idea from Måns Rullgård http://tinyurl.com/68mvk3 - Lock(); + lock(); XEvent xev; Atom wm_state = XAtom( "_NET_WM_STATE" ); @@ -71,11 +71,11 @@ void X11Impl::maximizeWindow() { XFlush(mDisplay); - Unlock(); + unlock(); } bool X11Impl::isWindowMaximized() { - Lock(); + lock(); //bool minimized = false; bool maximizedhorz = false; @@ -117,7 +117,7 @@ bool X11Impl::isWindowMaximized() { XFlush(mDisplay); - Unlock(); + unlock(); if( maximizedhorz && maximizedvert ) { return true; @@ -127,45 +127,45 @@ bool X11Impl::isWindowMaximized() { } void X11Impl::hideWindow() { - Lock(); + lock(); XUnmapWindow( mDisplay, mX11Window ); - Unlock(); + unlock(); } void X11Impl::raiseWindow() { - Lock(); + lock(); XRaiseWindow( mDisplay, mX11Window ); - Unlock(); + unlock(); } void X11Impl::showWindow() { - Lock(); + lock(); XMapRaised( mDisplay, mX11Window ); - Unlock(); + unlock(); } void X11Impl::moveWindow( int left, int top ) { - Lock(); + lock(); XMoveWindow( mDisplay, mX11Window, left, top ); XFlush( mDisplay ); - Unlock(); + unlock(); } void X11Impl::setContext( eeWindowContex Context ) { - Lock(); + lock(); glXMakeCurrent( mDisplay, mX11Window, Context ); - Unlock(); + unlock(); } Vector2i X11Impl::getPosition() { @@ -181,20 +181,20 @@ void X11Impl::showMouseCursor() { if ( !mCursorHidden ) return; - Lock(); + lock(); XDefineCursor( mDisplay, mMainWindow, mCursorCurrent ); mCursorHidden = false; - Unlock(); + unlock(); } void X11Impl::hideMouseCursor() { if ( mCursorHidden ) return; - Lock(); + lock(); if ( mCursorInvisible == None ) { unsigned long gcmask; @@ -225,7 +225,7 @@ void X11Impl::hideMouseCursor() { mCursorHidden = true; - Unlock(); + unlock(); } Cursor * X11Impl::createMouseCursor( Texture * tex, const Vector2i& hotspot, const std::string& name ) { @@ -244,21 +244,21 @@ void X11Impl::setMouseCursor( Cursor * cursor ) { mCursorCurrent = reinterpret_cast( cursor )->GetCursor(); if ( !mCursorHidden ) { - Lock(); + lock(); XDefineCursor( mDisplay, mMainWindow, mCursorCurrent ); - Unlock(); + unlock(); } } void X11Impl::restoreCursor() { if ( !mCursorHidden ) { - Lock(); + lock(); XDefineCursor( mDisplay, mMainWindow, mCursorCurrent ); - Unlock(); + unlock(); } else { hideMouseCursor(); } @@ -287,7 +287,7 @@ void X11Impl::setSystemMouseCursor( Cursor::SysType syscursor ) { XFreeCursor( mDisplay, mCursorSystemLast ); } - Lock(); + lock(); mCursorCurrent = XCreateFontCursor( mDisplay, cursor_shape ); mCursorSystemLast = mCursorCurrent; @@ -296,19 +296,19 @@ void X11Impl::setSystemMouseCursor( Cursor::SysType syscursor ) { XDefineCursor( mDisplay, mMainWindow, mCursorCurrent ); } - Unlock(); + unlock(); } -eeWindowHandle X11Impl::GetDisplay() const { +eeWindowHandle X11Impl::getDisplay() const { return mDisplay; } -void X11Impl::Lock() { +void X11Impl::lock() { if ( NULL != mLock ) mLock(); } -void X11Impl::Unlock() { +void X11Impl::unlock() { if ( NULL != mUnlock ) mUnlock(); } diff --git a/src/eepp/window/platform/x11/x11impl.hpp b/src/eepp/window/platform/x11/x11impl.hpp index 529bc8cca..f1446c117 100644 --- a/src/eepp/window/platform/x11/x11impl.hpp +++ b/src/eepp/window/platform/x11/x11impl.hpp @@ -54,11 +54,11 @@ class X11Impl : public PlatformImpl { void restoreCursor(); - eeWindowHandle GetDisplay() const; + eeWindowHandle getDisplay() const; - void Lock(); + void lock(); - void Unlock(); + void unlock(); eeWindowContex getWindowContext(); protected: diff --git a/src/examples/http_request/http_request.cpp b/src/examples/http_request/http_request.cpp index 2dec6548e..3d3cdb6ec 100644 --- a/src/examples/http_request/http_request.cpp +++ b/src/examples/http_request/http_request.cpp @@ -17,6 +17,7 @@ void printResponseHeaders( Http::Response& response ) { EE_MAIN_FUNC int main (int argc, char * argv []) { args::ArgumentParser parser("HTTP request program example"); args::HelpFlag help(parser, "help", "Display this help menu", {'h', "help"}); + args::Flag compressed(parser, "compressed", "Request compressed response", {"compressed"}); args::ValueFlag postData(parser, "data", "HTTP POST data", {'d', "data"}); args::ValueFlagList headers(parser, "header", "Pass custom header(s) to server", {'H', "header"}); args::Flag includeHead(parser, "include", "Include protocol response headers in the output", {'i',"include"}); @@ -136,6 +137,11 @@ EE_MAIN_FUNC int main (int argc, char * argv []) { http.setProxy( URI( proxy.Get() ) ); } + // Request a compressed response + if ( compressed ) { + request.setCompressedResponse( true ); + } + if ( !output ) { // Send the request Http::Response response = http.sendRequest(request); @@ -147,21 +153,22 @@ EE_MAIN_FUNC int main (int argc, char * argv []) { printResponseHeaders(response); if ( status == Http::Response::Ok ) { - std::cout << response.getBody() << std::endl; + std::cout << response.getBody(); } else { std::cout << "Error " << status << std::endl << response.getStatusDescription() << std::endl; - std::cout << response.getBody() << std::endl; + std::cout << response.getBody(); } } else { std::string path( output.Get() ); // If output path is a directory guess a file name if ( FileSystem::isDirectory( path ) ) { + FileSystem::dirPathAddSlashAtEnd( path ); + std::string lastPathSegment = uri.getLastPathSegment(); // If there's a path end segment if ( !lastPathSegment.empty() ) { - FileSystem::dirPathAddSlashAtEnd( path ); // Save with the path end segment name if ( !FileSystem::fileExists( path + lastPathSegment ) ) {