From c1135cf0063016ce9abacc23d987becaaef5aa9a Mon Sep 17 00:00:00 2001 From: Will Whitty Date: Sun, 9 May 2021 14:05:32 +0300 Subject: [PATCH] Work on porting HTTPRequest compression to 3.3 Fix doc issues Use memcpy Bind RESULT_BODY_DECOMPRESS_FAILED Docs update --- core/io/compression.cpp | 86 ++++++++++++++++++++++++++ core/io/compression.h | 3 + core/variant_call.cpp | 19 ++++++ doc/classes/HTTPRequest.xml | 42 +++++++++++-- doc/classes/PoolByteArray.xml | 14 +++++ scene/main/http_request.cpp | 113 ++++++++++++++++++++++++++++++++-- scene/main/http_request.h | 14 ++++- 7 files changed, 279 insertions(+), 12 deletions(-) diff --git a/core/io/compression.cpp b/core/io/compression.cpp index 0255d3230fc..d5022b37a61 100644 --- a/core/io/compression.cpp +++ b/core/io/compression.cpp @@ -179,8 +179,94 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p ERR_FAIL_V(-1); } +/** + This will handle both Gzip and Deflat streams. It will automatically allocate the output buffer into the provided p_dst_vect Vector. + This is required for compressed data who's final uncompressed size is unknown, as is the case for HTTP response bodies. + This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer. +*/ +int Compression::decompress_dynamic(PoolVector *p_dst, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) { + int ret; + uint8_t *dst = nullptr; + int out_mark = 0; + z_stream strm; + + ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR); + + // This function only supports GZip and Deflate + int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16; + ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO); + + // Initialize the stream + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + + int err = inflateInit2(&strm, window_bits); + ERR_FAIL_COND_V(err != Z_OK, -1); + + // Setup the stream inputs + strm.next_in = (Bytef *)p_src; + strm.avail_in = p_src_size; + + // Ensure the destination buffer is empty + p_dst->resize(0); + + // decompress until deflate stream ends or end of file + do { + // Add another chunk size to the output buffer + // This forces a copy of the whole buffer + p_dst->resize(p_dst->size() + gzip_chunk); + // Get pointer to the actual output buffer + dst = p_dst->write().ptr(); + + // Set the stream to the new output stream + // Since it was copied, we need to reset the stream to the new buffer + strm.next_out = &(dst[out_mark]); + strm.avail_out = gzip_chunk; + + // run inflate() on input until output buffer is full and needs to be resized + // or input runs out + do { + ret = inflate(&strm, Z_SYNC_FLUSH); + + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; + case Z_DATA_ERROR: + case Z_MEM_ERROR: + case Z_STREAM_ERROR: + WARN_PRINT(strm.msg); + (void)inflateEnd(&strm); + p_dst->resize(0); + return ret; + } + } while (strm.avail_out > 0 && strm.avail_in > 0); + + out_mark += gzip_chunk; + + // Encorce max output size + if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) { + (void)inflateEnd(&strm); + p_dst->resize(0); + return Z_BUF_ERROR; + } + } while (ret != Z_STREAM_END); + + // If all done successfully, resize the output if it's larger than the actual output + if (ret == Z_STREAM_END && (unsigned long)p_dst->size() > strm.total_out) { + p_dst->resize(strm.total_out); + } + + // clean up and return + (void)inflateEnd(&strm); + return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR; +} + int Compression::zlib_level = Z_DEFAULT_COMPRESSION; int Compression::gzip_level = Z_DEFAULT_COMPRESSION; int Compression::zstd_level = 3; bool Compression::zstd_long_distance_matching = false; int Compression::zstd_window_log_size = 27; // ZSTD_WINDOWLOG_LIMIT_DEFAULT +int Compression::gzip_chunk = 16384; diff --git a/core/io/compression.h b/core/io/compression.h index c6f62c5fc05..537fad77191 100644 --- a/core/io/compression.h +++ b/core/io/compression.h @@ -31,6 +31,7 @@ #ifndef COMPRESSION_H #define COMPRESSION_H +#include "core/pool_vector.h" #include "core/typedefs.h" class Compression { @@ -40,6 +41,7 @@ public: static int zstd_level; static bool zstd_long_distance_matching; static int zstd_window_log_size; + static int gzip_chunk; enum Mode { MODE_FASTLZ, @@ -51,6 +53,7 @@ public: static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD); static int get_max_compressed_buffer_size(int p_src_size, Mode p_mode = MODE_ZSTD); static int decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD); + static int decompress_dynamic(PoolVector *p_dst, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode); Compression(); }; diff --git a/core/variant_call.cpp b/core/variant_call.cpp index 23713e4bfc9..4126447fe74 100644 --- a/core/variant_call.cpp +++ b/core/variant_call.cpp @@ -641,6 +641,24 @@ struct _VariantCall { r_ret = decompressed; } + static void _call_PoolByteArray_decompress_dynamic(Variant &r_ret, Variant &p_self, const Variant **p_args) { + PoolByteArray *ba = reinterpret_cast(p_self._data._mem); + PoolByteArray *decompressed = memnew(PoolByteArray); + int max_output_size = (int)(*p_args[0]); + Compression::Mode mode = (Compression::Mode)(int)(*p_args[1]); + + decompressed->resize(1024); + int result = Compression::decompress_dynamic(decompressed, max_output_size, ba->read().ptr(), ba->size(), mode); + + if (result == OK) { + r_ret = decompressed; + } else { + decompressed->resize(0); + r_ret = decompressed; + ERR_FAIL_MSG("Decompression failed."); + } + } + static void _call_PoolByteArray_hex_encode(Variant &r_ret, Variant &p_self, const Variant **p_args) { PoolByteArray *ba = reinterpret_cast(p_self._data._mem); if (ba->size() == 0) { @@ -1892,6 +1910,7 @@ void register_variant_methods() { ADDFUNC0R(POOL_BYTE_ARRAY, STRING, PoolByteArray, hex_encode, varray()); ADDFUNC1R(POOL_BYTE_ARRAY, POOL_BYTE_ARRAY, PoolByteArray, compress, INT, "compression_mode", varray(0)); ADDFUNC2R(POOL_BYTE_ARRAY, POOL_BYTE_ARRAY, PoolByteArray, decompress, INT, "buffer_size", INT, "compression_mode", varray(0)); + ADDFUNC2R(POOL_BYTE_ARRAY, POOL_BYTE_ARRAY, PoolByteArray, decompress_dynamic, INT, "max_output_size", INT, "compression_mode", varray(0)); ADDFUNC0R(POOL_INT_ARRAY, INT, PoolIntArray, size, varray()); ADDFUNC0R(POOL_INT_ARRAY, BOOL, PoolIntArray, empty, varray()); diff --git a/doc/classes/HTTPRequest.xml b/doc/classes/HTTPRequest.xml index 9afdd6f1352..aaf2a2fb0d6 100644 --- a/doc/classes/HTTPRequest.xml +++ b/doc/classes/HTTPRequest.xml @@ -64,6 +64,10 @@ add_child(texture_rect) texture_rect.texture = texture [/codeblock] + [b]Gzipped response bodies[/b] + HttpRequest will automatically handle decompression of response bodies. + A "Accept-Encoding" header will be automatically added to each of your requests, unless one is already specified. + Any response with a "Content-Encoding: gzip" header will automatically be decompressed and delivered to you as a uncompressed bytes. [b]Note:[/b] When performing HTTP requests from a project exported to HTML5, keep in mind the remote server may not allow requests from foreign origins due to [url=https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS]CORS[/url]. If you host the server in question, you should modify its backend to allow requests from foreign origins by adding the [code]Access-Control-Allow-Origin: *[/code] HTTP header. [b]Note:[/b] SSL/TLS support is currently limited to TLS 1.0, TLS 1.1, and TLS 1.2. Attempting to connect to a TLS 1.3-only server will return an error. @@ -120,10 +124,34 @@ [b]Note:[/b] The [code]request_data[/code] parameter is ignored if [code]method[/code] is [constant HTTPClient.METHOD_GET]. This is because GET methods can't contain request data. As a workaround, you can pass request data as a query string in the URL. See [method String.http_escape] for an example. + + + + + + + + + + + + + + + Creates request on the underlying [HTTPClient] using a raw array of bytes for the request body. If there is no configuration errors, it tries to connect using [method HTTPClient.connect_to_host] and passes parameters onto [method HTTPClient.request]. + Returns [constant OK] if request is successfully created. (Does not imply that the server has responded), [constant ERR_UNCONFIGURED] if not in the tree, [constant ERR_BUSY] if still processing previous request, [constant ERR_INVALID_PARAMETER] if given string is not a valid URL format, or [constant ERR_CANT_CONNECT] if not using thread and the [HTTPClient] cannot connect to host. + + + + If [code]true[/code], this header will be added to each request: [code]Accept-Encoding: gzip, deflate[/code] telling servers that it's okay to compress response bodies. + Any Reponse body declaring a [code]Content-Encoding[/code] of either [code]gzip[/code] or [code]deflate[/code] will then be automatically decompressed, and the uncompressed bytes will be delivered via [code]request_completed[/code]. + If the user has specified their own [code]Accept-Encoding[/code] header, then no header will be added regaurdless of [code]accept_gzip[/code]. + If [code]false[/code] no header will be added, and no decompression will be performed on response bodies. The raw bytes of the response body will be returned via [code]request_completed[/code]. + - Maximum allowed size for response bodies. + Maximum allowed size for response bodies. If the response body is compressed, this will be used as the maximum allowed size for the decompressed body. The size of the buffer used and maximum bytes to read per iteration. See [member HTTPClient.read_chunk_size]. @@ -177,22 +205,24 @@ Request does not have a response (yet). + + Request exceeded its maximum size limit, see [member body_size_limit]. - + Request failed (currently unused). - + HTTPRequest couldn't open the download file. - + HTTPRequest couldn't write to the download file. - + Request reached its maximum redirect limit, see [member max_redirects]. - + diff --git a/doc/classes/PoolByteArray.xml b/doc/classes/PoolByteArray.xml index b2bf7296a7f..5d7be7a0762 100644 --- a/doc/classes/PoolByteArray.xml +++ b/doc/classes/PoolByteArray.xml @@ -53,6 +53,20 @@ Returns a new [PoolByteArray] with the data decompressed. Set [code]buffer_size[/code] to the size of the uncompressed data. Set the compression mode using one of [enum File.CompressionMode]'s constants. + + + + + + + + + Returns a new [PoolByteArray] with the data decompressed. Set the compression mode using one of [enum File.CompressionMode]'s constants. [b]This method only accepts gzip and deflate compression modes.[/b] + This method is potentially slower than [code]decompress[/code], as it may have to re-allocate it's output buffer multiple times while decompressing, where as [code]decompress[/code] knows it's output buffer size from the begining. + + GZIP has a maximal compression ratio of 1032:1, meaning it's very possible for a small compressed payload to decompress to a potentially very large output. To guard against this, you may provide a maximum size this function is allowed to allocate in bytes via [code]max_output_size[/code]. Passing -1 will allow for unbounded output. If any positive value is passed, and the decompression exceeds that ammount in bytes, then an error will be returned. + + diff --git a/scene/main/http_request.cpp b/scene/main/http_request.cpp index f17a31ed031..ad17afae133 100644 --- a/scene/main/http_request.cpp +++ b/scene/main/http_request.cpp @@ -29,6 +29,8 @@ /*************************************************************************/ #include "http_request.h" +#include "core/io/compression.h" +#include "core/ustring.h" void HTTPRequest::_redirect_request(const String &p_new_url) { } @@ -65,7 +67,50 @@ Error HTTPRequest::_parse_url(const String &p_url) { return OK; } +bool HTTPRequest::has_header(const Vector &p_headers, const String &p_header_name) { + bool exists = false; + + String lower_case_header_name = p_header_name.to_lower(); + for (int i = 0; i < p_headers.size() && !exists; i++) { + String sanitized = p_headers[i].strip_edges().to_lower(); + if (sanitized.begins_with(lower_case_header_name)) { + exists = true; + } + } + + return exists; +} + +String HTTPRequest::get_header_value(const PoolStringArray &p_headers, const String &p_header_name) { + String value = ""; + + String lowwer_case_header_name = p_header_name.to_lower(); + for (int i = 0; i < p_headers.size(); i++) { + if (p_headers[i].find(":", 0) >= 0) { + Vector parts = p_headers[i].split(":", false, 1); + if (parts[0].strip_edges().to_lower() == lowwer_case_header_name) { + value = parts[1].strip_edges(); + break; + } + } + } + + return value; +} + Error HTTPRequest::request(const String &p_url, const Vector &p_custom_headers, bool p_ssl_validate_domain, HTTPClient::Method p_method, const String &p_request_data) { + // Copy the string into a raw buffer + PoolVector raw_data; + + CharString charstr = p_request_data.utf8(); + size_t len = charstr.length(); + raw_data.resize(len); + memcpy(raw_data.write().ptr(), charstr.ptr(), len); + + return request_raw(p_url, p_custom_headers, p_ssl_validate_domain, p_method, raw_data); +} + +Error HTTPRequest::request_raw(const String &p_url, const Vector &p_custom_headers, bool p_ssl_validate_domain, HTTPClient::Method p_method, const PoolVector &p_request_data_raw) { ERR_FAIL_COND_V(!is_inside_tree(), ERR_UNCONFIGURED); ERR_FAIL_COND_V_MSG(requesting, ERR_BUSY, "HTTPRequest is processing a request. Wait for completion or cancel it before attempting a new one."); @@ -85,7 +130,14 @@ Error HTTPRequest::request(const String &p_url, const Vector &p_custom_h headers = p_custom_headers; - request_data = p_request_data; + if (accept_gzip) { + // If the user has specified a different Accept-Encoding, don't overwrite it + if (!has_header(headers, "Accept-Encoding")) { + headers.push_back("Accept-Encoding: gzip, deflate"); + } + } + + request_data = p_request_data_raw; requesting = true; @@ -269,7 +321,7 @@ bool HTTPRequest::_update_connection() { } else { // Did not request yet, do request - Error err = client->request(method, request_string, headers, request_data); + Error err = client->request_raw(method, request_string, headers, request_data); if (err != OK) { call_deferred("_request_done", RESULT_CONNECTION_ERROR, 0, PoolStringArray(), PoolByteArray()); return true; @@ -367,9 +419,47 @@ bool HTTPRequest::_update_connection() { ERR_FAIL_V(false); } -void HTTPRequest::_request_done(int p_status, int p_code, const PoolStringArray &headers, const PoolByteArray &p_data) { +void HTTPRequest::_request_done(int p_status, int p_code, const PoolStringArray &p_headers, const PoolByteArray &p_data) { cancel_request(); - emit_signal("request_completed", p_status, p_code, headers, p_data); + + // Determine if the request body is compressed + bool is_compressed; + String content_encoding = get_header_value(p_headers, "Content-Encoding").to_lower(); + Compression::Mode mode; + if (content_encoding == "gzip") { + mode = Compression::Mode::MODE_GZIP; + is_compressed = true; + } else if (content_encoding == "deflate") { + mode = Compression::Mode::MODE_DEFLATE; + is_compressed = true; + } else { + is_compressed = false; + } + + const PoolByteArray *data = NULL; + + if (accept_gzip && is_compressed && p_data.size() > 0) { + // Decompress request body + PoolByteArray *decompressed = memnew(PoolByteArray); + int result = Compression::decompress_dynamic(decompressed, body_size_limit, p_data.read().ptr(), p_data.size(), mode); + if (result == OK) { + data = decompressed; + } else if (result == -5) { + WARN_PRINT("Decompressed size of HTTP response body exceeded body_size_limit"); + p_status = RESULT_BODY_SIZE_LIMIT_EXCEEDED; + // Just return the raw data if we failed to decompress it + data = &p_data; + } else { + WARN_PRINT("Failed to decompress HTTP response body"); + p_status = RESULT_BODY_DECOMPRESS_FAILED; + // Just return the raw data if we failed to decompress it + data = &p_data; + } + } else { + data = &p_data; + } + + emit_signal("request_completed", p_status, p_code, p_headers, *data); } void HTTPRequest::_notification(int p_what) { @@ -391,6 +481,14 @@ void HTTPRequest::_notification(int p_what) { } } +void HTTPRequest::set_accept_gzip(bool p_gzip) { + accept_gzip = p_gzip; +} + +bool HTTPRequest::is_accepting_gzip() const { + return accept_gzip; +} + void HTTPRequest::set_use_threads(bool p_use) { ERR_FAIL_COND(get_http_client_status() != HTTPClient::STATUS_DISCONNECTED); use_threads.set_to(p_use); @@ -465,6 +563,7 @@ void HTTPRequest::_timeout() { void HTTPRequest::_bind_methods() { ClassDB::bind_method(D_METHOD("request", "url", "custom_headers", "ssl_validate_domain", "method", "request_data"), &HTTPRequest::request, DEFVAL(PoolStringArray()), DEFVAL(true), DEFVAL(HTTPClient::METHOD_GET), DEFVAL(String())); + ClassDB::bind_method(D_METHOD("request_raw", "url", "custom_headers", "ssl_validate_domain", "method", "request_data_raw"), &HTTPRequest::request_raw, DEFVAL(PoolStringArray()), DEFVAL(true), DEFVAL(HTTPClient::METHOD_GET), DEFVAL(PoolVector())); ClassDB::bind_method(D_METHOD("cancel_request"), &HTTPRequest::cancel_request); ClassDB::bind_method(D_METHOD("get_http_client_status"), &HTTPRequest::get_http_client_status); @@ -472,6 +571,9 @@ void HTTPRequest::_bind_methods() { ClassDB::bind_method(D_METHOD("set_use_threads", "enable"), &HTTPRequest::set_use_threads); ClassDB::bind_method(D_METHOD("is_using_threads"), &HTTPRequest::is_using_threads); + ClassDB::bind_method(D_METHOD("set_accept_gzip", "enable"), &HTTPRequest::set_accept_gzip); + ClassDB::bind_method(D_METHOD("is_accepting_gzip"), &HTTPRequest::is_accepting_gzip); + ClassDB::bind_method(D_METHOD("set_body_size_limit", "bytes"), &HTTPRequest::set_body_size_limit); ClassDB::bind_method(D_METHOD("get_body_size_limit"), &HTTPRequest::get_body_size_limit); @@ -498,6 +600,7 @@ void HTTPRequest::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::STRING, "download_file", PROPERTY_HINT_FILE), "set_download_file", "get_download_file"); ADD_PROPERTY(PropertyInfo(Variant::INT, "download_chunk_size", PROPERTY_HINT_RANGE, "256,16777216"), "set_download_chunk_size", "get_download_chunk_size"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "use_threads"), "set_use_threads", "is_using_threads"); + ADD_PROPERTY(PropertyInfo(Variant::BOOL, "accept_gzip"), "set_accept_gzip", "is_accepting_gzip"); ADD_PROPERTY(PropertyInfo(Variant::INT, "body_size_limit", PROPERTY_HINT_RANGE, "-1,2000000000"), "set_body_size_limit", "get_body_size_limit"); ADD_PROPERTY(PropertyInfo(Variant::INT, "max_redirects", PROPERTY_HINT_RANGE, "-1,64"), "set_max_redirects", "get_max_redirects"); ADD_PROPERTY(PropertyInfo(Variant::INT, "timeout", PROPERTY_HINT_RANGE, "0,86400"), "set_timeout", "get_timeout"); @@ -512,6 +615,7 @@ void HTTPRequest::_bind_methods() { BIND_ENUM_CONSTANT(RESULT_CONNECTION_ERROR); BIND_ENUM_CONSTANT(RESULT_SSL_HANDSHAKE_ERROR); BIND_ENUM_CONSTANT(RESULT_NO_RESPONSE); + BIND_ENUM_CONSTANT(RESULT_BODY_DECOMPRESS_FAILED); BIND_ENUM_CONSTANT(RESULT_BODY_SIZE_LIMIT_EXCEEDED); BIND_ENUM_CONSTANT(RESULT_REQUEST_FAILED); BIND_ENUM_CONSTANT(RESULT_DOWNLOAD_FILE_CANT_OPEN); @@ -528,6 +632,7 @@ HTTPRequest::HTTPRequest() { got_response = false; validate_ssl = false; use_ssl = false; + accept_gzip = true; response_code = 0; request_sent = false; requesting = false; diff --git a/scene/main/http_request.h b/scene/main/http_request.h index 360057c0881..3be0a5b6cf5 100644 --- a/scene/main/http_request.h +++ b/scene/main/http_request.h @@ -51,6 +51,7 @@ public: RESULT_SSL_HANDSHAKE_ERROR, RESULT_NO_RESPONSE, RESULT_BODY_SIZE_LIMIT_EXCEEDED, + RESULT_BODY_DECOMPRESS_FAILED, RESULT_REQUEST_FAILED, RESULT_DOWNLOAD_FILE_CANT_OPEN, RESULT_DOWNLOAD_FILE_WRITE_ERROR, @@ -69,13 +70,14 @@ private: bool validate_ssl; bool use_ssl; HTTPClient::Method method; - String request_data; + PoolVector request_data; bool request_sent; Ref client; PoolByteArray body; SafeFlag use_threads; + bool accept_gzip; bool got_response; int response_code; PoolVector response_headers; @@ -103,12 +105,15 @@ private: Error _parse_url(const String &p_url); Error _request(); + bool has_header(const Vector &p_headers, const String &p_header_name); + String get_header_value(const PoolStringArray &p_headers, const String &header_name); + SafeFlag thread_done; SafeFlag thread_request_quit; Thread thread; - void _request_done(int p_status, int p_code, const PoolStringArray &headers, const PoolByteArray &p_data); + void _request_done(int p_status, int p_code, const PoolStringArray &p_headers, const PoolByteArray &p_data); static void _thread_func(void *p_userdata); protected: @@ -117,12 +122,17 @@ protected: public: Error request(const String &p_url, const Vector &p_custom_headers = Vector(), bool p_ssl_validate_domain = true, HTTPClient::Method p_method = HTTPClient::METHOD_GET, const String &p_request_data = ""); //connects to a full url and perform request + Error request_raw(const String &p_url, const Vector &p_custom_headers = Vector(), bool p_ssl_validate_domain = true, HTTPClient::Method p_method = HTTPClient::METHOD_GET, const PoolVector &p_request_data_raw = PoolVector()); //connects to a full url and perform request void cancel_request(); + HTTPClient::Status get_http_client_status() const; void set_use_threads(bool p_use); bool is_using_threads() const; + void set_accept_gzip(bool p_gzip); + bool is_accepting_gzip() const; + void set_download_file(const String &p_file); String get_download_file() const;