asgardius/virtualx-engine
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Merge pull request #63740 from Faless/net/4.x_http_request_decompress

Browse source 
[HTTP] Implement streaming decompression.
This commit is contained in:
Rémi Verschelde 2022-09-20 22:29:16 +02:00 • committed by GitHub
commit ae2d9be0fe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 396 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

209
core/io/stream_peer_gzip.cpp Normal file
View file

@ -0,0 +1,209 @@
/*************************************************************************/
/* stream_peer_gzip.cpp */
/*************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/*************************************************************************/
/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur. */
/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md). */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/*************************************************************************/
#include "core/io/stream_peer_gzip.h"
#include "core/io/zip_io.h"
#include <zlib.h>
void StreamPeerGZIP::_bind_methods() {
ClassDB::bind_method(D_METHOD("start_compression", "use_deflate", "buffer_size"), &StreamPeerGZIP::start_compression, DEFVAL(false), DEFVAL(65535));
ClassDB::bind_method(D_METHOD("start_decompression", "use_deflate", "buffer_size"), &StreamPeerGZIP::start_decompression, DEFVAL(false), DEFVAL(65535));
ClassDB::bind_method(D_METHOD("finish"), &StreamPeerGZIP::finish);
ClassDB::bind_method(D_METHOD("clear"), &StreamPeerGZIP::clear);
}
StreamPeerGZIP::StreamPeerGZIP() {
}
StreamPeerGZIP::~StreamPeerGZIP() {
_close();
}
void StreamPeerGZIP::_close() {
if (ctx) {
z_stream *strm = (z_stream *)ctx;
if (compressing) {
deflateEnd(strm);
} else {
inflateEnd(strm);
}
memfree(strm);
ctx = nullptr;
}
}
void StreamPeerGZIP::clear() {
_close();
rb.clear();
buffer.clear();
}
Error StreamPeerGZIP::start_compression(bool p_is_deflate, int buffer_size) {
return _start(true, p_is_deflate, buffer_size);
}
Error StreamPeerGZIP::start_decompression(bool p_is_deflate, int buffer_size) {
return _start(false, p_is_deflate, buffer_size);
}
Error StreamPeerGZIP::_start(bool p_compress, bool p_is_deflate, int buffer_size) {
ERR_FAIL_COND_V(ctx != nullptr, ERR_ALREADY_IN_USE);
clear();
compressing = p_compress;
rb.resize(nearest_shift(buffer_size - 1));
buffer.resize(1024);
// Create ctx.
ctx = memalloc(sizeof(z_stream));
z_stream &strm = *(z_stream *)ctx;
strm.next_in = Z_NULL;
strm.avail_in = 0;
strm.zalloc = zipio_alloc;
strm.zfree = zipio_free;
strm.opaque = Z_NULL;
int window_bits = p_is_deflate ? 15 : (15 + 16);
int err = Z_OK;
int level = Z_DEFAULT_COMPRESSION;
if (compressing) {
err = deflateInit2(&strm, level, Z_DEFLATED, window_bits, 8, Z_DEFAULT_STRATEGY);
} else {
err = inflateInit2(&strm, window_bits);
}
ERR_FAIL_COND_V(err != Z_OK, FAILED);
return OK;
}
Error StreamPeerGZIP::_process(uint8_t *p_dst, int p_dst_size, const uint8_t *p_src, int p_src_size, int &r_consumed, int &r_out, bool p_close) {
ERR_FAIL_COND_V(!ctx, ERR_UNCONFIGURED);
z_stream &strm = *(z_stream *)ctx;
strm.avail_in = p_src_size;
strm.avail_out = p_dst_size;
strm.next_in = (Bytef *)p_src;
strm.next_out = (Bytef *)p_dst;
int flush = p_close ? Z_FINISH : Z_NO_FLUSH;
if (compressing) {
int err = deflate(&strm, flush);
ERR_FAIL_COND_V(err != (p_close ? Z_STREAM_END : Z_OK), FAILED);
} else {
int err = inflate(&strm, flush);
ERR_FAIL_COND_V(err != Z_OK && err != Z_STREAM_END, FAILED);
}
r_out = p_dst_size - strm.avail_out;
r_consumed = p_src_size - strm.avail_in;
return OK;
}
Error StreamPeerGZIP::put_data(const uint8_t *p_data, int p_bytes) {
int wrote = 0;
Error err = put_partial_data(p_data, p_bytes, wrote);
if (err != OK) {
return err;
}
ERR_FAIL_COND_V(p_bytes != wrote, ERR_OUT_OF_MEMORY);
return OK;
}
Error StreamPeerGZIP::put_partial_data(const uint8_t *p_data, int p_bytes, int &r_sent) {
ERR_FAIL_COND_V(!ctx, ERR_UNCONFIGURED);
ERR_FAIL_COND_V(p_bytes < 0, ERR_INVALID_PARAMETER);
// Ensure we have enough space in temporary buffer.
if (buffer.size() < p_bytes) {
buffer.resize(p_bytes);
}
r_sent = 0;
while (r_sent < p_bytes && rb.space_left() > 1024) { // Keep the ring buffer size meaningful.
int sent = 0;
int to_write = 0;
// Compress or decompress
Error err = _process(buffer.ptrw(), MIN(buffer.size(), rb.space_left()), p_data + r_sent, p_bytes - r_sent, sent, to_write);
if (err != OK) {
return err;
}
// When decompressing, we might need to do another round.
r_sent += sent;
// We can't write more than this buffer is full.
if (sent == 0 && to_write == 0) {
return OK;
}
if (to_write) {
// Copy to ring buffer.
int wrote = rb.write(buffer.ptr(), to_write);
ERR_FAIL_COND_V(wrote != to_write, ERR_BUG);
}
}
return OK;
}
Error StreamPeerGZIP::get_data(uint8_t *p_buffer, int p_bytes) {
int received = 0;
Error err = get_partial_data(p_buffer, p_bytes, received);
if (err != OK) {
return err;
}
ERR_FAIL_COND_V(p_bytes != received, ERR_UNAVAILABLE);
return OK;
}
Error StreamPeerGZIP::get_partial_data(uint8_t *p_buffer, int p_bytes, int &r_received) {
ERR_FAIL_COND_V(p_bytes < 0, ERR_INVALID_PARAMETER);
r_received = MIN(p_bytes, rb.data_left());
if (r_received == 0) {
return OK;
}
int received = rb.read(p_buffer, r_received);
ERR_FAIL_COND_V(received != r_received, ERR_BUG);
return OK;
}
int StreamPeerGZIP::get_available_bytes() const {
return rb.data_left();
}
Error StreamPeerGZIP::finish() {
ERR_FAIL_COND_V(!ctx || !compressing, ERR_UNAVAILABLE);
// Ensure we have enough space in temporary buffer.
if (buffer.size() < 1024) {
buffer.resize(1024); // 1024 should be more than enough.
}
int consumed = 0;
int to_write = 0;
Error err = _process(buffer.ptrw(), 1024, nullptr, 0, consumed, to_write, true); // compress
if (err != OK) {
return err;
}
int wrote = rb.write(buffer.ptr(), to_write);
ERR_FAIL_COND_V(wrote != to_write, ERR_OUT_OF_MEMORY);
return OK;
}

76
core/io/stream_peer_gzip.h Normal file
View file

@ -0,0 +1,76 @@
/*************************************************************************/
/* stream_peer_gzip.h */
/*************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/*************************************************************************/
/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur. */
/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md). */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/*************************************************************************/
#ifndef STREAM_PEER_GZIP_H
#define STREAM_PEER_GZIP_H
#include "core/io/stream_peer.h"
#include "core/core_bind.h"
#include "core/io/compression.h"
#include "core/templates/ring_buffer.h"
class StreamPeerGZIP : public StreamPeer {
GDCLASS(StreamPeerGZIP, StreamPeer);
private:
void *ctx = nullptr; // Will hold our z_stream instance.
bool compressing = true;
RingBuffer<uint8_t> rb;
Vector<uint8_t> buffer;
Error _process(uint8_t *p_dst, int p_dst_size, const uint8_t *p_src, int p_src_size, int &r_consumed, int &r_out, bool p_close = false);
void _close();
Error _start(bool p_compress, bool p_is_deflate, int buffer_size = 65535);
protected:
static void _bind_methods();
public:
Error start_compression(bool p_is_deflate, int buffer_size = 65535);
Error start_decompression(bool p_is_deflate, int buffer_size = 65535);
Error finish();
void clear();
virtual Error put_data(const uint8_t *p_data, int p_bytes) override;
virtual Error put_partial_data(const uint8_t *p_data, int p_bytes, int &r_sent) override;
virtual Error get_data(uint8_t *p_buffer, int p_bytes) override;
virtual Error get_partial_data(uint8_t *p_buffer, int p_bytes, int &r_received) override;
virtual int get_available_bytes() const override;
StreamPeerGZIP();
~StreamPeerGZIP();
};
#endif // STREAM_PEER_GZIP_H

2
core/register_core_types.cpp
View file

@ -58,6 +58,7 @@
#include "core/io/resource_format_binary.h" #include "core/io/resource_format_binary.h"
#include "core/io/resource_importer.h" #include "core/io/resource_importer.h"
#include "core/io/resource_uid.h" #include "core/io/resource_uid.h"
#include "core/io/stream_peer_gzip.h"
#include "core/io/stream_peer_tls.h" #include "core/io/stream_peer_tls.h"
#include "core/io/tcp_server.h" #include "core/io/tcp_server.h"
#include "core/io/translation_loader_po.h" #include "core/io/translation_loader_po.h"
@ -184,6 +185,7 @@ void register_core_types() {
GDREGISTER_ABSTRACT_CLASS(StreamPeer); GDREGISTER_ABSTRACT_CLASS(StreamPeer);
GDREGISTER_CLASS(StreamPeerExtension); GDREGISTER_CLASS(StreamPeerExtension);
GDREGISTER_CLASS(StreamPeerBuffer); GDREGISTER_CLASS(StreamPeerBuffer);
GDREGISTER_CLASS(StreamPeerGZIP);
GDREGISTER_CLASS(StreamPeerTCP); GDREGISTER_CLASS(StreamPeerTCP);
GDREGISTER_CLASS(TCPServer); GDREGISTER_CLASS(TCPServer);

42
doc/classes/StreamPeerGZIP.xml Normal file
View file

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8" ?>
<class name="StreamPeerGZIP" inherits="StreamPeer" is_experimental="true" version="4.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../class.xsd">
<brief_description>
Stream peer handling GZIP and deflate compression/decompresison.
</brief_description>
<description>
This class allows to compress or decompress data using GZIP/deflate in a streaming fashion. This is particularly useful when compressing or decompressing files that has to be sent through the network without having to allocate them all in memory.
After starting the stream via [method start_compression] (or [method start_decompression]), calling [method StreamPeer.put_partial_data] on this stream will compress (or decompress) the data, writing it to the internal buffer. Calling [method StreamPeer.get_available_bytes] will return the pending bytes in the internal buffer, and [method StreamPeer.get_partial_data] will retrieve the compressed (or decompressed) bytes from it. When the stream is over, you must call [method finish] to ensure the internal buffer is properly flushed (make sure to call [method StreamPeer.get_available_bytes] on last time to check if more data needs to be read after that).
</description>
<tutorials>
</tutorials>
<methods>
<method name="clear">
<return type="void" />
<description>
Clears this stream, resetting the internal state.
</description>
</method>
<method name="finish">
<return type="int" enum="Error" />
<description>
Finalizes the stream, compressing or decompressing any buffered chunk left.
</description>
</method>
<method name="start_compression">
<return type="int" enum="Error" />
<param index="0" name="use_deflate" type="bool" default="false" />
<param index="1" name="buffer_size" type="int" default="65535" />
<description>
Start the stream in compression mode with the given [param buffer_size], if [param use_deflate] is [code]true[/code] uses deflate instead of GZIP.
</description>
</method>
<method name="start_decompression">
<return type="int" enum="Error" />
<param index="0" name="use_deflate" type="bool" default="false" />
<param index="1" name="buffer_size" type="int" default="65535" />
<description>
Start the stream in decompression mode with the given [param buffer_size], if [param use_deflate] is [code]true[/code] uses deflate instead of GZIP.
</description>
</method>
</methods>
</class>

125
scene/main/http_request.cpp
View file

@ -32,9 +32,6 @@
#include "core/io/compression.h" #include "core/io/compression.h"
#include "scene/main/timer.h" #include "scene/main/timer.h"
void HTTPRequest::_redirect_request(const String &p_new_url) {
}
Error HTTPRequest::_request() { Error HTTPRequest::_request() {
return client->connect_to_host(url, port, use_tls, validate_tls); return client->connect_to_host(url, port, use_tls, validate_tls);
} }
@ -48,6 +45,7 @@ Error HTTPRequest::_parse_url(const String &p_url) {
body_len = -1; body_len = -1;
body.clear(); body.clear();
downloaded.set(0); downloaded.set(0);
final_body_size.set(0);
redirections = 0; redirections = 0;
String scheme; String scheme;
@ -153,7 +151,7 @@ Error HTTPRequest::request_raw(const String &p_url, const Vector<String> &p_cust
client->set_blocking_mode(false); client->set_blocking_mode(false);
err = _request(); err = _request();
if (err != OK) { if (err != OK) {
call_deferred(SNAME("_request_done"), RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray()); _defer_done(RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray());
return ERR_CANT_CONNECT; return ERR_CANT_CONNECT;
} }
@ -169,7 +167,7 @@ void HTTPRequest::_thread_func(void *p_userdata) {
Error err = hr->_request(); Error err = hr->_request();
if (err != OK) { if (err != OK) {
hr->call_deferred(SNAME("_request_done"), RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray()); hr->_defer_done(RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray());
} else { } else {
while (!hr->thread_request_quit.is_set()) { while (!hr->thread_request_quit.is_set()) {
bool exit = hr->_update_connection(); bool exit = hr->_update_connection();
@ -198,6 +196,7 @@ void HTTPRequest::cancel_request() {
} }
file.unref(); file.unref();
decompressor.unref();
client->close(); client->close();
body.clear(); body.clear();
got_response = false; got_response = false;
@ -208,7 +207,7 @@ void HTTPRequest::cancel_request() {
bool HTTPRequest::_handle_response(bool *ret_value) { bool HTTPRequest::_handle_response(bool *ret_value) {
if (!client->has_response()) { if (!client->has_response()) {
call_deferred(SNAME("_request_done"), RESULT_NO_RESPONSE, 0, PackedStringArray(), PackedByteArray()); _defer_done(RESULT_NO_RESPONSE, 0, PackedStringArray(), PackedByteArray());
*ret_value = true; *ret_value = true;
return true; return true;
} }
@ -219,6 +218,9 @@ bool HTTPRequest::_handle_response(bool *ret_value) {
client->get_response_headers(&rheaders); client->get_response_headers(&rheaders);
response_headers.clear(); response_headers.clear();
downloaded.set(0); downloaded.set(0);
final_body_size.set(0);
decompressor.unref();
for (const String &E : rheaders) { for (const String &E : rheaders) {
response_headers.push_back(E); response_headers.push_back(E);
} }
@ -227,7 +229,7 @@ bool HTTPRequest::_handle_response(bool *ret_value) {
// Handle redirect. // Handle redirect.
if (max_redirects >= 0 && redirections >= max_redirects) { if (max_redirects >= 0 && redirections >= max_redirects) {
call_deferred(SNAME("_request_done"), RESULT_REDIRECT_LIMIT_REACHED, response_code, response_headers, PackedByteArray()); _defer_done(RESULT_REDIRECT_LIMIT_REACHED, response_code, response_headers, PackedByteArray());
*ret_value = true; *ret_value = true;
return true; return true;
} }
@ -259,6 +261,7 @@ bool HTTPRequest::_handle_response(bool *ret_value) {
body_len = -1; body_len = -1;
body.clear(); body.clear();
downloaded.set(0); downloaded.set(0);
final_body_size.set(0);
redirections = new_redirs; redirections = new_redirs;
*ret_value = false; *ret_value = false;
return true; return true;
@ -266,13 +269,26 @@ bool HTTPRequest::_handle_response(bool *ret_value) {
} }
} }
// Check if we need to start streaming decompression.
String content_encoding;
if (accept_gzip) {
content_encoding = get_header_value(response_headers, "Content-Encoding").to_lower();
}
if (content_encoding == "gzip") {
decompressor.instantiate();
decompressor->start_decompression(false, get_download_chunk_size() * 2);
} else if (content_encoding == "deflate") {
decompressor.instantiate();
decompressor->start_decompression(true, get_download_chunk_size() * 2);
}
return false; return false;
} }
bool HTTPRequest::_update_connection() { bool HTTPRequest::_update_connection() {
switch (client->get_status()) { switch (client->get_status()) {
case HTTPClient::STATUS_DISCONNECTED: { case HTTPClient::STATUS_DISCONNECTED: {
call_deferred(SNAME("_request_done"), RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray()); _defer_done(RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray());
return true; // End it, since it's disconnected. return true; // End it, since it's disconnected.
} break; } break;
case HTTPClient::STATUS_RESOLVING: { case HTTPClient::STATUS_RESOLVING: {
@ -281,7 +297,7 @@ bool HTTPRequest::_update_connection() {
return false; return false;
} break; } break;
case HTTPClient::STATUS_CANT_RESOLVE: { case HTTPClient::STATUS_CANT_RESOLVE: {
call_deferred(SNAME("_request_done"), RESULT_CANT_RESOLVE, 0, PackedStringArray(), PackedByteArray()); _defer_done(RESULT_CANT_RESOLVE, 0, PackedStringArray(), PackedByteArray());
return true; return true;
} break; } break;
@ -291,7 +307,7 @@ bool HTTPRequest::_update_connection() {
return false; return false;
} break; // Connecting to IP. } break; // Connecting to IP.
case HTTPClient::STATUS_CANT_CONNECT: { case HTTPClient::STATUS_CANT_CONNECT: {
call_deferred(SNAME("_request_done"), RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray()); _defer_done(RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray());
return true; return true;
} break; } break;
@ -306,16 +322,16 @@ bool HTTPRequest::_update_connection() {
return ret_value; return ret_value;
} }
call_deferred(SNAME("_request_done"), RESULT_SUCCESS, response_code, response_headers, PackedByteArray()); _defer_done(RESULT_SUCCESS, response_code, response_headers, PackedByteArray());
return true; return true;
} }
if (body_len < 0) { if (body_len < 0) {
// Chunked transfer is done. // Chunked transfer is done.
call_deferred(SNAME("_request_done"), RESULT_SUCCESS, response_code, response_headers, body); _defer_done(RESULT_SUCCESS, response_code, response_headers, body);
return true; return true;
} }
call_deferred(SNAME("_request_done"), RESULT_CHUNKED_BODY_SIZE_MISMATCH, response_code, response_headers, PackedByteArray()); _defer_done(RESULT_CHUNKED_BODY_SIZE_MISMATCH, response_code, response_headers, PackedByteArray());
return true; return true;
// Request might have been done. // Request might have been done.
} else { } else {
@ -324,7 +340,7 @@ bool HTTPRequest::_update_connection() {
int size = request_data.size(); int size = request_data.size();
Error err = client->request(method, request_string, headers, size > 0 ? request_data.ptr() : nullptr, size); Error err = client->request(method, request_string, headers, size > 0 ? request_data.ptr() : nullptr, size);
if (err != OK) { if (err != OK) {
call_deferred(SNAME("_request_done"), RESULT_CONNECTION_ERROR, 0, PackedStringArray(), PackedByteArray()); _defer_done(RESULT_CONNECTION_ERROR, 0, PackedStringArray(), PackedByteArray());
return true; return true;
} }
@ -347,7 +363,7 @@ bool HTTPRequest::_update_connection() {
} }
if (!client->is_response_chunked() && client->get_response_body_length() == 0) { if (!client->is_response_chunked() && client->get_response_body_length() == 0) {
call_deferred(SNAME("_request_done"), RESULT_SUCCESS, response_code, response_headers, PackedByteArray()); _defer_done(RESULT_SUCCESS, response_code, response_headers, PackedByteArray());
return true; return true;
} }
@ -356,14 +372,14 @@ bool HTTPRequest::_update_connection() {
body_len = client->get_response_body_length(); body_len = client->get_response_body_length();
if (body_size_limit >= 0 && body_len > body_size_limit) { if (body_size_limit >= 0 && body_len > body_size_limit) {
call_deferred(SNAME("_request_done"), RESULT_BODY_SIZE_LIMIT_EXCEEDED, response_code, response_headers, PackedByteArray()); _defer_done(RESULT_BODY_SIZE_LIMIT_EXCEEDED, response_code, response_headers, PackedByteArray());
return true; return true;
} }
if (!download_to_file.is_empty()) { if (!download_to_file.is_empty()) {
file = FileAccess::open(download_to_file, FileAccess::WRITE); file = FileAccess::open(download_to_file, FileAccess::WRITE);
if (file.is_null()) { if (file.is_null()) {
call_deferred(SNAME("_request_done"), RESULT_DOWNLOAD_FILE_CANT_OPEN, response_code, response_headers, PackedByteArray()); _defer_done(RESULT_DOWNLOAD_FILE_CANT_OPEN, response_code, response_headers, PackedByteArray());
return true; return true;
} }
} }
@ -375,14 +391,33 @@ bool HTTPRequest::_update_connection() {
} }
PackedByteArray chunk = client->read_response_body_chunk(); PackedByteArray chunk = client->read_response_body_chunk();
downloaded.add(chunk.size());
// Decompress chunk if needed.
if (decompressor.is_valid()) {
Error err = decompressor->put_data(chunk.ptr(), chunk.size());
if (err == OK) {
chunk.resize(decompressor->get_available_bytes());
err = decompressor->get_data(chunk.ptrw(), chunk.size());
}
if (err != OK) {
_defer_done(RESULT_BODY_DECOMPRESS_FAILED, response_code, response_headers, PackedByteArray());
return true;
}
}
final_body_size.add(chunk.size());
if (body_size_limit >= 0 && final_body_size.get() > body_size_limit) {
_defer_done(RESULT_BODY_SIZE_LIMIT_EXCEEDED, response_code, response_headers, PackedByteArray());
return true;
}
if (chunk.size()) { if (chunk.size()) {
downloaded.add(chunk.size());
if (file.is_valid()) { if (file.is_valid()) {
const uint8_t *r = chunk.ptr(); const uint8_t *r = chunk.ptr();
file->store_buffer(r, chunk.size()); file->store_buffer(r, chunk.size());
if (file->get_error() != OK) { if (file->get_error() != OK) {
call_deferred(SNAME("_request_done"), RESULT_DOWNLOAD_FILE_WRITE_ERROR, response_code, response_headers, PackedByteArray()); _defer_done(RESULT_DOWNLOAD_FILE_WRITE_ERROR, response_code, response_headers, PackedByteArray());
return true; return true;
} }
} else { } else {
@ -390,19 +425,14 @@ bool HTTPRequest::_update_connection() {
} }
} }
if (body_size_limit >= 0 && downloaded.get() > body_size_limit) {
call_deferred(SNAME("_request_done"), RESULT_BODY_SIZE_LIMIT_EXCEEDED, response_code, response_headers, PackedByteArray());
return true;
}
if (body_len >= 0) { if (body_len >= 0) {
if (downloaded.get() == body_len) { if (downloaded.get() == body_len) {
call_deferred(SNAME("_request_done"), RESULT_SUCCESS, response_code, response_headers, body); _defer_done(RESULT_SUCCESS, response_code, response_headers, body);
return true; return true;
} }
} else if (client->get_status() == HTTPClient::STATUS_DISCONNECTED) { } else if (client->get_status() == HTTPClient::STATUS_DISCONNECTED) {
// We read till EOF, with no errors. Request is done. // We read till EOF, with no errors. Request is done.
call_deferred(SNAME("_request_done"), RESULT_SUCCESS, response_code, response_headers, body); _defer_done(RESULT_SUCCESS, response_code, response_headers, body);
return true; return true;
} }
@ -410,11 +440,11 @@ bool HTTPRequest::_update_connection() {
} break; // Request resulted in body: break which must be read. } break; // Request resulted in body: break which must be read.
case HTTPClient::STATUS_CONNECTION_ERROR: { case HTTPClient::STATUS_CONNECTION_ERROR: {
call_deferred(SNAME("_request_done"), RESULT_CONNECTION_ERROR, 0, PackedStringArray(), PackedByteArray()); _defer_done(RESULT_CONNECTION_ERROR, 0, PackedStringArray(), PackedByteArray());
return true; return true;
} break; } break;
case HTTPClient::STATUS_TLS_HANDSHAKE_ERROR: { case HTTPClient::STATUS_TLS_HANDSHAKE_ERROR: {
call_deferred(SNAME("_request_done"), RESULT_TLS_HANDSHAKE_ERROR, 0, PackedStringArray(), PackedByteArray()); _defer_done(RESULT_TLS_HANDSHAKE_ERROR, 0, PackedStringArray(), PackedByteArray());
return true; return true;
} break; } break;
} }
@ -422,41 +452,13 @@ bool HTTPRequest::_update_connection() {
ERR_FAIL_V(false); ERR_FAIL_V(false);
} }
void HTTPRequest::_defer_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data) {
call_deferred(SNAME("_request_done"), p_status, p_code, p_headers, p_data);
}
void HTTPRequest::_request_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data) { void HTTPRequest::_request_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data) {
cancel_request(); cancel_request();
// Determine if the request body is compressed.
bool is_compressed;
String content_encoding = get_header_value(p_headers, "Content-Encoding").to_lower();
Compression::Mode mode;
if (content_encoding == "gzip") {
mode = Compression::Mode::MODE_GZIP;
is_compressed = true;
} else if (content_encoding == "deflate") {
mode = Compression::Mode::MODE_DEFLATE;
is_compressed = true;
} else {
is_compressed = false;
}
if (accept_gzip && is_compressed && p_data.size() > 0) {
// Decompress request body
PackedByteArray decompressed;
int result = Compression::decompress_dynamic(&decompressed, body_size_limit, p_data.ptr(), p_data.size(), mode);
if (result == OK) {
emit_signal(SNAME("request_completed"), p_status, p_code, p_headers, decompressed);
return;
} else if (result == -5) {
WARN_PRINT("Decompressed size of HTTP response body exceeded body_size_limit");
p_status = RESULT_BODY_SIZE_LIMIT_EXCEEDED;
// Just return the raw data if we failed to decompress it.
} else {
WARN_PRINT("Failed to decompress HTTP response body");
p_status = RESULT_BODY_DECOMPRESS_FAILED;
// Just return the raw data if we failed to decompress it.
}
}
emit_signal(SNAME("request_completed"), p_status, p_code, p_headers, p_data); emit_signal(SNAME("request_completed"), p_status, p_code, p_headers, p_data);
} }
@ -566,7 +568,7 @@ double HTTPRequest::get_timeout() {
void HTTPRequest::_timeout() { void HTTPRequest::_timeout() {
cancel_request(); cancel_request();
call_deferred(SNAME("_request_done"), RESULT_TIMEOUT, 0, PackedStringArray(), PackedByteArray()); _defer_done(RESULT_TIMEOUT, 0, PackedStringArray(), PackedByteArray());
} }
void HTTPRequest::_bind_methods() { void HTTPRequest::_bind_methods() {
@ -594,7 +596,6 @@ void HTTPRequest::_bind_methods() {
ClassDB::bind_method(D_METHOD("get_downloaded_bytes"), &HTTPRequest::get_downloaded_bytes); ClassDB::bind_method(D_METHOD("get_downloaded_bytes"), &HTTPRequest::get_downloaded_bytes);
ClassDB::bind_method(D_METHOD("get_body_size"), &HTTPRequest::get_body_size); ClassDB::bind_method(D_METHOD("get_body_size"), &HTTPRequest::get_body_size);
ClassDB::bind_method(D_METHOD("_redirect_request"), &HTTPRequest::_redirect_request);
ClassDB::bind_method(D_METHOD("_request_done"), &HTTPRequest::_request_done); ClassDB::bind_method(D_METHOD("_request_done"), &HTTPRequest::_request_done);
ClassDB::bind_method(D_METHOD("set_timeout", "timeout"), &HTTPRequest::set_timeout); ClassDB::bind_method(D_METHOD("set_timeout", "timeout"), &HTTPRequest::set_timeout);

4
scene/main/http_request.h
View file

@ -32,6 +32,7 @@
#define HTTP_REQUEST_H #define HTTP_REQUEST_H
#include "core/io/http_client.h" #include "core/io/http_client.h"
#include "core/io/stream_peer_gzip.h"
#include "core/os/thread.h" #include "core/os/thread.h"
#include "core/templates/safe_refcount.h" #include "core/templates/safe_refcount.h"
#include "scene/main/node.h" #include "scene/main/node.h"
@ -84,10 +85,12 @@ private:
String download_to_file; String download_to_file;
Ref<StreamPeerGZIP> decompressor;
Ref<FileAccess> file; Ref<FileAccess> file;
int body_len = -1; int body_len = -1;
SafeNumeric<int> downloaded; SafeNumeric<int> downloaded;
SafeNumeric<int> final_body_size;
int body_size_limit = -1; int body_size_limit = -1;
int redirections = 0; int redirections = 0;
@ -113,6 +116,7 @@ private:
Thread thread; Thread thread;
void _defer_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data);
void _request_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data); void _request_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data);
static void _thread_func(void *p_userdata); static void _thread_func(void *p_userdata);