HTTP cleanup & better defaults

This commit is contained in:
mhilbrunner 2017-12-12 20:02:25 +01:00
parent 55962ce28f
commit 966c054fc9
5 changed files with 131 additions and 132 deletions

View file

@ -30,27 +30,53 @@
#include "http_client.h"
#include "io/stream_peer_ssl.h"
const char *HTTPClient::_methods[METHOD_MAX] = {
"GET",
"HEAD",
"POST",
"PUT",
"DELETE",
"OPTIONS",
"TRACE",
"CONNECT",
"PATCH"
};
#ifndef JAVASCRIPT_ENABLED
Error HTTPClient::connect_to_host(const String &p_host, int p_port, bool p_ssl, bool p_verify_host) {
close();
conn_port = p_port;
conn_host = p_host;
if (conn_host.begins_with("http://")) {
ssl = p_ssl;
ssl_verify_host = p_verify_host;
String host_lower = conn_host.to_lower();
if (host_lower.begins_with("http://")) {
conn_host = conn_host.replace_first("http://", "");
} else if (conn_host.begins_with("https://")) {
//use https
} else if (host_lower.begins_with("https://")) {
ssl = true;
conn_host = conn_host.replace_first("https://", "");
}
ssl = p_ssl;
ssl_verify_host = p_verify_host;
ERR_FAIL_COND_V(conn_host.length() < HOST_MIN_LEN, ERR_INVALID_PARAMETER);
if (conn_port < 0) {
if (ssl) {
conn_port = PORT_HTTPS;
} else {
conn_port = PORT_HTTP;
}
}
connection = tcp_connection;
if (conn_host.is_valid_ip_address()) {
//is ip
// Host contains valid IP
Error err = tcp_connection->connect_to_host(IP_Address(conn_host), p_port);
if (err) {
status = STATUS_CANT_CONNECT;
@ -59,7 +85,7 @@ Error HTTPClient::connect_to_host(const String &p_host, int p_port, bool p_ssl,
status = STATUS_CONNECTING;
} else {
//is hostname
// Host contains hostname and needs to be resolved to IP
resolving = IP::get_singleton()->resolve_hostname_queue_item(conn_host);
status = STATUS_RESOLVING;
}
@ -82,24 +108,13 @@ Ref<StreamPeer> HTTPClient::get_connection() const {
Error HTTPClient::request_raw(Method p_method, const String &p_url, const Vector<String> &p_headers, const PoolVector<uint8_t> &p_body) {
ERR_FAIL_INDEX_V(p_method, METHOD_MAX, ERR_INVALID_PARAMETER);
ERR_FAIL_COND_V(!p_url.begins_with("/"), ERR_INVALID_PARAMETER);
ERR_FAIL_COND_V(status != STATUS_CONNECTED, ERR_INVALID_PARAMETER);
ERR_FAIL_COND_V(connection.is_null(), ERR_INVALID_DATA);
static const char *_methods[METHOD_MAX] = {
"GET",
"HEAD",
"POST",
"PUT",
"DELETE",
"OPTIONS",
"TRACE",
"CONNECT",
"PATCH"
};
String request = String(_methods[p_method]) + " " + p_url + " HTTP/1.1\r\n";
if ((ssl && conn_port == 443) || (!ssl && conn_port == 80)) {
// don't append the standard ports
if ((ssl && conn_port == PORT_HTTPS) || (!ssl && conn_port == PORT_HTTP)) {
// Don't append the standard ports
request += "Host: " + conn_host + "\r\n";
} else {
request += "Host: " + conn_host + ":" + itos(conn_port) + "\r\n";
@ -113,17 +128,20 @@ Error HTTPClient::request_raw(Method p_method, const String &p_url, const Vector
}
if (add_clen) {
request += "Content-Length: " + itos(p_body.size()) + "\r\n";
//should it add utf8 encoding? not sure
// Should it add utf8 encoding?
}
request += "\r\n";
CharString cs = request.utf8();
PoolVector<uint8_t> data;
//Maybe this goes faster somehow?
for (int i = 0; i < cs.length(); i++) {
data.append(cs[i]);
data.resize(cs.length());
{
PoolVector<uint8_t>::Write data_write = data.write();
for (int i = 0; i < cs.length(); i++) {
data_write[i] = cs[i];
}
}
data.append_array(p_body);
PoolVector<uint8_t>::Read r = data.read();
@ -143,24 +161,13 @@ Error HTTPClient::request_raw(Method p_method, const String &p_url, const Vector
Error HTTPClient::request(Method p_method, const String &p_url, const Vector<String> &p_headers, const String &p_body) {
ERR_FAIL_INDEX_V(p_method, METHOD_MAX, ERR_INVALID_PARAMETER);
ERR_FAIL_COND_V(!p_url.begins_with("/"), ERR_INVALID_PARAMETER);
ERR_FAIL_COND_V(status != STATUS_CONNECTED, ERR_INVALID_PARAMETER);
ERR_FAIL_COND_V(connection.is_null(), ERR_INVALID_DATA);
static const char *_methods[METHOD_MAX] = {
"GET",
"HEAD",
"POST",
"PUT",
"DELETE",
"OPTIONS",
"TRACE",
"CONNECT",
"PATCH"
};
String request = String(_methods[p_method]) + " " + p_url + " HTTP/1.1\r\n";
if ((ssl && conn_port == 443) || (!ssl && conn_port == 80)) {
// don't append the standard ports
if ((ssl && conn_port == PORT_HTTPS) || (!ssl && conn_port == PORT_HTTP)) {
// Don't append the standard ports
request += "Host: " + conn_host + "\r\n";
} else {
request += "Host: " + conn_host + ":" + itos(conn_port) + "\r\n";
@ -174,7 +181,7 @@ Error HTTPClient::request(Method p_method, const String &p_url, const Vector<Str
}
if (add_clen) {
request += "Content-Length: " + itos(p_body.utf8().length()) + "\r\n";
//should it add utf8 encoding? not sure
// Should it add utf8 encoding?
}
request += "\r\n";
request += p_body;
@ -253,7 +260,7 @@ Error HTTPClient::poll() {
IP::ResolverStatus rstatus = IP::get_singleton()->get_resolve_item_status(resolving);
switch (rstatus) {
case IP::RESOLVER_STATUS_WAITING:
return OK; //still resolving
return OK; // Still resolving
case IP::RESOLVER_STATUS_DONE: {
@ -285,7 +292,7 @@ Error HTTPClient::poll() {
switch (s) {
case StreamPeerTCP::STATUS_CONNECTING: {
return OK; //do none
return OK;
} break;
case StreamPeerTCP::STATUS_CONNECTED: {
if (ssl) {
@ -296,7 +303,6 @@ Error HTTPClient::poll() {
status = STATUS_SSL_HANDSHAKE_ERROR;
return ERR_CANT_CONNECT;
}
//print_line("SSL! TURNED ON!");
connection = ssl;
}
status = STATUS_CONNECTED;
@ -312,7 +318,7 @@ Error HTTPClient::poll() {
}
} break;
case STATUS_CONNECTED: {
//request something please
// Connection established, requests can now be made
return OK;
} break;
case STATUS_REQUESTING: {
@ -328,7 +334,7 @@ Error HTTPClient::poll() {
}
if (rec == 0)
return OK; //keep trying!
return OK; // Still requesting, keep trying!
response_str.push_back(byte);
int rs = response_str.size();
@ -336,11 +342,10 @@ Error HTTPClient::poll() {
(rs >= 2 && response_str[rs - 2] == '\n' && response_str[rs - 1] == '\n') ||
(rs >= 4 && response_str[rs - 4] == '\r' && response_str[rs - 3] == '\n' && response_str[rs - 2] == '\r' && response_str[rs - 1] == '\n')) {
//end of response, parse.
// End of response, parse.
response_str.push_back(0);
String response;
response.parse_utf8((const char *)response_str.ptr());
//print_line("END OF RESPONSE? :\n"+response+"\n------");
Vector<String> responses = response.split("\n");
body_size = 0;
chunked = false;
@ -363,7 +368,6 @@ Error HTTPClient::poll() {
if (s.begins_with("transfer-encoding:")) {
String encoding = header.substr(header.find(":") + 1, header.length()).strip_edges();
//print_line("TRANSFER ENCODING: "+encoding);
if (encoding == "chunked") {
chunked = true;
}
@ -381,14 +385,14 @@ Error HTTPClient::poll() {
if (body_size == 0 && !chunked) {
status = STATUS_CONNECTED; //ask for something again?
status = STATUS_CONNECTED; // Ready for new requests
} else {
status = STATUS_BODY;
}
return OK;
}
}
//wait for response
// Wait for response
return OK;
} break;
case STATUS_DISCONNECTED: {
@ -424,7 +428,7 @@ PoolByteArray HTTPClient::read_response_body_chunk() {
while (true) {
if (chunk_left == 0) {
//reading len
// Reading length
uint8_t b;
int rec = 0;
err = _get_http_data(&b, 1, rec);
@ -467,7 +471,7 @@ PoolByteArray HTTPClient::read_response_body_chunk() {
}
if (len == 0) {
//end!
// End reached!
status = STATUS_CONNECTED;
chunk.clear();
return PoolByteArray();
@ -525,7 +529,7 @@ PoolByteArray HTTPClient::read_response_body_chunk() {
to_read -= rec;
_offset += rec;
} else {
if (to_read > 0) //ended up reading less
if (to_read > 0) // Ended up reading less
ret.resize(_offset);
break;
}
@ -540,7 +544,7 @@ PoolByteArray HTTPClient::read_response_body_chunk() {
close();
if (err == ERR_FILE_EOF) {
status = STATUS_DISCONNECTED; //server disconnected
status = STATUS_DISCONNECTED; // Server disconnected
} else {
status = STATUS_CONNECTION_ERROR;
@ -593,7 +597,7 @@ HTTPClient::HTTPClient() {
tcp_connection = StreamPeerTCP::create_ref();
resolving = IP::RESOLVER_INVALID_ID;
status = STATUS_DISCONNECTED;
conn_port = 80;
conn_port = -1;
body_size = 0;
chunked = false;
body_left = 0;
@ -653,7 +657,7 @@ PoolStringArray HTTPClient::_get_response_headers() {
void HTTPClient::_bind_methods() {
ClassDB::bind_method(D_METHOD("connect_to_host", "host", "port", "use_ssl", "verify_host"), &HTTPClient::connect_to_host, DEFVAL(false), DEFVAL(true));
ClassDB::bind_method(D_METHOD("connect_to_host", "host", "port", "use_ssl", "verify_host"), &HTTPClient::connect_to_host, DEFVAL(-1), DEFVAL(false), DEFVAL(true));
ClassDB::bind_method(D_METHOD("set_connection", "connection"), &HTTPClient::set_connection);
ClassDB::bind_method(D_METHOD("get_connection"), &HTTPClient::get_connection);
ClassDB::bind_method(D_METHOD("request_raw", "method", "url", "headers", "body"), &HTTPClient::request_raw);
@ -689,13 +693,13 @@ void HTTPClient::_bind_methods() {
BIND_ENUM_CONSTANT(METHOD_MAX);
BIND_ENUM_CONSTANT(STATUS_DISCONNECTED);
BIND_ENUM_CONSTANT(STATUS_RESOLVING); //resolving hostname (if passed a hostname)
BIND_ENUM_CONSTANT(STATUS_RESOLVING); // Resolving hostname (if hostname was passed in)
BIND_ENUM_CONSTANT(STATUS_CANT_RESOLVE);
BIND_ENUM_CONSTANT(STATUS_CONNECTING); //connecting to ip
BIND_ENUM_CONSTANT(STATUS_CONNECTING); // Connecting to IP
BIND_ENUM_CONSTANT(STATUS_CANT_CONNECT);
BIND_ENUM_CONSTANT(STATUS_CONNECTED); //connected ); requests only accepted here
BIND_ENUM_CONSTANT(STATUS_REQUESTING); // request in progress
BIND_ENUM_CONSTANT(STATUS_BODY); // request resulted in body ); which must be read
BIND_ENUM_CONSTANT(STATUS_CONNECTED); // Connected, now accepting requests
BIND_ENUM_CONSTANT(STATUS_REQUESTING); // Request in progress
BIND_ENUM_CONSTANT(STATUS_BODY); // Request resulted in body which must be read
BIND_ENUM_CONSTANT(STATUS_CONNECTION_ERROR);
BIND_ENUM_CONSTANT(STATUS_SSL_HANDSHAKE_ERROR);

View file

@ -133,19 +133,29 @@ public:
enum Status {
STATUS_DISCONNECTED,
STATUS_RESOLVING, //resolving hostname (if passed a hostname)
STATUS_RESOLVING, // Resolving hostname (if passed a hostname)
STATUS_CANT_RESOLVE,
STATUS_CONNECTING, //connecting to ip
STATUS_CONNECTING, // Connecting to IP
STATUS_CANT_CONNECT,
STATUS_CONNECTED, //connected, requests only accepted here
STATUS_REQUESTING, // request in progress
STATUS_BODY, // request resulted in body, which must be read
STATUS_CONNECTED, // Connected, requests can be made
STATUS_REQUESTING, // Request in progress
STATUS_BODY, // Request resulted in body, which must be read
STATUS_CONNECTION_ERROR,
STATUS_SSL_HANDSHAKE_ERROR,
};
private:
static const char *_methods[METHOD_MAX];
static const int HOST_MIN_LEN = 4;
enum Port {
PORT_HTTP = 80,
PORT_HTTPS = 443,
};
#ifndef JAVASCRIPT_ENABLED
Status status;
IP::ResolverID resolving;
@ -182,8 +192,7 @@ private:
static void _bind_methods();
public:
//Error connect_and_get(const String& p_url,bool p_verify_host=true); //connects to a full url and perform request
Error connect_to_host(const String &p_host, int p_port, bool p_ssl = false, bool p_verify_host = true);
Error connect_to_host(const String &p_host, int p_port = -1, bool p_ssl = false, bool p_verify_host = true);
void set_connection(const Ref<StreamPeer> &p_connection);
Ref<StreamPeer> get_connection() const;
@ -201,9 +210,9 @@ public:
Error get_response_headers(List<String> *r_response);
int get_response_body_length() const;
PoolByteArray read_response_body_chunk(); // can't get body as partial text because of most encodings UTF8, gzip, etc.
PoolByteArray read_response_body_chunk(); // Can't get body as partial text because of most encodings UTF8, gzip, etc.
void set_blocking_mode(bool p_enable); //useful mostly if running in a thread
void set_blocking_mode(bool p_enable); // Useful mostly if running in a thread
bool is_blocking_mode_enabled() const;
void set_read_chunk_size(int p_size);

View file

@ -37,16 +37,31 @@ Error HTTPClient::connect_to_host(const String &p_host, int p_port, bool p_ssl,
WARN_PRINT("Disabling HTTPClient's host verification is not supported for the HTML5 platform, host will be verified");
}
port = p_port;
use_tls = p_ssl;
host = p_host;
if (host.begins_with("http://")) {
String host_lower = conn_host.to_lower();
if (host_lower.begins_with("http://")) {
host.replace_first("http://", "");
} else if (host.begins_with("https://")) {
} else if (host_lower.begins_with("https://")) {
use_tls = true;
host.replace_first("https://", "");
}
ERR_FAIL_COND_V(host.length() < HOST_MIN_LEN, ERR_INVALID_PARAMETER);
if (port < 0) {
if (use_tls) {
port = PORT_HTTPS;
} else {
port = PORT_HTTP;
}
}
status = host.is_valid_ip_address() ? STATUS_CONNECTING : STATUS_RESOLVING;
port = p_port;
use_tls = p_ssl;
return OK;
}
@ -68,17 +83,7 @@ Error HTTPClient::prepare_request(Method p_method, const String &p_url, const Ve
ERR_FAIL_COND_V(status != STATUS_CONNECTED, ERR_INVALID_PARAMETER);
ERR_FAIL_COND_V(host.empty(), ERR_UNCONFIGURED);
ERR_FAIL_COND_V(port < 0, ERR_UNCONFIGURED);
static const char *_methods[HTTPClient::METHOD_MAX] = {
"GET",
"HEAD",
"POST",
"PUT",
"DELETE",
"OPTIONS",
"TRACE",
"CONNECT"
};
ERR_FAIL_COND_V(!p_url.begins_with("/"), ERR_INVALID_PARAMETER);
String url = (use_tls ? "https://" : "http://") + host + ":" + itos(port) + "/" + p_url;
godot_xhr_reset(xhr_id);

View file

@ -36,7 +36,6 @@ void HTTPRequest::_redirect_request(const String &p_new_url) {
Error HTTPRequest::_request() {
//print_line("Requesting:\n\tURL: "+url+"\n\tString: "+request_string+"\n\tPort: "+itos(port)+"\n\tSSL: "+itos(use_ssl)+"\n\tValidate SSL: "+itos(validate_ssl));
return client->connect_to_host(url, port, use_ssl, validate_ssl);
}
@ -54,37 +53,32 @@ Error HTTPRequest::_parse_url(const String &p_url) {
downloaded = 0;
redirections = 0;
//print_line("1 url: "+url);
if (url.begins_with("http://")) {
String url_lower = url.to_lower();
if (url_lower.begins_with("http://")) {
url = url.substr(7, url.length() - 7);
//print_line("no SSL");
} else if (url.begins_with("https://")) {
} else if (url_lower.begins_with("https://")) {
url = url.substr(8, url.length() - 8);
use_ssl = true;
port = 443;
//print_line("yes SSL");
} else {
ERR_EXPLAIN("Malformed URL");
ERR_FAIL_V(ERR_INVALID_PARAMETER);
}
//print_line("2 url: "+url);
if (url.length() < 1) {
ERR_EXPLAIN("URL too short");
ERR_FAIL_V(ERR_INVALID_PARAMETER);
}
int slash_pos = url.find("/");
if (slash_pos != -1) {
request_string = url.substr(slash_pos, url.length());
url = url.substr(0, slash_pos);
//print_line("request string: "+request_string);
} else {
request_string = "/";
//print_line("no request");
}
//print_line("3 url: "+url);
int colon_pos = url.find(":");
if (colon_pos != -1) {
port = url.substr(colon_pos + 1, url.length()).to_int();
@ -92,8 +86,6 @@ Error HTTPRequest::_parse_url(const String &p_url) {
ERR_FAIL_COND_V(port < 1 || port > 65535, ERR_INVALID_PARAMETER);
}
//print_line("4 url: "+url);
return OK;
}
@ -198,10 +190,8 @@ void HTTPRequest::cancel_request() {
}
client->close();
body.resize(0);
//downloaded=0;
got_response = false;
response_code = -1;
//body_len=-1;
request_sent = false;
requesting = false;
}
@ -221,12 +211,12 @@ bool HTTPRequest::_handle_response(bool *ret_value) {
response_headers.resize(0);
downloaded = 0;
for (List<String>::Element *E = rheaders.front(); E; E = E->next()) {
//print_line("HEADER: "+E->get());
response_headers.push_back(E->get());
}
if (response_code == 301 || response_code == 302) {
//redirect
// Handle redirect
if (max_redirects >= 0 && redirections >= max_redirects) {
call_deferred("_request_done", RESULT_REDIRECT_LIMIT_REACHED, response_code, response_headers, PoolByteArray());
@ -242,15 +232,13 @@ bool HTTPRequest::_handle_response(bool *ret_value) {
}
}
//print_line("NEW LOCATION: "+new_request);
if (new_request != "") {
//process redirect
// Process redirect
client->close();
int new_redirs = redirections + 1; //because _request() will clear it
int new_redirs = redirections + 1; // Because _request() will clear it
Error err;
if (new_request.begins_with("http")) {
//new url, request all again
// New url, request all again
err = _parse_url(new_request);
} else {
request_string = new_request;
@ -258,7 +246,6 @@ bool HTTPRequest::_handle_response(bool *ret_value) {
err = _request();
//print_line("new connection: "+itos(err));
if (err == OK) {
request_sent = false;
got_response = false;
@ -280,11 +267,11 @@ bool HTTPRequest::_update_connection() {
switch (client->get_status()) {
case HTTPClient::STATUS_DISCONNECTED: {
call_deferred("_request_done", RESULT_CANT_CONNECT, 0, PoolStringArray(), PoolByteArray());
return true; //end it, since it's doing something
return true; // End it, since it's doing something
} break;
case HTTPClient::STATUS_RESOLVING: {
client->poll();
//must wait
// Must wait
return false;
} break;
case HTTPClient::STATUS_CANT_RESOLVE: {
@ -294,9 +281,9 @@ bool HTTPRequest::_update_connection() {
} break;
case HTTPClient::STATUS_CONNECTING: {
client->poll();
//must wait
// Must wait
return false;
} break; //connecting to ip
} break; // Connecting to IP
case HTTPClient::STATUS_CANT_CONNECT: {
call_deferred("_request_done", RESULT_CANT_CONNECT, 0, PoolStringArray(), PoolByteArray());
@ -309,7 +296,7 @@ bool HTTPRequest::_update_connection() {
if (!got_response) {
//no body
// No body
bool ret_value;
@ -320,16 +307,16 @@ bool HTTPRequest::_update_connection() {
return true;
}
if (got_response && body_len < 0) {
//chunked transfer is done
// Chunked transfer is done
call_deferred("_request_done", RESULT_SUCCESS, response_code, response_headers, body);
return true;
}
call_deferred("_request_done", RESULT_CHUNKED_BODY_SIZE_MISMATCH, response_code, response_headers, PoolByteArray());
return true;
//request migh have been done
// Request migh have been done
} else {
//did not request yet, do request
// Did not request yet, do request
Error err = client->request(method, request_string, headers, request_data);
if (err != OK) {
@ -340,13 +327,13 @@ bool HTTPRequest::_update_connection() {
request_sent = true;
return false;
}
} break; //connected: { } break requests only accepted here
} break; // Connected: break requests only accepted here
case HTTPClient::STATUS_REQUESTING: {
//must wait, it's requesting
// Must wait, still requesting
client->poll();
return false;
} break; // request in progress
} break; // Request in progress
case HTTPClient::STATUS_BODY: {
if (!got_response) {
@ -363,7 +350,7 @@ bool HTTPRequest::_update_connection() {
}
if (client->is_response_chunked()) {
body_len = -1; //no body len because chunked, change your webserver configuration if you want body len
body_len = -1; // No body len because chunked, change your webserver configuration if you want body len
} else {
body_len = client->get_response_body_length();
@ -383,7 +370,6 @@ bool HTTPRequest::_update_connection() {
}
}
//print_line("BODY: "+itos(body.size()));
client->poll();
PoolByteArray chunk = client->read_response_body_chunk();
@ -411,15 +397,11 @@ bool HTTPRequest::_update_connection() {
call_deferred("_request_done", RESULT_SUCCESS, response_code, response_headers, body);
return true;
}
/*if (body.size()>=body_len) {
call_deferred("_request_done",RESULT_BODY_SIZE_MISMATCH,response_code,response_headers,ByteArray());
return true;
}*/
}
return false;
} break; // request resulted in body: { } break which must be read
} break; // Request resulted in body: break which must be read
case HTTPClient::STATUS_CONNECTION_ERROR: {
call_deferred("_request_done", RESULT_CONNECTION_ERROR, 0, PoolStringArray(), PoolByteArray());
return true;
@ -449,7 +431,7 @@ void HTTPRequest::_notification(int p_what) {
if (done) {
set_process_internal(false);
//cancel_request(); called from _request done now
// cancel_request(); called from _request done now
}
}
@ -543,7 +525,7 @@ void HTTPRequest::_bind_methods() {
ADD_PROPERTY(PropertyInfo(Variant::BOOL, "use_threads"), "set_use_threads", "is_using_threads");
ADD_PROPERTY(PropertyInfo(Variant::INT, "body_size_limit", PROPERTY_HINT_RANGE, "-1,2000000000"), "set_body_size_limit", "get_body_size_limit");
ADD_PROPERTY(PropertyInfo(Variant::INT, "max_redirects", PROPERTY_HINT_RANGE, "-1,1024"), "set_max_redirects", "get_max_redirects");
ADD_PROPERTY(PropertyInfo(Variant::INT, "max_redirects", PROPERTY_HINT_RANGE, "-1,64"), "set_max_redirects", "get_max_redirects");
ADD_SIGNAL(MethodInfo("request_completed", PropertyInfo(Variant::INT, "result"), PropertyInfo(Variant::INT, "response_code"), PropertyInfo(Variant::POOL_STRING_ARRAY, "headers"), PropertyInfo(Variant::POOL_BYTE_ARRAY, "body")));

View file

@ -42,7 +42,6 @@ class HTTPRequest : public Node {
public:
enum Result {
RESULT_SUCCESS,
//RESULT_NO_BODY,
RESULT_CHUNKED_BODY_SIZE_MISMATCH,
RESULT_CANT_CONNECT,
RESULT_CANT_RESOLVE,