diff --git a/client/exported_funcs.txt b/client/exported_funcs.txt index f925f95..478d8ab 100644 --- a/client/exported_funcs.txt +++ b/client/exported_funcs.txt @@ -1,5 +1,7 @@ init_curl start_request +cleanup_handle +create_handle tick_request active_requests @@ -7,10 +9,13 @@ get_version get_cacert get_error_str +http_set_options +http_get_info + +websocket_set_options recv_from_websocket send_to_websocket close_websocket -cleanup_handle get_result_size get_result_buffer get_result_code diff --git a/client/javascript/main.js b/client/javascript/main.js index 767b903..2cecac2 100644 --- a/client/javascript/main.js +++ b/client/javascript/main.js @@ -43,21 +43,10 @@ function check_loaded(check_websocket) { } } -//low level interface with c code -function perform_request(url, params, js_data_callback, js_end_callback, js_headers_callback, body=null) { - let params_str = JSON.stringify(params); +function create_handle(url, js_data_callback, js_end_callback, js_headers_callback) { let end_callback_ptr; let data_callback_ptr; let headers_callback_ptr; - let url_ptr = allocate_str(url); - let params_ptr = allocate_str(params_str); - - let body_ptr = null; - let body_length = 0; - if (body) { //assume body is an int8array - body_ptr = allocate_array(body); - body_length = body.length; - } function end_callback(error) { Module.removeFunction(end_callback_ptr); @@ -74,29 +63,23 @@ function perform_request(url, params, js_data_callback, js_end_callback, js_head js_data_callback(chunk); } - function headers_callback(response_json_ptr) { - let response_json = UTF8ToString(response_json_ptr); - let response_info = JSON.parse(response_json); - - if (body_ptr) _free(body_ptr); - _free(url_ptr); - _free(response_json_ptr); - - //if the response status is 0, an error occurred, - //but we don't know what it is yet - if (response_info.status !== 0) { - js_headers_callback(response_info); - } + function headers_callback() { + js_headers_callback(); } end_callback_ptr = Module.addFunction(end_callback, "vi"); - headers_callback_ptr = Module.addFunction(headers_callback, "vi"); + headers_callback_ptr = Module.addFunction(headers_callback, "v"); data_callback_ptr = Module.addFunction(data_callback, "vii"); - let http_handle = _start_request(url_ptr, params_ptr, data_callback_ptr, end_callback_ptr, headers_callback_ptr, body_ptr, body_length); - _free(params_ptr); + let http_handle = c_func(_create_handle, [url, data_callback_ptr, end_callback_ptr, headers_callback_ptr]); - active_requests ++; + return http_handle; +} + +function start_request(http_handle) { + _start_request(http_handle); _tick_request(); + active_requests ++; + if (!event_loop) { event_loop = setInterval(() => { if (_active_requests() || active_requests) { @@ -108,8 +91,6 @@ function perform_request(url, params, js_data_callback, js_end_callback, js_head } }, 0); } - - return http_handle; } function create_response(response_data, response_info) { @@ -217,8 +198,9 @@ function perform_request_async(url, params, body) { } } } - function headers_callback(response_info) { - response_obj = create_response(stream, response_info); + function headers_callback() { + let response_json = c_func_str(_http_get_info, [http_handle]); + response_obj = create_response(stream, JSON.parse(response_json)); resolve(response_obj); } function finish_callback(error) { @@ -232,8 +214,13 @@ function perform_request_async(url, params, body) { } //this will only fail if the stream is already errored or closed, which isn't a problem catch {} } + + let body_length = body ? body.length : 0; + let params_json = JSON.stringify(params); - http_handle = perform_request(url, params, data_callback, finish_callback, headers_callback, body); + http_handle = create_handle(url, data_callback, finish_callback, headers_callback); + c_func(_http_set_options, [http_handle, params_json, body, body_length]); + start_request(http_handle); }); } diff --git a/client/javascript/util.js b/client/javascript/util.js index c5557e6..c71db82 100644 --- a/client/javascript/util.js +++ b/client/javascript/util.js @@ -71,4 +71,32 @@ function data_to_array(data) { } throw "invalid data type to be sent"; +} + +//c function wrapper +function c_func(target, args=[]) { + let str_pointers = []; + for (let i = 0; i < args.length; i++) { + if (typeof args[i] !== "string") { + continue; + } + let ptr = allocate_str(args[i]); + args[i] = ptr; + str_pointers.push(ptr); + } + + let ret = target(...args); + for (let ptr of str_pointers) { + _free(ptr); + } + + return ret; +} + +//additional wrapper to free any returned strings +function c_func_str(target, args=[]) { + let ptr = c_func(target, args); + let str = UTF8ToString(ptr); + _free(ptr); + return str; } \ No newline at end of file diff --git a/client/javascript/websocket.js b/client/javascript/websocket.js index 54780de..222f795 100644 --- a/client/javascript/websocket.js +++ b/client/javascript/websocket.js @@ -22,11 +22,8 @@ class CurlWebSocket { } connect() { - let response_info; let data_callback = () => {}; - let headers_callback = (info) => { - response_info = info; - } + let headers_callback = () => {}; let finish_callback = (error) => { if (error === 0) { this.connected = true; @@ -39,7 +36,7 @@ class CurlWebSocket { else { this.cleanup(error); } - } + }; let request_options = { headers: this.options.headers || {} }; @@ -49,7 +46,11 @@ class CurlWebSocket { if (this.options.verbose) { request_options._libcurl_verbose = 1; } - this.http_handle = perform_request(this.url, request_options, data_callback, finish_callback, headers_callback, null); + + this.http_handle = create_handle(this.url, data_callback, finish_callback, headers_callback); + c_func(_http_set_options, [this.http_handle, JSON.stringify(request_options), null, 0]); + _websocket_set_options(this.http_handle); + start_request(this.http_handle); } recv() { diff --git a/client/libcurl/http.c b/client/libcurl/http.c new file mode 100644 index 0000000..6e46711 --- /dev/null +++ b/client/libcurl/http.c @@ -0,0 +1,114 @@ +#include +#include + +#include "cjson/cJSON.h" +#include "curl/curl.h" + +#include "types.h" +#include "util.h" + +void http_set_options(CURL* http_handle, const char* json_params, const char* body, int body_length) { + struct RequestInfo *request_info = get_handle_info(http_handle); + + //some default options + curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(http_handle, CURLOPT_ACCEPT_ENCODING, ""); + curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); + + //parse json options + cJSON* request_json = cJSON_Parse(json_params); + cJSON* item = NULL; + struct curl_slist* headers_list = malloc(sizeof(struct curl_slist)); + + cJSON_ArrayForEach(item, request_json) { + char* key = item->string; + + if (strcmp(key, "_libcurl_verbose") == 0) { + curl_easy_setopt(http_handle, CURLOPT_VERBOSE, 1L); + } + + if (strcmp(key, "_connect_only") == 0) { + curl_easy_setopt(http_handle, CURLOPT_CONNECT_ONLY, 1L); + curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); + curl_easy_setopt(http_handle, CURLOPT_SSL_ENABLE_ALPN, 0L); + request_info->prevent_cleanup = 1; + } + + if (strcmp(key, "method") == 0 && cJSON_IsString(item)) { + curl_easy_setopt(http_handle, CURLOPT_CUSTOMREQUEST, item->valuestring); + } + + if (strcmp(key, "headers") == 0 && cJSON_IsObject(item)) { + cJSON* header = NULL; + + cJSON_ArrayForEach(header, item) { + if (!cJSON_IsString(header)) continue; + int header_length = strlen(header->string) + strlen(header->valuestring) + 2; + char* header_str = malloc(header_length+1); + header_str[header_length] = 0; + + sprintf(header_str, "%s: %s", header->string, header->valuestring); + headers_list = curl_slist_append(headers_list, header_str); + free(header_str); + } + + curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, headers_list); + } + + if (strcmp(key, "redirect") == 0 && cJSON_IsString(item)) { + if (strcmp(item->valuestring, "error") == 0 || strcmp(item->valuestring, "manual") == 0) { + curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 0); + } + } + } + cJSON_Delete(request_json); + + //add post data if specified + if (body != NULL) { + curl_easy_setopt(http_handle, CURLOPT_POSTFIELDS, body); + curl_easy_setopt(http_handle, CURLOPT_POSTFIELDSIZE, body_length); + } + + request_info->headers_list = headers_list; +} + +char* http_get_info(CURL* http_handle) { + struct RequestInfo *request_info = get_handle_info(http_handle); + + //create new json object with response info + cJSON* response_json = cJSON_CreateObject(); + + long response_code; + curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code); + cJSON* status_item = cJSON_CreateNumber(response_code); + cJSON_AddItemToObject(response_json, "status", status_item); + + char* response_url; + curl_easy_getinfo(http_handle, CURLINFO_EFFECTIVE_URL, &response_url); + cJSON* url_item = cJSON_CreateString(response_url); + cJSON_AddItemToObject(response_json, "url", url_item); + + cJSON* headers_item = cJSON_CreateArray(); + struct curl_header *prev_header = NULL; + struct curl_header *header = NULL; + while ((header = curl_easy_nextheader(http_handle, CURLH_HEADER, -1, prev_header))) { + cJSON* header_key_entry = cJSON_CreateString(header->name); + cJSON* header_value_entry = cJSON_CreateString(header->value); + cJSON* header_pair_item = cJSON_CreateArray(); + cJSON_AddItemToArray(header_pair_item, header_key_entry); + cJSON_AddItemToArray(header_pair_item, header_value_entry); + cJSON_AddItemToArray(headers_item, header_pair_item); + prev_header = header; + } + cJSON_AddItemToObject(response_json, "headers", headers_item); + + long redirect_count; + curl_easy_getinfo(http_handle, CURLINFO_REDIRECT_COUNT, &redirect_count); + cJSON* redirects_item = cJSON_CreateBool(redirect_count > 0); + cJSON_AddItemToObject(response_json, "redirected", redirects_item); + + char* response_json_str = cJSON_Print(response_json); + cJSON_Delete(response_json); + + return response_json_str; +} \ No newline at end of file diff --git a/client/libcurl/main.c b/client/libcurl/main.c index 8b958b5..16419be 100644 --- a/client/libcurl/main.c +++ b/client/libcurl/main.c @@ -5,8 +5,6 @@ #include "curl/curl.h" #include "curl/easy.h" -#include "curl/header.h" -#include "cjson/cJSON.h" #include "curl/multi.h" #include "cacert.h" @@ -16,22 +14,19 @@ void finish_request(CURLMsg *curl_msg); void forward_headers(struct RequestInfo *request_info); -#define ERROR_REDIRECT_DISALLOWED -1 - CURLM *multi_handle; int request_active = 0; struct curl_blob cacert_blob; -size_t write_function(void *data, size_t size, size_t nmemb, struct RequestInfo *request_info) { +size_t write_function(char *data, size_t size, size_t nmemb, struct RequestInfo *request_info) { + //this should be in the write callback rather than curl's header callback because + //the write function will only be called after redirects if (!request_info->headers_received) { forward_headers(request_info); } size_t real_size = size * nmemb; - char* chunk = malloc(real_size); - memcpy(chunk, data, real_size); - (*request_info->data_callback)(chunk, real_size); - free(chunk); + (*request_info->data_callback)(data, real_size); return real_size; } @@ -53,164 +48,46 @@ void tick_request() { } } -CURL* start_request(const char* url, const char* json_params, DataCallback data_callback, EndCallback end_callback, HeadersCallback headers_callback, const char* body, int body_length) { +CURL* create_handle(const char* url, DataCallback data_callback, EndCallback end_callback, HeadersCallback headers_callback) { CURL *http_handle = curl_easy_init(); - int abort_on_redirect = 0; - int prevent_cleanup = 0; - - curl_easy_setopt(http_handle, CURLOPT_URL, url); - curl_easy_setopt(http_handle, CURLOPT_CAINFO_BLOB , cacert_blob); - - //some default options - curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(http_handle, CURLOPT_ACCEPT_ENCODING, ""); - curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); - - //if url is a websocket, tell curl that we should handle the connection manually - if (starts_with(url, "wss://") || starts_with(url, "ws://")) { - curl_easy_setopt(http_handle, CURLOPT_CONNECT_ONLY, 2L); - prevent_cleanup = 1; - } - - //parse json options - cJSON* request_json = cJSON_Parse(json_params); - cJSON* item = NULL; - struct curl_slist* headers_list = NULL; - - cJSON_ArrayForEach(item, request_json) { - char* key = item->string; - - if (strcmp(key, "_libcurl_verbose") == 0) { - curl_easy_setopt(http_handle, CURLOPT_VERBOSE, 1L); - } - - if (strcmp(key, "_connect_only") == 0) { - curl_easy_setopt(http_handle, CURLOPT_CONNECT_ONLY, 1L); - curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); - curl_easy_setopt(http_handle, CURLOPT_SSL_ENABLE_ALPN, 0L); - prevent_cleanup = 1; - } - - if (strcmp(key, "method") == 0 && cJSON_IsString(item)) { - curl_easy_setopt(http_handle, CURLOPT_CUSTOMREQUEST, item->valuestring); - } - - if (strcmp(key, "headers") == 0 && cJSON_IsObject(item)) { - cJSON* header = NULL; - - cJSON_ArrayForEach(header, item) { - if (!cJSON_IsString(header)) continue; - int header_length = strlen(header->string) + strlen(header->valuestring) + 2; - char* header_str = malloc(header_length+1); - header_str[header_length] = 0; - - sprintf(header_str, "%s: %s", header->string, header->valuestring); - headers_list = curl_slist_append(headers_list, header_str); - free(header_str); - } - - curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, headers_list); - } - - if (strcmp(key, "redirect") == 0 && cJSON_IsString(item)) { - if (strcmp(item->valuestring, "error") == 0) { - abort_on_redirect = 1; - curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 0); - } - else if (strcmp(item->valuestring, "manual") == 0) { - curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 0); - } - } - } - cJSON_Delete(request_json); - - //add post data if specified - if (body != NULL) { - curl_easy_setopt(http_handle, CURLOPT_POSTFIELDS, body); - curl_easy_setopt(http_handle, CURLOPT_POSTFIELDSIZE, body_length); - } //create request metadata struct struct RequestInfo *request_info = malloc(sizeof(struct RequestInfo)); request_info->http_handle = http_handle; - request_info->abort_on_redirect = abort_on_redirect; request_info->curl_msg = NULL; - request_info->headers_list = headers_list; - request_info->prevent_cleanup = prevent_cleanup; + request_info->prevent_cleanup = 0; request_info->headers_received = 0; request_info->end_callback = end_callback; request_info->data_callback = data_callback; request_info->headers_callback = headers_callback; + curl_easy_setopt(http_handle, CURLOPT_PRIVATE, request_info); + curl_easy_setopt(http_handle, CURLOPT_URL, url); + curl_easy_setopt(http_handle, CURLOPT_CAINFO_BLOB , cacert_blob); + //callbacks to pass the response data back to js curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, &write_function); - curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, data_callback); - - curl_easy_setopt(http_handle, CURLOPT_PRIVATE, request_info); curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, request_info); - curl_multi_add_handle(multi_handle, http_handle); - return http_handle; } +void start_request(CURL* http_handle) { + curl_multi_add_handle(multi_handle, http_handle); +} + void forward_headers(struct RequestInfo *request_info) { request_info->headers_received = 1; - CURL *http_handle = request_info->http_handle; - - //create new json object with response info - cJSON* response_json = cJSON_CreateObject(); - - long response_code; - curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code); - cJSON* status_item = cJSON_CreateNumber(response_code); - cJSON_AddItemToObject(response_json, "status", status_item); - - char* response_url; - curl_easy_getinfo(http_handle, CURLINFO_EFFECTIVE_URL, &response_url); - cJSON* url_item = cJSON_CreateString(response_url); - cJSON_AddItemToObject(response_json, "url", url_item); - - cJSON* headers_item = cJSON_CreateArray(); - struct curl_header *prev_header = NULL; - struct curl_header *header = NULL; - while ((header = curl_easy_nextheader(http_handle, CURLH_HEADER, -1, prev_header))) { - cJSON* header_key_entry = cJSON_CreateString(header->name); - cJSON* header_value_entry = cJSON_CreateString(header->value); - cJSON* header_pair_item = cJSON_CreateArray(); - cJSON_AddItemToArray(header_pair_item, header_key_entry); - cJSON_AddItemToArray(header_pair_item, header_value_entry); - cJSON_AddItemToArray(headers_item, header_pair_item); - prev_header = header; - } - cJSON_AddItemToObject(response_json, "headers", headers_item); - - long redirect_count; - curl_easy_getinfo(http_handle, CURLINFO_REDIRECT_COUNT, &redirect_count); - cJSON* redirects_item = cJSON_CreateBool(redirect_count > 0); - cJSON_AddItemToObject(response_json, "redirected", redirects_item); - - char* response_json_str = cJSON_Print(response_json); - cJSON_Delete(response_json); - - (*request_info->headers_callback)(response_json_str); + (*request_info->headers_callback)(); } void finish_request(CURLMsg *curl_msg) { - //get initial request info from the http handle - struct RequestInfo *request_info; CURL *http_handle = curl_msg->easy_handle; - curl_easy_getinfo(http_handle, CURLINFO_PRIVATE, &request_info); - if (!request_info->headers_received) { - forward_headers(request_info); - } + struct RequestInfo *request_info = get_handle_info(http_handle); int error = (int) curl_msg->data.result; - long response_code; - curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code); - - if (request_info->abort_on_redirect && response_code / 100 == 3) { - error = ERROR_REDIRECT_DISALLOWED; + if (!request_info->headers_received && error == 0) { + forward_headers(request_info); } //clean up curl @@ -224,6 +101,13 @@ void finish_request(CURLMsg *curl_msg) { free(request_info); } +void cleanup_handle(CURL* http_handle) { + struct RequestInfo *request_info = get_handle_info(http_handle); + curl_multi_remove_handle(multi_handle, http_handle); + curl_easy_cleanup(http_handle); + free(request_info); +} + unsigned char* get_cacert() { return _cacert_pem; } @@ -231,6 +115,9 @@ unsigned char* get_cacert() { void init_curl() { curl_global_init(CURL_GLOBAL_DEFAULT); multi_handle = curl_multi_init(); + + //emscripten has a fairly low file descriptor limit which means + //we must limit the total number of active tcp connections curl_multi_setopt(multi_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, 50L); curl_multi_setopt(multi_handle, CURLMOPT_MAXCONNECTS, 40L); diff --git a/client/libcurl/types.h b/client/libcurl/types.h index b270c0d..3a4e7b1 100644 --- a/client/libcurl/types.h +++ b/client/libcurl/types.h @@ -2,11 +2,10 @@ typedef void(*DataCallback)(char* chunk_ptr, int chunk_size); typedef void(*EndCallback)(int error); -typedef void(*HeadersCallback)(char* response_json); +typedef void(*HeadersCallback)(); struct RequestInfo { CURL* http_handle; - int abort_on_redirect; int prevent_cleanup; int headers_received; struct CURLMsg *curl_msg; diff --git a/client/libcurl/util.c b/client/libcurl/util.c index 3aadde4..80d03f4 100644 --- a/client/libcurl/util.c +++ b/client/libcurl/util.c @@ -38,4 +38,10 @@ char* get_version() { const char* get_error_str(CURLcode error_code) { return curl_easy_strerror(error_code); +} + +struct RequestInfo *get_handle_info(CURL* http_handle) { + struct RequestInfo *request_info; + curl_easy_getinfo(http_handle, CURLINFO_PRIVATE, &request_info); + return request_info; } \ No newline at end of file diff --git a/client/libcurl/util.h b/client/libcurl/util.h index fdb4982..f171ae2 100644 --- a/client/libcurl/util.h +++ b/client/libcurl/util.h @@ -1 +1,5 @@ -int starts_with(const char *a, const char *b); \ No newline at end of file +#include "curl/curl.h" + +int starts_with(const char *a, const char *b); + +struct RequestInfo* get_handle_info(CURL* http_handle); \ No newline at end of file diff --git a/client/libcurl/websocket.c b/client/libcurl/websocket.c index 49d2ddf..e9ac026 100644 --- a/client/libcurl/websocket.c +++ b/client/libcurl/websocket.c @@ -4,6 +4,7 @@ #include "curl/websockets.h" #include "types.h" +#include "util.h" extern CURLM* multi_handle; @@ -36,14 +37,10 @@ void close_websocket(CURL* http_handle) { curl_ws_send(http_handle, "", 0, &sent, 0, CURLWS_CLOSE); } -//clean up the http handle associated with the websocket, since the main loop can't do this automatically -void cleanup_handle(CURL* http_handle) { - struct RequestInfo *request_info; - curl_easy_getinfo(http_handle, CURLINFO_PRIVATE, &request_info); - - curl_multi_remove_handle(multi_handle, http_handle); - curl_easy_cleanup(http_handle); - free(request_info); +void websocket_set_options(CURL* http_handle) { + struct RequestInfo *request_info = get_handle_info(http_handle); + curl_easy_setopt(http_handle, CURLOPT_CONNECT_ONLY, 2L); + request_info->prevent_cleanup = 1; } int get_result_size (const struct WSResult* result) { diff --git a/client/libcurl/ws.c b/client/libcurl/ws.c new file mode 100644 index 0000000..e69de29 diff --git a/client/package.json b/client/package.json index 0b8bdcd..0f97216 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "libcurl.js", - "version": "0.5.3", + "version": "0.6.0-dev", "description": "An experimental port of libcurl to WebAssembly for use in the browser.", "main": "libcurl.mjs", "scripts": {