refactor request code

client/exported_funcs.txt

@@ -1,5 +1,7 @@
 init_curl
 start_request
+cleanup_handle
+create_handle
 tick_request
 active_requests
 
@@ -7,10 +9,13 @@ get_version
 get_cacert
 get_error_str
 
+http_set_options
+http_get_info
+
+websocket_set_options
 recv_from_websocket
 send_to_websocket
 close_websocket
-cleanup_handle
 get_result_size
 get_result_buffer
 get_result_code

client/javascript/main.js

@@ -43,21 +43,10 @@ function check_loaded(check_websocket) {
   }
 }
 
-//low level interface with c code
+function create_handle(url, js_data_callback, js_end_callback, js_headers_callback) {
-function perform_request(url, params, js_data_callback, js_end_callback, js_headers_callback, body=null) {
-  let params_str = JSON.stringify(params);
   let end_callback_ptr;
   let data_callback_ptr;
   let headers_callback_ptr;
-  let url_ptr = allocate_str(url);
-  let params_ptr = allocate_str(params_str);
-
-  let body_ptr = null;
-  let body_length = 0;
-  if (body) { //assume body is an int8array
-    body_ptr = allocate_array(body);
-    body_length = body.length;
-  }
 
   function end_callback(error) {
     Module.removeFunction(end_callback_ptr);
@@ -74,29 +63,23 @@ function perform_request(url, params, js_data_callback, js_end_callback, js_head
     js_data_callback(chunk);
   }
 
-  function headers_callback(response_json_ptr) {
+  function headers_callback() {
-    let response_json = UTF8ToString(response_json_ptr);
+    js_headers_callback();
-    let response_info = JSON.parse(response_json);
-    
-    if (body_ptr) _free(body_ptr);
-    _free(url_ptr);
-    _free(response_json_ptr);
-
-    //if the response status is 0, an error occurred,
-    //but we don't know what it is yet
-    if (response_info.status !== 0) {
-      js_headers_callback(response_info);
-    }    
   }
 
   end_callback_ptr = Module.addFunction(end_callback, "vi");
-  headers_callback_ptr = Module.addFunction(headers_callback, "vi");
+  headers_callback_ptr = Module.addFunction(headers_callback, "v");
   data_callback_ptr = Module.addFunction(data_callback, "vii");
-  let http_handle = _start_request(url_ptr, params_ptr, data_callback_ptr, end_callback_ptr, headers_callback_ptr, body_ptr, body_length);
+  let http_handle = c_func(_create_handle, [url, data_callback_ptr, end_callback_ptr, headers_callback_ptr]);
-  _free(params_ptr);
   
-  active_requests ++;
+  return http_handle;
+}
+
+function start_request(http_handle) {
+  _start_request(http_handle);
   _tick_request();
+  active_requests ++;
+
   if (!event_loop) {
     event_loop = setInterval(() => {
       if (_active_requests() || active_requests) {
@@ -108,8 +91,6 @@ function perform_request(url, params, js_data_callback, js_end_callback, js_head
       }
     }, 0);
   }
-
-  return http_handle;
 }
 
 function create_response(response_data, response_info) {
@@ -217,8 +198,9 @@ function perform_request_async(url, params, body) {
         }
       }
     }
-    function headers_callback(response_info) {
+    function headers_callback() {
-      response_obj = create_response(stream, response_info);
+      let response_json = c_func_str(_http_get_info, [http_handle]);
+      response_obj = create_response(stream, JSON.parse(response_json));
       resolve(response_obj);
     }
     function finish_callback(error) {
@@ -232,8 +214,13 @@ function perform_request_async(url, params, body) {
       } //this will only fail if the stream is already errored or closed, which isn't a problem
       catch {}
     }
+
+    let body_length = body ? body.length : 0;
+    let params_json = JSON.stringify(params);
     
-    http_handle = perform_request(url, params, data_callback, finish_callback, headers_callback, body);
+    http_handle = create_handle(url, data_callback, finish_callback, headers_callback);
+    c_func(_http_set_options, [http_handle, params_json, body, body_length]);
+    start_request(http_handle);
   });
 }
 

client/javascript/util.js

@@ -71,4 +71,32 @@ function data_to_array(data) {
   }
 
   throw "invalid data type to be sent";
+}
+
+//c function wrapper
+function c_func(target, args=[]) {
+  let str_pointers = [];
+  for (let i = 0; i < args.length; i++) {
+    if (typeof args[i] !== "string") {
+      continue;
+    }
+    let ptr = allocate_str(args[i]);
+    args[i] = ptr;
+    str_pointers.push(ptr);
+  }
+
+  let ret = target(...args);
+  for (let ptr of str_pointers) {
+    _free(ptr);
+  }
+
+  return ret;
+}
+
+//additional wrapper to free any returned strings
+function c_func_str(target, args=[]) {
+  let ptr = c_func(target, args);
+  let str = UTF8ToString(ptr);
+  _free(ptr);
+  return str;
 }

client/javascript/websocket.js

@@ -22,11 +22,8 @@ class CurlWebSocket {
   }
 
   connect() {
-    let response_info;
     let data_callback = () => {};
-    let headers_callback = (info) => {
+    let headers_callback = () => {};
-      response_info = info;
-    }
     let finish_callback = (error) => {
       if (error === 0) {
         this.connected = true;
@@ -39,7 +36,7 @@ class CurlWebSocket {
       else {
         this.cleanup(error);
       }
-    }
+    };
     let request_options = {
       headers: this.options.headers || {}
     };
@@ -49,7 +46,11 @@ class CurlWebSocket {
     if (this.options.verbose) {
       request_options._libcurl_verbose = 1;
     }
-    this.http_handle = perform_request(this.url, request_options, data_callback, finish_callback, headers_callback, null);
+
+    this.http_handle = create_handle(this.url, data_callback, finish_callback, headers_callback);
+    c_func(_http_set_options, [this.http_handle, JSON.stringify(request_options), null, 0]);
+    _websocket_set_options(this.http_handle);
+    start_request(this.http_handle);
   }
 
   recv() {

client/libcurl/http.c

@@ -0,0 +1,114 @@
+#include <stdlib.h>
+#include <string.h>
+
+#include "cjson/cJSON.h"
+#include "curl/curl.h"
+
+#include "types.h"
+#include "util.h"
+
+void http_set_options(CURL* http_handle, const char* json_params, const char* body, int body_length) {
+  struct RequestInfo *request_info = get_handle_info(http_handle);
+
+  //some default options
+  curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1);
+  curl_easy_setopt(http_handle, CURLOPT_ACCEPT_ENCODING, "");
+  curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+
+  //parse json options
+  cJSON* request_json = cJSON_Parse(json_params);
+  cJSON* item = NULL;
+  struct curl_slist* headers_list = malloc(sizeof(struct curl_slist));
+
+  cJSON_ArrayForEach(item, request_json) {
+    char* key = item->string;
+
+    if (strcmp(key, "_libcurl_verbose") == 0) {
+      curl_easy_setopt(http_handle, CURLOPT_VERBOSE, 1L);
+    }
+
+    if (strcmp(key, "_connect_only") == 0) {
+      curl_easy_setopt(http_handle, CURLOPT_CONNECT_ONLY, 1L);
+      curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
+      curl_easy_setopt(http_handle, CURLOPT_SSL_ENABLE_ALPN, 0L);
+      request_info->prevent_cleanup = 1;
+    }
+
+    if (strcmp(key, "method") == 0 && cJSON_IsString(item)) {
+      curl_easy_setopt(http_handle, CURLOPT_CUSTOMREQUEST, item->valuestring);
+    }
+    
+    if (strcmp(key, "headers") == 0 && cJSON_IsObject(item)) {
+      cJSON* header = NULL;
+
+      cJSON_ArrayForEach(header, item) {
+        if (!cJSON_IsString(header)) continue;
+        int header_length = strlen(header->string) + strlen(header->valuestring) + 2;
+        char* header_str = malloc(header_length+1);
+        header_str[header_length] = 0;
+
+        sprintf(header_str, "%s: %s", header->string, header->valuestring);
+        headers_list = curl_slist_append(headers_list, header_str);
+        free(header_str);
+      }
+
+      curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, headers_list);
+    }
+
+    if (strcmp(key, "redirect") == 0 && cJSON_IsString(item)) {
+      if (strcmp(item->valuestring, "error") == 0 || strcmp(item->valuestring, "manual") == 0) {
+        curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 0);
+      }
+    }
+  }
+  cJSON_Delete(request_json);
+
+  //add post data if specified
+  if (body != NULL) {
+    curl_easy_setopt(http_handle, CURLOPT_POSTFIELDS, body);
+    curl_easy_setopt(http_handle, CURLOPT_POSTFIELDSIZE, body_length);
+  }
+
+  request_info->headers_list = headers_list;
+}
+
+char* http_get_info(CURL* http_handle) {
+  struct RequestInfo *request_info = get_handle_info(http_handle);
+
+  //create new json object with response info
+  cJSON* response_json = cJSON_CreateObject();
+
+  long response_code;
+  curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
+  cJSON* status_item = cJSON_CreateNumber(response_code);
+  cJSON_AddItemToObject(response_json, "status", status_item);
+
+  char* response_url;
+  curl_easy_getinfo(http_handle, CURLINFO_EFFECTIVE_URL, &response_url);
+  cJSON* url_item = cJSON_CreateString(response_url);
+  cJSON_AddItemToObject(response_json, "url", url_item);
+
+  cJSON* headers_item = cJSON_CreateArray();
+  struct curl_header *prev_header = NULL;
+  struct curl_header *header = NULL;
+  while ((header = curl_easy_nextheader(http_handle, CURLH_HEADER, -1, prev_header))) {
+    cJSON* header_key_entry = cJSON_CreateString(header->name);
+    cJSON* header_value_entry = cJSON_CreateString(header->value);
+    cJSON* header_pair_item = cJSON_CreateArray();
+    cJSON_AddItemToArray(header_pair_item, header_key_entry);
+    cJSON_AddItemToArray(header_pair_item, header_value_entry);
+    cJSON_AddItemToArray(headers_item, header_pair_item);
+    prev_header = header;
+  }
+  cJSON_AddItemToObject(response_json, "headers", headers_item);
+
+  long redirect_count;
+  curl_easy_getinfo(http_handle, CURLINFO_REDIRECT_COUNT, &redirect_count);
+  cJSON* redirects_item = cJSON_CreateBool(redirect_count > 0);
+  cJSON_AddItemToObject(response_json, "redirected", redirects_item);
+
+  char* response_json_str = cJSON_Print(response_json);
+  cJSON_Delete(response_json);
+
+  return response_json_str;
+}

client/libcurl/main.c

@@ -5,8 +5,6 @@
 
 #include "curl/curl.h"
 #include "curl/easy.h"
-#include "curl/header.h"
-#include "cjson/cJSON.h"
 #include "curl/multi.h"
 
 #include "cacert.h"
@@ -16,22 +14,19 @@
 void finish_request(CURLMsg *curl_msg);
 void forward_headers(struct RequestInfo *request_info);
 
-#define ERROR_REDIRECT_DISALLOWED -1
-
 CURLM *multi_handle;
 int request_active = 0;
 struct curl_blob cacert_blob;
 
-size_t write_function(void *data, size_t size, size_t nmemb, struct RequestInfo *request_info) {
+size_t write_function(char *data, size_t size, size_t nmemb, struct RequestInfo *request_info) {
+  //this should be in the write callback rather than curl's header callback because
+  //the write function will only be called after redirects
   if (!request_info->headers_received) {
     forward_headers(request_info);
   }
 
   size_t real_size = size * nmemb;
-  char* chunk = malloc(real_size);
+  (*request_info->data_callback)(data, real_size);
-  memcpy(chunk, data, real_size);
-  (*request_info->data_callback)(chunk, real_size);
-  free(chunk);
   return real_size;
 }
 
@@ -53,164 +48,46 @@ void tick_request() {
   }
 }
 
-CURL* start_request(const char* url, const char* json_params, DataCallback data_callback, EndCallback end_callback, HeadersCallback headers_callback, const char* body, int body_length) {
+CURL* create_handle(const char* url, DataCallback data_callback, EndCallback end_callback, HeadersCallback headers_callback) {
   CURL *http_handle = curl_easy_init();  
-  int abort_on_redirect = 0;
-  int prevent_cleanup = 0;
- 
-  curl_easy_setopt(http_handle, CURLOPT_URL, url);
-  curl_easy_setopt(http_handle, CURLOPT_CAINFO_BLOB , cacert_blob);
-
-  //some default options
-  curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1);
-  curl_easy_setopt(http_handle, CURLOPT_ACCEPT_ENCODING, "");
-  curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-
-  //if url is a websocket, tell curl that we should handle the connection manually
-  if (starts_with(url, "wss://") || starts_with(url, "ws://")) {
-    curl_easy_setopt(http_handle, CURLOPT_CONNECT_ONLY, 2L);
-    prevent_cleanup = 1;
-  }
-
-  //parse json options
-  cJSON* request_json = cJSON_Parse(json_params);
-  cJSON* item = NULL;
-  struct curl_slist* headers_list = NULL;
-
-  cJSON_ArrayForEach(item, request_json) {
-    char* key = item->string;
-
-    if (strcmp(key, "_libcurl_verbose") == 0) {
-      curl_easy_setopt(http_handle, CURLOPT_VERBOSE, 1L);
-    }
-
-    if (strcmp(key, "_connect_only") == 0) {
-      curl_easy_setopt(http_handle, CURLOPT_CONNECT_ONLY, 1L);
-      curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
-      curl_easy_setopt(http_handle, CURLOPT_SSL_ENABLE_ALPN, 0L);
-      prevent_cleanup = 1;
-    }
-
-    if (strcmp(key, "method") == 0 && cJSON_IsString(item)) {
-      curl_easy_setopt(http_handle, CURLOPT_CUSTOMREQUEST, item->valuestring);
-    }
-    
-    if (strcmp(key, "headers") == 0 && cJSON_IsObject(item)) {
-      cJSON* header = NULL;
-
-      cJSON_ArrayForEach(header, item) {
-        if (!cJSON_IsString(header)) continue;
-        int header_length = strlen(header->string) + strlen(header->valuestring) + 2;
-        char* header_str = malloc(header_length+1);
-        header_str[header_length] = 0;
-
-        sprintf(header_str, "%s: %s", header->string, header->valuestring);
-        headers_list = curl_slist_append(headers_list, header_str);
-        free(header_str);
-      }
-
-      curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, headers_list);
-    }
-
-    if (strcmp(key, "redirect") == 0 && cJSON_IsString(item)) {
-      if (strcmp(item->valuestring, "error") == 0) {
-        abort_on_redirect = 1;
-        curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 0);
-      }
-      else if (strcmp(item->valuestring, "manual") == 0) {
-        curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 0);
-      }
-    }
-  }
-  cJSON_Delete(request_json);
-  
-  //add post data if specified
-  if (body != NULL) {
-    curl_easy_setopt(http_handle, CURLOPT_POSTFIELDS, body);
-    curl_easy_setopt(http_handle, CURLOPT_POSTFIELDSIZE, body_length);
-  }
 
   //create request metadata struct
   struct RequestInfo *request_info = malloc(sizeof(struct RequestInfo));
   request_info->http_handle = http_handle;
-  request_info->abort_on_redirect = abort_on_redirect;
   request_info->curl_msg = NULL;
-  request_info->headers_list = headers_list;
+  request_info->prevent_cleanup = 0;
-  request_info->prevent_cleanup = prevent_cleanup;
   request_info->headers_received = 0;
   request_info->end_callback = end_callback;
   request_info->data_callback = data_callback;
   request_info->headers_callback = headers_callback;
 
+  curl_easy_setopt(http_handle, CURLOPT_PRIVATE, request_info);
+  curl_easy_setopt(http_handle, CURLOPT_URL, url);
+  curl_easy_setopt(http_handle, CURLOPT_CAINFO_BLOB , cacert_blob);
+
   //callbacks to pass the response data back to js
   curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, &write_function);
-  curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, data_callback);
-
-  curl_easy_setopt(http_handle, CURLOPT_PRIVATE, request_info);
   curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, request_info);
   
-  curl_multi_add_handle(multi_handle, http_handle);
-
   return http_handle;
 }
 
+void start_request(CURL* http_handle) {
+  curl_multi_add_handle(multi_handle, http_handle);
+}
+
 void forward_headers(struct RequestInfo *request_info) {
   request_info->headers_received = 1;
-  CURL *http_handle = request_info->http_handle;
+  (*request_info->headers_callback)();
-
-  //create new json object with response info
-  cJSON* response_json = cJSON_CreateObject();
-
-  long response_code;
-  curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
-  cJSON* status_item = cJSON_CreateNumber(response_code);
-  cJSON_AddItemToObject(response_json, "status", status_item);
-
-  char* response_url;
-  curl_easy_getinfo(http_handle, CURLINFO_EFFECTIVE_URL, &response_url);
-  cJSON* url_item = cJSON_CreateString(response_url);
-  cJSON_AddItemToObject(response_json, "url", url_item);
-
-  cJSON* headers_item = cJSON_CreateArray();
-  struct curl_header *prev_header = NULL;
-  struct curl_header *header = NULL;
-  while ((header = curl_easy_nextheader(http_handle, CURLH_HEADER, -1, prev_header))) {
-    cJSON* header_key_entry = cJSON_CreateString(header->name);
-    cJSON* header_value_entry = cJSON_CreateString(header->value);
-    cJSON* header_pair_item = cJSON_CreateArray();
-    cJSON_AddItemToArray(header_pair_item, header_key_entry);
-    cJSON_AddItemToArray(header_pair_item, header_value_entry);
-    cJSON_AddItemToArray(headers_item, header_pair_item);
-    prev_header = header;
-  }
-  cJSON_AddItemToObject(response_json, "headers", headers_item);
-
-  long redirect_count;
-  curl_easy_getinfo(http_handle, CURLINFO_REDIRECT_COUNT, &redirect_count);
-  cJSON* redirects_item = cJSON_CreateBool(redirect_count > 0);
-  cJSON_AddItemToObject(response_json, "redirected", redirects_item);
-
-  char* response_json_str = cJSON_Print(response_json);
-  cJSON_Delete(response_json);
-
-  (*request_info->headers_callback)(response_json_str);
 }
 
 void finish_request(CURLMsg *curl_msg) {
-  //get initial request info from the http handle
-  struct RequestInfo *request_info;
   CURL *http_handle = curl_msg->easy_handle;
-  curl_easy_getinfo(http_handle, CURLINFO_PRIVATE, &request_info);
+  struct RequestInfo *request_info = get_handle_info(http_handle);
-  if (!request_info->headers_received) {
-    forward_headers(request_info);
-  }
 
   int error = (int) curl_msg->data.result;
-  long response_code;
+  if (!request_info->headers_received && error == 0) {
-  curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
+    forward_headers(request_info);
-
-  if (request_info->abort_on_redirect && response_code / 100 == 3) {
-    error = ERROR_REDIRECT_DISALLOWED;
   }
 
   //clean up curl
@@ -224,6 +101,13 @@ void finish_request(CURLMsg *curl_msg) {
   free(request_info);
 }
 
+void cleanup_handle(CURL* http_handle) {
+  struct RequestInfo *request_info = get_handle_info(http_handle);
+  curl_multi_remove_handle(multi_handle, http_handle);
+  curl_easy_cleanup(http_handle);
+  free(request_info);
+}
+
 unsigned char* get_cacert() {
   return _cacert_pem;
 }
@@ -231,6 +115,9 @@ unsigned char* get_cacert() {
 void init_curl() {
   curl_global_init(CURL_GLOBAL_DEFAULT);
   multi_handle = curl_multi_init();
+
+  //emscripten has a fairly low file descriptor limit which means
+  //we must limit the total number of active tcp connections
   curl_multi_setopt(multi_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, 50L);
   curl_multi_setopt(multi_handle, CURLMOPT_MAXCONNECTS, 40L);
   

client/libcurl/types.h

@@ -2,11 +2,10 @@
 
 typedef void(*DataCallback)(char* chunk_ptr, int chunk_size);
 typedef void(*EndCallback)(int error);
-typedef void(*HeadersCallback)(char* response_json);
+typedef void(*HeadersCallback)();
 
 struct RequestInfo {
   CURL* http_handle;
-  int abort_on_redirect;
   int prevent_cleanup;
   int headers_received;
   struct CURLMsg *curl_msg;

client/libcurl/util.c

@@ -38,4 +38,10 @@ char* get_version() {
 
 const char* get_error_str(CURLcode error_code) {
   return curl_easy_strerror(error_code);
+}
+
+struct RequestInfo *get_handle_info(CURL* http_handle) {
+  struct RequestInfo *request_info;
+  curl_easy_getinfo(http_handle, CURLINFO_PRIVATE, &request_info);
+  return request_info;
 }

client/libcurl/util.h

@@ -1 +1,5 @@
-int starts_with(const char *a, const char *b);
+#include "curl/curl.h"
+
+int starts_with(const char *a, const char *b);
+
+struct RequestInfo* get_handle_info(CURL* http_handle);

client/libcurl/websocket.c

@@ -4,6 +4,7 @@
 #include "curl/websockets.h"
 
 #include "types.h"
+#include "util.h"
 
 extern CURLM* multi_handle;
 
@@ -36,14 +37,10 @@ void close_websocket(CURL* http_handle) {
   curl_ws_send(http_handle, "", 0, &sent, 0, CURLWS_CLOSE);
 }
 
-//clean up the http handle associated with the websocket, since the main loop can't do this automatically
+void websocket_set_options(CURL* http_handle) {
-void cleanup_handle(CURL* http_handle) {
+  struct RequestInfo *request_info = get_handle_info(http_handle);
-  struct RequestInfo *request_info;
+  curl_easy_setopt(http_handle, CURLOPT_CONNECT_ONLY, 2L);
-  curl_easy_getinfo(http_handle, CURLINFO_PRIVATE, &request_info);
+  request_info->prevent_cleanup = 1;
-
-  curl_multi_remove_handle(multi_handle, http_handle);
-  curl_easy_cleanup(http_handle);
-  free(request_info);
 }
 
 int get_result_size (const struct WSResult* result) {

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libcurl.js",
-  "version": "0.5.3",
+  "version": "0.6.0-dev",
   "description": "An experimental port of libcurl to WebAssembly for use in the browser.",
   "main": "libcurl.mjs",
   "scripts": {