working response streaming

CHANGELOG.md

@@ -0,0 +1,67 @@
+# Libcurl.js Changelog:
+
+## v0.5.0 (3/8/24):
+- Added support for readable streams in the response
+
+## v0.4.2 (3/7/24):
+- Expose a function to get error strings
+
+## v0.4.1 (3/7/24):
+- Fix handling of duplicate HTTP headers
+
+## v0.4.0 (3/7/24):
+- Add TLS socket support
+- Add function to get the CA cert bundle
+- Re-add wsproxy support
+- Add custom network transport support
+- Split WebSocket API into simple `libcurl.CurlWebSocket` and `libcurl.WebSocket` polyfill
+- Refactor WebSocket API code
+
+## v0.3.9 (3/3/24):
+- Fix running libcurl.js inside a web worker
+
+## v0.3.8 (2/28/24):
+- Update Wisp client and server
+- Expose Wisp client API functions
+
+## v0.3.7 (2/27/24):
+- Pin C library versions to stable
+- Load the CA certs directly from memory instead of from the Emscripten virtual filesystem
+
+## v0.3.6 (2/26/24):
+- Fix ES6 module syntax
+
+## v0.3.4 (2/24/24):
+- Limit max TCP connections to 50
+
+## v0.3.3 (2/4/24):
+- Fix a memory leak with WebSockets
+
+## v0.3.2 (2/4/24):
+- Fix handling of 204 and 205 response codes
+- Add verbose option to WebSocket API
+- Fix conversion of request payloads
+
+## v0.3.1 (2/3/24):
+- Add a copyright notice to the JS bundle
+
+## v0.3.0 (2/3/24):
+- Add API to get the libcurl.js version and C library versions
+- Add checks to ensure that the Websocket proxy URL has been set 
+
+## v0.2.0 (2/2/24):
+- Add an option to redirect the verbose curl output.
+- Use separate callbacks for stdout and stderr.
+
+## v0.1.2 (2/1/23):
+- Fix bundling the WASM into a single file
+- Add unit tests
+
+## v0.1.1 (1/29/23):
+- Don't set a default websocket proxy URL
+
+## v0.1.0 (1/28/23):
+- Initial release on NPM
+- Add Github Actions for automatic builds
+- Add WebSocket support
+- Add Fetch API support 

client/javascript/main.js

@@ -43,10 +43,11 @@ function check_loaded(check_websocket) {
 }
 
 //low level interface with c code
-function perform_request(url, params, js_data_callback, js_end_callback, body=null) {
+function perform_request(url, params, js_data_callback, js_end_callback, js_headers_callback, body=null) {
   let params_str = JSON.stringify(params);
   let end_callback_ptr;
   let data_callback_ptr;
+  let headers_callback_ptr;
   let url_ptr = allocate_str(url);
   let params_ptr = allocate_str(params_str);
 
@@ -57,29 +58,36 @@ function perform_request(url, params, js_data_callback, js_end_callback, body=nu
     body_length = body.length;
   }
 
-  let end_callback = (error, response_json_ptr) => {
-    let response_json = UTF8ToString(response_json_ptr);
-    let response_info = JSON.parse(response_json);
-
+  function end_callback(error) {
     Module.removeFunction(end_callback_ptr);
     Module.removeFunction(data_callback_ptr);
-    if (body_ptr) _free(body_ptr);
-    _free(url_ptr);
-    _free(response_json_ptr);
+    Module.removeFunction(headers_callback_ptr);
     
     active_requests --;
-    js_end_callback(error, response_info);
+    js_end_callback(error);
   }
 
-  let data_callback = (chunk_ptr, chunk_size) => {
+  function data_callback(chunk_ptr, chunk_size) {
     let data = Module.HEAPU8.subarray(chunk_ptr, chunk_ptr + chunk_size);
     let chunk = new Uint8Array(data);
     js_data_callback(chunk);
   }
 
-  end_callback_ptr = Module.addFunction(end_callback, "vii");
+  function headers_callback(response_json_ptr) {
+    let response_json = UTF8ToString(response_json_ptr);
+    let response_info = JSON.parse(response_json);
+
+    if (body_ptr) _free(body_ptr);
+    _free(url_ptr);
+    _free(response_json_ptr);
+
+    js_headers_callback(response_info);
+  }
+
+  end_callback_ptr = Module.addFunction(end_callback, "vi");
+  headers_callback_ptr = Module.addFunction(headers_callback, "vi");
   data_callback_ptr = Module.addFunction(data_callback, "vii");
-  let http_handle = _start_request(url_ptr, params_ptr, data_callback_ptr, end_callback_ptr, body_ptr, body_length);
+  let http_handle = _start_request(url_ptr, params_ptr, data_callback_ptr, end_callback_ptr, headers_callback_ptr, body_ptr, body_length);
   _free(params_ptr);
   
   active_requests ++;
@@ -167,24 +175,33 @@ async function create_options(params) {
 //wrap perform_request in a promise
 function perform_request_async(url, params, body) {
   return new Promise((resolve, reject) => {
-    let chunks = [];
-    let data_callback = (new_data) => {
-      chunks.push(new_data);
+    let stream_controller;
+    let stream = new ReadableStream({
+      start(controller) {
+        stream_controller = controller;
+      }
+    });
+
+    function data_callback(new_data) {
+      stream_controller.enqueue(new_data);
     };
 
-    let finish_callback = (error, response_info) => {
+    function headers_callback(response_info) {
+      let response_obj = create_response(stream, response_info);
+      resolve(response_obj);
+    }
+
+    function finish_callback(error) {
       if (error != 0) {
         let error_str = `Request failed with error code ${error}: ${get_error_str(error)}`;
         if (error != 0) error_msg(error_str);
         reject(error_str);
         return;
       }
-      let response_data = merge_arrays(chunks);
-      chunks = null;
-      let response_obj = create_response(response_data, response_info);
-      resolve(response_obj);
+      stream_controller.close();
     }
-    perform_request(url, params, data_callback, finish_callback, body);
+    
+    perform_request(url, params, data_callback, finish_callback, headers_callback, body);
   });
 }
 

client/libcurl/main.c

@@ -14,6 +14,7 @@
 #include "types.h"
 
 void finish_request(CURLMsg *curl_msg);
+void forward_headers(struct RequestInfo *request_info);
 
 #define ERROR_REDIRECT_DISALLOWED -1
 
@@ -21,11 +22,16 @@ CURLM *multi_handle;
 int request_active = 0;
 struct curl_blob cacert_blob;
 
-size_t write_function(void *data, size_t size, size_t nmemb, DataCallback data_callback) {
+size_t write_function(void *data, size_t size, size_t nmemb, struct RequestInfo *request_info) {
+  if (!request_info->headers_received) {
+    request_info->headers_received = 1;
+    forward_headers(request_info);
+  }
+
   size_t real_size = size * nmemb;
   char* chunk = malloc(real_size);
   memcpy(chunk, data, real_size);
-  data_callback(chunk, real_size);
+  (*request_info->data_callback)(chunk, real_size);
   free(chunk);
   return real_size;
 }
@@ -48,7 +54,7 @@ void tick_request() {
   }
 }
 
-CURL* start_request(const char* url, const char* json_params, DataCallback data_callback, EndCallback end_callback, const char* body, int body_length) {
+CURL* start_request(const char* url, const char* json_params, DataCallback data_callback, EndCallback end_callback, HeadersCallback headers_callback, const char* body, int body_length) {
   CURL *http_handle = curl_easy_init();  
   int abort_on_redirect = 0;
   int prevent_cleanup = 0;
@@ -56,10 +62,6 @@ CURL* start_request(const char* url, const char* json_params, DataCallback data_
   curl_easy_setopt(http_handle, CURLOPT_URL, url);
   curl_easy_setopt(http_handle, CURLOPT_CAINFO_BLOB , cacert_blob);
 
-  //callbacks to pass the response data back to js
-  curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, &write_function);
-  curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, data_callback);
-
   //some default options
   curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1);
   curl_easy_setopt(http_handle, CURLOPT_ACCEPT_ENCODING, "");
@@ -129,36 +131,38 @@ CURL* start_request(const char* url, const char* json_params, DataCallback data_
     curl_easy_setopt(http_handle, CURLOPT_POSTFIELDSIZE, body_length);
   }
 
+  //create request metadata struct
   struct RequestInfo *request_info = malloc(sizeof(struct RequestInfo));
+  request_info->http_handle = http_handle;
   request_info->abort_on_redirect = abort_on_redirect;
   request_info->curl_msg = NULL;
   request_info->headers_list = headers_list;
-  request_info->end_callback = end_callback;
   request_info->prevent_cleanup = prevent_cleanup;
+  request_info->headers_received = 0;
+  request_info->end_callback = end_callback;
+  request_info->data_callback = data_callback;
+  request_info->headers_callback = headers_callback;
+
+  //callbacks to pass the response data back to js
+  curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, &write_function);
+  curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, data_callback);
+
   curl_easy_setopt(http_handle, CURLOPT_PRIVATE, request_info);
+  curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, request_info);
   
   curl_multi_add_handle(multi_handle, http_handle);
 
   return http_handle;
 }
 
-void finish_request(CURLMsg *curl_msg) {
-  //get initial request info from the http handle
-  struct RequestInfo *request_info;
-  CURL *http_handle = curl_msg->easy_handle;
-  curl_easy_getinfo(http_handle, CURLINFO_PRIVATE, &request_info);
-
-  int error = (int) curl_msg->data.result;
-  long response_code;
-  curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
-
-  if (request_info->abort_on_redirect && response_code / 100 == 3) {
-    error = ERROR_REDIRECT_DISALLOWED;
-  }
+void forward_headers(struct RequestInfo *request_info) {
+  CURL *http_handle = request_info->http_handle;
 
   //create new json object with response info
   cJSON* response_json = cJSON_CreateObject();
 
+  long response_code;
+  curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
   cJSON* status_item = cJSON_CreateNumber(response_code);
   cJSON_AddItemToObject(response_json, "status", status_item);
 
@@ -189,9 +193,26 @@ void finish_request(CURLMsg *curl_msg) {
   char* response_json_str = cJSON_Print(response_json);
   cJSON_Delete(response_json);
 
+  (*request_info->headers_callback)(response_json_str);
+}
+
+void finish_request(CURLMsg *curl_msg) {
+  //get initial request info from the http handle
+  struct RequestInfo *request_info;
+  CURL *http_handle = curl_msg->easy_handle;
+  curl_easy_getinfo(http_handle, CURLINFO_PRIVATE, &request_info);
+
+  int error = (int) curl_msg->data.result;
+  long response_code;
+  curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
+
+  if (request_info->abort_on_redirect && response_code / 100 == 3) {
+    error = ERROR_REDIRECT_DISALLOWED;
+  }
+
   //clean up curl
   curl_slist_free_all(request_info->headers_list);
-  (*request_info->end_callback)(error, response_json_str);
+  (*request_info->end_callback)(error);
   if (request_info->prevent_cleanup) {
     return;
   }

client/libcurl/types.h

@@ -1,12 +1,19 @@
+#include "curl/curl.h"
+
 typedef void(*DataCallback)(char* chunk_ptr, int chunk_size);
-typedef void(*EndCallback)(int error, char* response_json);
+typedef void(*EndCallback)(int error);
+typedef void(*HeadersCallback)(char* response_json);
 
 struct RequestInfo {
+  CURL* http_handle;
   int abort_on_redirect;
   int prevent_cleanup;
+  int headers_received;
   struct CURLMsg *curl_msg;
   struct curl_slist* headers_list;
+  DataCallback data_callback;
   EndCallback end_callback;
+  HeadersCallback headers_callback;
 };
 
 struct WSResult {

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libcurl.js",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "description": "An experimental port of libcurl to WebAssembly for use in the browser.",
   "main": "libcurl.mjs",
   "scripts": {