support all request body types

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## v0.5.0 (3/8/24):
 - Added support for streaming HTTP responses via a readable stream
+- Improve compatibility for older browsers
+- Support for all types of fetch request bodies
 
 ## v0.4.2 (3/7/24):
 - Expose a function to get error strings

README.md

@@ -17,6 +17,7 @@ This is an experimental port of [libcurl](https://curl.se/libcurl/) to WebAssemb
   * [Getting Version Info](#getting-version-info)
   * [Getting the CA Certificates Bundle](#getting-the-ca-certificates-bundle)
 - [Proxy Server](#proxy-server)
+- [Project Structure](#project-structure)
 - [Copyright](#copyright)
   * [Copyright Notice](#copyright-notice)
 
@@ -223,6 +224,17 @@ server/run.sh --static=./client
 
 For a full list of server arguments, see the [wisp-server-python documentation](https://github.com/MercuryWorkshop/wisp-server-python).
 
+## Project Structure:
+- `client` - Contains all the client-side code.
+  - `fragments` - Various patches for the JS that emscripten produces. The script which does the patching can be found at `client/tools/patch_js.py`.
+  - `javascript` - All the code for the Javascript API, and for interfacing with the compiled C code.
+  - `libcurl` - The C code that interfaces with the libcurl library and gets compiled by emscripten.
+  - `tests` - Unit tests and the scripts for running them.
+  - `tools` - Helper shell scripts for the build process, and for compiling the various C libraries.
+  - `wisp_client` - A submodule for the Wisp client library.
+- `server` - Contains all the server-side code for running the websocket proxy server. 
+  - `wisp_sever` - A submodule for the Python Wisp server.
+
 ## Copyright:
 This project is licensed under the GNU AGPL v3.
 

client/javascript/copyright.js

@@ -1,15 +1,10 @@
-const copyright_notice = `ading2210/libcurl.js - A port of libcurl to WASM for use in the browser.
-Copyright (C) 2023 ading2210
+const copyright_notice = `libcurl.js is licensed under the GNU AGPL v3. You can find the license text and source code at the project's git repository: https://github.com/ading2210/libcurl.js
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.`;
+Several C libraries are used, and their licenses are listed below:
+- libcurl: curl License (https://curl.se/docs/copyright.html)
+- openssl: Apache License 2.0 (https://github.com/openssl/openssl/blob/master/LICENSE.txt)
+- cjson: MIT License (https://github.com/DaveGamble/cJSON/blob/master/LICENSE)
+- zlib: zlib License (https://www.zlib.net/zlib_license.html)
+- brotli: MIT License (https://github.com/google/brotli/blob/master/LICENSE)
+- nghttp2: MIT License (https://github.com/nghttp2/nghttp2/blob/master/COPYING)
+`;

client/javascript/main.js

@@ -107,17 +107,6 @@ function perform_request(url, params, js_data_callback, js_end_callback, js_head
   return http_handle;
 }
 
-function merge_arrays(arrays) {
-  let total_len = arrays.reduce((acc, val) => acc + val.length, 0);
-  let new_array = new Uint8Array(total_len);
-  let offset = 0;
-  for (let array of arrays) {
-    new_array.set(array, offset);
-    offset += array.length;
-  }
-  return new_array;
-}
-
 function create_response(response_data, response_info) {
   response_info.ok = response_info.status >= 200 && response_info.status < 300;
   response_info.statusText = status_messages[response_info.status] || "";
@@ -151,23 +140,31 @@ function create_response(response_data, response_info) {
   return response_obj;
 }
 
-
 async function create_options(params) {
   let body = null;
-  if (params.body) {
-    body = await data_to_array(params.body);
-    params.body = true;
+  let request_obj = new Request("/", params);
+  let array_buffer = await request_obj.arrayBuffer();
+  if (array_buffer.byteLength > 0) {
+    body = new Uint8Array(array_buffer);
   }
   
-  if (!params.headers) params.headers = {};
-  params.headers = new HeadersDict(params.headers);
+  let headers = params.headers || {};
+  if (params.headers instanceof Headers) {
+    for(let [key, value] of headers) {
+      headers[key] = value;
+    }
+  }
+  params.headers = new HeadersDict(headers);
 
-  if (params.referer) {
-    params.headers["Referer"] = params.referer;
+  if (params.referrer) {
+    params.headers["Referer"] = params.referrer;
   }
   if (!params.headers["User-Agent"]) {
     params.headers["User-Agent"] = navigator.userAgent;
   }
+  if (body) {
+    params.headers["Content-Type"] = request_obj.headers.get("Content-Type");
+  }
 
   return body;
 }

client/javascript/util.js

@@ -39,18 +39,13 @@ function get_error_str(error_code) {
   return UTF8ToString(error_ptr);
 }
 
-//convert any data to a uint8array
-async function data_to_array(data) {
+//convert various data types to a uint8array (blobs excluded)
+function data_to_array(data) {
   let data_array = null;
   if (typeof data === "string") {
     data_array = new TextEncoder().encode(data);
   }
 
-  else if (data instanceof Blob) {
-    let array_buffer = await data.arrayBuffer();
-    data_array = new Uint8Array(array_buffer);
-  }
-
   //any typedarray
   else if (data instanceof ArrayBuffer) {
     //dataview objects
@@ -67,16 +62,9 @@ async function data_to_array(data) {
     }
   }
 
-  else if (data instanceof ReadableStream) {
-    let chunks = [];
-    for await (let chunk of data) {
-      chunks.push(chunk);
-    }
-    data_array = merge_arrays(chunks);
-  }
-
   else {
     throw "invalid data type to be sent";
   }
+
   return data_array;
 }

client/javascript/ws_polyfill.js

@@ -73,7 +73,6 @@ class FakeWebSocket extends EventTarget {
   }
 
   send(data) {
-    let is_text = typeof data === "string";
     if (this.status === this.CONNECTING) {
       throw new DOMException("websocket not ready yet");
     }
@@ -81,15 +80,18 @@ class FakeWebSocket extends EventTarget {
       return;
     }
 
-    (async () => {
-      if (is_text) {
-        this.socket.send(data);
-      }
-      else {
-        let data_array = await data_to_array(data);
-        this.send(data_array);
-      }
-    })();
+    if (data instanceof Blob) {
+      (async () => {
+        let array_buffer = await data.arrayBuffer();
+        this.socket.send(new Uint8Array(array_buffer));
+      })();
+    }
+    else if (typeof data === "string") {
+      this.socket.send(data);
+    }
+    else {
+      this.socket.send(data_to_array(data));
+    }
   }
 
   close() {