From 563a2b7310bdc92847dab3ccb1b500122f55590e Mon Sep 17 00:00:00 2001 From: ading2210 Date: Thu, 18 Jan 2024 14:39:39 -0500 Subject: [PATCH] improve fetch api compatibility --- README.md | 10 ++++-- client/build.sh | 2 +- client/main.js | 81 ++++++++++++++++++++++++++++++++++++---------- client/wisp_client | 2 +- server/run.sh | 5 ++- server/wisp_server | 2 +- 6 files changed, 79 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 5cbddb6..a8f56db 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,12 @@ let r = await libcurl.fetch("https://ading.dev"); console.log(await r.text()); ``` +Most of the standard Fetch API's features are supported, with the exception of: +- CORS enforcement +- `FormData` or `URLSearchParams` as the request body +- Sending credentials/cookies automatically +- Caching + ### Changing the Websocket URL: You can change the URL of the websocket proxy by using `libcurl.set_websocket`. ```js @@ -53,8 +59,8 @@ The proxy server consists of a standard [Wisp](https://github.com/MercuryWorksho To host the proxy server, run the following commands: ``` git clone https://github.com/ading2210/libcurl.js --recursive -cd libcurl.js/server -./run.sh +cd libcurl.js +STATIC=$(pwd)/client server/run.sh ``` You can use the `HOST` and `PORT` environment variables to control the hostname and port that the proxy server listens on. diff --git a/client/build.sh b/client/build.sh index 9ddff64..0c1a7f1 100755 --- a/client/build.sh +++ b/client/build.sh @@ -11,7 +11,7 @@ FRAGMENTS_DIR="fragments" WRAPPER_SOURCE="main.js" WISP_CLIENT="wisp_client" -EXPORTED_FUNCS="_init_curl,_start_request,_tick_request,_active_requests" +EXPORTED_FUNCS="_init_curl,_start_request,_tick_request,_active_requests,_free" RUNTIME_METHODS="addFunction,removeFunction,allocate,ALLOC_NORMAL" COMPILER_OPTIONS="-o $MODULE_FILE -lcurl -lssl -lcrypto -lcjson -lz -lbrotlidec -lbrotlicommon -lnghttp2 -I $INCLUDE_DIR -L $LIB_DIR" EMSCRIPTEN_OPTIONS="-lwebsocket.js -sASSERTIONS=1 -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sEXPORTED_FUNCTIONS=$EXPORTED_FUNCS -sEXPORTED_RUNTIME_METHODS=$RUNTIME_METHODS" diff --git a/client/main.js b/client/main.js index 827ebb4..28eab38 100644 --- a/client/main.js +++ b/client/main.js @@ -7,12 +7,12 @@ window.libcurl = (function() { //extra client code goes here /* __extra_libraries__ */ -const websocket_url = `wss://${location.hostname}/ws/`; +var websocket_url = `wss://${location.hostname}/ws/`; var event_loop = null; -var active_requests = 0 +var active_requests = 0; //a case insensitive dictionary for request headers -class Headers { +class HeadersDict { constructor(obj) { for (let key in obj) { this[key] = obj[key]; @@ -124,6 +124,7 @@ function create_response(response_data, response_info) { let response_obj = new Response(response_data, response_info); for (let key in response_info) { + if (key == "headers") continue; Object.defineProperty(response_obj, key, { writable: false, value: response_info[key] @@ -132,20 +133,56 @@ function create_response(response_data, response_info) { return response_obj; } -function create_options(params) { +async function parse_body(data) { + let data_array = null; + if (typeof data === "string") { + data_array = new TextEncoder().encode(data); + } + + else if (data instanceof Blob) { + let array_buffer = await data.arrayBuffer(); + data_array = new Uint8Array(array_buffer); + } + + //any typedarray + else if (data instanceof ArrayBuffer) { + //dataview objects + if (ArrayBuffer.isView(data) && data instanceof DataView) { + data_array = new Uint8Array(data.buffer); + } + //regular typed arrays + else if (ArrayBuffer.isView(data)) { + data_array = Uint8Array.from(data); + } + //regular arraybuffers + else { + data_array = new Uint8Array(data); + } + } + + else if (data instanceof ReadableStream) { + let chunks = []; + for await (let chunk of data) { + chunks.push(chunk); + } + data_array = merge_arrays(chunks); + } + + else { + throw "invalid data type to be sent"; + } + return data_array; +} + +async function create_options(params) { let body = null; if (params.body) { - if (is_str(params.body)) { - body = new TextEncoder().encode(params.body); - } - else { - body = Uint8Array.from(params); - } + body = await parse_body(params.body); params.body = true; } if (!params.headers) params.headers = {}; - params.headers = new Headers(params.headers); + params.headers = new HeadersDict(params.headers); if (params.referer) { params.headers["Referer"] = params.referer; @@ -157,9 +194,8 @@ function create_options(params) { return body; } -function libcurl_fetch(url, params={}) { - let body = create_options(params); - +//wrap perform_request in a promise +function perform_request_async(url, params, body) { return new Promise((resolve, reject) => { let chunks = []; let data_callback = (new_data) => { @@ -176,11 +212,21 @@ function libcurl_fetch(url, params={}) { resolve(response_obj); } perform_request(url, params, data_callback, finish_callback, body); - }) + }); +} + +async function libcurl_fetch(url, params={}) { + let body = await create_options(params); + return await perform_request_async(url, params, body); } function set_websocket_url(url) { - Module.websocket.url = url; + if (!Module.websocket) { + document.addEventListener("libcurl_load", () => { + set_websocket_url(url); + }); + } + else Module.websocket.url = url; } function main() { @@ -195,7 +241,8 @@ function main() { Module.onRuntimeInitialized = main; return { fetch: libcurl_fetch, - set_websocket: set_websocket_url + set_websocket: set_websocket_url, + wisp: _wisp_connections } })() \ No newline at end of file diff --git a/client/wisp_client b/client/wisp_client index 79ebf15..0df32b3 160000 --- a/client/wisp_client +++ b/client/wisp_client @@ -1 +1 @@ -Subproject commit 79ebf1517a17cfbeb427a3c6bea788c3e30704f6 +Subproject commit 0df32b3910780f4d91fc6f55ab7aab2dc726a770 diff --git a/server/run.sh b/server/run.sh index 4f3cfeb..a2cc59f 100755 --- a/server/run.sh +++ b/server/run.sh @@ -4,7 +4,10 @@ set -e -cd wisp_server +SCRIPT_PATH=$(realpath $0) +BASE_PATH=$(dirname $SCRIPT_PATH) + +cd $BASE_PATH/wisp_server if [ ! -d ".venv" ]; then python3 -m venv .venv fi diff --git a/server/wisp_server b/server/wisp_server index a74dcab..138ef0f 160000 --- a/server/wisp_server +++ b/server/wisp_server @@ -1 +1 @@ -Subproject commit a74dcabec6d8564a2e245421e710fff7928a99fb +Subproject commit 138ef0f73027b3d237ec84bee7ea43b4f30c8b74