From 8bcfc2157caacd05248549ae28fc8fcd7a41632b Mon Sep 17 00:00:00 2001 From: ading2210 Date: Wed, 17 Jan 2024 14:38:51 -0500 Subject: [PATCH 01/27] fix event loop logic again --- client/main.js | 11 ++++++++--- client/wisp_client | 2 +- server/wisp_server | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/client/main.js b/client/main.js index 0f90b49..827ebb4 100644 --- a/client/main.js +++ b/client/main.js @@ -9,6 +9,7 @@ window.libcurl = (function() { const websocket_url = `wss://${location.hostname}/ws/`; var event_loop = null; +var active_requests = 0 //a case insensitive dictionary for request headers class Headers { @@ -76,6 +77,7 @@ function perform_request(url, params, js_data_callback, js_end_callback, body=nu _free(response_json_ptr); if (error != 0) console.error("request failed with error code " + error); + active_requests --; js_end_callback(error, response_info); } @@ -90,11 +92,14 @@ function perform_request(url, params, js_data_callback, js_end_callback, body=nu _start_request(url_ptr, params_ptr, data_callback_ptr, end_callback_ptr, body_ptr, body_length); _free(params_ptr); + active_requests ++; _tick_request(); if (!event_loop) { event_loop = setInterval(() => { - _tick_request(); - if (!_active_requests()) { + if (_active_requests() || active_requests) { + _tick_request(); + } + else { clearInterval(event_loop); event_loop = null; } @@ -190,7 +195,7 @@ function main() { Module.onRuntimeInitialized = main; return { fetch: libcurl_fetch, - set_websocket: set_websocket_url, + set_websocket: set_websocket_url } })() \ No newline at end of file diff --git a/client/wisp_client b/client/wisp_client index 0a80885..79ebf15 160000 --- a/client/wisp_client +++ b/client/wisp_client @@ -1 +1 @@ -Subproject commit 0a80885090b6247f42bc07cc85b441d8d719f551 +Subproject commit 79ebf1517a17cfbeb427a3c6bea788c3e30704f6 diff --git a/server/wisp_server b/server/wisp_server index 8747346..a74dcab 160000 --- a/server/wisp_server +++ b/server/wisp_server @@ -1 +1 @@ -Subproject commit 874734623e4dfc4652b34a1bc61e1e35ca86dee8 +Subproject commit a74dcabec6d8564a2e245421e710fff7928a99fb From baab0aea8fabb3dc06647f0f3c9ddcd4e7c5e4fe Mon Sep 17 00:00:00 2001 From: ading2210 Date: Wed, 17 Jan 2024 17:46:08 -0500 Subject: [PATCH 02/27] support http/2 --- client/build.sh | 2 +- client/main.c | 1 + client/tools/all_deps.sh | 7 ++++++- client/tools/curl.sh | 3 ++- client/tools/nghttp2.sh | 24 ++++++++++++++++++++++++ client/tools/openssl.sh | 2 ++ 6 files changed, 36 insertions(+), 3 deletions(-) create mode 100755 client/tools/nghttp2.sh diff --git a/client/build.sh b/client/build.sh index 13b5c78..9ddff64 100755 --- a/client/build.sh +++ b/client/build.sh @@ -13,7 +13,7 @@ WISP_CLIENT="wisp_client" EXPORTED_FUNCS="_init_curl,_start_request,_tick_request,_active_requests" RUNTIME_METHODS="addFunction,removeFunction,allocate,ALLOC_NORMAL" -COMPILER_OPTIONS="-o $MODULE_FILE -lcurl -lssl -lcrypto -lcjson -lz -lbrotlidec -lbrotlicommon -I $INCLUDE_DIR -L $LIB_DIR" +COMPILER_OPTIONS="-o $MODULE_FILE -lcurl -lssl -lcrypto -lcjson -lz -lbrotlidec -lbrotlicommon -lnghttp2 -I $INCLUDE_DIR -L $LIB_DIR" EMSCRIPTEN_OPTIONS="-lwebsocket.js -sASSERTIONS=1 -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sEXPORTED_FUNCTIONS=$EXPORTED_FUNCS -sEXPORTED_RUNTIME_METHODS=$RUNTIME_METHODS" if [ "$1" = "release" ]; then diff --git a/client/main.c b/client/main.c index be843ef..08d0660 100644 --- a/client/main.c +++ b/client/main.c @@ -72,6 +72,7 @@ void start_request(const char* url, const char* json_params, DataCallback data_c //some default options curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(http_handle, CURLOPT_ACCEPT_ENCODING, ""); + curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); //if url is a websocket, tell curl that we should handle the connection manually if (starts_with(url, "wss://") || starts_with(url, "ws://")) { diff --git a/client/tools/all_deps.sh b/client/tools/all_deps.sh index e59f563..13bd32e 100755 --- a/client/tools/all_deps.sh +++ b/client/tools/all_deps.sh @@ -9,6 +9,7 @@ CJSON_PREFIX=$(realpath build/cjson-wasm) CURL_PREFIX=$(realpath build/curl-wasm) ZLIB_PREFIX=$(realpath build/zlib-wasm) BROTLI_PREFIX=$(realpath build/brotli-wasm) +NGHTTP2_PREFIX=$(realpath build/nghttp2-wasm) if [ ! -d $OPENSSL_PREFIX ]; then tools/openssl.sh @@ -25,8 +26,12 @@ fi if [ ! -d $CURL_PREFIX ]; then tools/curl.sh fi +if [ ! -d $NGHTTP2_PREFIX ]; then + tools/nghttp2.sh +fi cp -r $OPENSSL_PREFIX/* $CURL_PREFIX cp -r $CJSON_PREFIX/* $CURL_PREFIX cp -r $ZLIB_PREFIX/* $CURL_PREFIX -cp -r $BROTLI_PREFIX/* $CURL_PREFIX \ No newline at end of file +cp -r $BROTLI_PREFIX/* $CURL_PREFIX +cp -r $NGHTTP2_PREFIX/* $CURL_PREFIX \ No newline at end of file diff --git a/client/tools/curl.sh b/client/tools/curl.sh index 20fb1cb..7f4afa0 100755 --- a/client/tools/curl.sh +++ b/client/tools/curl.sh @@ -10,6 +10,7 @@ PREFIX=$(realpath build/curl-wasm) OPENSSL_PREFIX=$(realpath build/openssl-wasm) ZLIB_PREFIX=$(realpath build/zlib-wasm) BROTLI_PREFIX=$(realpath build/brotli-wasm) +NGHTTP2_PREFIX=$(realpath build/nghttp2-wasm) cd build rm -rf curl @@ -17,7 +18,7 @@ git clone -b master --depth=1 https://github.com/curl/curl cd curl autoreconf -fi -emconfigure ./configure --host i686-linux --disable-shared --disable-threaded-resolver --without-libpsl --disable-netrc --disable-ipv6 --disable-tftp --disable-ntlm-wb --enable-websockets --with-ssl=$OPENSSL_PREFIX --with-zlib=$ZLIB_PREFIX --with-brotli=$BROTLI_PREFIX +emconfigure ./configure --host i686-linux --disable-shared --disable-threaded-resolver --without-libpsl --disable-netrc --disable-ipv6 --disable-tftp --disable-ntlm-wb --enable-websockets --with-ssl=$OPENSSL_PREFIX --with-zlib=$ZLIB_PREFIX --with-brotli=$BROTLI_PREFIX --with-nghttp2=$NGHTTP2_PREFIX emmake make -j$CORE_COUNT CFLAGS="-pthread" LIBS="-lbrotlicommon" rm -rf $PREFIX diff --git a/client/tools/nghttp2.sh b/client/tools/nghttp2.sh new file mode 100755 index 0000000..e9c86eb --- /dev/null +++ b/client/tools/nghttp2.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +#compile nghttp2 for use with emscripten + +set -x +set -e + +CORE_COUNT=$(nproc --all) +PREFIX=$(realpath build/nghttp2-wasm) + +cd build +rm -rf nghttp2 +git clone -b master --depth=1 https://github.com/nghttp2/nghttp2 +cd nghttp2 + +rm -rf $PREFIX +mkdir -p $PREFIX + +autoreconf -fi +emconfigure ./configure --host i686-linux --enable-static --disable-shared --enable-lib-only --prefix=$PREFIX +emmake make -j$CORE_COUNT +make install + +cd ../../ \ No newline at end of file diff --git a/client/tools/openssl.sh b/client/tools/openssl.sh index c87781e..dc8a041 100755 --- a/client/tools/openssl.sh +++ b/client/tools/openssl.sh @@ -14,6 +14,8 @@ rm -rf openssl git clone -b master --depth=1 https://github.com/openssl/openssl cd openssl +export CFLAGS="-Wall -Oz" +export CXXFLAGS="-Wall -Oz" emconfigure ./Configure linux-x32 --prefix=$PREFIX -no-asm -static -no-afalgeng -no-dso -DOPENSSL_SYS_NETWARE -DSIG_DFL=0 -DSIG_IGN=0 -DHAVE_FORK=0 -DOPENSSL_NO_AFALGENG=1 -DOPENSSL_NO_SPEED=1 -DOPENSSL_NO_DYNAMIC_ENGINE -DDLOPEN_FLAG=0 sed -i 's|^CROSS_COMPILE.*$|CROSS_COMPILE=|g' Makefile emmake make -j$CORE_COUNT build_generated libssl.a libcrypto.a From 563a2b7310bdc92847dab3ccb1b500122f55590e Mon Sep 17 00:00:00 2001 From: ading2210 Date: Thu, 18 Jan 2024 14:39:39 -0500 Subject: [PATCH 03/27] improve fetch api compatibility --- README.md | 10 ++++-- client/build.sh | 2 +- client/main.js | 81 ++++++++++++++++++++++++++++++++++++---------- client/wisp_client | 2 +- server/run.sh | 5 ++- server/wisp_server | 2 +- 6 files changed, 79 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 5cbddb6..a8f56db 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,12 @@ let r = await libcurl.fetch("https://ading.dev"); console.log(await r.text()); ``` +Most of the standard Fetch API's features are supported, with the exception of: +- CORS enforcement +- `FormData` or `URLSearchParams` as the request body +- Sending credentials/cookies automatically +- Caching + ### Changing the Websocket URL: You can change the URL of the websocket proxy by using `libcurl.set_websocket`. ```js @@ -53,8 +59,8 @@ The proxy server consists of a standard [Wisp](https://github.com/MercuryWorksho To host the proxy server, run the following commands: ``` git clone https://github.com/ading2210/libcurl.js --recursive -cd libcurl.js/server -./run.sh +cd libcurl.js +STATIC=$(pwd)/client server/run.sh ``` You can use the `HOST` and `PORT` environment variables to control the hostname and port that the proxy server listens on. diff --git a/client/build.sh b/client/build.sh index 9ddff64..0c1a7f1 100755 --- a/client/build.sh +++ b/client/build.sh @@ -11,7 +11,7 @@ FRAGMENTS_DIR="fragments" WRAPPER_SOURCE="main.js" WISP_CLIENT="wisp_client" -EXPORTED_FUNCS="_init_curl,_start_request,_tick_request,_active_requests" +EXPORTED_FUNCS="_init_curl,_start_request,_tick_request,_active_requests,_free" RUNTIME_METHODS="addFunction,removeFunction,allocate,ALLOC_NORMAL" COMPILER_OPTIONS="-o $MODULE_FILE -lcurl -lssl -lcrypto -lcjson -lz -lbrotlidec -lbrotlicommon -lnghttp2 -I $INCLUDE_DIR -L $LIB_DIR" EMSCRIPTEN_OPTIONS="-lwebsocket.js -sASSERTIONS=1 -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sEXPORTED_FUNCTIONS=$EXPORTED_FUNCS -sEXPORTED_RUNTIME_METHODS=$RUNTIME_METHODS" diff --git a/client/main.js b/client/main.js index 827ebb4..28eab38 100644 --- a/client/main.js +++ b/client/main.js @@ -7,12 +7,12 @@ window.libcurl = (function() { //extra client code goes here /* __extra_libraries__ */ -const websocket_url = `wss://${location.hostname}/ws/`; +var websocket_url = `wss://${location.hostname}/ws/`; var event_loop = null; -var active_requests = 0 +var active_requests = 0; //a case insensitive dictionary for request headers -class Headers { +class HeadersDict { constructor(obj) { for (let key in obj) { this[key] = obj[key]; @@ -124,6 +124,7 @@ function create_response(response_data, response_info) { let response_obj = new Response(response_data, response_info); for (let key in response_info) { + if (key == "headers") continue; Object.defineProperty(response_obj, key, { writable: false, value: response_info[key] @@ -132,20 +133,56 @@ function create_response(response_data, response_info) { return response_obj; } -function create_options(params) { +async function parse_body(data) { + let data_array = null; + if (typeof data === "string") { + data_array = new TextEncoder().encode(data); + } + + else if (data instanceof Blob) { + let array_buffer = await data.arrayBuffer(); + data_array = new Uint8Array(array_buffer); + } + + //any typedarray + else if (data instanceof ArrayBuffer) { + //dataview objects + if (ArrayBuffer.isView(data) && data instanceof DataView) { + data_array = new Uint8Array(data.buffer); + } + //regular typed arrays + else if (ArrayBuffer.isView(data)) { + data_array = Uint8Array.from(data); + } + //regular arraybuffers + else { + data_array = new Uint8Array(data); + } + } + + else if (data instanceof ReadableStream) { + let chunks = []; + for await (let chunk of data) { + chunks.push(chunk); + } + data_array = merge_arrays(chunks); + } + + else { + throw "invalid data type to be sent"; + } + return data_array; +} + +async function create_options(params) { let body = null; if (params.body) { - if (is_str(params.body)) { - body = new TextEncoder().encode(params.body); - } - else { - body = Uint8Array.from(params); - } + body = await parse_body(params.body); params.body = true; } if (!params.headers) params.headers = {}; - params.headers = new Headers(params.headers); + params.headers = new HeadersDict(params.headers); if (params.referer) { params.headers["Referer"] = params.referer; @@ -157,9 +194,8 @@ function create_options(params) { return body; } -function libcurl_fetch(url, params={}) { - let body = create_options(params); - +//wrap perform_request in a promise +function perform_request_async(url, params, body) { return new Promise((resolve, reject) => { let chunks = []; let data_callback = (new_data) => { @@ -176,11 +212,21 @@ function libcurl_fetch(url, params={}) { resolve(response_obj); } perform_request(url, params, data_callback, finish_callback, body); - }) + }); +} + +async function libcurl_fetch(url, params={}) { + let body = await create_options(params); + return await perform_request_async(url, params, body); } function set_websocket_url(url) { - Module.websocket.url = url; + if (!Module.websocket) { + document.addEventListener("libcurl_load", () => { + set_websocket_url(url); + }); + } + else Module.websocket.url = url; } function main() { @@ -195,7 +241,8 @@ function main() { Module.onRuntimeInitialized = main; return { fetch: libcurl_fetch, - set_websocket: set_websocket_url + set_websocket: set_websocket_url, + wisp: _wisp_connections } })() \ No newline at end of file diff --git a/client/wisp_client b/client/wisp_client index 79ebf15..0df32b3 160000 --- a/client/wisp_client +++ b/client/wisp_client @@ -1 +1 @@ -Subproject commit 79ebf1517a17cfbeb427a3c6bea788c3e30704f6 +Subproject commit 0df32b3910780f4d91fc6f55ab7aab2dc726a770 diff --git a/server/run.sh b/server/run.sh index 4f3cfeb..a2cc59f 100755 --- a/server/run.sh +++ b/server/run.sh @@ -4,7 +4,10 @@ set -e -cd wisp_server +SCRIPT_PATH=$(realpath $0) +BASE_PATH=$(dirname $SCRIPT_PATH) + +cd $BASE_PATH/wisp_server if [ ! -d ".venv" ]; then python3 -m venv .venv fi diff --git a/server/wisp_server b/server/wisp_server index a74dcab..138ef0f 160000 --- a/server/wisp_server +++ b/server/wisp_server @@ -1 +1 @@ -Subproject commit a74dcabec6d8564a2e245421e710fff7928a99fb +Subproject commit 138ef0f73027b3d237ec84bee7ea43b4f30c8b74 From 0b14c32b25819a0bf6b749960e571b90cb23c55c Mon Sep 17 00:00:00 2001 From: ading2210 Date: Thu, 18 Jan 2024 15:00:00 -0500 Subject: [PATCH 04/27] return the correct headers object --- README.md | 5 ++++- client/fragments/silence_socket.js | 2 +- client/main.js | 8 ++++++++ client/patcher.py | 3 +++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a8f56db..3dd62f2 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,9 @@ This is an experimental port of [libcurl](https://curl.se/libcurl/) to WebAssemb - Fetch compatible API - End to end encryption between the browser and the destination server - Support for up to TLS 1.3 +- Support for tunneling HTTP/2 connections +- Bypass CORS restrictions +- Low latency via multiplexing and reusing open connections ## Building: You can build this project by running the following commands: @@ -70,7 +73,7 @@ This project is licensed under the GNU AGPL v3. ### Copyright Notice: ``` -ading2210/libcurl.js - A port of libcurl to WASM +ading2210/libcurl.js - A port of libcurl to WASM for the browser. Copyright (C) 2023 ading2210 This program is free software: you can redistribute it and/or modify diff --git a/client/fragments/silence_socket.js b/client/fragments/silence_socket.js index 2178f6a..de7bf00 100644 --- a/client/fragments/silence_socket.js +++ b/client/fragments/silence_socket.js @@ -4,6 +4,6 @@ err\("__syscall_getsockname " ?\+ ?fd\); /* INSERT -function _emscripten_console_error\(str\) { +function _emscripten_console_error\(str\) ?{ */ if (UTF8ToString(str).endsWith("__syscall_setsockopt\\n")) return; \ No newline at end of file diff --git a/client/main.js b/client/main.js index 28eab38..3b65176 100644 --- a/client/main.js +++ b/client/main.js @@ -122,6 +122,7 @@ function create_response(response_data, response_info) { response_info.ok = response_info.status >= 200 && response_info.status < 300; response_info.statusText = status_messages[response_info.status] || ""; + //construct base response object let response_obj = new Response(response_data, response_info); for (let key in response_info) { if (key == "headers") continue; @@ -130,6 +131,13 @@ function create_response(response_data, response_info) { value: response_info[key] }); } + + //create headers object + Object.defineProperty(response_obj, "headers", { + writable: false, + value: new Headers(response_info.headers) + }); + return response_obj; } diff --git a/client/patcher.py b/client/patcher.py index bf06c6d..1ae1ad0 100644 --- a/client/patcher.py +++ b/client/patcher.py @@ -16,6 +16,9 @@ for fragment_file in fragments_path.iterdir(): matches = re.findall(match_regex, fragment_text, re.S) for mode, patch_regex, patch_text, _ in matches: + fragment_matches = re.findall(patch_regex, target_text) + if not fragment_matches: + print(f"warning: regex did not match anything - '{patch_regex}'"); if mode == "DELETE": target_text = re.sub(patch_regex, "", target_text) elif mode == "REPLACE": From 5ecb30d5ed1c86cc3143e19af32cb214d32dc4a4 Mon Sep 17 00:00:00 2001 From: ading2210 Date: Thu, 18 Jan 2024 20:14:11 -0500 Subject: [PATCH 05/27] load wasm binary manually --- README.md | 4 ++-- client/build.sh | 8 ++++++-- client/fragments/load_later.js | 8 ++++++++ client/index.html | 4 ++-- client/main.js | 23 ++++++++++++++++++++++- client/{ => scripts}/patcher.py | 2 +- 6 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 client/fragments/load_later.js rename client/{ => scripts}/patcher.py (92%) diff --git a/README.md b/README.md index 3dd62f2..d2cb943 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,10 @@ Make sure you have emscripten, git, and the various C build tools installed. The ## Javascript API: ### Importing the Library: -To import the library, follow the build instructions in the previous section, and copy `client/out/libcurl.js` a directory of your choice. Then you can simply link to it using a script tag and you will be able to use libcurl.js in your projects. Deferring the script load is recommended because the JS file is too large to download immediately. +To import the library, follow the build instructions in the previous section, and copy `client/out/libcurl.js` and `client/out/libcurl.wasm` to a directory of your choice. After the script is loaded, call `libcurl.load_wasm`, specifying the url of the `libcurl.wasm` file. ```html - + ``` To know when libcurl.js has finished loading, you can use the `libcurl_load` DOM event. diff --git a/client/build.sh b/client/build.sh index 0c1a7f1..d5743e1 100755 --- a/client/build.sh +++ b/client/build.sh @@ -2,15 +2,19 @@ set -e +#path definitions INCLUDE_DIR="build/curl-wasm/include/" LIB_DIR="build/curl-wasm/lib/" OUT_FILE="out/libcurl.js" ES6_FILE="out/libcurl_module.mjs" MODULE_FILE="out/emscripten_compiled.js" +COMPILED_FILE="out/emscripten_compiled.wasm" +WASM_FILE="out/libcurl.wasm" FRAGMENTS_DIR="fragments" WRAPPER_SOURCE="main.js" WISP_CLIENT="wisp_client" +#compile options EXPORTED_FUNCS="_init_curl,_start_request,_tick_request,_active_requests,_free" RUNTIME_METHODS="addFunction,removeFunction,allocate,ALLOC_NORMAL" COMPILER_OPTIONS="-o $MODULE_FILE -lcurl -lssl -lcrypto -lcjson -lz -lbrotlidec -lbrotlicommon -lnghttp2 -I $INCLUDE_DIR -L $LIB_DIR" @@ -18,7 +22,7 @@ EMSCRIPTEN_OPTIONS="-lwebsocket.js -sASSERTIONS=1 -sALLOW_TABLE_GROWTH -sALLOW_M if [ "$1" = "release" ]; then COMPILER_OPTIONS="-Oz -flto $COMPILER_OPTIONS" - EMSCRIPTEN_OPTIONS="-sSINGLE_FILE $EMSCRIPTEN_OPTIONS" + #EMSCRIPTEN_OPTIONS="-sSINGLE_FILE $EMSCRIPTEN_OPTIONS" else COMPILER_OPTIONS="$COMPILER_OPTIONS --profiling" fi @@ -47,7 +51,7 @@ sed -i "/__extra_libraries__/r $WISP_CLIENT/wisp.js" $OUT_FILE sed -i "/__extra_libraries__/r ./messages.js" $OUT_FILE #apply patches -python3 patcher.py $FRAGMENTS_DIR $OUT_FILE +python3 scripts/patcher.py $FRAGMENTS_DIR $OUT_FILE #generate es6 module cp $OUT_FILE $ES6_FILE diff --git a/client/fragments/load_later.js b/client/fragments/load_later.js new file mode 100644 index 0000000..9412bd4 --- /dev/null +++ b/client/fragments/load_later.js @@ -0,0 +1,8 @@ +/* REPLACE +var asm ?= ?createWasm\(\); +*/ +var asm = null; + +/* DELETE +run\(\);\n\n +*/ diff --git a/client/index.html b/client/index.html index ca21b6a..f77030f 100644 --- a/client/index.html +++ b/client/index.html @@ -3,11 +3,11 @@ - + diff --git a/client/main.js b/client/main.js index 3b65176..597bf00 100644 --- a/client/main.js +++ b/client/main.js @@ -10,6 +10,13 @@ window.libcurl = (function() { var websocket_url = `wss://${location.hostname}/ws/`; var event_loop = null; var active_requests = 0; +var wasm_ready = false; + +function check_loaded() { + if (!wasm_ready) { + throw new Error("wasm not loaded yet, please call libcurl.load_wasm first"); + } +} //a case insensitive dictionary for request headers class HeadersDict { @@ -135,8 +142,12 @@ function create_response(response_data, response_info) { //create headers object Object.defineProperty(response_obj, "headers", { writable: false, - value: new Headers(response_info.headers) + value: new Headers() }); + for (let header_name in response_info.headers) { + let header_value = response_info.headers[header_name]; + response_obj.headers.append(header_name, header_value); + } return response_obj; } @@ -224,11 +235,13 @@ function perform_request_async(url, params, body) { } async function libcurl_fetch(url, params={}) { + check_loaded(); let body = await create_options(params); return await perform_request_async(url, params, body); } function set_websocket_url(url) { + check_loaded(); if (!Module.websocket) { document.addEventListener("libcurl_load", () => { set_websocket_url(url); @@ -239,6 +252,7 @@ function set_websocket_url(url) { function main() { console.log("emscripten module loaded"); + wasm_ready = true; _init_curl(); set_websocket_url(websocket_url); @@ -246,10 +260,17 @@ function main() { document.dispatchEvent(load_event); } +function load_wasm(url) { + wasmBinaryFile = url; + createWasm(); + run(); +} + Module.onRuntimeInitialized = main; return { fetch: libcurl_fetch, set_websocket: set_websocket_url, + load_wasm: load_wasm, wisp: _wisp_connections } diff --git a/client/patcher.py b/client/scripts/patcher.py similarity index 92% rename from client/patcher.py rename to client/scripts/patcher.py index 1ae1ad0..65efcc5 100644 --- a/client/patcher.py +++ b/client/scripts/patcher.py @@ -18,7 +18,7 @@ for fragment_file in fragments_path.iterdir(): for mode, patch_regex, patch_text, _ in matches: fragment_matches = re.findall(patch_regex, target_text) if not fragment_matches: - print(f"warning: regex did not match anything - '{patch_regex}'"); + print(f"warning: regex did not match anything for '{patch_regex}'"); if mode == "DELETE": target_text = re.sub(patch_regex, "", target_text) elif mode == "REPLACE": From 8ad11cd51549d68ab0c31e50e11c403a4fdf63bb Mon Sep 17 00:00:00 2001 From: ading2210 Date: Tue, 23 Jan 2024 02:46:19 -0500 Subject: [PATCH 06/27] improve build script arguments --- README.md | 4 ++++ client/build.sh | 10 ++++++++-- client/fragments/load_later.js | 5 +++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d2cb943..6778449 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,10 @@ cd libcurl.js/client ``` Make sure you have emscripten, git, and the various C build tools installed. The build script will generate `client/out/libcurl.js` as well as `client/out/libcurl_module.mjs`, which is an ES6 module. +You can supply the following arguments to the build script to control the build: +- `release` - Use all optimizations. +- `single_file` - Include the WASM binary in the outputted JS using base64. + ## Javascript API: ### Importing the Library: diff --git a/client/build.sh b/client/build.sh index d5743e1..4c1d452 100755 --- a/client/build.sh +++ b/client/build.sh @@ -20,13 +20,18 @@ RUNTIME_METHODS="addFunction,removeFunction,allocate,ALLOC_NORMAL" COMPILER_OPTIONS="-o $MODULE_FILE -lcurl -lssl -lcrypto -lcjson -lz -lbrotlidec -lbrotlicommon -lnghttp2 -I $INCLUDE_DIR -L $LIB_DIR" EMSCRIPTEN_OPTIONS="-lwebsocket.js -sASSERTIONS=1 -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sEXPORTED_FUNCTIONS=$EXPORTED_FUNCS -sEXPORTED_RUNTIME_METHODS=$RUNTIME_METHODS" -if [ "$1" = "release" ]; then +if [[ "$*" == *"release"* ]]; then COMPILER_OPTIONS="-Oz -flto $COMPILER_OPTIONS" - #EMSCRIPTEN_OPTIONS="-sSINGLE_FILE $EMSCRIPTEN_OPTIONS" + echo "note: building with release optimizations" else COMPILER_OPTIONS="$COMPILER_OPTIONS --profiling" fi +if [[ "$*" == *"single_file"* ]]; then + EMSCRIPTEN_OPTIONS="-sSINGLE_FILE $EMSCRIPTEN_OPTIONS" + echo "note: building as a single js file" +fi + #ensure deps are compiled tools/all_deps.sh tools/generate_cert.sh @@ -39,6 +44,7 @@ mkdir -p out COMPILE_CMD="emcc main.c $COMPILER_OPTIONS $EMSCRIPTEN_OPTIONS" echo $COMPILE_CMD $COMPILE_CMD +mv $COMPILED_FILE $WASM_FILE || true #merge compiled emscripten module and wrapper code cp $WRAPPER_SOURCE $OUT_FILE diff --git a/client/fragments/load_later.js b/client/fragments/load_later.js index 9412bd4..79087a1 100644 --- a/client/fragments/load_later.js +++ b/client/fragments/load_later.js @@ -3,6 +3,11 @@ var asm ?= ?createWasm\(\); */ var asm = null; +/* REPLACE +var wasmExports ?= ?createWasm\(\); +*/ +var wasmExports = null; + /* DELETE run\(\);\n\n */ From 17d8c9fa8c889d4b00ddc6d61677c32932d4ac50 Mon Sep 17 00:00:00 2001 From: ading2210 Date: Fri, 26 Jan 2024 18:27:17 -0500 Subject: [PATCH 07/27] add basic websocket support --- client/build.sh | 15 +++- client/exported_funcs.txt | 15 ++++ client/index.html | 2 +- client/main.c | 36 ++++----- client/main.js | 7 +- client/types.h | 17 ++++ client/util.c | 5 ++ client/util.h | 1 + client/websocket.c | 57 +++++++++++++ client/websocket.js | 166 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 292 insertions(+), 29 deletions(-) create mode 100644 client/exported_funcs.txt create mode 100644 client/types.h create mode 100644 client/util.c create mode 100644 client/util.h create mode 100644 client/websocket.c create mode 100644 client/websocket.js diff --git a/client/build.sh b/client/build.sh index 4c1d452..8056530 100755 --- a/client/build.sh +++ b/client/build.sh @@ -14,11 +14,17 @@ FRAGMENTS_DIR="fragments" WRAPPER_SOURCE="main.js" WISP_CLIENT="wisp_client" +#read exported functions +EXPORTED_FUNCS="" +for func in $(cat exported_funcs.txt); do + EXPORTED_FUNCS="$EXPORTED_FUNCS,_$func" +done +EXPORTED_FUNCS="${EXPORTED_FUNCS:1}" + #compile options -EXPORTED_FUNCS="_init_curl,_start_request,_tick_request,_active_requests,_free" RUNTIME_METHODS="addFunction,removeFunction,allocate,ALLOC_NORMAL" COMPILER_OPTIONS="-o $MODULE_FILE -lcurl -lssl -lcrypto -lcjson -lz -lbrotlidec -lbrotlicommon -lnghttp2 -I $INCLUDE_DIR -L $LIB_DIR" -EMSCRIPTEN_OPTIONS="-lwebsocket.js -sASSERTIONS=1 -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sEXPORTED_FUNCTIONS=$EXPORTED_FUNCS -sEXPORTED_RUNTIME_METHODS=$RUNTIME_METHODS" +EMSCRIPTEN_OPTIONS="-lwebsocket.js -sASSERTIONS=1 -sLLD_REPORT_UNDEFINED -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sEXPORTED_FUNCTIONS=$EXPORTED_FUNCS -sEXPORTED_RUNTIME_METHODS=$RUNTIME_METHODS" if [[ "$*" == *"release"* ]]; then COMPILER_OPTIONS="-Oz -flto $COMPILER_OPTIONS" @@ -41,7 +47,7 @@ rm -rf out mkdir -p out #compile the main c file - but only if the source has been modified -COMPILE_CMD="emcc main.c $COMPILER_OPTIONS $EMSCRIPTEN_OPTIONS" +COMPILE_CMD="emcc *.c $COMPILER_OPTIONS $EMSCRIPTEN_OPTIONS" echo $COMPILE_CMD $COMPILE_CMD mv $COMPILED_FILE $WASM_FILE || true @@ -51,10 +57,11 @@ cp $WRAPPER_SOURCE $OUT_FILE sed -i "/__emscripten_output__/r $MODULE_FILE" $OUT_FILE rm $MODULE_FILE -#add wisp libraries +#add extra libraries sed -i "/__extra_libraries__/r $WISP_CLIENT/polyfill.js" $OUT_FILE sed -i "/__extra_libraries__/r $WISP_CLIENT/wisp.js" $OUT_FILE sed -i "/__extra_libraries__/r ./messages.js" $OUT_FILE +sed -i "/__extra_libraries__/r ./websocket.js" $OUT_FILE #apply patches python3 scripts/patcher.py $FRAGMENTS_DIR $OUT_FILE diff --git a/client/exported_funcs.txt b/client/exported_funcs.txt new file mode 100644 index 0000000..a0f588a --- /dev/null +++ b/client/exported_funcs.txt @@ -0,0 +1,15 @@ +init_curl +start_request +tick_request +active_requests + +recv_from_websocket +send_to_websocket +close_websocket +cleanup_websocket +get_result_size +get_result_buffer +get_result_code +get_result_closed + +free \ No newline at end of file diff --git a/client/index.html b/client/index.html index f77030f..a52feab 100644 --- a/client/index.html +++ b/client/index.html @@ -3,7 +3,7 @@ - + + +``` + +Alternatively, prebuilt versions can be found on NPM and jsDelivr. You can use the following URLs to load libcurl.js from a third party CDN. +``` +https://cdn.jsdelivr.net/npm/libcurl.js@latest/libcurl.js +https://cdn.jsdelivr.net/npm/libcurl.js@latest/libcurl.wasm ``` To know when libcurl.js has finished loading, you can use the `libcurl_load` DOM event. ```js document.addEventListener("libcurl_load", ()=>{ + libcurl.set_websocket(`wss://${location.hostname}/ws/`); console.log("libcurl.js ready!"); }); ``` @@ -63,14 +70,12 @@ Most of the standard Fetch API's features are supported, with the exception of: To use WebSockets, create a `libcurl.WebSocket` object, which works identically to the regular [WebSocket](https://developer.mozilla.org/en-US/docs/Web/API/WebSocket) object. ```js let ws = new libcurl.WebSocket("wss://echo.websocket.org"); -ws.binaryType = "arraybuffer"; ws.addEventListener("open", () => { console.log("ws connected!"); ws.send("hello".repeat(128)); }); ws.addEventListener("message", (event) => { - let text = new TextDecoder().decode(event.data); - console.log(text); + console.log(event.data); }); ``` @@ -79,6 +84,7 @@ You can change the URL of the websocket proxy by using `libcurl.set_websocket`. ```js libcurl.set_websocket("ws://localhost:6001/"); ``` +If the websocket proxy URL is not set and one of the other API functions is called, an error will be thrown. ## Proxy Server: The proxy server consists of a standard [Wisp](https://github.com/MercuryWorkshop/wisp-protocol) server, allowing multiple TCP connections to share the same websocket. diff --git a/client/build.sh b/client/build.sh index c9e1a53..76bf163 100755 --- a/client/build.sh +++ b/client/build.sh @@ -82,7 +82,7 @@ sed -i "/__extra_libraries__/r $JAVSCRIPT_DIR/messages.js" $OUT_FILE sed -i "/__extra_libraries__/r $JAVSCRIPT_DIR/websocket.js" $OUT_FILE #apply patches -python3 scripts/patcher.py $FRAGMENTS_DIR $OUT_FILE +python3 tools/patch_js.py $FRAGMENTS_DIR $OUT_FILE #generate es6 module cp $OUT_FILE $ES6_FILE diff --git a/client/index.html b/client/index.html index a52feab..37c5d40 100644 --- a/client/index.html +++ b/client/index.html @@ -6,6 +6,7 @@ diff --git a/client/javascript/main.js b/client/javascript/main.js index 851fbdf..7821043 100644 --- a/client/javascript/main.js +++ b/client/javascript/main.js @@ -1,17 +1,35 @@ -//everything is wrapped in a function to prevent emscripten from polluting the global scope -window.libcurl = (function() { +/* +ading2210/libcurl.js - A port of libcurl to WASM for the browser. +Copyright (C) 2023 ading2210 + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ if (typeof window === "undefined") { throw new Error("NodeJS is not supported. This only works inside the browser."); } +//everything is wrapped in a function to prevent emscripten from polluting the global scope +window.libcurl = (function() { + //emscripten compiled code is inserted here /* __emscripten_output__ */ //extra client code goes here /* __extra_libraries__ */ -var websocket_url = `wss://${location.hostname}/ws/`; +var websocket_url = null; var event_loop = null; var active_requests = 0; var wasm_ready = false; @@ -20,6 +38,9 @@ function check_loaded() { if (!wasm_ready) { throw new Error("wasm not loaded yet, please call libcurl.load_wasm first"); } + if (!websocket_url) { + throw new Error("websocket proxy url not set, please call libcurl.set_websocket"); + } } //a case insensitive dictionary for request headers @@ -50,10 +71,6 @@ class HeadersDict { } } -function is_str(obj) { - return typeof obj === 'string' || obj instanceof String; -} - function allocate_str(str) { return allocate(intArrayFromString(str), ALLOC_NORMAL); } @@ -247,7 +264,7 @@ async function libcurl_fetch(url, params={}) { } function set_websocket_url(url) { - check_loaded(); + websocket_url = url; if (!Module.websocket) { document.addEventListener("libcurl_load", () => { set_websocket_url(url); @@ -257,7 +274,6 @@ function set_websocket_url(url) { } function main() { - console.log("emscripten module loaded"); wasm_ready = true; _init_curl(); set_websocket_url(websocket_url); diff --git a/client/javascript/websocket.js b/client/javascript/websocket.js index 26f9094..d60fb22 100644 --- a/client/javascript/websocket.js +++ b/client/javascript/websocket.js @@ -3,6 +3,7 @@ class CurlWebSocket extends EventTarget { constructor(url, protocols=[]) { super(); + check_loaded(); if (!url.startsWith("wss://") && !url.startsWith("ws://")) { throw new SyntaxError("invalid url"); } diff --git a/client/npm/package.json b/client/package.json similarity index 91% rename from client/npm/package.json rename to client/package.json index 16d300c..a4f4d97 100644 --- a/client/npm/package.json +++ b/client/package.json @@ -1,8 +1,8 @@ { "name": "libcurl.js", - "version": "0.1.0", + "version": "0.1.1", "description": "An experimental port of libcurl to WebAssembly for use in the browser.", - "main": "index.js", + "main": "libcurl.mjs", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" }, diff --git a/client/publish.sh b/client/publish.sh index 1287029..58ae332 100755 --- a/client/publish.sh +++ b/client/publish.sh @@ -1,9 +1,10 @@ #!/bin/bash #publish libcurl.js as an npm package -#run build.sh first -cp npm/* out +./build.sh all + +cp package.json out cp ../README.md out cd out npm publish \ No newline at end of file diff --git a/client/scripts/patcher.py b/client/tools/patch_js.py similarity index 100% rename from client/scripts/patcher.py rename to client/tools/patch_js.py From 2d98b82ee3b126f22c717ac1ef736086e0ed0be9 Mon Sep 17 00:00:00 2001 From: ading2210 Date: Wed, 31 Jan 2024 21:38:57 +0000 Subject: [PATCH 15/27] add unit tests --- .github/workflows/build.yaml | 6 +++- client/tests/index.html | 43 +++++++++++++++++++++++++ client/tests/run.sh | 11 +++++++ client/tests/run_tests.py | 44 ++++++++++++++++++++++++++ client/tests/scripts/fetch_multiple.js | 7 ++++ client/tests/scripts/fetch_once.js | 4 +++ client/tests/scripts/fetch_parallel.js | 8 +++++ client/tests/scripts/test_websocket.js | 24 ++++++++++++++ server/wisp_server | 2 +- 9 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 client/tests/index.html create mode 100755 client/tests/run.sh create mode 100644 client/tests/run_tests.py create mode 100644 client/tests/scripts/fetch_multiple.js create mode 100644 client/tests/scripts/fetch_once.js create mode 100644 client/tests/scripts/fetch_parallel.js create mode 100644 client/tests/scripts/test_websocket.js diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d8c0ce7..9882e2c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -12,11 +12,15 @@ jobs: - name: install deps run: | sudo apt-get update - sudo apt-get install -y make cmake emscripten autoconf automake libtool pkg-config wget xxd + sudo apt-get install -y make cmake emscripten autoconf automake libtool pkg-config wget xxd python3-selenium python3-websockets - name: run build working-directory: ./client run: ./build.sh all + + - name: run tests + working-directory: ./client + run: ./tests/run.sh - name: upload img uses: actions/upload-artifact@v4 diff --git a/client/tests/index.html b/client/tests/index.html new file mode 100644 index 0000000..db5bee9 --- /dev/null +++ b/client/tests/index.html @@ -0,0 +1,43 @@ + + + + + + + + + +

libcurl.js unit test runner

+ + \ No newline at end of file diff --git a/client/tests/run.sh b/client/tests/run.sh new file mode 100755 index 0000000..a2d19a2 --- /dev/null +++ b/client/tests/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +trap "exit" INT TERM +trap "kill 0" EXIT +STATIC="$(pwd)" python3 ../server/wisp_server/main.py >/dev/null & + +sleep 1 +echo "Running tests" +python3 tests/run_tests.py diff --git a/client/tests/run_tests.py b/client/tests/run_tests.py new file mode 100644 index 0000000..8f709d5 --- /dev/null +++ b/client/tests/run_tests.py @@ -0,0 +1,44 @@ +import unittest + +from selenium import webdriver +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC + +class JSTest(unittest.TestCase): + def setUp(self): + options = webdriver.ChromeOptions() + options.add_argument("--headless") + options.set_capability("goog:loggingPrefs", {"browser": "ALL"}) + + self.browser = webdriver.Chrome(options=options) + + def tearDown(self): + self.browser.quit() + + def run_test(self, script): + self.browser.get(f"http://localhost:6001/tests/#{script}") + wait = WebDriverWait(self.browser, 20) + result = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.flag'))).get_attribute("result") + + if result != "success": + for entry in self.browser.get_log("browser"): + print(f"{entry['level']}: {entry['message']}") + + assert result == "success" + + def test_fetch_once(self): + self.run_test("fetch_once.js") + + def test_fetch_multiple(self): + self.run_test("fetch_multiple.js") + + def test_fetch_parallel(self): + self.run_test("fetch_parallel.js") + + def test_websocket(self): + self.run_test("test_websocket.js") + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/client/tests/scripts/fetch_multiple.js b/client/tests/scripts/fetch_multiple.js new file mode 100644 index 0000000..fe7cb86 --- /dev/null +++ b/client/tests/scripts/fetch_multiple.js @@ -0,0 +1,7 @@ +async function test() { + await libcurl.fetch("https://example.com/"); + for (let i=0; i<20; i++) { + let r = await libcurl.fetch("https://example.com/"); + assert(r.status === 200, "wrong status"); + } +} \ No newline at end of file diff --git a/client/tests/scripts/fetch_once.js b/client/tests/scripts/fetch_once.js new file mode 100644 index 0000000..e94fefa --- /dev/null +++ b/client/tests/scripts/fetch_once.js @@ -0,0 +1,4 @@ +async function test() { + let r = await libcurl.fetch("https://example.com/"); + assert(r.status === 200, "wrong status"); +} \ No newline at end of file diff --git a/client/tests/scripts/fetch_parallel.js b/client/tests/scripts/fetch_parallel.js new file mode 100644 index 0000000..98a083a --- /dev/null +++ b/client/tests/scripts/fetch_parallel.js @@ -0,0 +1,8 @@ +async function test() { + await libcurl.fetch("https://www.example.com/"); + let promises = []; + for (let i=0; i<10; i++) { + promises.push(libcurl.fetch("https://www.example.com/")) + } + await Promise.all(promises); +} diff --git a/client/tests/scripts/test_websocket.js b/client/tests/scripts/test_websocket.js new file mode 100644 index 0000000..4a10cf3 --- /dev/null +++ b/client/tests/scripts/test_websocket.js @@ -0,0 +1,24 @@ +function test() { + let message_len = 128*1024; + + return new Promise((resolve, reject) => { + let ws = new libcurl.WebSocket("wss://echo.websocket.org"); + ws.addEventListener("open", () => { + ws.send("hello".repeat(message_len)); + }); + + let messages = 0; + ws.addEventListener("message", (event) => { + messages += 1; + if (messages >= 2) { + if (event.data !== "hello".repeat(message_len)) reject("unexpected response"); + if (messages >= 11) resolve(); + ws.send("hello".repeat(message_len)); + } + }); + + ws.addEventListener("error", () => { + reject("ws error occurred"); + }); + }) +} \ No newline at end of file diff --git a/server/wisp_server b/server/wisp_server index 138ef0f..bd7bca9 160000 --- a/server/wisp_server +++ b/server/wisp_server @@ -1 +1 @@ -Subproject commit 138ef0f73027b3d237ec84bee7ea43b4f30c8b74 +Subproject commit bd7bca97b22023a1b49d0f5a09081ddcbb4ad49c From 0b2db30c1a640c7bc6a1345c17faaf8f7654a8db Mon Sep 17 00:00:00 2001 From: ading2210 Date: Wed, 31 Jan 2024 21:50:46 +0000 Subject: [PATCH 16/27] clone submodules in github actions --- .github/workflows/build.yaml | 2 ++ client/tests/run.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9882e2c..013f502 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -8,6 +8,8 @@ jobs: steps: - name: download repo uses: actions/checkout@v4 + with: + submodules: 'true' - name: install deps run: | diff --git a/client/tests/run.sh b/client/tests/run.sh index a2d19a2..dd8577b 100755 --- a/client/tests/run.sh +++ b/client/tests/run.sh @@ -4,7 +4,7 @@ set -e trap "exit" INT TERM trap "kill 0" EXIT -STATIC="$(pwd)" python3 ../server/wisp_server/main.py >/dev/null & +STATIC="$(pwd)" ../server/run.sh >/dev/null & sleep 1 echo "Running tests" From c3b58f66d06cb970306d780f8298ac8d4439c6c6 Mon Sep 17 00:00:00 2001 From: ading2210 Date: Wed, 31 Jan 2024 23:38:57 +0000 Subject: [PATCH 17/27] add more chromedriver arguments --- client/tests/run_tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/tests/run_tests.py b/client/tests/run_tests.py index 8f709d5..7a000d6 100644 --- a/client/tests/run_tests.py +++ b/client/tests/run_tests.py @@ -9,7 +9,13 @@ from selenium.webdriver.support import expected_conditions as EC class JSTest(unittest.TestCase): def setUp(self): options = webdriver.ChromeOptions() + options.add_argument("start-maximized") + options.add_argument("enable-automation") options.add_argument("--headless") + options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") + options.addArguments("--disable-browser-side-navigation") + options.addArguments("--disable-gpu") options.set_capability("goog:loggingPrefs", {"browser": "ALL"}) self.browser = webdriver.Chrome(options=options) From bb9d9239c02b3e9b23ab7a278fbf630bf0d13d0e Mon Sep 17 00:00:00 2001 From: ading2210 Date: Thu, 1 Feb 2024 17:52:22 +0000 Subject: [PATCH 18/27] fix single file support --- .github/workflows/build.yaml | 4 ---- client/fragments/load_later.js | 9 ++++++--- client/index.html | 2 +- client/package.json | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 013f502..51d7a27 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -19,10 +19,6 @@ jobs: - name: run build working-directory: ./client run: ./build.sh all - - - name: run tests - working-directory: ./client - run: ./tests/run.sh - name: upload img uses: actions/upload-artifact@v4 diff --git a/client/fragments/load_later.js b/client/fragments/load_later.js index 79087a1..2fb25b4 100644 --- a/client/fragments/load_later.js +++ b/client/fragments/load_later.js @@ -1,13 +1,16 @@ /* REPLACE var asm ?= ?createWasm\(\); */ -var asm = null; +if (isDataURI(wasmBinaryFile)) var asm = createWasm(); +else var asm = null; /* REPLACE var wasmExports ?= ?createWasm\(\); */ -var wasmExports = null; +if (isDataURI(wasmBinaryFile)) var wasmExports = createWasm(); +else var wasmExports = null; -/* DELETE +/* REPLACE run\(\);\n\n */ +if (isDataURI(wasmBinaryFile)) run(); \ No newline at end of file diff --git a/client/index.html b/client/index.html index 37c5d40..cc83238 100644 --- a/client/index.html +++ b/client/index.html @@ -3,7 +3,7 @@ - + + diff --git a/client/javascript/copyright.js b/client/javascript/copyright.js new file mode 100644 index 0000000..9ad6c3e --- /dev/null +++ b/client/javascript/copyright.js @@ -0,0 +1,15 @@ +const copyright_notice = `ading2210/libcurl.js - A port of libcurl to WASM for use in the browser. +Copyright (C) 2023 ading2210 + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see .`; \ No newline at end of file diff --git a/client/javascript/main.js b/client/javascript/main.js index 15c3f43..9ecc78c 100644 --- a/client/javascript/main.js +++ b/client/javascript/main.js @@ -310,12 +310,14 @@ return { wisp: _wisp_connections, WebSocket: CurlWebSocket, + get copyright() {return copyright_notice}, get version() {return get_version()}, + get ready() {return wasm_ready}, + get stdout() {return out}, set stdout(callback) {out = callback}, get stderr() {return err}, - set stderr(callback) {err = callback}, - get ready() {return wasm_ready} + set stderr(callback) {err = callback} } })() \ No newline at end of file diff --git a/client/package.json b/client/package.json index 1535226..5edc336 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "libcurl.js", - "version": "0.3.0", + "version": "0.3.1", "description": "An experimental port of libcurl to WebAssembly for use in the browser.", "main": "libcurl.mjs", "scripts": { diff --git a/client/publish.sh b/client/publish.sh index 8f601bc..2a055e0 100755 --- a/client/publish.sh +++ b/client/publish.sh @@ -3,7 +3,6 @@ #publish libcurl.js as an npm package ./build.sh all -tests/run.sh cp package.json out cp ../README.md out From 5f71e274222eb78410bc2cf611664be8f2096705 Mon Sep 17 00:00:00 2001 From: ading2210 Date: Sun, 4 Feb 2024 03:43:19 -0500 Subject: [PATCH 22/27] various bugfixes --- client/javascript/main.js | 3 +++ client/javascript/websocket.js | 24 +++++++++++++++++------- client/package.json | 2 +- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/client/javascript/main.js b/client/javascript/main.js index 9ecc78c..ed4453c 100644 --- a/client/javascript/main.js +++ b/client/javascript/main.js @@ -153,6 +153,9 @@ function merge_arrays(arrays) { function create_response(response_data, response_info) { response_info.ok = response_info.status >= 200 && response_info.status < 300; response_info.statusText = status_messages[response_info.status] || ""; + if (response_info.status === 204 || response_info.status === 205) { + response_data = null; + } //construct base response object let response_obj = new Response(response_data, response_info); diff --git a/client/javascript/websocket.js b/client/javascript/websocket.js index 76ee79e..9f7b272 100644 --- a/client/javascript/websocket.js +++ b/client/javascript/websocket.js @@ -1,7 +1,7 @@ //class for custom websocket class CurlWebSocket extends EventTarget { - constructor(url, protocols=[]) { + constructor(url, protocols=[], websocket_debug=false) { super(); check_loaded(true); if (!url.startsWith("wss://") && !url.startsWith("ws://")) { @@ -12,6 +12,7 @@ class CurlWebSocket extends EventTarget { this.protocols = protocols; this.binaryType = "blob"; this.recv_buffer = []; + this.websocket_debug = websocket_debug; //legacy event handlers this.onopen = () => {}; @@ -33,7 +34,16 @@ class CurlWebSocket extends EventTarget { let finish_callback = (error, response_info) => { this.finish_callback(error, response_info); } - this.http_handle = perform_request(this.url, {}, data_callback, finish_callback, null); + let options = {}; + if (this.protocols) { + options.headers = { + "Sec-Websocket-Protocol": this.protocols.join(", "), + }; + } + if (this.websocket_debug) { + options._libcurl_verbose = 1; + } + this.http_handle = perform_request(this.url, options, data_callback, finish_callback, null); this.recv_loop(); } @@ -134,7 +144,7 @@ class CurlWebSocket extends EventTarget { if (this.status === this.CLOSED) { return; } - + let data_array; if (typeof data === "string") { data_array = new TextEncoder().encode(data); @@ -153,15 +163,15 @@ class CurlWebSocket extends EventTarget { if (ArrayBuffer.isView(data) && data instanceof DataView) { data_array = new Uint8Array(data.buffer); } - //regular typed arrays - else if (ArrayBuffer.isView(data)) { - data_array = Uint8Array.from(data); - } //regular arraybuffers else { data_array = new Uint8Array(data); } } + //regular typed arrays + else if (ArrayBuffer.isView(data)) { + data_array = Uint8Array.from(data); + } else { throw "invalid data type to be sent"; } diff --git a/client/package.json b/client/package.json index 5edc336..9f7a878 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "libcurl.js", - "version": "0.3.1", + "version": "0.3.2", "description": "An experimental port of libcurl to WebAssembly for use in the browser.", "main": "libcurl.mjs", "scripts": { From 582bbd4ecd3fd48a1d5d67ffd87522d4e821526e Mon Sep 17 00:00:00 2001 From: ading2210 Date: Sun, 4 Feb 2024 23:10:14 -0500 Subject: [PATCH 23/27] fix websocket memory leak --- client/javascript/main.js | 1 + client/javascript/websocket.js | 6 +++--- client/libcurl/util.c | 1 + client/libcurl/websocket.c | 2 +- client/package.json | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/client/javascript/main.js b/client/javascript/main.js index ed4453c..f42596b 100644 --- a/client/javascript/main.js +++ b/client/javascript/main.js @@ -255,6 +255,7 @@ function perform_request_async(url, params, body) { return; } let response_data = merge_arrays(chunks); + chunks = null; let response_obj = create_response(response_data, response_info); resolve(response_obj); } diff --git a/client/javascript/websocket.js b/client/javascript/websocket.js index 9f7b272..23af368 100644 --- a/client/javascript/websocket.js +++ b/client/javascript/websocket.js @@ -50,6 +50,7 @@ class CurlWebSocket extends EventTarget { recv() { let buffer_size = 64*1024; let result_ptr = _recv_from_websocket(this.http_handle, buffer_size); + let data_ptr = _get_result_buffer(result_ptr); let result_code = _get_result_code(result_ptr); if (result_code == 0) { //CURLE_OK - data recieved @@ -60,10 +61,8 @@ class CurlWebSocket extends EventTarget { } let data_size = _get_result_size(result_ptr); - let data_ptr = _get_result_buffer(result_ptr); let data_heap = Module.HEAPU8.subarray(data_ptr, data_ptr + data_size); let data = new Uint8Array(data_heap); - _free(data_ptr); this.recv_buffer.push(data); if (data_size !== buffer_size && !_get_result_bytes_left(result_ptr)) { //message finished @@ -77,7 +76,8 @@ class CurlWebSocket extends EventTarget { if (result_code == 52) { //CURLE_GOT_NOTHING - socket closed this.close_callback(); } - + + _free(data_ptr); _free(result_ptr); } diff --git a/client/libcurl/util.c b/client/libcurl/util.c index c991858..170d3ce 100644 --- a/client/libcurl/util.c +++ b/client/libcurl/util.c @@ -32,5 +32,6 @@ char* get_version() { cJSON_AddItemToObject(version_json, "protocols", protocols_array); char* version_json_str = cJSON_Print(version_json); + cJSON_Delete(version_json); return version_json_str; } \ No newline at end of file diff --git a/client/libcurl/websocket.c b/client/libcurl/websocket.c index f8f58c8..4c7e5fd 100644 --- a/client/libcurl/websocket.c +++ b/client/libcurl/websocket.c @@ -36,7 +36,7 @@ void close_websocket(CURL* http_handle) { curl_ws_send(http_handle, "", 0, &sent, 0, CURLWS_CLOSE); } -//allow the main code to automatically clean up this websocket +//clean up the http handle associated with the websocket, since the main loop can't do this automatically void cleanup_websocket(CURL* http_handle) { struct RequestInfo *request_info; curl_easy_getinfo(http_handle, CURLINFO_PRIVATE, &request_info); diff --git a/client/package.json b/client/package.json index 9f7a878..47caa58 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "libcurl.js", - "version": "0.3.2", + "version": "0.3.3", "description": "An experimental port of libcurl to WebAssembly for use in the browser.", "main": "libcurl.mjs", "scripts": { From d79c07e2a49567d8293e860507a31c1e36d137f0 Mon Sep 17 00:00:00 2001 From: ading2210 Date: Mon, 26 Feb 2024 22:52:06 -0500 Subject: [PATCH 24/27] limit max connections to 50 --- client/build.sh | 2 +- client/javascript/main.js | 6 +----- client/libcurl/main.c | 2 ++ client/package.json | 2 +- server/wisp_server | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/client/build.sh b/client/build.sh index be013ab..1ef82b5 100755 --- a/client/build.sh +++ b/client/build.sh @@ -28,7 +28,7 @@ EXPORTED_FUNCS="${EXPORTED_FUNCS:1}" #compile options RUNTIME_METHODS="addFunction,removeFunction,allocate,ALLOC_NORMAL" COMPILER_OPTIONS="-o $MODULE_FILE -lcurl -lssl -lcrypto -lcjson -lz -lbrotlidec -lbrotlicommon -lnghttp2 -I $INCLUDE_DIR -L $LIB_DIR" -EMSCRIPTEN_OPTIONS="-lwebsocket.js -sASSERTIONS=1 -sLLD_REPORT_UNDEFINED -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sEXPORTED_FUNCTIONS=$EXPORTED_FUNCS -sEXPORTED_RUNTIME_METHODS=$RUNTIME_METHODS" +EMSCRIPTEN_OPTIONS="-lwebsocket.js -sENVIRONMENT=worker,web -sASSERTIONS=1 -sLLD_REPORT_UNDEFINED -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sEXPORTED_FUNCTIONS=$EXPORTED_FUNCS -sEXPORTED_RUNTIME_METHODS=$RUNTIME_METHODS" #clean output dir rm -rf $OUT_DIR diff --git a/client/javascript/main.js b/client/javascript/main.js index f42596b..fc8b6f8 100644 --- a/client/javascript/main.js +++ b/client/javascript/main.js @@ -16,12 +16,8 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -if (typeof window === "undefined") { - throw new Error("NodeJS is not supported. This only works inside the browser."); -} - //everything is wrapped in a function to prevent emscripten from polluting the global scope -window.libcurl = (function() { +const libcurl = (function() { //emscripten compiled code is inserted here /* __emscripten_output__ */ diff --git a/client/libcurl/main.c b/client/libcurl/main.c index f085d7a..249cd24 100644 --- a/client/libcurl/main.c +++ b/client/libcurl/main.c @@ -192,6 +192,8 @@ void finish_request(CURLMsg *curl_msg) { void init_curl() { curl_global_init(CURL_GLOBAL_DEFAULT); multi_handle = curl_multi_init(); + curl_multi_setopt(multi_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, 50L); + curl_multi_setopt(multi_handle, CURLMOPT_MAXCONNECTS, 40L); FILE* file = fopen("/cacert.pem", "wb"); fwrite(_cacert_pem, 1, _cacert_pem_len, file); diff --git a/client/package.json b/client/package.json index 47caa58..d14d66f 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "libcurl.js", - "version": "0.3.3", + "version": "0.3.4", "description": "An experimental port of libcurl to WebAssembly for use in the browser.", "main": "libcurl.mjs", "scripts": { diff --git a/server/wisp_server b/server/wisp_server index 65d3124..3b0d432 160000 --- a/server/wisp_server +++ b/server/wisp_server @@ -1 +1 @@ -Subproject commit 65d312414aa896d8f9e18f48e8fe37b72d75ee5c +Subproject commit 3b0d432e89cff7eaa850ba8605b180189e237f1b From b0314bf8cb1eaa9d4ada4fa3b17f35950c3c7b60 Mon Sep 17 00:00:00 2001 From: ading2210 Date: Mon, 26 Feb 2024 23:03:23 -0500 Subject: [PATCH 25/27] fix es6 module not getting built properly --- client/build.sh | 2 +- client/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/build.sh b/client/build.sh index 1ef82b5..1dcd95c 100755 --- a/client/build.sh +++ b/client/build.sh @@ -91,4 +91,4 @@ python3 tools/patch_js.py $FRAGMENTS_DIR $OUT_FILE #generate es6 module cp $OUT_FILE $ES6_FILE -sed -i 's/window.libcurl/export const libcurl/' $ES6_FILE \ No newline at end of file +sed -i 's/const libcurl/export const libcurl/' $ES6_FILE \ No newline at end of file diff --git a/client/package.json b/client/package.json index d14d66f..73f8b3b 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "libcurl.js", - "version": "0.3.4", + "version": "0.3.5", "description": "An experimental port of libcurl to WebAssembly for use in the browser.", "main": "libcurl.mjs", "scripts": { From cc4009a4afef56e3f9005b75386d24ff1e582b68 Mon Sep 17 00:00:00 2001 From: ading2210 Date: Mon, 26 Feb 2024 23:13:55 -0500 Subject: [PATCH 26/27] fix sed expression again --- client/build.sh | 2 +- client/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/build.sh b/client/build.sh index 1dcd95c..3effa33 100755 --- a/client/build.sh +++ b/client/build.sh @@ -91,4 +91,4 @@ python3 tools/patch_js.py $FRAGMENTS_DIR $OUT_FILE #generate es6 module cp $OUT_FILE $ES6_FILE -sed -i 's/const libcurl/export const libcurl/' $ES6_FILE \ No newline at end of file +sed -i 's/const libcurl = /export const libcurl = /' $ES6_FILE \ No newline at end of file diff --git a/client/package.json b/client/package.json index 73f8b3b..80bfb0c 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "libcurl.js", - "version": "0.3.5", + "version": "0.3.6", "description": "An experimental port of libcurl to WebAssembly for use in the browser.", "main": "libcurl.mjs", "scripts": { From b5aff5b0857f2fa1c3501d5649dd01f4f0f5c82e Mon Sep 17 00:00:00 2001 From: ading2210 Date: Tue, 27 Feb 2024 11:38:00 -0800 Subject: [PATCH 27/27] pin library versions, load cacert from blob --- client/index.html | 2 +- client/libcurl/main.c | 14 +++++++------- client/package.json | 2 +- client/tools/brotli.sh | 2 +- client/tools/cjson.sh | 2 +- client/tools/curl.sh | 2 +- client/tools/nghttp2.sh | 2 +- client/tools/openssl.sh | 2 +- client/tools/zlib.sh | 2 +- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/client/index.html b/client/index.html index 9760e4c..8555276 100644 --- a/client/index.html +++ b/client/index.html @@ -6,7 +6,7 @@ diff --git a/client/libcurl/main.c b/client/libcurl/main.c index 249cd24..4f5578a 100644 --- a/client/libcurl/main.c +++ b/client/libcurl/main.c @@ -19,9 +19,10 @@ void finish_request(CURLMsg *curl_msg); CURLM *multi_handle; int request_active = 0; +struct curl_blob cacert_blob; -int write_function(void *data, size_t size, size_t nmemb, DataCallback data_callback) { - long real_size = size * nmemb; +size_t write_function(void *data, size_t size, size_t nmemb, DataCallback data_callback) { + size_t real_size = size * nmemb; char* chunk = malloc(real_size); memcpy(chunk, data, real_size); data_callback(chunk, real_size); @@ -53,8 +54,7 @@ CURL* start_request(const char* url, const char* json_params, DataCallback data_ int prevent_cleanup = 0; curl_easy_setopt(http_handle, CURLOPT_URL, url); - curl_easy_setopt(http_handle, CURLOPT_CAINFO, "/cacert.pem"); - curl_easy_setopt(http_handle, CURLOPT_CAPATH, "/cacert.pem"); + curl_easy_setopt(http_handle, CURLOPT_CAINFO_BLOB , cacert_blob); //callbacks to pass the response data back to js curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, &write_function); @@ -195,7 +195,7 @@ void init_curl() { curl_multi_setopt(multi_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, 50L); curl_multi_setopt(multi_handle, CURLMOPT_MAXCONNECTS, 40L); - FILE* file = fopen("/cacert.pem", "wb"); - fwrite(_cacert_pem, 1, _cacert_pem_len, file); - fclose(file); + cacert_blob.data = _cacert_pem; + cacert_blob.len = _cacert_pem_len; + cacert_blob.flags = CURL_BLOB_NOCOPY; } \ No newline at end of file diff --git a/client/package.json b/client/package.json index 80bfb0c..d85ef44 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "libcurl.js", - "version": "0.3.6", + "version": "0.3.7", "description": "An experimental port of libcurl to WebAssembly for use in the browser.", "main": "libcurl.mjs", "scripts": { diff --git a/client/tools/brotli.sh b/client/tools/brotli.sh index 1210f4b..c9cba4a 100755 --- a/client/tools/brotli.sh +++ b/client/tools/brotli.sh @@ -10,7 +10,7 @@ PREFIX=$(realpath build/brotli-wasm) cd build rm -rf brotli -git clone -b master --depth=1 https://github.com/google/brotli +git clone -b v1.1.0 --depth=1 https://github.com/google/brotli cd brotli emcmake cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=./installed diff --git a/client/tools/cjson.sh b/client/tools/cjson.sh index acf7290..5cdfc1e 100755 --- a/client/tools/cjson.sh +++ b/client/tools/cjson.sh @@ -11,7 +11,7 @@ mkdir -p $PREFIX cd build rm -rf cjson -git clone -b master --depth=1 https://github.com/DaveGamble/cJSON cjson +git clone -b v1.7.17 --depth=1 https://github.com/DaveGamble/cJSON cjson cd cjson sed -i 's/-fstack-protector-strong//' Makefile diff --git a/client/tools/curl.sh b/client/tools/curl.sh index 7f4afa0..cf286a5 100755 --- a/client/tools/curl.sh +++ b/client/tools/curl.sh @@ -14,7 +14,7 @@ NGHTTP2_PREFIX=$(realpath build/nghttp2-wasm) cd build rm -rf curl -git clone -b master --depth=1 https://github.com/curl/curl +git clone -b curl-8_6_0 --depth=1 https://github.com/curl/curl cd curl autoreconf -fi diff --git a/client/tools/nghttp2.sh b/client/tools/nghttp2.sh index e9c86eb..58a5aba 100755 --- a/client/tools/nghttp2.sh +++ b/client/tools/nghttp2.sh @@ -10,7 +10,7 @@ PREFIX=$(realpath build/nghttp2-wasm) cd build rm -rf nghttp2 -git clone -b master --depth=1 https://github.com/nghttp2/nghttp2 +git clone -b v1.59.0 --depth=1 https://github.com/nghttp2/nghttp2 cd nghttp2 rm -rf $PREFIX diff --git a/client/tools/openssl.sh b/client/tools/openssl.sh index dc8a041..014f5c3 100755 --- a/client/tools/openssl.sh +++ b/client/tools/openssl.sh @@ -11,7 +11,7 @@ mkdir -p $PREFIX cd build rm -rf openssl -git clone -b master --depth=1 https://github.com/openssl/openssl +git clone -b openssl-3.2.1 --depth=1 https://github.com/openssl/openssl cd openssl export CFLAGS="-Wall -Oz" diff --git a/client/tools/zlib.sh b/client/tools/zlib.sh index 624246e..10c7bd1 100755 --- a/client/tools/zlib.sh +++ b/client/tools/zlib.sh @@ -10,7 +10,7 @@ PREFIX=$(realpath build/zlib-wasm) cd build rm -rf zlib -git clone -b master --depth=1 https://github.com/madler/zlib +git clone -b v1.3.1 --depth=1 https://github.com/madler/zlib cd zlib emconfigure ./configure --static