mercuryworkshop-mirror/libcurl.js
1
0
Fork 0
mirror of https://github.com/ading2210/libcurl.js.git synced 2025-05-13 14:30:02 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into wolfssl-testing

Browse source
This commit is contained in:
ading2210 2024-01-17 01:09:34 -05:00
commit 01622283f5
17 changed files with 221 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,4 +1,5 @@
/client/build
/client/out
/client/fragments/tmp
/server/.venv
/server/websockify

6
.gitmodules vendored Normal file
View file

@ -0,0 +1,6 @@
[submodule "client/wisp_client"]
path = client/wisp_client
url = https://github.com/MercuryWorkshop/wisp-client-js
[submodule "server/wisp_server"]
path = server/wisp_server
url = https://github.com/MercuryWorkshop/wisp-server-python

8
README.md
View file

@ -10,7 +10,7 @@ This is an experimental port of [libcurl](https://curl.se/libcurl/) to WebAssemb
## Building:
You can build this project by running the following commands:
```
git clone https://github.com/ading2210/libcurl.js
git clone https://github.com/ading2210/libcurl.js --recursive
cd libcurl.js/client
./build.sh
```
@ -48,16 +48,16 @@ libcurl.set_websocket("ws://localhost:6001/");
```
## Proxy Server:
The proxy server consists of a [SOCKS5 proxy server](https://github.com/Amaindex/asyncio-socks-server) behind a [websocket TCP proxy](https://github.com/novnc/websockify).
The proxy server consists of a standard [Wisp](https://github.com/MercuryWorkshop/wisp-protocol) server, allowing multiple TCP connections to share the same websocket.
To host the proxy server, run the following commands:
```
git clone https://github.com/ading2210/libcurl.js
git clone https://github.com/ading2210/libcurl.js --recursive
cd libcurl.js/server
./run.sh
```
You can use the `PORT` and `SOCKS5_PORT` environment variables to control which ports the websocket proxy and the SOCKS5 server run on.
You can use the `HOST` and `PORT` environment variables to control the hostname and port that the proxy server listens on.
## Copyright:
This project is licensed under the GNU AGPL v3.

23
client/build.sh
View file

@ -7,15 +7,20 @@ LIB_DIR="build/curl-wasm/lib/"
OUT_FILE="out/libcurl.js"
ES6_FILE="out/libcurl_module.mjs"
MODULE_FILE="out/emscripten_compiled.js"
FRAGMENTS_DIR="fragments"
WRAPPER_SOURCE="main.js"
WISP_CLIENT="wisp_client"
EXPORTED_FUNCS="_load_certs,_perform_request"
EXPORTED_FUNCS="_init_curl,_start_request,_tick_request,_active_requests"
RUNTIME_METHODS="addFunction,removeFunction,allocate,ALLOC_NORMAL"
COMPILER_OPTIONS="-o $MODULE_FILE -lcurl -lwolfssl -lcjson -lz -lbrotlidec -lbrotlicommon -I $INCLUDE_DIR -L $LIB_DIR"
EMSCRIPTEN_OPTIONS="-lwebsocket.js -sSINGLE_FILE -sASYNCIFY -sALLOW_TABLE_GROWTH -sEXPORTED_FUNCTIONS=$EXPORTED_FUNCS -sEXPORTED_RUNTIME_METHODS=$RUNTIME_METHODS"
EMSCRIPTEN_OPTIONS="-lwebsocket.js -sASSERTIONS=1 -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sEXPORTED_FUNCTIONS=$EXPORTED_FUNCS -sEXPORTED_RUNTIME_METHODS=$RUNTIME_METHODS"
if [ "$1" = "release" ]; then
COMPILER_OPTIONS="-O3 $COMPILER_OPTIONS"
COMPILER_OPTIONS="-Oz -flto $COMPILER_OPTIONS"
EMSCRIPTEN_OPTIONS="-sSINGLE_FILE $EMSCRIPTEN_OPTIONS"
else
COMPILER_OPTIONS="$COMPILER_OPTIONS --profiling"
fi
#ensure deps are compiled
@ -31,15 +36,19 @@ COMPILE_CMD="emcc main.c $COMPILER_OPTIONS $EMSCRIPTEN_OPTIONS"
echo $COMPILE_CMD
$COMPILE_CMD
#patch the output to work around some emscripten bugs
sed -i 's/err("__syscall_getsockname " \?+ \?fd);//' $MODULE_FILE
sed -i 's/function _emscripten_console_error(str) {/& if(UTF8ToString(str).endsWith("__syscall_setsockopt\\n")) return;/' $MODULE_FILE
#merge compiled emscripten module and wrapper code
cp $WRAPPER_SOURCE $OUT_FILE
sed -i "/__emscripten_output__/r $MODULE_FILE" $OUT_FILE
rm $MODULE_FILE
#add wisp libraries
sed -i "/__extra_libraries__/r $WISP_CLIENT/polyfill.js" $OUT_FILE
sed -i "/__extra_libraries__/r $WISP_CLIENT/wisp.js" $OUT_FILE
sed -i "/__extra_libraries__/r ./messages.js" $OUT_FILE
#apply patches
python3 patcher.py $FRAGMENTS_DIR $OUT_FILE
#generate es6 module
cp $OUT_FILE $ES6_FILE
sed -i 's/window.libcurl/export const libcurl/' $ES6_FILE

5
client/fragments/force_wsproxy.js Normal file
View file

@ -0,0 +1,5 @@
/* INSERT
var ?opts ?= ?undefined;
*/
var parts=addr.split("/");
url = url + parts[0] + ":" + port;

9
client/fragments/silence_socket.js Normal file
View file

@ -0,0 +1,9 @@
/* DELETE
err\("__syscall_getsockname " ?\+ ?fd\);
*/
/* INSERT
function _emscripten_console_error\(str\) {
*/
if (UTF8ToString(str).endsWith("__syscall_setsockopt\\n")) return;

4
client/fragments/wisp_support.js Normal file
View file

@ -0,0 +1,4 @@
/* REPLACE
new WebSocketConstructor
*/
new WispWebSocket

108
client/main.c
View file

@ -8,12 +8,28 @@
#include "curl/header.h"
#include "cjson/cJSON.h"
#include "cacert.h"
#include "curl/multi.h"
typedef void(*DataCallback)(char* chunk_ptr, int chunk_size);
typedef void(*EndCallback)(int error, char* response_json);
void finish_request(CURLMsg *curl_msg);
#define ERROR_REDIRECT_DISALLOWED -1
CURLM *multi_handle;
int request_active = 0;
struct RequestInfo {
int abort_on_redirect;
struct CURLMsg *curl_msg;
struct curl_slist* headers_list;
EndCallback end_callback;
};
int starts_with(const char *a, const char *b) {
return strncmp(a, b, strlen(b)) == 0;
}
int write_function(void *data, size_t size, size_t nmemb, DataCallback data_callback) {
long real_size = size * nmemb;
char* chunk = malloc(real_size);
@ -23,24 +39,31 @@ int write_function(void *data, size_t size, size_t nmemb, DataCallback data_call
return real_size;
}
void perform_request(const char* url, const char* json_params, DataCallback data_callback, EndCallback end_callback, const char* body, int body_length) {
printf("downloading %s\n", url);
int active_requests() {
return request_active;
}
CURL *http_handle;
CURLM *multi_handle;
int still_running = 1;
void tick_request() {
CURLMcode mc;
struct CURLMsg *curl_msg;
request_active = 1;
mc = curl_multi_perform(multi_handle, &request_active);
int msgq = 0;
curl_msg = curl_multi_info_read(multi_handle, &msgq);
if (curl_msg && curl_msg->msg == CURLMSG_DONE) {
finish_request(curl_msg);
}
}
void start_request(const char* url, const char* json_params, DataCallback data_callback, EndCallback end_callback, const char* body, int body_length) {
CURL *http_handle = curl_easy_init();
int abort_on_redirect = 0;
char error_buffer[CURL_ERROR_SIZE];
curl_global_init(CURL_GLOBAL_DEFAULT);
http_handle = curl_easy_init();
curl_easy_setopt(http_handle, CURLOPT_URL, url);
curl_easy_setopt(http_handle, CURLOPT_PROXY, "socks5h://127.0.0.1:1234");
curl_easy_setopt(http_handle, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
curl_easy_setopt(http_handle, CURLOPT_CAINFO, "/cacert.pem");
curl_easy_setopt(http_handle, CURLOPT_CAPATH, "/cacert.pem");
curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER, error_buffer);
//callbacks to pass the response data back to js
curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, &write_function);
@ -50,6 +73,11 @@ void perform_request(const char* url, const char* json_params, DataCallback data
curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(http_handle, CURLOPT_ACCEPT_ENCODING, "");
//if url is a websocket, tell curl that we should handle the connection manually
if (starts_with(url, "wss://") || starts_with(url, "ws://")) {
curl_easy_setopt(http_handle, CURLOPT_CONNECT_ONLY, 2L);
}
//parse json options
cJSON* request_json = cJSON_Parse(json_params);
cJSON* item = NULL;
@ -101,46 +129,33 @@ void perform_request(const char* url, const char* json_params, DataCallback data
curl_easy_setopt(http_handle, CURLOPT_POSTFIELDSIZE, body_length);
}
multi_handle = curl_multi_init();
struct RequestInfo *request_info = malloc(sizeof(struct RequestInfo));
request_info->abort_on_redirect = abort_on_redirect;
request_info->curl_msg = NULL;
request_info->headers_list = headers_list;
request_info->end_callback = end_callback;
curl_easy_setopt(http_handle, CURLOPT_PRIVATE, request_info);
curl_multi_add_handle(multi_handle, http_handle);
}
CURLMcode mc;
struct CURLMsg *m;
error_buffer[0] = 0;
void finish_request(CURLMsg *curl_msg) {
//get initial request info from the http handle
struct RequestInfo *request_info;
CURL *http_handle = curl_msg->easy_handle;
curl_easy_getinfo(http_handle, CURLINFO_PRIVATE, &request_info);
do {
mc = curl_multi_perform(multi_handle, &still_running);
if(!mc)
mc = curl_multi_poll(multi_handle, NULL, 0, 1000, NULL);
if(mc) {
fprintf(stderr, "curl_multi_poll() failed, code %d.\n", (int)mc);
break;
}
int msgq = 0;
m = curl_multi_info_read(multi_handle, &msgq);
//ensure we dont block the main thread
emscripten_sleep(0);
} while(still_running);
int error = (int) m->data.result;
int error = (int) curl_msg->data.result;
long response_code;
curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
if (abort_on_redirect && response_code / 100 == 3) {
if (request_info->abort_on_redirect && response_code / 100 == 3) {
error = ERROR_REDIRECT_DISALLOWED;
}
//create new json object with response info
cJSON* response_json = cJSON_CreateObject();
cJSON* error_item = cJSON_CreateString(error_buffer);
cJSON_AddItemToObject(response_json, "error", error_item);
cJSON* status_item = cJSON_CreateNumber(response_code);
cJSON_AddItemToObject(response_json, "status", status_item);
@ -168,13 +183,11 @@ void perform_request(const char* url, const char* json_params, DataCallback data
cJSON_Delete(response_json);
//clean up curl
curl_slist_free_all(headers_list);
curl_slist_free_all(request_info->headers_list);
curl_multi_remove_handle(multi_handle, http_handle);
curl_easy_cleanup(http_handle);
curl_multi_cleanup(multi_handle);
curl_global_cleanup();
(*end_callback)(error, response_json_str);
(*request_info->end_callback)(error, response_json_str);
free(request_info);
}
char* copy_bytes(const char* ptr, const int size) {
@ -183,7 +196,10 @@ char* copy_bytes(const char* ptr, const int size) {
return new_ptr;
}
void load_certs() {
void init_curl() {
curl_global_init(CURL_GLOBAL_DEFAULT);
multi_handle = curl_multi_init();
FILE *file = fopen("/cacert.pem", "wb");
fwrite(_cacert_pem, 1, _cacert_pem_len, file);
fclose(file);

85
client/main.js
View file

@ -4,73 +4,11 @@ window.libcurl = (function() {
//emscripten compiled code is inserted here
/* __emscripten_output__ */
const websocket_url = `wss://${location.hostname}/ws`;
//extra client code goes here
/* __extra_libraries__ */
const status_messages = {
100: "Continue",
101: "Switching Protocols",
102: "Processing",
103: "Early Hints",
200: "OK",
201: "Created",
202: "Accepted",
203: "Non-Authoritative Information",
204: "No Content",
205: "Reset Content",
206: "Partial Content",
207: "Multi-Status",
208: "Already Reported",
226: "IM Used",
300: "Multiple Choices",
301: "Moved Permanently",
302: "Found",
303: "See Other",
304: "Not Modified",
305: "Use Proxy",
306: "Switch Proxy",
307: "Temporary Redirect",
308: "Permanent Redirect",
400: "Bad Request",
401: "Unauthorized",
402: "Payment Required",
403: "Forbidden",
404: "Not Found",
405: "Method Not Allowed",
406: "Not Acceptable",
407: "Proxy Authentication Required",
408: "Request Timeout",
409: "Conflict",
410: "Gone",
411: "Length Required",
412: "Precondition Failed",
413: "Payload Too Large",
414: "URI Too Long",
415: "Unsupported Media Type",
416: "Range Not Satisfiable",
417: "Expectation Failed",
418: "I'm a teapot",
421: "Misdirected Request",
422: "Unprocessable Content",
423: "Locked",
424: "Failed Dependency",
425: "Too Early",
426: "Upgrade Required",
428: "Precondition Required",
429: "Too Many Requests",
431: "Request Header Fields Too Large",
451: "Unavailable For Legal Reasons",
500: "Internal Server Error",
501: "Not Implemented",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
505: "HTTP Version Not Supported",
506: "Variant Also Negotiates",
507: "Insufficient Storage",
508: "Loop Detected",
510: "Not Extended",
511: "Network Authentication Required"
}
const websocket_url = `wss://${location.hostname}/ws/`;
var event_loop = null;
//a case insensitive dictionary for request headers
class Headers {
@ -148,8 +86,19 @@ function perform_request(url, params, js_data_callback, js_end_callback, body=nu
end_callback_ptr = Module.addFunction(end_callback, "vii");
data_callback_ptr = Module.addFunction(data_callback, "vii");
_perform_request(url_ptr, params_ptr, data_callback_ptr, end_callback_ptr, body_ptr, body_length);
_start_request(url_ptr, params_ptr, data_callback_ptr, end_callback_ptr, body_ptr, body_length);
_free(params_ptr);
_tick_request();
if (!event_loop) {
event_loop = setInterval(() => {
_tick_request();
if (!_active_requests()) {
clearInterval(event_loop);
event_loop = null;
}
}, 0);
}
}
function merge_arrays(arrays) {
@ -231,7 +180,7 @@ function set_websocket_url(url) {
function main() {
console.log("emscripten module loaded");
_load_certs();
_init_curl();
set_websocket_url(websocket_url);
let load_event = new Event("libcurl_load");

65
client/messages.js Normal file
View file

@ -0,0 +1,65 @@
const status_messages = {
100: "Continue",
101: "Switching Protocols",
102: "Processing",
103: "Early Hints",
200: "OK",
201: "Created",
202: "Accepted",
203: "Non-Authoritative Information",
204: "No Content",
205: "Reset Content",
206: "Partial Content",
207: "Multi-Status",
208: "Already Reported",
226: "IM Used",
300: "Multiple Choices",
301: "Moved Permanently",
302: "Found",
303: "See Other",
304: "Not Modified",
305: "Use Proxy",
306: "Switch Proxy",
307: "Temporary Redirect",
308: "Permanent Redirect",
400: "Bad Request",
401: "Unauthorized",
402: "Payment Required",
403: "Forbidden",
404: "Not Found",
405: "Method Not Allowed",
406: "Not Acceptable",
407: "Proxy Authentication Required",
408: "Request Timeout",
409: "Conflict",
410: "Gone",
411: "Length Required",
412: "Precondition Failed",
413: "Payload Too Large",
414: "URI Too Long",
415: "Unsupported Media Type",
416: "Range Not Satisfiable",
417: "Expectation Failed",
418: "I'm a teapot",
421: "Misdirected Request",
422: "Unprocessable Content",
423: "Locked",
424: "Failed Dependency",
425: "Too Early",
426: "Upgrade Required",
428: "Precondition Required",
429: "Too Many Requests",
431: "Request Header Fields Too Large",
451: "Unavailable For Legal Reasons",
500: "Internal Server Error",
501: "Not Implemented",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
505: "HTTP Version Not Supported",
506: "Variant Also Negotiates",
507: "Insufficient Storage",
508: "Loop Detected",
510: "Not Extended",
511: "Network Authentication Required"
}

26
client/patcher.py Normal file
View file

@ -0,0 +1,26 @@
import re
import sys
import pathlib
match_regex = r'/\* (.+?)\n(.+?)\n\*/\n(.*?)(\n\n|$)'
fragments_dir = sys.argv[1]
target_dir = sys.argv[2]
fragments_path = pathlib.Path(fragments_dir)
target_path = pathlib.Path(target_dir)
target_text = target_path.read_text()
for fragment_file in fragments_path.iterdir():
print(f"applying patch from {fragment_file.name}")
fragment_text = fragment_file.read_text()
matches = re.findall(match_regex, fragment_text, re.S)
for mode, patch_regex, patch_text, _ in matches:
if mode == "DELETE":
target_text = re.sub(patch_regex, "", target_text)
elif mode == "REPLACE":
target_text = re.sub(patch_regex, patch_text, target_text)
elif mode == "INSERT":
target_text = re.sub("("+patch_regex+")", r'\1'+patch_text, target_text)
target_path.write_text(target_text)

2
client/tools/curl.sh
View file

@ -17,7 +17,7 @@ git clone -b master --depth=1 https://github.com/curl/curl
cd curl
autoreconf -fi
emconfigure ./configure --host i686-linux --disable-shared --disable-threaded-resolver --without-libpsl --disable-netrc --disable-ipv6 --disable-tftp --disable-ntlm-wb --with-wolfssl=$WOLFSSL_PREFIX --with-zlib=$ZLIB_PREFIX --with-brotli=$BROTLI_PREFIX
emconfigure ./configure --host i686-linux --disable-shared --disable-threaded-resolver --without-libpsl --disable-netrc --disable-ipv6 --disable-tftp --disable-ntlm-wb --enable-websockets --with-wolfssl=$WOLFSSL_PREFIX --with-zlib=$ZLIB_PREFIX --with-brotli=$BROTLI_PREFIX
emmake make -j$CORE_COUNT CFLAGS="-Os -pthread" LIBS="-lbrotlicommon"
rm -rf $PREFIX

1
client/wisp_client Submodule

@ -0,0 +1 @@
Subproject commit 0a80885090b6247f42bc07cc85b441d8d719f551

45
server/main.py
View file

@ -1,45 +0,0 @@
import logging
import os
from asyncio_socks_server.app import SocksServer
from websockify.websocketproxy import WebSocketProxy
#start a socks5 proxy as well as websockify
def setup_logging(prefix):
stderr_handler = logging.StreamHandler()
stderr_handler.setLevel(logging.DEBUG)
log_formatter = logging.Formatter(prefix + "%(message)s")
stderr_handler.setFormatter(log_formatter)
root = logging.getLogger()
root.addHandler(stderr_handler)
root.setLevel(logging.INFO)
def start_websockify(listen_port, proxy_port):
options = {
"listen_host": "127.0.0.1",
"listen_port": int(listen_port),
"target_host": "127.0.0.1",
"target_port": int(proxy_port)
}
server = WebSocketProxy(**options)
server.start_server()
def start_socks(proxy_port):
socks_app = SocksServer(
LISTEN_HOST="127.0.0.1",
LISTEN_PORT=int(proxy_port)
)
socks_app.run()
if __name__ == "__main__":
listen_port = os.environ.get("PORT") or 6001
proxy_port = os.environ.get("SOCKS5_PORT") or 6002
pid = os.fork()
if pid == 0:
setup_logging("[websockify] ")
start_websockify(listen_port, proxy_port)
else:
start_socks(proxy_port)

2
server/requirements.txt
View file

@ -1,2 +0,0 @@
websockify
asyncio-socks-server

8
server/run.sh
View file

@ -4,16 +4,14 @@
set -e
cd wisp_server
if [ ! -d ".venv" ]; then
python3 -m venv .venv
fi
source .venv/bin/activate
if ! python3 -c "import asyncio_socks_server, websockify" 2> /dev/null; then
pip3 install asyncio-socks-server
git clone https://github.com/novnc/websockify -b master --depth=1
pip3 install ./websockify
rm -rf websockify
if ! python3 -c "import websockets" 2> /dev/null; then
pip3 install -r requirements.txt
fi
python3 main.py

1
server/wisp_server Submodule

@ -0,0 +1 @@
Subproject commit 874734623e4dfc4652b34a1bc61e1e35ca86dee8