Merge branch 'main' into wolfssl-testing

.gitignore

@@ -1,4 +1,5 @@
 /client/build
 /client/out
+/client/fragments/tmp
 /server/.venv
 /server/websockify

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "client/wisp_client"]
+	path = client/wisp_client
+	url = https://github.com/MercuryWorkshop/wisp-client-js
+[submodule "server/wisp_server"]
+	path = server/wisp_server
+	url = https://github.com/MercuryWorkshop/wisp-server-python

README.md

@@ -10,7 +10,7 @@ This is an experimental port of [libcurl](https://curl.se/libcurl/) to WebAssemb
 ## Building:
 You can build this project by running the following commands:
 ```
-git clone https://github.com/ading2210/libcurl.js
+git clone https://github.com/ading2210/libcurl.js --recursive
 cd libcurl.js/client
 ./build.sh
 ```
@@ -48,16 +48,16 @@ libcurl.set_websocket("ws://localhost:6001/");
 ```
 
 ## Proxy Server:
-The proxy server consists of a [SOCKS5 proxy server](https://github.com/Amaindex/asyncio-socks-server) behind a [websocket TCP proxy](https://github.com/novnc/websockify). 
+The proxy server consists of a standard [Wisp](https://github.com/MercuryWorkshop/wisp-protocol) server, allowing multiple TCP connections to share the same websocket.
 
 To host the proxy server, run the following commands:
 ```
-git clone https://github.com/ading2210/libcurl.js
+git clone https://github.com/ading2210/libcurl.js --recursive
 cd libcurl.js/server
 ./run.sh
 ```
 
-You can use the `PORT` and `SOCKS5_PORT` environment variables to control which ports the websocket proxy and the SOCKS5 server run on.
+You can use the `HOST` and `PORT` environment variables to control the hostname and port that the proxy server listens on.
 
 ## Copyright:
 This project is licensed under the GNU AGPL v3.

client/build.sh

@@ -7,15 +7,20 @@ LIB_DIR="build/curl-wasm/lib/"
 OUT_FILE="out/libcurl.js"
 ES6_FILE="out/libcurl_module.mjs"
 MODULE_FILE="out/emscripten_compiled.js"
+FRAGMENTS_DIR="fragments"
 WRAPPER_SOURCE="main.js"
+WISP_CLIENT="wisp_client"
 
-EXPORTED_FUNCS="_load_certs,_perform_request"
+EXPORTED_FUNCS="_init_curl,_start_request,_tick_request,_active_requests"
 RUNTIME_METHODS="addFunction,removeFunction,allocate,ALLOC_NORMAL"
 COMPILER_OPTIONS="-o $MODULE_FILE -lcurl -lwolfssl -lcjson -lz -lbrotlidec -lbrotlicommon -I $INCLUDE_DIR -L $LIB_DIR"
-EMSCRIPTEN_OPTIONS="-lwebsocket.js -sSINGLE_FILE -sASYNCIFY -sALLOW_TABLE_GROWTH -sEXPORTED_FUNCTIONS=$EXPORTED_FUNCS -sEXPORTED_RUNTIME_METHODS=$RUNTIME_METHODS"
+EMSCRIPTEN_OPTIONS="-lwebsocket.js -sASSERTIONS=1 -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sEXPORTED_FUNCTIONS=$EXPORTED_FUNCS -sEXPORTED_RUNTIME_METHODS=$RUNTIME_METHODS"
 
 if [ "$1" = "release" ]; then
-  COMPILER_OPTIONS="-O3 $COMPILER_OPTIONS"
+  COMPILER_OPTIONS="-Oz -flto $COMPILER_OPTIONS"
+  EMSCRIPTEN_OPTIONS="-sSINGLE_FILE $EMSCRIPTEN_OPTIONS"
+else
+  COMPILER_OPTIONS="$COMPILER_OPTIONS --profiling"
 fi
 
 #ensure deps are compiled
@@ -31,15 +36,19 @@ COMPILE_CMD="emcc main.c $COMPILER_OPTIONS $EMSCRIPTEN_OPTIONS"
 echo $COMPILE_CMD
 $COMPILE_CMD
 
-#patch the output to work around some emscripten bugs
-sed -i 's/err("__syscall_getsockname " \?+ \?fd);//' $MODULE_FILE
-sed -i 's/function _emscripten_console_error(str) {/& if(UTF8ToString(str).endsWith("__syscall_setsockopt\\n")) return;/' $MODULE_FILE
-
 #merge compiled emscripten module and wrapper code
 cp $WRAPPER_SOURCE $OUT_FILE
 sed -i "/__emscripten_output__/r $MODULE_FILE" $OUT_FILE
 rm $MODULE_FILE
 
+#add wisp libraries
+sed -i "/__extra_libraries__/r $WISP_CLIENT/polyfill.js" $OUT_FILE
+sed -i "/__extra_libraries__/r $WISP_CLIENT/wisp.js" $OUT_FILE
+sed -i "/__extra_libraries__/r ./messages.js" $OUT_FILE
+
+#apply patches
+python3 patcher.py $FRAGMENTS_DIR $OUT_FILE
+
 #generate es6 module
 cp $OUT_FILE $ES6_FILE
 sed -i 's/window.libcurl/export const libcurl/' $ES6_FILE

client/fragments/force_wsproxy.js

@@ -0,0 +1,5 @@
+/* INSERT
+var ?opts ?= ?undefined;
+*/
+var parts=addr.split("/");
+url = url + parts[0] + ":" + port;

client/fragments/silence_socket.js

@@ -0,0 +1,9 @@
+/* DELETE
+err\("__syscall_getsockname " ?\+ ?fd\);
+*/
+
+
+/* INSERT
+function _emscripten_console_error\(str\) {
+*/
+if (UTF8ToString(str).endsWith("__syscall_setsockopt\\n")) return;

client/fragments/wisp_support.js

@@ -0,0 +1,4 @@
+/* REPLACE
+new WebSocketConstructor
+*/
+new WispWebSocket

client/main.c

@@ -8,12 +8,28 @@
 #include "curl/header.h"
 #include "cjson/cJSON.h"
 #include "cacert.h"
+#include "curl/multi.h"
 
 typedef void(*DataCallback)(char* chunk_ptr, int chunk_size);
 typedef void(*EndCallback)(int error, char* response_json);
+void finish_request(CURLMsg *curl_msg);
 
 #define ERROR_REDIRECT_DISALLOWED -1
 
+CURLM *multi_handle;
+int request_active = 0;
+
+struct RequestInfo {
+  int abort_on_redirect;
+  struct CURLMsg *curl_msg;
+  struct curl_slist* headers_list;
+  EndCallback end_callback;
+};
+
+int starts_with(const char *a, const char *b) {
+  return strncmp(a, b, strlen(b)) == 0;
+}
+
 int write_function(void *data, size_t size, size_t nmemb, DataCallback data_callback) {
   long real_size = size * nmemb;
   char* chunk = malloc(real_size);
@@ -23,24 +39,31 @@ int write_function(void *data, size_t size, size_t nmemb, DataCallback data_call
   return real_size;
 }
 
-void perform_request(const char* url, const char* json_params, DataCallback data_callback, EndCallback end_callback, const char* body, int body_length) {
+int active_requests() {
-  printf("downloading %s\n", url);
+  return request_active;
+}
 
-  CURL *http_handle;
+void tick_request() {
-  CURLM *multi_handle;
+  CURLMcode mc;
-  int still_running = 1;
+  struct CURLMsg *curl_msg;
+  request_active = 1;
+  
+  mc = curl_multi_perform(multi_handle, &request_active);
+
+  int msgq = 0;
+  curl_msg = curl_multi_info_read(multi_handle, &msgq);
+  if (curl_msg && curl_msg->msg == CURLMSG_DONE) {
+    finish_request(curl_msg);
+  }
+}
+
+void start_request(const char* url, const char* json_params, DataCallback data_callback, EndCallback end_callback, const char* body, int body_length) {
+  CURL *http_handle = curl_easy_init();  
   int abort_on_redirect = 0;
-  char error_buffer[CURL_ERROR_SIZE];
- 
-  curl_global_init(CURL_GLOBAL_DEFAULT);
-  http_handle = curl_easy_init();
  
   curl_easy_setopt(http_handle, CURLOPT_URL, url);
-  curl_easy_setopt(http_handle, CURLOPT_PROXY, "socks5h://127.0.0.1:1234");
-  curl_easy_setopt(http_handle, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
   curl_easy_setopt(http_handle, CURLOPT_CAINFO, "/cacert.pem");
   curl_easy_setopt(http_handle, CURLOPT_CAPATH, "/cacert.pem");
-  curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER, error_buffer);
 
   //callbacks to pass the response data back to js
   curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, &write_function);
@@ -50,6 +73,11 @@ void perform_request(const char* url, const char* json_params, DataCallback data
   curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1);
   curl_easy_setopt(http_handle, CURLOPT_ACCEPT_ENCODING, "");
 
+  //if url is a websocket, tell curl that we should handle the connection manually
+  if (starts_with(url, "wss://") || starts_with(url, "ws://")) {
+    curl_easy_setopt(http_handle, CURLOPT_CONNECT_ONLY, 2L);
+  }
+
   //parse json options
   cJSON* request_json = cJSON_Parse(json_params);
   cJSON* item = NULL;
@@ -100,47 +128,34 @@ void perform_request(const char* url, const char* json_params, DataCallback data
     curl_easy_setopt(http_handle, CURLOPT_POSTFIELDS, body);
     curl_easy_setopt(http_handle, CURLOPT_POSTFIELDSIZE, body_length);
   }
+
+  struct RequestInfo *request_info = malloc(sizeof(struct RequestInfo));
+  request_info->abort_on_redirect = abort_on_redirect;
+  request_info->curl_msg = NULL;
+  request_info->headers_list = headers_list;
+  request_info->end_callback = end_callback;
+  curl_easy_setopt(http_handle, CURLOPT_PRIVATE, request_info);
   
-  multi_handle = curl_multi_init();
   curl_multi_add_handle(multi_handle, http_handle);
-  
+}
-  CURLMcode mc;
-  struct CURLMsg *m;
-  error_buffer[0] = 0;
 
-  do {
+void finish_request(CURLMsg *curl_msg) {
-    mc = curl_multi_perform(multi_handle, &still_running);
+  //get initial request info from the http handle
- 
+  struct RequestInfo *request_info;
-    if(!mc)
+  CURL *http_handle = curl_msg->easy_handle;
-      mc = curl_multi_poll(multi_handle, NULL, 0, 1000, NULL);
+  curl_easy_getinfo(http_handle, CURLINFO_PRIVATE, &request_info);
- 
-    if(mc) {
-      fprintf(stderr, "curl_multi_poll() failed, code %d.\n", (int)mc);
-      break;
-    }
 
-    int msgq = 0;
+  int error = (int) curl_msg->data.result;
-    m = curl_multi_info_read(multi_handle, &msgq);
-
-    //ensure we dont block the main thread
-    emscripten_sleep(0);
- 
-  } while(still_running);
-  
-  int error = (int) m->data.result;
   long response_code;
   curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
 
-  if (abort_on_redirect && response_code / 100 == 3) {
+  if (request_info->abort_on_redirect && response_code / 100 == 3) {
     error = ERROR_REDIRECT_DISALLOWED;
   }
 
   //create new json object with response info
   cJSON* response_json = cJSON_CreateObject();
 
-  cJSON* error_item = cJSON_CreateString(error_buffer);
-  cJSON_AddItemToObject(response_json, "error", error_item);
-
   cJSON* status_item = cJSON_CreateNumber(response_code);
   cJSON_AddItemToObject(response_json, "status", status_item);
 
@@ -168,13 +183,11 @@ void perform_request(const char* url, const char* json_params, DataCallback data
   cJSON_Delete(response_json);
   
   //clean up curl
-  curl_slist_free_all(headers_list);
+  curl_slist_free_all(request_info->headers_list);
   curl_multi_remove_handle(multi_handle, http_handle);
   curl_easy_cleanup(http_handle);
-  curl_multi_cleanup(multi_handle);
+  (*request_info->end_callback)(error, response_json_str);
-  curl_global_cleanup();
+  free(request_info);
-
-  (*end_callback)(error, response_json_str);
 }
 
 char* copy_bytes(const char* ptr, const int size) {
@@ -183,7 +196,10 @@ char* copy_bytes(const char* ptr, const int size) {
   return new_ptr;
 }
 
-void load_certs() {
+void init_curl() {
+  curl_global_init(CURL_GLOBAL_DEFAULT);
+  multi_handle = curl_multi_init();
+  
   FILE *file = fopen("/cacert.pem", "wb");
   fwrite(_cacert_pem, 1, _cacert_pem_len, file);
   fclose(file);

client/main.js

@@ -4,73 +4,11 @@ window.libcurl = (function() {
 //emscripten compiled code is inserted here
 /* __emscripten_output__ */
 
-const websocket_url = `wss://${location.hostname}/ws`;
+//extra client code goes here
+/* __extra_libraries__ */
 
-const status_messages = {
+const websocket_url = `wss://${location.hostname}/ws/`;
-  100: "Continue",
+var event_loop = null;
-  101: "Switching Protocols",
-  102: "Processing",
-  103: "Early Hints",
-  200: "OK",
-  201: "Created",
-  202: "Accepted",
-  203: "Non-Authoritative Information",
-  204: "No Content",
-  205: "Reset Content",
-  206: "Partial Content",
-  207: "Multi-Status",
-  208: "Already Reported",
-  226: "IM Used",
-  300: "Multiple Choices",
-  301: "Moved Permanently",
-  302: "Found",
-  303: "See Other",
-  304: "Not Modified",
-  305: "Use Proxy",
-  306: "Switch Proxy",
-  307: "Temporary Redirect",
-  308: "Permanent Redirect",
-  400: "Bad Request",
-  401: "Unauthorized",
-  402: "Payment Required",
-  403: "Forbidden",
-  404: "Not Found",
-  405: "Method Not Allowed",
-  406: "Not Acceptable",
-  407: "Proxy Authentication Required",
-  408: "Request Timeout",
-  409: "Conflict",
-  410: "Gone",
-  411: "Length Required",
-  412: "Precondition Failed",
-  413: "Payload Too Large",
-  414: "URI Too Long",
-  415: "Unsupported Media Type",
-  416: "Range Not Satisfiable",
-  417: "Expectation Failed",
-  418: "I'm a teapot",
-  421: "Misdirected Request",
-  422: "Unprocessable Content",
-  423: "Locked",
-  424: "Failed Dependency",
-  425: "Too Early",
-  426: "Upgrade Required",
-  428: "Precondition Required",
-  429: "Too Many Requests",
-  431: "Request Header Fields Too Large",
-  451: "Unavailable For Legal Reasons",
-  500: "Internal Server Error",
-  501: "Not Implemented",
-  502: "Bad Gateway",
-  503: "Service Unavailable",
-  504: "Gateway Timeout",
-  505: "HTTP Version Not Supported",
-  506: "Variant Also Negotiates",
-  507: "Insufficient Storage",
-  508: "Loop Detected",
-  510: "Not Extended",
-  511: "Network Authentication Required"
-}
 
 //a case insensitive dictionary for request headers
 class Headers {
@@ -148,8 +86,19 @@ function perform_request(url, params, js_data_callback, js_end_callback, body=nu
 
   end_callback_ptr = Module.addFunction(end_callback, "vii");
   data_callback_ptr = Module.addFunction(data_callback, "vii");
-  _perform_request(url_ptr, params_ptr, data_callback_ptr, end_callback_ptr, body_ptr, body_length);
+  _start_request(url_ptr, params_ptr, data_callback_ptr, end_callback_ptr, body_ptr, body_length);
   _free(params_ptr);
+  
+  _tick_request();
+  if (!event_loop) {
+    event_loop = setInterval(() => {
+      _tick_request();
+      if (!_active_requests()) {
+        clearInterval(event_loop);
+        event_loop = null;
+      }
+    }, 0);
+  }
 }
 
 function merge_arrays(arrays) {
@@ -231,7 +180,7 @@ function set_websocket_url(url) {
 
 function main() {
   console.log("emscripten module loaded");
-  _load_certs();
+  _init_curl();
   set_websocket_url(websocket_url);
 
   let load_event = new Event("libcurl_load");

client/messages.js

@@ -0,0 +1,65 @@
+const status_messages = {
+  100: "Continue",
+  101: "Switching Protocols",
+  102: "Processing",
+  103: "Early Hints",
+  200: "OK",
+  201: "Created",
+  202: "Accepted",
+  203: "Non-Authoritative Information",
+  204: "No Content",
+  205: "Reset Content",
+  206: "Partial Content",
+  207: "Multi-Status",
+  208: "Already Reported",
+  226: "IM Used",
+  300: "Multiple Choices",
+  301: "Moved Permanently",
+  302: "Found",
+  303: "See Other",
+  304: "Not Modified",
+  305: "Use Proxy",
+  306: "Switch Proxy",
+  307: "Temporary Redirect",
+  308: "Permanent Redirect",
+  400: "Bad Request",
+  401: "Unauthorized",
+  402: "Payment Required",
+  403: "Forbidden",
+  404: "Not Found",
+  405: "Method Not Allowed",
+  406: "Not Acceptable",
+  407: "Proxy Authentication Required",
+  408: "Request Timeout",
+  409: "Conflict",
+  410: "Gone",
+  411: "Length Required",
+  412: "Precondition Failed",
+  413: "Payload Too Large",
+  414: "URI Too Long",
+  415: "Unsupported Media Type",
+  416: "Range Not Satisfiable",
+  417: "Expectation Failed",
+  418: "I'm a teapot",
+  421: "Misdirected Request",
+  422: "Unprocessable Content",
+  423: "Locked",
+  424: "Failed Dependency",
+  425: "Too Early",
+  426: "Upgrade Required",
+  428: "Precondition Required",
+  429: "Too Many Requests",
+  431: "Request Header Fields Too Large",
+  451: "Unavailable For Legal Reasons",
+  500: "Internal Server Error",
+  501: "Not Implemented",
+  502: "Bad Gateway",
+  503: "Service Unavailable",
+  504: "Gateway Timeout",
+  505: "HTTP Version Not Supported",
+  506: "Variant Also Negotiates",
+  507: "Insufficient Storage",
+  508: "Loop Detected",
+  510: "Not Extended",
+  511: "Network Authentication Required"
+}

client/patcher.py

@@ -0,0 +1,26 @@
+import re
+import sys
+import pathlib
+
+match_regex = r'/\* (.+?)\n(.+?)\n\*/\n(.*?)(\n\n|$)'
+
+fragments_dir = sys.argv[1]
+target_dir = sys.argv[2]
+fragments_path = pathlib.Path(fragments_dir)
+target_path = pathlib.Path(target_dir)
+target_text = target_path.read_text()
+
+for fragment_file in fragments_path.iterdir():
+  print(f"applying patch from {fragment_file.name}")
+  fragment_text = fragment_file.read_text()
+  matches = re.findall(match_regex, fragment_text, re.S)
+
+  for mode, patch_regex, patch_text, _ in matches:
+    if mode == "DELETE":
+      target_text = re.sub(patch_regex, "", target_text)
+    elif mode == "REPLACE":
+      target_text = re.sub(patch_regex, patch_text, target_text)
+    elif mode == "INSERT":
+      target_text = re.sub("("+patch_regex+")", r'\1'+patch_text, target_text)
+
+target_path.write_text(target_text)

client/tools/curl.sh

@@ -17,7 +17,7 @@ git clone -b master --depth=1 https://github.com/curl/curl
 cd curl
 
 autoreconf -fi
-emconfigure ./configure --host i686-linux --disable-shared --disable-threaded-resolver --without-libpsl --disable-netrc --disable-ipv6 --disable-tftp --disable-ntlm-wb --with-wolfssl=$WOLFSSL_PREFIX --with-zlib=$ZLIB_PREFIX --with-brotli=$BROTLI_PREFIX
+emconfigure ./configure --host i686-linux --disable-shared --disable-threaded-resolver --without-libpsl --disable-netrc --disable-ipv6 --disable-tftp --disable-ntlm-wb --enable-websockets --with-wolfssl=$WOLFSSL_PREFIX --with-zlib=$ZLIB_PREFIX --with-brotli=$BROTLI_PREFIX
 emmake make -j$CORE_COUNT CFLAGS="-Os -pthread" LIBS="-lbrotlicommon"
 
 rm -rf $PREFIX

client/wisp_client

@@ -0,0 +1 @@
+Subproject commit 0a80885090b6247f42bc07cc85b441d8d719f551

server/main.py

@@ -1,45 +0,0 @@
-import logging
-import os
-
-from asyncio_socks_server.app import SocksServer
-from websockify.websocketproxy import WebSocketProxy
-
-#start a socks5 proxy as well as websockify
-
-def setup_logging(prefix):
-  stderr_handler = logging.StreamHandler()
-  stderr_handler.setLevel(logging.DEBUG)
-  log_formatter = logging.Formatter(prefix + "%(message)s")
-  stderr_handler.setFormatter(log_formatter)
-  root = logging.getLogger()
-  root.addHandler(stderr_handler)
-  root.setLevel(logging.INFO)
-
-def start_websockify(listen_port, proxy_port):
-  options = {
-    "listen_host": "127.0.0.1", 
-    "listen_port": int(listen_port), 
-    "target_host": "127.0.0.1", 
-    "target_port": int(proxy_port)
-  }
-
-  server = WebSocketProxy(**options)
-  server.start_server()
-
-def start_socks(proxy_port):
-  socks_app = SocksServer(
-    LISTEN_HOST="127.0.0.1",
-    LISTEN_PORT=int(proxy_port)
-  )
-  socks_app.run()
-
-if __name__ == "__main__":
-  listen_port = os.environ.get("PORT") or 6001
-  proxy_port = os.environ.get("SOCKS5_PORT") or 6002
-
-  pid = os.fork()
-  if pid == 0:
-    setup_logging("[websockify] ")
-    start_websockify(listen_port, proxy_port)
-  else:
-    start_socks(proxy_port)

server/requirements.txt

@@ -1,2 +0,0 @@
-websockify
-asyncio-socks-server

server/run.sh

@@ -4,16 +4,14 @@
 
 set -e
 
+cd wisp_server
 if [ ! -d ".venv" ]; then
   python3 -m venv .venv
 fi
 source .venv/bin/activate
 
-if ! python3 -c "import asyncio_socks_server, websockify" 2> /dev/null; then
+if ! python3 -c "import websockets" 2> /dev/null; then
-  pip3 install asyncio-socks-server
+  pip3 install -r requirements.txt
-  git clone https://github.com/novnc/websockify -b master --depth=1
-  pip3 install ./websockify
-  rm -rf websockify
 fi
 
 python3 main.py

server/wisp_server

@@ -0,0 +1 @@
+Subproject commit 874734623e4dfc4652b34a1bc61e1e35ca86dee8