From 2a072ecee0830ef8abed1a25f8b55c13a0b03af5 Mon Sep 17 00:00:00 2001 From: ading2210 Date: Sun, 17 Mar 2024 17:50:58 -0400 Subject: [PATCH] ftp works but it blocks the thread --- CHANGELOG.md | 4 ++ README.md | 10 ++++- client/build.sh | 1 + client/exported_funcs.txt | 1 + client/javascript/ftp.js | 65 ++++++++++++++++----------- client/javascript/http.js | 66 ++++++++------------------- client/javascript/main.js | 80 ++------------------------------- client/javascript/session.js | 71 +++++++++++++++++++++++++++-- client/javascript/tls_socket.js | 5 +-- client/libcurl/ftp.c | 15 +++++-- client/libcurl/request.c | 27 +++++------ client/libcurl/types.h | 4 +- 12 files changed, 165 insertions(+), 184 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4322d63..766c69b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Libcurl.js Changelog: +## v0.6.0 (3/14/24): +- Refactor JS and C code +- Allow for multiple sessions with separate connection pools + ## v0.5.3 (3/9/24): - Update Wisp client and server, which improves error handling - Expose the wisp-client-js version in the API diff --git a/README.md b/README.md index 9c7d54f..62336b7 100644 --- a/README.md +++ b/README.md @@ -97,9 +97,15 @@ Most of the standard Fetch API's features are supported, with the exception of: - Sending credentials/cookies automatically - Caching -The response may contain multiple HTTP headers with the same name, which the `Headers` object isn't able to properly represent. If this matters to you, use `response.raw_headers`, which is an array of key value pairs, instead of `response.headers`. There is support for streaming the response body using a `ReadableStream`, as well as canceling requests using an `AbortSignal`. +The response may contain multiple HTTP headers with the same name, which the `Headers` object isn't able to properly represent. If this matters to you, use `response.raw_headers`, which is an array of key value pairs, instead of `response.headers`. There is support for streaming the response body using a `ReadableStream`, as well as canceling requests using an `AbortSignal`. All requests made using this method share the same connection pool, which has a limit of 50 active TCP connections. -Also note that there is a hard limit of 50 active TCP connections due to emscripten limitations. +### Creating New HTTP Sessions: +To create new sessions for HTTP requests, use the `libcurl.HTTPSession` class. The constructor for this class takes the following arguments: +- `options` - An optional object with various settings. + +The valid HTTP session settings are: +- `enable_cookies` - A boolean which indicate whether or not cookies should be persisted within the session. +- `cookie_jar` - A blob containing the data in the cookie jar file. This should have been exported from a previous session. ### Creating WebSocket Connections: To use WebSockets, create a `libcurl.CurlWebSocket` object, which takes the following arguments: diff --git a/client/build.sh b/client/build.sh index fec001a..8cb15d8 100755 --- a/client/build.sh +++ b/client/build.sh @@ -83,6 +83,7 @@ sed -i "s/__wisp_version__/$WISP_VERSION/" $OUT_FILE #js files are inserted in reverse order +sed -i "/__extra_libraries__/r $JAVSCRIPT_DIR/ftp.js" $OUT_FILE sed -i "/__extra_libraries__/r $JAVSCRIPT_DIR/tls_socket.js" $OUT_FILE sed -i "/__extra_libraries__/r $JAVSCRIPT_DIR/ws_polyfill.js" $OUT_FILE sed -i "/__extra_libraries__/r $JAVSCRIPT_DIR/websocket.js" $OUT_FILE diff --git a/client/exported_funcs.txt b/client/exported_funcs.txt index d867ac1..240c534 100644 --- a/client/exported_funcs.txt +++ b/client/exported_funcs.txt @@ -31,5 +31,6 @@ recv_from_socket send_to_socket ftp_set_options +ftp_set_cmd free \ No newline at end of file diff --git a/client/javascript/ftp.js b/client/javascript/ftp.js index 51c0d5d..6978905 100644 --- a/client/javascript/ftp.js +++ b/client/javascript/ftp.js @@ -1,44 +1,55 @@ -//unfinished! - -class FTPSession { +class FTPSession extends CurlSession { constructor(url, options={}) { - if (!url.startsWith("ftp://") || !url.startsWith("ftps://")) { + if (!url.startsWith("ftp://") && !url.startsWith("ftps://")) { throw "invalid url protocol"; } + super(); this.url = url; - this.cwd = new URL(url).pathname; this.options = options; - this.http_handle = null; } - do_request(url) { + send_cmd(cmd) { return new Promise((resolve, reject) => { - let http_handle; - let data_callback = (data) => {this.data_callback(data)}; - let finish_callback = (error) => { - _cleanup_handle(http_handle); - if (error) { - reject(); - } - else { - resolve(); - } - }; - let headers_callback = () => {this.headers_callback()}; + let request_ptr; + let chunks = []; - http_handle = create_request(url, data_callback, finish_callback, headers_callback); - _ftp_set_options(http_handle, url, 1); - start_request(http_handle); + let data_callback = () => {}; + let finish_callback = (error) => { + this.remove_request(request_ptr); + if (error) { + reject(`Sending FTP command failed with error ${error}: ${get_error_str(error)}`); + } + } + let headers_callback = (chunk) => { + chunks.push(chunk); + console.log(chunk); + } + + request_ptr = this.create_request(this.url, data_callback, finish_callback, headers_callback); + c_func(_ftp_set_cmd, [request_ptr, cmd]); }); } - async download(path) { - let url = new URL(path, this.url); - _ftp_set_options(this.http_handle, url, 0); - } + download(path) { + let url = new URL(path, this.url).href; + console.log(url); - cleanup() { + return new Promise((resolve, reject) => { + let request_ptr; + let finish_callback = (error) => { + this.remove_request(request_ptr); + if (error) { + reject(`FTP request failed with error ${error}: ${get_error_str(error)}`); + } + }; + let headers_callback = (stream) => { + resolve(stream); + }; + request_ptr = this.stream_response(url, headers_callback, finish_callback); + _ftp_set_options(request_ptr); + this.start_request(request_ptr); + }); } } \ No newline at end of file diff --git a/client/javascript/http.js b/client/javascript/http.js index 8e1bc89..25db8cb 100644 --- a/client/javascript/http.js +++ b/client/javascript/http.js @@ -1,72 +1,42 @@ class HTTPSession extends CurlSession { - constructor() { + constructor(options) { super(); + this.options = options; this.set_connections(50, 40); } request_async(url, params, body) { return new Promise((resolve, reject) => { - let stream_controller; let http_handle; - let response_obj; - let aborted = false; - //handle abort signals - if (params.signal instanceof AbortSignal) { - params.signal.addEventListener("abort", () => { - if (aborted) return; - aborted = true; - _cleanup_handle(http_handle); - if (!response_obj) { - reject(new DOMException("The operation was aborted.")); - } - else { - stream_controller.error("The operation was aborted."); - } - }); - } - - let stream = new ReadableStream({ - start(controller) { - stream_controller = controller; - } - }); - - let data_callback = (new_data) => { - try { - stream_controller.enqueue(new_data); - } - catch (e) { - //the readable stream has been closed elsewhere, so cancel the request - if (e instanceof TypeError) { - _cleanup_handle(http_handle); - } - else { - throw e; - } - } - } - let headers_callback = () => { + let headers_callback = (stream) => { let response_json = c_func_str(_http_get_info, [http_handle]); - response_obj = this.constructor.create_response(stream, JSON.parse(response_json)); - resolve(response_obj); + let response = this.constructor.create_response(stream, JSON.parse(response_json)); + + if (params.redirect === "error" && response.status >= 300 && response.status < 400) { + finish_callback(-2); + return; + } + resolve(response); } let finish_callback = (error) => { - if (error != 0) { + if (error > 0) { error_msg(`Request "${url}" failed with error code ${error}: ${get_error_str(error)}`); reject(`Request failed with error code ${error}: ${get_error_str(error)}`); } - try { - stream_controller.close(); - } //this will only fail if the stream is already errored or closed, which isn't a problem - catch {} + else if (error === -1) { + reject(new DOMException("The operation was aborted.")); + } + else if (error === -2) { + reject("Request failed because redirects were disallowed."); + } this.remove_request(http_handle); } let body_length = body ? body.length : 0; let params_json = JSON.stringify(params); - http_handle = this.create_request(url, data_callback, finish_callback, headers_callback); + http_handle = this.stream_response(url, headers_callback, finish_callback, params.signal); c_func(_http_set_options, [http_handle, params_json, body, body_length]); this.start_request(http_handle); }); diff --git a/client/javascript/main.js b/client/javascript/main.js index c4f6623..d6b82bf 100644 --- a/client/javascript/main.js +++ b/client/javascript/main.js @@ -41,82 +41,6 @@ function check_loaded(check_websocket) { throw new Error("websocket proxy url not set, please call libcurl.set_websocket"); } } - -//wrap perform_request in a promise -function perform_request_async(url, params, body) { - return new Promise((resolve, reject) => { - let stream_controller; - let http_handle; - let response_obj; - let aborted = false; - - //handle abort signals - if (params.signal instanceof AbortSignal) { - params.signal.addEventListener("abort", () => { - if (aborted) return; - aborted = true; - _cleanup_handle(http_handle); - if (!response_obj) { - reject(new DOMException("The operation was aborted.")); - } - else { - stream_controller.error("The operation was aborted."); - } - }); - } - - let stream = new ReadableStream({ - start(controller) { - stream_controller = controller; - } - }); - - function data_callback(new_data) { - try { - stream_controller.enqueue(new_data); - } - catch (e) { - //the readable stream has been closed elsewhere, so cancel the request - if (e instanceof TypeError) { - _cleanup_handle(http_handle); - } - else { - throw e; - } - } - } - function headers_callback() { - let response_json = c_func_str(_http_get_info, [http_handle]); - response_obj = create_response(stream, JSON.parse(response_json)); - resolve(response_obj); - } - function finish_callback(error) { - if (error != 0) { - error_msg(`Request "${url}" failed with error code ${error}: ${get_error_str(error)}`); - reject(`Request failed with error code ${error}: ${get_error_str(error)}`); - return; - } - try { - stream_controller.close(); - } //this will only fail if the stream is already errored or closed, which isn't a problem - catch {} - } - - let body_length = body ? body.length : 0; - let params_json = JSON.stringify(params); - - http_handle = create_request(url, data_callback, finish_callback, headers_callback); - c_func(_http_set_options, [http_handle, params_json, body, body_length]); - start_request(http_handle); - }); -} - -async function libcurl_fetch(url, params={}) { - check_loaded(true); - let body = await create_options(params); - return await perform_request_async(url, params, body); -} - function set_websocket_url(url) { websocket_url = url; if (Module.websocket) { @@ -177,7 +101,9 @@ api = { WebSocket: WebSocket, CurlWebSocket: CurlWebSocket, TLSSocket: TLSSocket, - fetch: () => {throw "not ready"}, + HTTPSession: HTTPSession, + FTPSession: FTPSession, + fetch() {throw "not ready"}, get copyright() {return copyright_notice}, get version() {return get_version()}, diff --git a/client/javascript/session.js b/client/javascript/session.js index f156042..e2911f3 100644 --- a/client/javascript/session.js +++ b/client/javascript/session.js @@ -29,7 +29,6 @@ class CurlSession { let end_callback = (error) => { Module.removeFunction(end_callback_ptr); Module.removeFunction(data_callback_ptr); - Module.removeFunction(headers_callback_ptr); this.active_requests--; js_end_callback(error); @@ -41,12 +40,14 @@ class CurlSession { js_data_callback(chunk); } - let headers_callback = () => { - js_headers_callback(); + let headers_callback = (chunk_ptr, chunk_size) => { + let data = Module.HEAPU8.subarray(chunk_ptr, chunk_ptr + chunk_size); + let chunk = new Uint8Array(data); + js_headers_callback(chunk); } end_callback_ptr = Module.addFunction(end_callback, "vi"); - headers_callback_ptr = Module.addFunction(headers_callback, "v"); + headers_callback_ptr = Module.addFunction(headers_callback, "vii"); data_callback_ptr = Module.addFunction(data_callback, "vii"); let request_ptr = c_func(_create_request, [url, data_callback_ptr, end_callback_ptr, headers_callback_ptr]); @@ -78,6 +79,7 @@ class CurlSession { this.event_loop = setInterval(() => { let libcurl_active = _session_get_active(this.session_ptr); if (libcurl_active || this.active_requests) { + console.log("test"); _session_perform(this.session_ptr); } else { @@ -95,4 +97,65 @@ class CurlSession { _session_cleanup(this.session_ptr); this.session_ptr = null; } + + //wrap request callbacks using a readable stream and return the new callbacks + stream_response(url, headers_callback, end_callback, abort_signal) { + let stream_controller; + let aborted = false; + let headers_received = false; + + let stream = new ReadableStream({ + start(controller) { + stream_controller = controller; + } + }); + + if (abort_signal instanceof AbortSignal) { + abort_signal.addEventListener("abort", () => { + if (aborted) return; + aborted = true; + if (headers_received) { + stream_controller.error("The operation was aborted."); + } + real_abort_callback(); + }); + } + + let real_data_callback = (new_data) => { + if (!headers_received) { + headers_received = true; + headers_callback(stream); + } + + try { + stream_controller.enqueue(new_data); + } + catch (e) { + //the readable stream has been closed elsewhere, so cancel the request + if (e instanceof TypeError) { + finish_callback(-1); + } + else { + throw e; + } + } + } + + let real_end_callback = (error) => { + if (!headers_received && error === 0) { + headers_received = true; + headers_callback(stream); + } + + if (error != 0) { + try { + stream_controller.close(); + } + catch {} + } + end_callback(error); + } + + return this.create_request(url, real_data_callback, real_end_callback, () => {}); + } } \ No newline at end of file diff --git a/client/javascript/tls_socket.js b/client/javascript/tls_socket.js index a55cb4d..17ed256 100644 --- a/client/javascript/tls_socket.js +++ b/client/javascript/tls_socket.js @@ -22,11 +22,8 @@ class TLSSocket extends CurlSession { } connect() { - let response_info; let data_callback = () => {}; - let headers_callback = (info) => { - response_info = info; - } + let headers_callback = () => {}; let finish_callback = (error) => { if (error === 0) { this.connected = true; diff --git a/client/libcurl/ftp.c b/client/libcurl/ftp.c index a1cdacb..307ddda 100644 --- a/client/libcurl/ftp.c +++ b/client/libcurl/ftp.c @@ -3,7 +3,16 @@ #include "types.h" #include "util.h" -void ftp_set_options(CURL* http_handle, const char* url, int no_body) { - curl_easy_setopt(http_handle, CURLOPT_NOBODY, (long) no_body); - curl_easy_setopt(http_handle, CURLOPT_URL, url); +void ftp_set_options(CURL* easy_handle) { + curl_easy_setopt(easy_handle, CURLOPT_VERBOSE, 1L); +} + +void ftp_set_cmd(CURL* easy_handle, const char* cmd) { + struct curl_slist *cmd_list = NULL; + cmd_list = curl_slist_append(cmd_list, cmd); + + curl_easy_setopt(easy_handle, CURLOPT_QUOTE, cmd_list); + curl_easy_setopt(easy_handle, CURLOPT_NOBODY, 1L); + + curl_slist_free_all(cmd_list); } \ No newline at end of file diff --git a/client/libcurl/request.c b/client/libcurl/request.c index e90b857..f3ac7ac 100644 --- a/client/libcurl/request.c +++ b/client/libcurl/request.c @@ -17,18 +17,18 @@ void forward_headers(struct RequestInfo *request_info); struct curl_blob cacert_blob; size_t write_function(char *data, size_t size, size_t nmemb, struct RequestInfo *request_info) { - //this should be in the write callback rather than curl's header callback because - //the write function will only be called after redirects - if (!request_info->headers_received) { - forward_headers(request_info); - } - size_t real_size = size * nmemb; (*request_info->data_callback)(data, real_size); return real_size; } -CURL* create_request(const char* url, DataCallback data_callback, EndCallback end_callback, HeadersCallback headers_callback) { +size_t header_function(char *data, size_t size, size_t nmemb, struct RequestInfo *request_info) { + size_t real_size = size * nmemb; + (*request_info->headers_callback)(data, real_size); + return real_size; +} + +CURL* create_request(const char* url, DataCallback data_callback, EndCallback end_callback, DataCallback headers_callback) { CURL *http_handle = curl_easy_init(); //create request metadata struct @@ -36,7 +36,6 @@ CURL* create_request(const char* url, DataCallback data_callback, EndCallback en request_info->http_handle = http_handle; request_info->curl_msg = NULL; request_info->headers_list = NULL; - request_info->headers_received = 0; request_info->end_callback = end_callback; request_info->data_callback = data_callback; request_info->headers_callback = headers_callback; @@ -48,23 +47,19 @@ CURL* create_request(const char* url, DataCallback data_callback, EndCallback en //callbacks to pass the response data back to js curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, &write_function); curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, request_info); + + //callback which runs on every response header + curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION, &header_function); + curl_easy_setopt(http_handle, CURLOPT_HEADERDATA, request_info); return http_handle; } -void forward_headers(struct RequestInfo *request_info) { - request_info->headers_received = 1; - (*request_info->headers_callback)(); -} - void finish_request(CURLMsg *curl_msg) { CURL *http_handle = curl_msg->easy_handle; struct RequestInfo *request_info = get_request_info(http_handle); int error = (int) curl_msg->data.result; - if (!request_info->headers_received && error == 0) { - forward_headers(request_info); - } //clean up curl if (request_info->headers_list != NULL) { diff --git a/client/libcurl/types.h b/client/libcurl/types.h index 522c399..a0642bd 100644 --- a/client/libcurl/types.h +++ b/client/libcurl/types.h @@ -2,16 +2,14 @@ typedef void(*DataCallback)(char* chunk_ptr, int chunk_size); typedef void(*EndCallback)(int error); -typedef void(*HeadersCallback)(); struct RequestInfo { CURL* http_handle; - int headers_received; struct CURLMsg *curl_msg; struct curl_slist* headers_list; DataCallback data_callback; + DataCallback headers_callback; EndCallback end_callback; - HeadersCallback headers_callback; }; struct WSResult {