ftp works but it blocks the thread

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Libcurl.js Changelog:
 
+## v0.6.0 (3/14/24):
+- Refactor JS and C code
+- Allow for multiple sessions with separate connection pools
+
 ## v0.5.3 (3/9/24):
 - Update Wisp client and server, which improves error handling
 - Expose the wisp-client-js version in the API

README.md

@@ -97,9 +97,15 @@ Most of the standard Fetch API's features are supported, with the exception of:
 - Sending credentials/cookies automatically
 - Caching
 
-The response may contain multiple HTTP headers with the same name, which the `Headers` object isn't able to properly represent. If this matters to you, use `response.raw_headers`, which is an array of key value pairs, instead of `response.headers`. There is support for streaming the response body using a `ReadableStream`, as well as canceling requests using an `AbortSignal`.
+The response may contain multiple HTTP headers with the same name, which the `Headers` object isn't able to properly represent. If this matters to you, use `response.raw_headers`, which is an array of key value pairs, instead of `response.headers`. There is support for streaming the response body using a `ReadableStream`, as well as canceling requests using an `AbortSignal`. All requests made using this method share the same connection pool, which has a limit of 50 active TCP connections.
 
-Also note that there is a hard limit of 50 active TCP connections due to emscripten limitations. 
+### Creating New HTTP Sessions:
+To create new sessions for HTTP requests, use the `libcurl.HTTPSession` class. The constructor for this class takes the following arguments:
+- `options` - An optional object with various settings.
+
+The valid HTTP session settings are:
+- `enable_cookies` - A boolean which indicate whether or not cookies should be persisted within the session.
+- `cookie_jar` - A blob containing the data in the cookie jar file. This should have been exported from a previous session. 
 
 ### Creating WebSocket Connections:
 To use WebSockets, create a `libcurl.CurlWebSocket` object, which takes the following arguments:

client/build.sh

@@ -83,6 +83,7 @@ sed -i "s/__wisp_version__/$WISP_VERSION/" $OUT_FILE
 
 
 #js files are inserted in reverse order
+sed -i "/__extra_libraries__/r $JAVSCRIPT_DIR/ftp.js" $OUT_FILE
 sed -i "/__extra_libraries__/r $JAVSCRIPT_DIR/tls_socket.js" $OUT_FILE
 sed -i "/__extra_libraries__/r $JAVSCRIPT_DIR/ws_polyfill.js" $OUT_FILE
 sed -i "/__extra_libraries__/r $JAVSCRIPT_DIR/websocket.js" $OUT_FILE

client/exported_funcs.txt

@@ -31,5 +31,6 @@ recv_from_socket
 send_to_socket
 
 ftp_set_options
+ftp_set_cmd
 
 free

client/javascript/ftp.js

@@ -1,44 +1,55 @@
-//unfinished!
-
-class FTPSession {
+class FTPSession extends CurlSession {
   constructor(url, options={}) {
-    if (!url.startsWith("ftp://") || !url.startsWith("ftps://")) {
+    if (!url.startsWith("ftp://") && !url.startsWith("ftps://")) {
       throw "invalid url protocol";
     }
+    super();
 
     this.url = url;
-    this.cwd = new URL(url).pathname;
     this.options = options;
-    this.http_handle = null;
   }
 
-  do_request(url) {
+  send_cmd(cmd) {
     return new Promise((resolve, reject) => {
-      let http_handle;
-      let data_callback = (data) => {this.data_callback(data)};
-      let finish_callback = (error) => {
-        _cleanup_handle(http_handle);
-        if (error) {
-          reject();
-        }
-        else {
-          resolve();
-        }
-      };
-      let headers_callback = () => {this.headers_callback()};
+      let request_ptr;
+      let chunks = [];
 
-      http_handle = create_request(url, data_callback, finish_callback, headers_callback);
-      _ftp_set_options(http_handle, url, 1);
-      start_request(http_handle);
+      let data_callback = () => {};
+      let finish_callback = (error) => {
+        this.remove_request(request_ptr);
+        if (error) {
+          reject(`Sending FTP command failed with error ${error}: ${get_error_str(error)}`);
+        }
+      }
+      let headers_callback = (chunk) => {
+        chunks.push(chunk);
+        console.log(chunk);
+      }
+
+      request_ptr = this.create_request(this.url, data_callback, finish_callback, headers_callback);
+      c_func(_ftp_set_cmd, [request_ptr, cmd]);
     });
   }
 
-  async download(path) {
-    let url = new URL(path, this.url);
-    _ftp_set_options(this.http_handle, url, 0);
+  download(path) {
+    let url = new URL(path, this.url).href;
+    console.log(url);
+
+    return new Promise((resolve, reject) => {
+      let request_ptr;
+      let finish_callback = (error) => {
+        this.remove_request(request_ptr);
+        if (error) {
+          reject(`FTP request failed with error ${error}: ${get_error_str(error)}`);
         }
+      };
+      let headers_callback = (stream) => {
+        resolve(stream);
+      };
 
-  cleanup() {
-
+      request_ptr = this.stream_response(url, headers_callback, finish_callback);
+      _ftp_set_options(request_ptr);
+      this.start_request(request_ptr);
+    });
   }
 }

client/javascript/http.js

@@ -1,72 +1,42 @@
 class HTTPSession extends CurlSession {
-  constructor() {
+  constructor(options) {
     super();
+    this.options = options;
     this.set_connections(50, 40);
   }
 
   request_async(url, params, body) {
     return new Promise((resolve, reject) => {
-      let stream_controller;
       let http_handle;
-      let response_obj;
-      let aborted = false;
   
-      //handle abort signals
-      if (params.signal instanceof AbortSignal) {
-        params.signal.addEventListener("abort", () => {
-          if (aborted) return;
-          aborted = true;
-          _cleanup_handle(http_handle);
-          if (!response_obj) {
-            reject(new DOMException("The operation was aborted."));
-          }
-          else {
-            stream_controller.error("The operation was aborted.");
-          }
-        });
-      }
-  
-      let stream = new ReadableStream({
-        start(controller) {
-          stream_controller = controller;
-        }
-      });
-      
-      let data_callback = (new_data) => {
-        try {
-          stream_controller.enqueue(new_data);  
-        }
-        catch (e) {
-          //the readable stream has been closed elsewhere, so cancel the request
-          if (e instanceof TypeError) {
-            _cleanup_handle(http_handle);
-          }
-          else {
-            throw e;
-          }
-        }
-      }
-      let headers_callback = () => {
+      let headers_callback = (stream) => {
         let response_json = c_func_str(_http_get_info, [http_handle]);
-        response_obj = this.constructor.create_response(stream, JSON.parse(response_json));
-        resolve(response_obj);
+        let response = this.constructor.create_response(stream, JSON.parse(response_json));
+        
+        if (params.redirect === "error" && response.status >= 300 && response.status < 400) {
+          finish_callback(-2);
+          return;
+        }
+        resolve(response);
       }
       let finish_callback = (error) => {
-        if (error != 0) {
+        if (error > 0) {
           error_msg(`Request "${url}" failed with error code ${error}: ${get_error_str(error)}`);
           reject(`Request failed with error code ${error}: ${get_error_str(error)}`);
         }
-        try {
-          stream_controller.close();
-        } //this will only fail if the stream is already errored or closed, which isn't a problem
-        catch {}
+        else if (error === -1) {
+          reject(new DOMException("The operation was aborted."));
+        }
+        else if (error === -2) {
+          reject("Request failed because redirects were disallowed.");
+        }
         this.remove_request(http_handle);
       }
   
       let body_length = body ? body.length : 0;
       let params_json = JSON.stringify(params);
       
-      http_handle = this.create_request(url, data_callback, finish_callback, headers_callback);
+      http_handle = this.stream_response(url, headers_callback, finish_callback, params.signal);
       c_func(_http_set_options, [http_handle, params_json, body, body_length]);
       this.start_request(http_handle);
     });

client/javascript/main.js

@@ -41,82 +41,6 @@ function check_loaded(check_websocket) {
     throw new Error("websocket proxy url not set, please call libcurl.set_websocket");
   }
 }
-
-//wrap perform_request in a promise
-function perform_request_async(url, params, body) {
-  return new Promise((resolve, reject) => {
-    let stream_controller;
-    let http_handle;
-    let response_obj;
-    let aborted = false;
-
-    //handle abort signals
-    if (params.signal instanceof AbortSignal) {
-      params.signal.addEventListener("abort", () => {
-        if (aborted) return;
-        aborted = true;
-        _cleanup_handle(http_handle);
-        if (!response_obj) {
-          reject(new DOMException("The operation was aborted."));
-        }
-        else {
-          stream_controller.error("The operation was aborted.");
-        }
-      });
-    }
-
-    let stream = new ReadableStream({
-      start(controller) {
-        stream_controller = controller;
-      }
-    });
-    
-    function data_callback(new_data) {
-      try {
-        stream_controller.enqueue(new_data);  
-      }
-      catch (e) {
-        //the readable stream has been closed elsewhere, so cancel the request
-        if (e instanceof TypeError) {
-          _cleanup_handle(http_handle);
-        }
-        else {
-          throw e;
-        }
-      }
-    }
-    function headers_callback() {
-      let response_json = c_func_str(_http_get_info, [http_handle]);
-      response_obj = create_response(stream, JSON.parse(response_json));
-      resolve(response_obj);
-    }
-    function finish_callback(error) {
-      if (error != 0) {
-        error_msg(`Request "${url}" failed with error code ${error}: ${get_error_str(error)}`);
-        reject(`Request failed with error code ${error}: ${get_error_str(error)}`);
-        return;
-      }
-      try {
-        stream_controller.close();
-      } //this will only fail if the stream is already errored or closed, which isn't a problem
-      catch {}
-    }
-
-    let body_length = body ? body.length : 0;
-    let params_json = JSON.stringify(params);
-    
-    http_handle = create_request(url, data_callback, finish_callback, headers_callback);
-    c_func(_http_set_options, [http_handle, params_json, body, body_length]);
-    start_request(http_handle);
-  });
-}
-
-async function libcurl_fetch(url, params={}) {
-  check_loaded(true);
-  let body = await create_options(params);
-  return await perform_request_async(url, params, body);
-}
-
 function set_websocket_url(url) {
   websocket_url = url;
   if (Module.websocket) {
@@ -177,7 +101,9 @@ api = {
   WebSocket: WebSocket,
   CurlWebSocket: CurlWebSocket,
   TLSSocket: TLSSocket,
-  fetch: () => {throw "not ready"},
+  HTTPSession: HTTPSession,
+  FTPSession: FTPSession,
+  fetch() {throw "not ready"},
   
   get copyright() {return copyright_notice},
   get version() {return get_version()},

client/javascript/session.js

@@ -29,7 +29,6 @@ class CurlSession {
     let end_callback = (error) => {
       Module.removeFunction(end_callback_ptr);
       Module.removeFunction(data_callback_ptr);
-      Module.removeFunction(headers_callback_ptr);
       
       this.active_requests--;
       js_end_callback(error);
@@ -41,12 +40,14 @@ class CurlSession {
       js_data_callback(chunk);
     }
   
-    let headers_callback = () => {
-      js_headers_callback();
+    let headers_callback = (chunk_ptr, chunk_size) => {
+      let data = Module.HEAPU8.subarray(chunk_ptr, chunk_ptr + chunk_size);
+      let chunk = new Uint8Array(data);
+      js_headers_callback(chunk);
     }
   
     end_callback_ptr = Module.addFunction(end_callback, "vi");
-    headers_callback_ptr = Module.addFunction(headers_callback, "v");
+    headers_callback_ptr = Module.addFunction(headers_callback, "vii");
     data_callback_ptr = Module.addFunction(data_callback, "vii");
     let request_ptr = c_func(_create_request, [url, data_callback_ptr, end_callback_ptr, headers_callback_ptr]);
     
@@ -78,6 +79,7 @@ class CurlSession {
     this.event_loop = setInterval(() => {
       let libcurl_active = _session_get_active(this.session_ptr);
       if (libcurl_active || this.active_requests) {
+        console.log("test");
         _session_perform(this.session_ptr);
       }
       else {
@@ -95,4 +97,65 @@ class CurlSession {
     _session_cleanup(this.session_ptr);
     this.session_ptr = null;
   }
+
+  //wrap request callbacks using a readable stream and return the new callbacks
+  stream_response(url, headers_callback, end_callback, abort_signal) {
+    let stream_controller;
+    let aborted = false;
+    let headers_received = false;
+
+    let stream = new ReadableStream({
+      start(controller) {
+        stream_controller = controller;
+      }
+    });
+
+    if (abort_signal instanceof AbortSignal) {
+      abort_signal.addEventListener("abort", () => {
+        if (aborted) return;
+        aborted = true;
+        if (headers_received) {
+          stream_controller.error("The operation was aborted.");
+        }
+        real_abort_callback();
+      });
+    }
+
+    let real_data_callback = (new_data) => {
+      if (!headers_received) {
+        headers_received = true;
+        headers_callback(stream);
+      }
+
+      try {
+        stream_controller.enqueue(new_data);
+      }
+      catch (e) {
+        //the readable stream has been closed elsewhere, so cancel the request
+        if (e instanceof TypeError) {
+          finish_callback(-1);
+        }
+        else {
+          throw e;
+        }
+      }
+    }
+
+    let real_end_callback = (error) => {
+      if (!headers_received && error === 0) {
+        headers_received = true;
+        headers_callback(stream);
+      }
+
+      if (error != 0) {
+        try {
+          stream_controller.close();
+        }
+        catch {}
+      }
+      end_callback(error);
+    }
+
+    return this.create_request(url, real_data_callback, real_end_callback, () => {});
+  }
 }

client/javascript/tls_socket.js

@@ -22,11 +22,8 @@ class TLSSocket extends CurlSession {
   }
 
   connect() {
-    let response_info;
     let data_callback = () => {};
-    let headers_callback = (info) => {
-      response_info = info;
-    }
+    let headers_callback = () => {};
     let finish_callback = (error) => {
       if (error === 0) {
         this.connected = true;

client/libcurl/ftp.c

@@ -3,7 +3,16 @@
 #include "types.h"
 #include "util.h"
 
-void ftp_set_options(CURL* http_handle, const char* url, int no_body) {
-  curl_easy_setopt(http_handle, CURLOPT_NOBODY, (long) no_body);
-  curl_easy_setopt(http_handle, CURLOPT_URL, url);
+void ftp_set_options(CURL* easy_handle) {
+  curl_easy_setopt(easy_handle, CURLOPT_VERBOSE, 1L);
+}
+
+void ftp_set_cmd(CURL* easy_handle, const char* cmd) {
+  struct curl_slist *cmd_list = NULL;
+  cmd_list = curl_slist_append(cmd_list, cmd);
+  
+  curl_easy_setopt(easy_handle, CURLOPT_QUOTE, cmd_list);
+  curl_easy_setopt(easy_handle, CURLOPT_NOBODY, 1L);
+
+  curl_slist_free_all(cmd_list);
 }

client/libcurl/request.c

@@ -17,18 +17,18 @@ void forward_headers(struct RequestInfo *request_info);
 struct curl_blob cacert_blob;
 
 size_t write_function(char *data, size_t size, size_t nmemb, struct RequestInfo *request_info) {
-  //this should be in the write callback rather than curl's header callback because
-  //the write function will only be called after redirects
-  if (!request_info->headers_received) {
-    forward_headers(request_info);
-  }
-
   size_t real_size = size * nmemb;
   (*request_info->data_callback)(data, real_size);
   return real_size;
 }
 
-CURL* create_request(const char* url, DataCallback data_callback, EndCallback end_callback, HeadersCallback headers_callback) {
+size_t header_function(char *data, size_t size, size_t nmemb, struct RequestInfo *request_info) {
+  size_t real_size = size * nmemb;
+  (*request_info->headers_callback)(data, real_size);
+  return real_size;
+}
+
+CURL* create_request(const char* url, DataCallback data_callback, EndCallback end_callback, DataCallback headers_callback) {
   CURL *http_handle = curl_easy_init();  
 
   //create request metadata struct
@@ -36,7 +36,6 @@ CURL* create_request(const char* url, DataCallback data_callback, EndCallback en
   request_info->http_handle = http_handle;
   request_info->curl_msg = NULL;
   request_info->headers_list = NULL;
-  request_info->headers_received = 0;
   request_info->end_callback = end_callback;
   request_info->data_callback = data_callback;
   request_info->headers_callback = headers_callback;
@@ -49,12 +48,11 @@ CURL* create_request(const char* url, DataCallback data_callback, EndCallback en
   curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, &write_function);
   curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, request_info);
 
-  return http_handle;
-}
+  //callback which runs on every response header
+  curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION, &header_function);
+  curl_easy_setopt(http_handle, CURLOPT_HEADERDATA, request_info);
   
-void forward_headers(struct RequestInfo *request_info) {
-  request_info->headers_received = 1;
-  (*request_info->headers_callback)();
+  return http_handle;
 }
 
 void finish_request(CURLMsg *curl_msg) {
@@ -62,9 +60,6 @@ void finish_request(CURLMsg *curl_msg) {
   struct RequestInfo *request_info = get_request_info(http_handle);
 
   int error = (int) curl_msg->data.result;
-  if (!request_info->headers_received && error == 0) {
-    forward_headers(request_info);
-  }
 
   //clean up curl
   if (request_info->headers_list != NULL) {

client/libcurl/types.h

@@ -2,16 +2,14 @@
 
 typedef void(*DataCallback)(char* chunk_ptr, int chunk_size);
 typedef void(*EndCallback)(int error);
-typedef void(*HeadersCallback)();
 
 struct RequestInfo {
   CURL* http_handle;
-  int headers_received;
   struct CURLMsg *curl_msg;
   struct curl_slist* headers_list;
   DataCallback data_callback;
+  DataCallback headers_callback;
   EndCallback end_callback;
-  HeadersCallback headers_callback;
 };
 
 struct WSResult {