fix url handling and websockets

client/fragments/wisp_support.js

@@ -3,6 +3,8 @@ ws ?= ?new WebSocketConstructor\(url, ?opts\)
 */
 try {
   let transport;
+  let tls = url.includes("---tls-enabled---");
+  url = url.replace("---tls-enabled---", "");
   if (api.transport === "wisp") {
     transport = new WispWebSocket(url);
   }
@@ -15,8 +17,14 @@ try {
   else { //custom transports
     transport = new api.transport(url);
   }
+  //
+  if (tls) {
     ws = new tls_shim.ReclaimTLSSocket(url, transport);
   }
+  else {
+    ws = transport;
+  }
+}
 catch (e) {
   error_msg("Error while creating a TCP connection: " + e);
   throw e;

client/javascript/http.js

@@ -79,11 +79,15 @@ class HTTPSession extends CurlSession {
         http_handle = null;
       }
 
+      let url_obj = new URL(url);
+      let tls = url_obj.protocol === "https:";
+      params.headers["Host"] = url_obj.host;
+      
       body_ptr = body ? allocate_array(body) : null;
       let body_length = body ? body.length : 0;
       let params_json = JSON.stringify(params);
 
-      http_handle = this.stream_response(url, headers_callback, finish_callback, params.signal);
+      http_handle = this.stream_response(url, headers_callback, finish_callback, params.signal, tls);
       c_func(_http_set_options, [http_handle, params_json, body_ptr, body_length]);
       if (this.cookie_filename && params.credentials !== "omit") {
         c_func(_http_set_cookie_jar, [http_handle, this.cookie_filename]);
@@ -124,8 +128,21 @@ class HTTPSession extends CurlSession {
     }
     check_proxy(params.proxy);
 
+    let redirect_mode = params.redirect;
     let body = await this.constructor.create_options(params);
+    params.redirect = "manual";
+    if (redirect_mode === "manual")
       return await this.request_async(url, params, body);
+    
+    for (let i = 0; i < 20; i++) {
+      let r = await this.request_async(url, params, body);
+      if (r.status !== 201 && (r.status+"")[0] !== "3") 
+        return r;
+      if (redirect_mode === "error") 
+        throw new Error("Too many redirects");
+      url = new URL(r.headers.get("location"), url).href;
+    }
+    throw new Error("Too many redirects");
   }
 
   static create_response(response_data, response_info) {
@@ -134,6 +151,13 @@ class HTTPSession extends CurlSession {
     if (response_info.status === 204 || response_info.status === 205) {
       response_data = null;
     }
+    if (response_info.url.includes("---tls-enabled---")) {
+      let url_obj = new URL(response_info.url);
+      url_obj.hostname = url_obj.hostname.replace("---tls-enabled---", "");
+      url_obj.protocol = "https:";
+      if (url_obj.port === "443") url_obj.port = "";
+      response_info.url = url_obj.href;
+    }
 
     //construct base response object
     let response_obj = new Response(response_data, response_info);

client/javascript/session.js

@@ -51,7 +51,7 @@ class CurlSession {
     this.request_callbacks[request_id].headers(chunk);
   }
 
-  create_request(url, js_data_callback, js_end_callback, js_headers_callback) {
+  create_request(url, js_data_callback, js_end_callback, js_headers_callback, tls=false) {
     this.assert_ready();
     let request_id = this.last_request_id++;
     this.request_callbacks[request_id] = {
@@ -60,6 +60,18 @@ class CurlSession {
       headers: js_headers_callback
     }
 
+    //if tls is enabled, add a flag to the url
+    if (tls) {
+      let url_obj = new URL(url);
+      url_obj.hostname += "---tls-enabled---";
+      if (url_obj.protocol == "https:") url_obj.protocol = "http:";
+      if (url_obj.protocol == "wss:") url_obj.protocol = "ws:";
+      if (!url_obj.port) {
+        url_obj.href = url_obj.href.replace(url_obj.origin, url_obj.origin + ":443");
+      }
+      url = url_obj.href;
+    }
+  
     let request_ptr = c_func(_create_request, [
       url, request_id, this.data_callback_ptr, this.end_callback_ptr, this.headers_callback_ptr
     ]);
@@ -133,7 +145,7 @@ class CurlSession {
   }
 
   //wrap request callbacks using a readable stream and return the new callbacks
-  stream_response(url, headers_callback, end_callback, abort_signal) {
+  stream_response(url, headers_callback, end_callback, abort_signal, tls) {
     let stream_controller;
     let aborted = false;
     let headers_received = false;
@@ -190,6 +202,6 @@ class CurlSession {
       end_callback(error);
     }
 
-    return this.create_request(url, real_data_callback, real_end_callback, () => {});
+    return this.create_request(url, real_data_callback, real_end_callback, () => {}, tls);
   }
 }

client/javascript/tls_socket.js

@@ -43,7 +43,7 @@ class TLSSocket extends CurlSession {
       }
     }
 
-    this.http_handle = this.create_request(this.url, data_callback, finish_callback, headers_callback);
+    this.http_handle = this.create_request(this.url, data_callback, finish_callback, headers_callback, true);
     _tls_socket_set_options(this.http_handle, +this.options.verbose);
     if (this.options.proxy) {
       c_func_str(_request_set_proxy, [this.http_handle, this.options.proxy]);

client/javascript/websocket.js

@@ -56,7 +56,11 @@ class CurlWebSocket extends CurlSession {
       request_options._libcurl_verbose = 1;
     }
 
-    this.http_handle = this.create_request(this.url, data_callback, finish_callback, headers_callback);
+    let url_obj = new URL(this.url);
+    let tls = url_obj.protocol === "wss:";
+    request_options.headers["Host"] = url_obj.host;
+
+    this.http_handle = this.create_request(this.url, data_callback, finish_callback, headers_callback, tls);
     c_func(_http_set_options, [this.http_handle, JSON.stringify(request_options), null, 0]);
     _websocket_set_options(this.http_handle);
     if (this.options.proxy) {

client/tests/run_tests.py

@@ -18,6 +18,7 @@ class JSTest(unittest.TestCase):
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("--disable-browser-side-navigation")
     options.add_argument("--disable-gpu")
+    options.add_argument("--enable-experimental-web-platform-features")
     options.set_capability("goog:loggingPrefs", {"browser": "ALL"})
 
     try:

client/tests/scripts/redirect_out.js

@@ -7,5 +7,5 @@ async function test() {
   libcurl.stderr = out_callback;
   await libcurl.fetch("https://example.com/", {_libcurl_verbose: 1});
   console.log(output);
-  assert(output[0] === "* Host example.com:443 was resolved.", "unexpected output in stderr");
+  assert(output[0].includes("* Host example.com"), "unexpected output in stderr");
 }

client/tls/index.mjs

@@ -68,9 +68,6 @@ export class ReclaimTLSSocket extends EventTarget {
 
     //set up socket callbacks
     this.socket.binaryType = "arraybuffer";
-    this.socket.onopen = () => {
-      this.tls.startHandshake();
-    }
     this.socket.onmessage = (event) => {
       this.tls.handleReceivedBytes(new Uint8Array(event.data));
     }
@@ -82,6 +79,14 @@ export class ReclaimTLSSocket extends EventTarget {
       this.status = this.CLOSED;
       this.emit_event(new Event("error"));
     }
+    if (this.socket.readyState === this.socket.CONNECTING) {
+      this.socket.onopen = () => {
+        this.tls.startHandshake();
+      }  
+    }
+    else {
+      this.tls.startHandshake();
+    }
     this.status = this.CONNECTING;
   }