more efficient ca cert encoding

client/libcurl/certs.c

@@ -0,0 +1,61 @@
+#include <stdlib.h>
+#include <string.h>
+
+#include "curl/curl.h"
+#include "mbedtls/base64.h"
+
+#include "cacert.h"
+
+struct curl_blob cacert_blob;
+unsigned char* cacert_pem;
+int cacert_pem_len;
+
+unsigned char* get_cacert() {
+  return cacert_pem;
+}
+
+//generate a pem file from the stored binary certificates
+void generate_pem() {
+  char* begin_cert_str = "-----BEGIN CERTIFICATE-----\n";
+  int begin_cert_len = strlen(begin_cert_str);
+  char* end_cert_str = "\n-----END CERTIFICATE-----\n";
+  int end_cert_len = strlen(end_cert_str);
+
+  //calculate total length of the pem file
+  cacert_pem_len = 0;
+  for (int i = 0; i < cert_count; i++) {
+    int cert_len = cert_lengths[i];
+    int b64_len = ((4 * cert_len / 3) + 3) & ~3;
+    cacert_pem_len += begin_cert_len + end_cert_len + b64_len;
+  }
+  cacert_pem = malloc(cacert_pem_len);
+
+  //loop for base64 encoding each part
+  int offset = 0;
+  for (int i = 0; i < cert_count; i++) {
+    unsigned char* cert = _certs[i];
+    int cert_len = cert_lengths[i];
+    int b64_len = ((4 * cert_len / 3) + 3) & ~3;
+    
+    strcpy((char*) (cacert_pem + offset), begin_cert_str);
+    offset += begin_cert_len;
+
+    size_t olen;
+    mbedtls_base64_encode(cacert_pem + offset, b64_len+1, &olen, cert, cert_len);
+    offset += b64_len;
+
+    strcpy((char*) (cacert_pem + offset), end_cert_str);
+    offset += end_cert_len;
+  }
+  cacert_pem[cacert_pem_len-1] = '\0';
+}
+
+void init_curl() {
+  curl_global_init(CURL_GLOBAL_DEFAULT);
+  generate_pem();
+
+  //tell curl to use our generated pem file
+  cacert_blob.data = cacert_pem;
+  cacert_blob.len = cacert_pem_len;
+  cacert_blob.flags = CURL_BLOB_NOCOPY;
+}

client/libcurl/request.c

@@ -7,14 +7,13 @@
 #include "curl/easy.h"
 #include "curl/multi.h"
 
-#include "cacert.h"
 #include "util.h"
 #include "types.h"
 
 void finish_request(CURLMsg *curl_msg);
 void forward_headers(struct RequestInfo *request_info);
 
-struct curl_blob cacert_blob;
+extern struct curl_blob cacert_blob;
 
 size_t write_function(char *data, size_t size, size_t nmemb, struct RequestInfo *request_info) {
   size_t real_size = size * nmemb;
@@ -82,14 +81,3 @@ void finish_request(CURLMsg *curl_msg) {
 void request_set_proxy(CURL* http_handle, const char* proxy_url) {
   curl_easy_setopt(http_handle, CURLOPT_PROXY, proxy_url);
 }
-
-unsigned char* get_cacert() {
-  return _cacert_pem;
-}
-
-void init_curl() {
-  curl_global_init(CURL_GLOBAL_DEFAULT);
-  cacert_blob.data = _cacert_pem;
-  cacert_blob.len = _cacert_pem_len;
-  cacert_blob.flags = CURL_BLOB_NOCOPY;
-}

client/tools/gen_cert.py

@@ -0,0 +1,42 @@
+import sys
+import base64
+import re
+import hashlib
+
+with open(sys.argv[1]) as f:
+  pem_file = f.read()
+
+cert_regex = r'-----BEGIN CERTIFICATE-----\n(.+?)\n-----END CERTIFICATE-----'
+cert_template = "-----BEGIN CERTIFICATE-----\n{b64}\n-----END CERTIFICATE-----"
+certs_b64 = re.findall(cert_regex, pem_file, flags=re.S)
+certs_b64 = [s.replace("\n", "") for s in certs_b64]
+
+certs_str = "\n".join(cert_template.format(b64=s) for s in certs_b64)
+total_len = len(certs_str)
+print(hashlib.sha256(certs_str.encode()).hexdigest(), file=sys.stderr)
+
+header_part_template = """
+static uint8_t _cert_{num}[] = {array};
+"""
+header_end_template = """
+uint8_t* _certs[] = {certs_array};
+uint16_t cert_lengths[] = {lengths_array};
+uint16_t cert_count = {cert_count};
+"""
+
+header_file = "#include <stdint.h>"
+cert_lens = []
+cert_count = len(certs_b64)
+for i, cert_b64 in enumerate(certs_b64):
+  cert = base64.b64decode(cert_b64)
+  cert_lens.append(len(cert))
+  array_str = "{" + ",".join(hex(byte) for byte in cert) + "}"
+  header_file += header_part_template.format(num=i, array=array_str)
+
+header_file += header_end_template.format(
+  certs_array = "{" + ",".join(f"_cert_{i}" for i in range(cert_count)) + "}",
+  lengths_array = "{" + ",".join(str(i) for i in cert_lens) + "}",
+  cert_count=cert_count
+)
+
+print(header_file)

client/tools/generate_cert.sh

@@ -3,25 +3,16 @@
 #export ca certs to a c header file
 
 set -e
+set -x
 
-CURL_PREFIX=$(realpath build/curl-wasm)
+CURL_PREFIX="$(realpath build/curl-wasm)"
 CACERT_FILE="$(realpath build/cacert.pem)"
 CACERT_HEADER="$CURL_PREFIX/include/cacert.h"
 
-CACERT_DIR="$(dirname $CACERT_FILE)"
-REPLACE_STR="$(echo $CACERT_DIR | tr '/-' '_')"
+CACERT_DIR="$(dirname "$CACERT_FILE")"
+REPLACE_STR="$(echo "$CACERT_DIR" | tr '/-' '_')"
 
-if [ ! -f $CACERT_FILE ]; then
+if [ ! -f "$CACERT_FILE" ]; then
   wget "https://curl.se/ca/cacert.pem" -O "$CACERT_FILE"
-  #without this cert open.spotify.com does not work
-  #https://github.com/wolfSSL/wolfssl/issues/8137
-  new_cert="$(curl "https://www.certainly.com/certificates/Certainly_Intermediate_R1.pem")"
-  insert_before="Certainly Root E1"
-  replacement="$(printf "\n$new_cert\n\n$insert_before")"
-
-  cacert_str="$(cat "$CACERT_FILE")"
-  cacert_str="${cacert_str/"$insert_before"/"$replacement"}"
-  echo "$cacert_str" > $CACERT_FILE
+  python3 tools/gen_cert.py "$CACERT_FILE" > "$CACERT_HEADER"
 fi
-xxd -i $CACERT_FILE > $CACERT_HEADER
-sed -i "s/$REPLACE_STR//" $CACERT_HEADER