it's not done but I'm too scared of losing this work to not commit

This commit is contained in:
Spencer Pogorzelski 2023-08-11 19:14:46 -07:00
parent 636cdbf561
commit bcaf437fd3
6 changed files with 498 additions and 39 deletions

View file

@ -29,11 +29,26 @@ export class AdriftBareClient extends Client {
duplex: string | undefined,
signal: AbortSignal | undefined
): Promise<BareResponse> {
let rawResponse = await this.connection.httprequest({ a: "test data" });
if (
body !== null &&
typeof body !== "undefined" &&
typeof body !== "string"
) {
console.log({ body });
throw new Error("bare-client-custom passed an unexpected body type");
}
let rawResponse = await this.connection.httprequest({
method,
requestHeaders,
body,
remote,
cache,
duplex,
});
return new Response(JSON.stringify(rawResponse)) as BareResponse;
}
async connect(
connect(
remote: URL,
protocols: string[],
getRequestHeaders: GetRequestHeadersCallback,

View file

@ -1,6 +1,7 @@
import {
C2SRequestType,
C2SRequestTypes,
HTTPRequestPayload,
S2CRequestType,
S2CRequestTypes,
} from "../protocol";
@ -41,9 +42,11 @@ export default class Connection {
}
}
async send(data: ArrayBuffer | Blob, type: C2SRequestType): Promise<number> {
let requestID = this.counter++;
async send(
requestID: number,
data: ArrayBuffer | Blob,
type: C2SRequestType
): Promise<void> {
let header = new ArrayBuffer(2 + 1);
let view = new DataView(header);
@ -58,17 +61,15 @@ export default class Connection {
this.transport.send(buf);
console.log(buf);
return requestID;
}
httprequest(data: object): Promise<object> {
httprequest(data: HTTPRequestPayload): Promise<object> {
let json = JSON.stringify(data);
return new Promise(async (resolve) => {
let id = this.counter;
let id = ++this.counter;
this.callbacks[id] = resolve;
await this.send(new Blob([json]), C2SRequestTypes.HTTPRequest);
await this.send(id, new Blob([json]), C2SRequestTypes.HTTPRequest);
});
}
}

View file

@ -19,6 +19,7 @@
"express": "^4.18.2",
"express-ws": "^5.0.2",
"firebase": "^10.1.0",
"ipaddr.js": "^2.1.0",
"preact": "^10.16.0",
"ts-node": "^10.9.1",
"wrtc": "^0.4.7"
@ -26,6 +27,7 @@
"devDependencies": {
"@types/express": "^4.17.17",
"@types/express-ws": "^3.0.1",
"@types/node": "^20.4.10",
"nodemon": "^3.0.1"
}
}

59
pnpm-lock.yaml generated
View file

@ -29,12 +29,15 @@ dependencies:
firebase:
specifier: ^10.1.0
version: 10.1.0(react-native@0.72.3)
ipaddr.js:
specifier: ^2.1.0
version: 2.1.0
preact:
specifier: ^10.16.0
version: 10.16.0
ts-node:
specifier: ^10.9.1
version: 10.9.1(@types/node@20.4.9)(typescript@5.1.6)
version: 10.9.1(@types/node@20.4.10)(typescript@5.1.6)
wrtc:
specifier: ^0.4.7
version: 0.4.7
@ -46,6 +49,9 @@ devDependencies:
'@types/express-ws':
specifier: ^3.0.1
version: 3.0.1
'@types/node':
specifier: ^20.4.10
version: 20.4.10
nodemon:
specifier: ^3.0.1
version: 3.0.1
@ -2135,7 +2141,7 @@ packages:
engines: {node: ^8.13.0 || >=10.10.0}
dependencies:
'@grpc/proto-loader': 0.7.8
'@types/node': 20.4.9
'@types/node': 20.4.10
dev: false
/@grpc/proto-loader@0.6.13:
@ -2185,7 +2191,7 @@ packages:
dependencies:
'@jest/fake-timers': 29.6.2
'@jest/types': 29.6.1
'@types/node': 20.4.9
'@types/node': 20.4.10
jest-mock: 29.6.2
dev: false
@ -2195,7 +2201,7 @@ packages:
dependencies:
'@jest/types': 29.6.1
'@sinonjs/fake-timers': 10.3.0
'@types/node': 20.4.9
'@types/node': 20.4.10
jest-message-util: 29.6.2
jest-mock: 29.6.2
jest-util: 29.6.2
@ -2214,7 +2220,7 @@ packages:
dependencies:
'@types/istanbul-lib-coverage': 2.0.4
'@types/istanbul-reports': 3.0.1
'@types/node': 20.4.9
'@types/node': 20.4.10
'@types/yargs': 15.0.15
chalk: 4.1.2
dev: false
@ -2225,7 +2231,7 @@ packages:
dependencies:
'@types/istanbul-lib-coverage': 2.0.4
'@types/istanbul-reports': 3.0.1
'@types/node': 20.4.9
'@types/node': 20.4.10
'@types/yargs': 16.0.5
chalk: 4.1.2
dev: false
@ -2237,7 +2243,7 @@ packages:
'@jest/schemas': 29.6.0
'@types/istanbul-lib-coverage': 2.0.4
'@types/istanbul-reports': 3.0.1
'@types/node': 20.4.9
'@types/node': 20.4.10
'@types/yargs': 17.0.24
chalk: 4.1.2
dev: false
@ -2633,13 +2639,13 @@ packages:
resolution: {integrity: sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==}
dependencies:
'@types/connect': 3.4.35
'@types/node': 20.4.9
'@types/node': 20.4.10
dev: true
/@types/connect@3.4.35:
resolution: {integrity: sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==}
dependencies:
'@types/node': 20.4.9
'@types/node': 20.4.10
dev: true
/@types/escodegen@0.0.7:
@ -2649,7 +2655,7 @@ packages:
/@types/express-serve-static-core@4.17.35:
resolution: {integrity: sha512-wALWQwrgiB2AWTT91CB62b6Yt0sNHpznUXeZEcnPU3DRdlDIz74x8Qg1UUYKSVFi+va5vKOLYRBI1bRKiLLKIg==}
dependencies:
'@types/node': 20.4.9
'@types/node': 20.4.10
'@types/qs': 6.9.7
'@types/range-parser': 1.2.4
'@types/send': 0.17.1
@ -2704,8 +2710,8 @@ packages:
resolution: {integrity: sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==}
dev: true
/@types/node@20.4.9:
resolution: {integrity: sha512-8e2HYcg7ohnTUbHk8focoklEQYvemQmu9M/f43DZVx43kHn0tE3BY/6gSDxS7k0SprtS0NHvj+L80cGLnoOUcQ==}
/@types/node@20.4.10:
resolution: {integrity: sha512-vwzFiiy8Rn6E0MtA13/Cxxgpan/N6UeNYR9oUu6kuJWxu6zCk98trcDp8CBhbtaeuq9SykCmXkFr2lWLoPcvLg==}
/@types/qs@6.9.7:
resolution: {integrity: sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==}
@ -2719,7 +2725,7 @@ packages:
resolution: {integrity: sha512-Cwo8LE/0rnvX7kIIa3QHCkcuF21c05Ayb0ZfxPiv0W8VRiZiNW/WuRupHKpqqGVGf7SUA44QSOUKaEd9lIrd/Q==}
dependencies:
'@types/mime': 1.3.2
'@types/node': 20.4.9
'@types/node': 20.4.10
dev: true
/@types/serve-static@1.15.2:
@ -2727,7 +2733,7 @@ packages:
dependencies:
'@types/http-errors': 2.0.1
'@types/mime': 3.0.1
'@types/node': 20.4.9
'@types/node': 20.4.10
dev: true
/@types/stack-utils@2.0.1:
@ -2741,7 +2747,7 @@ packages:
/@types/ws@8.5.5:
resolution: {integrity: sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg==}
dependencies:
'@types/node': 20.4.9
'@types/node': 20.4.10
dev: true
/@types/yargs-parser@21.0.0:
@ -4082,6 +4088,11 @@ packages:
engines: {node: '>= 0.10'}
dev: false
/ipaddr.js@2.1.0:
resolution: {integrity: sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==}
engines: {node: '>= 10'}
dev: false
/is-arrayish@0.2.1:
resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==}
dev: false
@ -4189,7 +4200,7 @@ packages:
'@jest/environment': 29.6.2
'@jest/fake-timers': 29.6.2
'@jest/types': 29.6.1
'@types/node': 20.4.9
'@types/node': 20.4.10
jest-mock: 29.6.2
jest-util: 29.6.2
dev: false
@ -4219,7 +4230,7 @@ packages:
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
dependencies:
'@jest/types': 29.6.1
'@types/node': 20.4.9
'@types/node': 20.4.10
jest-util: 29.6.2
dev: false
@ -4233,7 +4244,7 @@ packages:
engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0}
dependencies:
'@jest/types': 27.5.1
'@types/node': 20.4.9
'@types/node': 20.4.10
chalk: 4.1.2
ci-info: 3.8.0
graceful-fs: 4.2.11
@ -4245,7 +4256,7 @@ packages:
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
dependencies:
'@jest/types': 29.6.1
'@types/node': 20.4.9
'@types/node': 20.4.10
chalk: 4.1.2
ci-info: 3.8.0
graceful-fs: 4.2.11
@ -4268,7 +4279,7 @@ packages:
resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==}
engines: {node: '>= 10.13.0'}
dependencies:
'@types/node': 20.4.9
'@types/node': 20.4.10
merge-stream: 2.0.0
supports-color: 8.1.1
dev: false
@ -5322,7 +5333,7 @@ packages:
'@protobufjs/pool': 1.1.0
'@protobufjs/utf8': 1.1.0
'@types/long': 4.0.2
'@types/node': 20.4.9
'@types/node': 20.4.10
long: 4.0.0
dev: false
@ -5341,7 +5352,7 @@ packages:
'@protobufjs/path': 1.1.2
'@protobufjs/pool': 1.1.0
'@protobufjs/utf8': 1.1.0
'@types/node': 20.4.9
'@types/node': 20.4.10
long: 5.2.3
dev: false
@ -6035,7 +6046,7 @@ packages:
resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==}
dev: false
/ts-node@10.9.1(@types/node@20.4.9)(typescript@5.1.6):
/ts-node@10.9.1(@types/node@20.4.10)(typescript@5.1.6):
resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==}
hasBin: true
peerDependencies:
@ -6054,7 +6065,7 @@ packages:
'@tsconfig/node12': 1.0.11
'@tsconfig/node14': 1.0.3
'@tsconfig/node16': 1.0.4
'@types/node': 20.4.9
'@types/node': 20.4.10
acorn: 8.10.0
acorn-walk: 8.2.0
arg: 4.1.3

View file

@ -15,3 +15,20 @@ export const S2CRequestTypes = {
WSDataBinary: 3,
} as const;
export type S2CRequestType = ObjectValues<typeof S2CRequestTypes>;
export type ProtoBareHeaders = Record<string, string | string[]>;
export type HTTPRequestPayload = {
method: string;
requestHeaders: ProtoBareHeaders;
body: string | null;
remote: URL;
cache: string | undefined;
duplex: string | undefined;
};
export type HTTPResponsePayload = {
status: number;
statusText: string;
headers: ProtoBareHeaders;
};

View file

@ -1,8 +1,27 @@
import { LookupAddress, LookupAllOptions, lookup } from "dns";
import dotenv from "dotenv";
import EventEmitter from "events";
import express from "express";
import expressWs from "express-ws";
import {
ClientRequest,
Agent as HTTPAgent,
IncomingMessage,
RequestOptions,
STATUS_CODES,
request as httpRequest,
} from "http";
import { Agent as HTTPSAgent, request as httpsRequest } from "https";
import { isValid, parse } from "ipaddr.js";
import { Readable } from "stream";
import * as wrtc from "wrtc";
import { C2SRequestTypes } from "../protocol";
import {
C2SRequestTypes,
HTTPRequestPayload,
HTTPResponsePayload,
ProtoBareHeaders,
S2CRequestTypes,
} from "../protocol";
const configuration = {
iceServers: [
@ -73,7 +92,11 @@ expressWs(app);
app.use(express.json());
app.use((req, res, next) => {
res.header("Access-Control-Allow-Origin", "*");
res.header("x-robots-tag", "noindex");
res.header("access-control-allow-headers", "*");
res.header("access-control-allow-origin", "*");
res.header("access-control-allow-methods", "*");
res.header("access-control-expose-headers", "*");
next();
});
@ -91,11 +114,289 @@ app.post("/connect", (req, res) => {
}
});
const forbiddenForwardHeaders: string[] = [
"connection",
"transfer-encoding",
"host",
"connection",
"origin",
"referer",
];
const forbiddenPassHeaders: string[] = [
"vary",
"connection",
"transfer-encoding",
"access-control-allow-headers",
"access-control-allow-methods",
"access-control-expose-headers",
"access-control-max-age",
"access-control-request-headers",
"access-control-request-method",
];
// common defaults
const defaultForwardHeaders: string[] = ["accept-encoding", "accept-language"];
const defaultPassHeaders: string[] = [
"content-encoding",
"content-length",
"last-modified",
];
// defaults if the client provides a cache key
const defaultCacheForwardHeaders: string[] = [
"if-modified-since",
"if-none-match",
"cache-control",
];
const defaultCachePassHeaders: string[] = ["cache-control", "etag"];
const cacheNotModified = 304;
export interface BareErrorBody {
code: string;
id: string;
message?: string;
stack?: string;
}
export class BareError extends Error {
status: number;
body: BareErrorBody;
constructor(status: number, body: BareErrorBody) {
super(body.message || body.code);
this.status = status;
this.body = body;
}
}
interface BareServerOptions {
logErrors: boolean;
/**
* Callback for filtering the remote URL.
* @returns Nothing
* @throws An error if the remote is bad.
*/
filterRemote?: (remote: Readonly<URL>) => Promise<void> | void;
/**
* DNS lookup
* May not get called when remote.host is an IP
* Use in combination with filterRemote to block IPs
*/
lookup: (
hostname: string,
options: LookupAllOptions,
callback: (
err: NodeJS.ErrnoException | null,
addresses: LookupAddress[],
family: number
) => void
) => void;
localAddress?: string;
family?: number;
httpAgent: HTTPAgent;
httpsAgent: HTTPSAgent;
database: Map<string, string>;
}
export interface Address {
address: string;
family: number;
}
/**
* Converts the address and family of a DNS lookup callback into an array if it wasn't already
*/
export function toAddressArray(address: string | Address[], family?: number) {
if (typeof address === "string")
return [
{
address,
family,
},
] as Address[];
else return address;
}
const options: BareServerOptions = {
logErrors: true,
filterRemote: (url) => {
// if the remote is an IP then it didn't go through the init.lookup hook
// isValid determines if this is so
if (isValid(url.hostname) && parse(url.hostname).range() !== "unicast")
throw new RangeError("Forbidden IP");
},
lookup: (hostname, options, callback) =>
lookup(hostname, options, (err, address, family) => {
if (
address &&
toAddressArray(address, family).some(
({ address }) => parse(address).range() !== "unicast"
)
)
callback(new RangeError("Forbidden IP"), [], -1);
else callback(err, address, family);
}),
httpAgent: new HTTPAgent({
keepAlive: true,
}),
httpsAgent: new HTTPSAgent({
keepAlive: true,
}),
database: new Map<string, string>(),
};
export interface MetaV1 {
v: 1;
response?: {
headers: ProtoBareHeaders;
};
}
export interface MetaV2 {
v: 2;
response?: { status: number; statusText: string; headers: ProtoBareHeaders };
sendHeaders: ProtoBareHeaders;
remote: string;
forwardHeaders: string[];
}
export default interface CommonMeta {
value: MetaV1 | MetaV2;
expires: number;
}
export class JSONDatabaseAdapter {
impl: Map<string, string>;
constructor(impl) {
this.impl = impl;
}
async get(key: string) {
const res = await this.impl.get(key);
if (typeof res === "string") return JSON.parse(res) as CommonMeta;
}
async set(key: string, value: CommonMeta) {
return await this.impl.set(key, JSON.stringify(value));
}
async has(key: string) {
return await this.impl.has(key);
}
async delete(key: string) {
return await this.impl.delete(key);
}
async *[Symbol.asyncIterator]() {
for (const [id, value] of await this.impl.entries()) {
yield [id, JSON.parse(value)] as [string, CommonMeta];
}
}
}
async function cleanupDatabase(database: Map<string, string>) {
const adapter = new JSONDatabaseAdapter(database);
for await (const [id, { expires }] of adapter)
if (expires < Date.now()) database.delete(id);
}
const interval = setInterval(() => cleanupDatabase(options.database), 1000);
function outgoingError<T>(error: T): T | BareError {
if (error instanceof Error) {
switch ((<Error & { code?: string }>error).code) {
case "ENOTFOUND":
return new BareError(500, {
code: "HOST_NOT_FOUND",
id: "request",
message: "The specified host could not be resolved.",
});
case "ECONNREFUSED":
return new BareError(500, {
code: "CONNECTION_REFUSED",
id: "response",
message: "The remote rejected the request.",
});
case "ECONNRESET":
return new BareError(500, {
code: "CONNECTION_RESET",
id: "response",
message: "The request was forcibly closed.",
});
case "ETIMEOUT":
return new BareError(500, {
code: "CONNECTION_TIMEOUT",
id: "response",
message: "The response timed out.",
});
}
}
return error;
}
export async function bareFetch(
request: HTTPRequestPayload,
signal: AbortSignal,
remote: URL,
options: BareServerOptions
): Promise<IncomingMessage> {
if (options.filterRemote) await options.filterRemote(remote);
const req: RequestOptions = {
method: request.method,
headers: request.requestHeaders,
setHost: false,
signal,
localAddress: options.localAddress,
family: options.family,
lookup: options.lookup,
};
let outgoing: ClientRequest;
// NodeJS will convert the URL into HTTP options automatically
// see https://github.com/nodejs/node/blob/e30e71665cab94118833cc536a43750703b19633/lib/internal/url.js#L1277
if (remote.protocol === "https:")
outgoing = httpsRequest(remote, {
...req,
agent: options.httpsAgent,
});
else if (remote.protocol === "http:")
outgoing = httpRequest(remote, {
...req,
agent: options.httpAgent,
});
else throw new RangeError(`Unsupported protocol: '${remote.protocol}'`);
if (request.body) Readable.from([request.body]).pipe(outgoing);
else outgoing.end();
return await new Promise((resolve, reject) => {
outgoing.on("response", (response: IncomingMessage) => {
resolve(response);
});
outgoing.on("upgrade", (req, socket) => {
reject("Remote did not send a response");
socket.destroy();
});
outgoing.on("error", (error: Error) => {
reject(outgoingError(error));
});
});
}
class Client {
send: (msg: Buffer) => void;
events: EventEmitter;
constructor(send) {
this.send = send;
this.events = new EventEmitter();
}
static parseMsgInit(
@ -117,7 +418,9 @@ class Client {
}
}
static parseHttpReqPayload(payloadRaw: Buffer) {
static parseHttpReqPayload(
payloadRaw: Buffer
): HTTPRequestPayload | undefined {
let payload;
try {
payload = JSON.parse(payloadRaw.toString());
@ -128,21 +431,131 @@ class Client {
throw e;
}
console.log({ payload });
return payload;
}
onMsg(msg: Buffer) {
static bareErrorToResponse(e: BareError): {
payload: HTTPResponsePayload;
body: Buffer;
} {
return {
payload: {
status: e.status,
statusText: STATUS_CODES[e.status] || "",
headers: {},
},
body: Buffer.from(JSON.stringify(e.body)),
};
}
async handleHTTPRequest(payload: HTTPRequestPayload): Promise<{
payload: HTTPResponsePayload;
body: Buffer;
}> {
const abort = new AbortController();
const onClose = () => {
abort.abort();
this.events.off("close", onClose);
};
this.events.on("close", onClose);
let resp: IncomingMessage;
try {
resp = await bareFetch(
payload,
abort.signal,
new URL(payload.remote),
options
);
} catch (e) {
if (e instanceof BareError) {
return Client.bareErrorToResponse(e);
}
this.events.off("close", onClose);
throw e;
}
this.events.off("close", onClose);
const buffers: any[] = [];
// node.js readable streams implement the async iterator protocol
for await (const data of resp) {
buffers.push(data);
}
const body = Buffer.concat(buffers);
return {
payload: {
status: resp.statusCode || 500,
statusText: resp.statusMessage || "",
headers: Object.fromEntries(
Object.entries(resp.headersDistinct).filter(([k, v]) => Boolean(v))
) as ProtoBareHeaders,
},
body,
};
}
sendHTTPResponse(seq: number, payload: HTTPResponsePayload, body: Buffer) {
const payloadBuffer = Buffer.from(JSON.stringify(payload));
const buf = Buffer.alloc(2 + 2 + 4 + payloadBuffer.length + body.length);
let cursor = 0;
cursor += buf.writeUInt16BE(seq, cursor);
cursor += buf.writeUInt16BE(S2CRequestTypes.HTTPResponse, cursor);
cursor += buf.writeUInt32BE(payloadBuffer.length, cursor);
cursor += payloadBuffer.copy(buf, cursor);
body.copy(buf, cursor);
this.send(buf);
}
async onMsg(msg: Buffer) {
const init = Client.parseMsgInit(msg);
if (!init) return;
const { cursor, seq, op } = init;
switch (op) {
case C2SRequestTypes.HTTPRequest:
Client.parseHttpReqPayload(msg.subarray(cursor));
let resp;
const reqPayload = Client.parseHttpReqPayload(msg.subarray(cursor));
if (!reqPayload) return;
try {
resp = await this.handleHTTPRequest(reqPayload);
} catch (e) {
if (options.logErrors) console.error(e);
let bareError;
if (e instanceof BareError) {
bareError = e;
} else if (e instanceof Error) {
bareError = new BareError(500, {
code: "UNKNOWN",
id: `error.${e.name}`,
message: e.message,
stack: e.stack,
});
} else {
bareError = new BareError(500, {
code: "UNKNOWN",
id: "error.Exception",
message: "Error: " + e,
stack: new Error(<string | undefined>e).stack,
});
}
resp = Client.bareErrorToResponse(bareError);
}
const { payload, body } = resp;
this.sendHTTPResponse(seq, payload, body);
break;
default:
// not implemented
break;
}
}
onClose() {
this.events.emit("close");
}
}
app.ws("/dev-ws", (ws, req) => {