feat(infra): Update infra and add support for teams to SST (#186)

## Description
- [x] Adds support for AWS SSO, which makes us (the team) able to use
SST and update the components independently
- [x] Splits the webpage into the landing page (Qwik), and Astro (the
console) in charge of playing. This allows us to pass in Environment
Variables to the console
- ~Migrates the docs from Nuxt to Nextjs, and connects them to SST. This
allows us to use Fumadocs _citation needed_ that's much more beautiful,
and supports OpenApi~
- Cloudflare pages with github integration is not working on our new CF
account. So we will have to push the pages deployment manually with
Github actions
- [x] Moves the current set up from my personal CF and AWS accounts to
dedicated Nestri accounts -

## Related Issues
<!-- List any related issues (e.g., "Closes #123", "Fixes #456") -->

## Type of Change

- [ ] Bug fix (non-breaking change)
- [x] New feature (non-breaking change)
- [ ] Breaking change (fix or feature that changes existing
functionality)
- [x] Documentation update
- [ ] Other (please describe):

## Checklist

- [x] I have updated relevant documentation
- [x] My code follows the project's coding style
- [x] My changes generate no new warnings/errors

## Notes for Reviewers
<!-- Point out areas you'd like reviewers to focus on, questions you
have, or decisions that need discussion -->
Please approve my PR 🥹


## Screenshots/Demo
<!-- If applicable, add screenshots or a GIF demo of your changes
(especially for UI changes) -->

## Additional Context
<!-- Add any other context about the pull request here -->
This commit is contained in:
Wanjohi
2025-02-27 18:52:05 +03:00
committed by GitHub
parent 237e016b2d
commit 457aac2258
138 changed files with 4218 additions and 2579 deletions

View File

@@ -1,54 +1,82 @@
import { authFingerprintKey } from "./auth";
import { bus } from "./bus";
import { database } from "./database";
import { domain } from "./dns";
import { secret } from "./secrets"
// import { party } from "./party"
import { gpuTaskDefinition, ecsCluster } from "./cluster";
import { email } from "./email";
import { secret } from "./secret";
sst.Linkable.wrap(random.RandomString, (resource) => ({
properties: {
value: resource.result,
},
}));
export const urls = new sst.Linkable("Urls", {
properties: {
api: "https://api." + domain,
auth: "https://auth." + domain,
site: $dev ? "http://localhost:4321" : "https://" + domain,
},
});
export const kv = new sst.cloudflare.Kv("CloudflareAuthKV")
export const authFingerprintKey = new random.RandomString(
"AuthFingerprintKey",
{
length: 32,
},
);
export const auth = new sst.cloudflare.Worker("Auth", {
link: [
kv,
urls,
authFingerprintKey,
secret.InstantAdminToken,
secret.InstantAppId,
secret.LoopsApiKey,
secret.GithubClientID,
secret.GithubClientSecret,
secret.DiscordClientID,
secret.DiscordClientSecret,
],
handler: "./packages/functions/src/auth.ts",
url: true,
domain: "auth." + domain
});
export const auth = new sst.aws.Auth("Auth", {
issuer: {
timeout: "3 minutes",
handler: "./packages/functions/src/auth.handler",
link: [
bus,
email,
database,
authFingerprintKey,
secret.PolarSecret,
secret.GithubClientID,
secret.DiscordClientID,
secret.GithubClientSecret,
secret.DiscordClientSecret,
],
permissions: [
{
actions: ["ses:SendEmail"],
resources: ["*"],
},
],
},
domain: {
name: "auth." + domain,
dns: sst.cloudflare.dns(),
},
})
export const api = new sst.cloudflare.Worker("Api", {
export const apiFunction = new sst.aws.Function("ApiFn", {
handler: "packages/functions/src/api/index.handler",
link: [
bus,
urls,
ecsCluster,
gpuTaskDefinition,
authFingerprintKey,
secret.LoopsApiKey,
secret.InstantAppId,
secret.AwsAccessKey,
secret.AwsSecretKey,
secret.InstantAdminToken,
database,
secret.PolarSecret,
],
url: true,
handler: "./packages/functions/src/api/index.ts",
domain: "api." + domain
timeout: "3 minutes",
streaming: !$dev,
url: true
})
export const api = new sst.aws.Router("Api", {
routes: {
"/*": apiFunction.url
},
domain: {
name: "api." + domain,
dns: sst.cloudflare.dns(),
},
})
export const outputs = {
auth: auth.url,
api: api.url
}
api: api.url,
};

View File

@@ -1,12 +0,0 @@
export const authFingerprintKey = new random.RandomString(
"AuthFingerprintKey",
{
length: 32,
},
);
sst.Linkable.wrap(random.RandomString, (resource) => ({
properties: {
value: resource.result,
},
}));

17
infra/bus.ts Normal file
View File

@@ -0,0 +1,17 @@
import { database } from "./database";
import { email } from "./email";
import { allSecrets } from "./secret";
export const bus = new sst.aws.Bus("Bus");
bus.subscribe("Event", {
handler: "./packages/functions/src/event/event.handler",
link: [database, email, ...allSecrets],
timeout: "5 minutes",
permissions: [
{
actions: ["ses:SendEmail"],
resources: ["*"],
},
],
});

View File

@@ -1,155 +0,0 @@
import { sshKey } from "./ssh";
import { authFingerprintKey } from "./auth";
export const ecsCluster = new aws.ecs.Cluster("NestriGPUCluster", {
name: "NestriGPUCluster",
});
const ecsInstanceRole = new aws.iam.Role("NestriGPUInstanceRole", {
name: "GPUAssumeRoleProd",
assumeRolePolicy: JSON.stringify({
Version: "2012-10-17",
Statement: [{
Action: "sts:AssumeRole",
Principal: {
Service: "ec2.amazonaws.com",
},
Effect: "Allow",
Sid: "",
}],
}),
});
new aws.iam.RolePolicyAttachment("NestriGPUInstancePolicyAttachment", {
role: ecsInstanceRole.name,
policyArn: "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role",
});
const ecsInstanceProfile = new aws.iam.InstanceProfile("NestriGPUInstanceProfile", {
role: ecsInstanceRole.name,
});
// const server = new aws.ec2.Instance("NestriGPU", {
// instanceType: aws.ec2.InstanceType.G4dn_XLarge,
// ami: "ami-046a6af96ef510bb6",//Fedora cloud
// keyName: sshKey.keyName,
// instanceMarketOptions: {
// marketType: "spot",
// spotOptions: {
// maxPrice: "0.2",
// spotInstanceType: "persistent",
// instanceInterruptionBehavior: "stop"
// },
// },
// iamInstanceProfile: ecsInstanceProfile,
// });
const logGroup = new aws.cloudwatch.LogGroup("NestriGPULogGroup", {
name: "/ecs/nestri-gpu-prod",
retentionInDays: 7,
});
// Create a Task Definition for the ECS service to test it
export const gpuTaskDefinition = new aws.ecs.TaskDefinition("NestriGPUTask", {
family: "NestriGPUTaskProd",
requiresCompatibilities: ["EC2"],
volumes: [
{
name: "host",
hostPath: "/mnt/"
// efsVolumeConfiguration: {
// fileSystemId: storage.id,
// authorizationConfig: { accessPointId: storage.accessPoint },
// transitEncryption: "ENABLED",
// }
}
],
containerDefinitions: authFingerprintKey.result.apply(v => JSON.stringify([{
"essential": true,
"name": "nestri",
"memory": 1024,
"cpu": 200,
"gpu": 1,
"image": "ghcr.io/nestrilabs/nestri/runner:nightly",
"environment": [
{
"name": "RESOLUTION",
"value": "1920x1080"
},
{
"name": "AUTH_FINGERPRINT",
"value": v
},
{
"name": "FRAMERATE",
"value": "60"
},
{
"name": "NESTRI_ROOM",
"value": "aws-testing"
},
{
"name": "RELAY_URL",
"value": "https://relay.dathorse.com"
},
{
"name": "NESTRI_PARAMS",
"value": "--verbose=true --video-codec=h264 --video-bitrate=4000 --video-bitrate-max=6000 --gpu-card-path=/dev/dri/card0"
},
],
"mountPoints": [{ "containerPath": "/home/nestri", "sourceVolume": "host" }],
"disableNetworking": false,
"linuxParameter": {
"sharedMemorySize": 5120
},
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "/ecs/nestri-gpu-prod",
"awslogs-region": "us-east-1",
"awslogs-stream-prefix": "nestri-gpu-task"
}
}
}]))
});
sst.Linkable.wrap(aws.ecs.TaskDefinition, (resource) => ({
properties: {
value: resource.arn,
},
}));
sst.Linkable.wrap(aws.ecs.Cluster, (resource) => ({
properties: {
value: resource.arn,
},
}));
// userData: $interpolate`#!/bin/bash
// sudo rm /etc/sysconfig/docker
// echo DAEMON_MAXFILES=1048576 | sudo tee -a /etc/sysconfig/docker
// echo DAEMON_PIDFILE_TIMEOUT=10 | sud o tee -a /etc/sysconfig/docker
// echo OPTIONS="--default-ulimit nofile=32768:65536" | sudo tee -a /etc/sysconfig/docker
// sudo tee "/etc/docker/daemon.json" > /dev/null <<EOF
// {
// "default-runtime": "nvidia",
// "runtimes": {
// "nvidia": {
// "path": "/usr/bin/nvidia-container-runtime",
// "runtimeArgs": []
// }
// }
// }
// EOF
// sudo systemctl restart docker
// echo ECS_CLUSTER='${ecsCluster.name}' | sudo tee -a /etc/ecs/ecs.config
// echo ECS_ENABLE_GPU_SUPPORT=true | sudo tee -a /etc/ecs/ecs.config
// echo ECS_CONTAINER_STOP_TIMEOUT=3h | sudo tee -a /etc/ecs/ecs.config
// echo ECS_ENABLE_SPOT_INSTANCE_DRAINING=true | sudo tee -a /etc/ecs/ecs.config
// `,
// This is used for requesting a container to be deployed on AWS
// const queue = new sst.aws.Queue("PartyQueue", { fifo: true });
// queue.subscribe({ handler: "packages/functions/src/party/subscriber.handler", permissions:{}, link:[taskF]})
// const authRes = $interpolate`${authFingerprintKey.result}`

39
infra/database.ts Normal file
View File

@@ -0,0 +1,39 @@
const dbProject = new neon.Project("Nestri", {
historyRetentionSeconds: 86400,
// name:"Nestri"
})
const dbBranchId = $app.stage !== "production" ?
new neon.Branch("DatabaseBranch", {
parentId: dbProject.defaultBranchId,
projectId: dbProject.id,
name: $app.stage,
}).id : dbProject.defaultBranchId
const dbEndpoint = new neon.Endpoint("NestriEndpoint", {
projectId: dbProject.id,
branchId: dbBranchId
})
const dbRole = new neon.Role("AdminRole", {
name: "admin",
branchId: dbBranchId,
projectId: dbProject.id,
})
const db = new neon.Database("NestriDatabase", {
branchId: dbBranchId,
projectId: dbProject.id,
ownerName: dbRole.name,
name: `nestri-${$app.stage}`,
})
export const database = new sst.Linkable("Database", {
properties: {
name: db.name,
user: dbRole.name,
host: dbEndpoint.host,
password: dbRole.password,
},
});

6
infra/email.ts Normal file
View File

@@ -0,0 +1,6 @@
import { domain } from "./dns";
export const email = new sst.aws.Email("Mail",{
sender: domain,
dns: sst.cloudflare.dns(),
})

View File

@@ -1,33 +0,0 @@
// This is for the websocket/MQTT endpoint that helps the API communicate with the container
// [API] <-> party <-websocket-> container
// The container is it's own this, and can listen to Websocket connections to start or stop a Steam Game
// import { authFingerprintKey } from "./auth";
// import { ecsCluster, gpuTaskDefinition } from "./cluster";
// export const party = new sst.aws.Realtime("Party", {
// authorizer: "packages/functions/src/party/authorizer.handler"
// });
// export const partyFn = new sst.aws.Function("NestriPartyFn", {
// handler: "packages/functions/src/party/create.handler",
// // link: [queue],
// link: [authFingerprintKey],
// environment: {
// TASK_DEFINITION: gpuTaskDefinition.arn,
// // AUTH_FINGERPRINT: authFingerprintKey.result,
// ECS_CLUSTER: ecsCluster.arn,
// },
// permissions: [
// {
// effect: "allow",
// actions: ["ecs:RunTask"],
// resources: [gpuTaskDefinition.arn]
// }
// ],
// url: true,
// });
// export const outputs = {
// partyFunction: partyFn.url
// }

View File

@@ -1,179 +0,0 @@
// const vpc = new sst.aws.Vpc("NestriRelayVpc", { az: 2 })
// import { subnet1, subnet2, securityGroup } from "./vpc"
// const taskExecutionRole = new aws.iam.Role('NestriRelayExecutionRole', {
// assumeRolePolicy: JSON.stringify({
// Version: '2012-10-17',
// Statement: [
// {
// Effect: 'Allow',
// Principal: {
// Service: 'ecs-tasks.amazonaws.com',
// },
// Action: 'sts:AssumeRole',
// },
// ],
// }),
// });
// const taskRole = new aws.iam.Role('NestriRelayTaskRole', {
// assumeRolePolicy: JSON.stringify({
// Version: '2012-10-17',
// Statement: [
// {
// Effect: 'Allow',
// Principal: {
// Service: 'ecs-tasks.amazonaws.com',
// },
// Action: 'sts:AssumeRole',
// },
// ],
// }),
// });
// new aws.cloudwatch.LogGroup('NestriRelayLogGroup', {
// name: '/ecs/nestri-relay',
// retentionInDays: 7,
// });
// new aws.iam.RolePolicyAttachment('NestriRelayExecutionRoleAttachment', {
// policyArn: 'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy',
// role: taskRole,
// });
// const logPolicy = new aws.iam.Policy('NestriRelayLogPolicy', {
// policy: JSON.stringify({
// Version: '2012-10-17',
// Statement: [
// {
// Effect: 'Allow',
// Action: ['logs:CreateLogStream', 'logs:PutLogEvents'],
// Resource: 'arn:aws:logs:*:*:*',
// },
// ],
// }),
// });
// new aws.iam.RolePolicyAttachment('NestriRelayTaskRoleAttachment', {
// policyArn: logPolicy.arn,
// role: taskExecutionRole,
// });
// const taskDefinition = new aws.ecs.TaskDefinition("NestriRelayTask", {
// family: "NestriRelay",
// cpu: "1024",
// memory: "2048",
// networkMode: "awsvpc",
// taskRoleArn: taskRole.arn,
// requiresCompatibilities: ["FARGATE"],
// executionRoleArn: taskExecutionRole.arn,
// containerDefinitions: JSON.stringify([{
// name: "nestri-relay",
// essential: true,
// memory: 2048,
// image: "ghcr.io/nestrilabs/nestri/relay:nightly",
// portMappings: [
// // HTTP port
// {
// protocol: "tcp",
// hostPort: 80,
// containerPort: 80,
// },
// // UDP port range (1,000 ports)
// {
// containerPortRange: "10000-11000",
// protocol: "udp",
// },
// ],
// "environment": [
// {
// name: "ENDPOINT_PORT",
// value: "80"
// },
// ],
// logConfiguration: {
// logDriver: 'awslogs',
// options: {
// 'awslogs-group': '/ecs/nestri-relay',
// 'awslogs-region': 'us-east-1',
// 'awslogs-stream-prefix': 'ecs',
// },
// },
// }]),
// });
// const relayCluster = new aws.ecs.Cluster('NestriRelay');
// new aws.ecs.Service('NestriRelayService', {
// name: 'NestriRelayService',
// cluster: relayCluster.arn,
// desiredCount: 1,
// launchType: 'FARGATE',
// taskDefinition: taskDefinition.arn,
// deploymentCircuitBreaker: {
// enable: true,
// rollback: true,
// },
// enableExecuteCommand: true,
// networkConfiguration: {
// assignPublicIp: true,
// subnets: [subnet1.id, subnet2.id],
// securityGroups: [securityGroup.id],
// },
// });
//FIXME: I cannot create Global Accelerators (Something to do with Quotas - Yet my account is fine)
// const usWest2 = new aws.Provider("GlobalAccelerator", { region: aws.Region.USWest2 })
// const accelerator = new aws.globalaccelerator.Accelerator('Accelerator', {
// name: 'NestriRelayAccelerator',
// enabled: true,
// ipAddressType: 'IPV4',
// }, { provider: usWest2 });
// const httpListener = new aws.globalaccelerator.Listener('TcpListener', {
// acceleratorArn: accelerator.id,
// clientAffinity: 'SOURCE_IP',
// protocol: 'TCP',
// portRanges: [{
// fromPort: 80,
// toPort: 80,
// }],
// }, { provider: usWest2 });
// const udpListener = new aws.globalaccelerator.Listener('UdpListener', {
// acceleratorArn: accelerator.id,
// clientAffinity: 'SOURCE_IP',
// protocol: 'UDP',
// portRanges: [{
// fromPort: 10000,
// toPort: 11000,
// }],
// }, { provider: usWest2 });
// new aws.globalaccelerator.EndpointGroup('TcpRelay', {
// listenerArn: httpListener.id,
// // healthCheckPath: '/',
// endpointGroupRegion: aws.Region.USEast1,
// endpointConfigurations: [{
// clientIpPreservationEnabled: true,
// endpointId: subnet1.id, //vpc.publicSubnets[0].apply(i => i),
// weight: 100,
// }],
// }, { provider: usWest2 });
// new aws.globalaccelerator.EndpointGroup('UdpRelay', {
// listenerArn: udpListener.id,
// // healthCheckPort: 80,
// // healthCheckPath: "/",
// endpointGroupRegion: aws.Region.USEast1,
// endpointConfigurations: [{
// clientIpPreservationEnabled: true,
// endpointId: subnet1.id,//vpc.publicSubnets[0].apply(i => i),
// weight: 100,
// }],
// }, { provider: usWest2 });
// export const outputs = {
// relay: accelerator.dnsName
// }

11
infra/secret.ts Normal file
View File

@@ -0,0 +1,11 @@
export const secret = {
// InstantAppId: new sst.Secret("InstantAppId"),
PolarSecret: new sst.Secret("PolarSecret", process.env.POLAR_API_KEY),
GithubClientID: new sst.Secret("GithubClientID"),
DiscordClientID: new sst.Secret("DiscordClientID"),
GithubClientSecret: new sst.Secret("GithubClientSecret"),
// InstantAdminToken: new sst.Secret("InstantAdminToken"),
DiscordClientSecret: new sst.Secret("DiscordClientSecret"),
};
export const allSecrets = Object.values(secret);

View File

@@ -1,13 +0,0 @@
export const secret = {
LoopsApiKey: new sst.Secret("LoopsApiKey"),
InstantAppId: new sst.Secret("InstantAppId"),
AwsSecretKey: new sst.Secret("AwsSecretKey"),
AwsAccessKey: new sst.Secret("AwsAccessKey"),
GithubClientID: new sst.Secret("GithubClientID"),
DiscordClientID: new sst.Secret("DiscordClientID"),
GithubClientSecret: new sst.Secret("GithubClientSecret"),
InstantAdminToken: new sst.Secret("InstantAdminToken"),
DiscordClientSecret: new sst.Secret("DiscordClientSecret"),
};
export const allSecrets = Object.values(secret);

View File

@@ -1,19 +0,0 @@
import { resolve } from "path";
import { writeFileSync } from "fs";
export const privateKey = new tls.PrivateKey("NestriGPUPrivateKey", {
algorithm: "RSA",
rsaBits: 4096,
});
// Just in case you want to SSH
export const sshKey = new aws.ec2.KeyPair("NestriGPUKey", {
keyName: "NestriGPUKeyProd",
publicKey: privateKey.publicKeyOpenssh
})
export const keyPath = privateKey.privateKeyOpenssh.apply((key) => {
const path = "key_ssh";
writeFileSync(path, key, { mode: 0o600 });
return resolve(path);
});

View File

@@ -1,2 +0,0 @@
export const isPermanentStage =
$app.stage === "production" || $app.stage === "dev";

View File

@@ -1,4 +0,0 @@
// export const vpc = new sst.aws.Vpc("Vpc")
// export const storage = new sst.aws.Efs("GameStorage",{ vpc })
// //

View File

@@ -1,103 +0,0 @@
// export const vpc = new aws.ec2.Vpc('NestriVpc', {
// cidrBlock: '172.16.0.0/16',
// });
// export const subnet1 = new aws.ec2.Subnet('NestriSubnet1', {
// vpcId: vpc.id,
// cidrBlock: '172.16.1.0/24',
// // cidrBlock: '110.0.12.0/22',
// availabilityZone: 'us-east-1a',
// });
// export const subnet2 = new aws.ec2.Subnet('NestriSubnet2', {
// vpcId: vpc.id,
// cidrBlock: '172.16.2.0/24',
// // cidrBlock: '10.0.20.0/22',
// availabilityZone: 'us-east-1b',
// });
// const internetGateway = new aws.ec2.InternetGateway('NestriInternetGateway', {
// vpcId: vpc.id,
// });
// const routeTable = new aws.ec2.RouteTable('NestriRouteTable', {
// vpcId: vpc.id,
// routes: [
// {
// cidrBlock: '0.0.0.0/0',
// gatewayId: internetGateway.id,
// },
// ],
// });
// new aws.ec2.RouteTableAssociation('NestriSubnet1RouteTable', {
// subnetId: subnet1.id,
// routeTableId: routeTable.id,
// });
// new aws.ec2.RouteTableAssociation('NestriSubnet2RouteTable', {
// subnetId: subnet2.id,
// routeTableId: routeTable.id,
// });
// // const vpc = new sst.aws.Vpc("NestriRelayVpc")
// export const securityGroup = new aws.ec2.SecurityGroup("NestriSecurityGroup", {
// vpcId: vpc.id,
// description: "Managed thru SST",
// ingress: [
// {
// protocol: "tcp",
// fromPort: 80,
// toPort: 80,
// cidrBlocks: ["0.0.0.0/0"],
// },
// {
// protocol: "udp",
// fromPort: 10000,
// toPort: 20000,
// cidrBlocks: ["0.0.0.0/0"],
// },
// ],
// egress: [
// {
// protocol: "-1",
// cidrBlocks: ["0.0.0.0/0"],
// fromPort: 0,
// toPort: 0
// }
// ]
// });
// const loadBalancer = new aws.lb.LoadBalancer('NestriVpcLoadBalancer', {
// name: 'NestriVpcLoadBalancer',
// internal: false,
// securityGroups: [securityGroup.id],
// subnets: vpc.publicSubnets
// });
// const targetGroup = new aws.lb.TargetGroup('NestriVpcTargetGroup', {
// name: 'NestriVpcTargetGroup',
// port: 80,
// protocol: 'HTTP',
// targetType: 'ip',
// vpcId: vpc.id,
// healthCheck: {
// path: '/',
// protocol: 'HTTP',
// },
// });
// new aws.lb.Listener('NestriVpcLoadBalancerListener', {
// loadBalancerArn: loadBalancer.arn,
// port: 80,
// protocol: 'HTTP',
// defaultActions: [
// {
// type: 'forward',
// targetGroupArn: targetGroup.arn,
// },
// ],
// });
// // export const subnets = [subnet1, subnet2]

20
infra/www.ts Normal file
View File

@@ -0,0 +1,20 @@
// This is the website part where people play and connect
import { auth, api } from "./api";
import { domain } from "./dns";
new sst.aws.StaticSite("Web", {
path: "./packages/www",
build: {
output: "./dist",
command: "bun run build",
},
domain: {
dns: sst.cloudflare.dns(),
name: "console." + domain
},
environment: {
VITE_API_URL: api.url,
VITE_AUTH_URL: auth.url,
VITE_STAGE: $app.stage,
},
})