github-mirrors/netris-nestri
1
0
Fork 0
mirror of https://github.com/nestriness/nestri.git synced 2025-12-12 08:45:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

⭐ feat(core): Implement Steam library sync with metadata extraction and image processing (#278)

Browse Source 
## Description
<!-- Briefly describe the purpose and scope of your changes -->


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Added AWS queue infrastructure and SQS handler for processing Steam
game libraries and images.
- Introduced event-driven handling for new credentials and game
additions, including image uploads to S3.
- Added client functions to fetch Steam user libraries, friends lists,
app info, and related images.
- Added new database columns and schema updates to track game
acquisition, playtime, and family sharing.
  - Added utility function for chunking arrays.
- Added new event notifications for library queue processing and game
creation.
  - Added new lookup functions for categories and teams by slug.
- Introduced a new Team API with endpoints to list and fetch teams by
slug.
  - Added a new Steam library page displaying game images.

- **Enhancements**
  - Improved game creation with event notifications and upsert logic.
  - Enhanced category and team retrieval with new lookup functions.
  - Renamed and refined image categories for clearer classification.
  - Expanded dependencies for image processing and AWS SDK integration.
- Improved image processing utilities with caching, ranking, and
metadata extraction.
  - Refined Steam client utilities for concurrency and error handling.

- **Bug Fixes**
- Fixed event publishing timing and removed deprecated credential
retrieval methods.

- **Chores**
- Updated infrastructure configurations with increased timeouts, memory,
and resource linking.
- Added new dependencies for image processing, caching, and AWS SDK
clients.
  - Refined internal code structure and imports for clarity.
  - Removed Steam provider and related UI components from the frontend.
- Disabled authentication providers and Steam-related routes in the
frontend.
  - Updated API fetch handler to accept environment bindings.

- **Refactor**
- Simplified query result handling and renamed functions for better
clarity.
- Removed outdated event handler in favor of consolidated event
subscriber.
- Consolidated and simplified database relationships and permission
queries.

- **Tests**
  - No explicit test changes included in this release.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Wanjohi
2025-05-17 00:51:18 +03:00
committed by GitHub
parent cc2065299d
commit e1a903a7c9
82 changed files with 7819 additions and 1002 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
packages/functions/package.json
View File

@@ -16,6 +16,9 @@
},
"dependencies": {
"@actor-core/bun": "^0.8.0",
"@actor-core/file-system": "^0.8.0",
"@aws-sdk/client-s3": "^3.806.0",
"@aws-sdk/client-sqs": "^3.806.0",
"@nestri/core": "workspace:",
"actor-core": "^0.8.0",
"hono": "^4.7.8",

1
packages/functions/src/api/friend.ts
View File

@@ -88,7 +88,6 @@ export namespace FriendApi {
return c.json({
data: friend
})
}
)
}

8
packages/functions/src/api/game.ts
View File

@@ -4,8 +4,8 @@ import { describeRoute } from "hono-openapi";
import { Game } from "@nestri/core/game/index";
import { Examples } from "@nestri/core/examples";
import { Library } from "@nestri/core/library/index";
import { ErrorResponses, notPublic, Result, validator } from "./utils";
import { ErrorCodes, VisibleError } from "@nestri/core/error";
import { ErrorResponses, notPublic, Result, validator } from "./utils";
export namespace GameApi {
export const route = new Hono()
@@ -14,20 +14,20 @@ export namespace GameApi {
describeRoute({
tags: ["Game"],
summary: "List games",
description: "List all the games on a user's library",
description: "List all the games on this user's library",
responses: {
200: {
content: {
"application/json": {
schema: Result(
Game.Info.array().openapi({
description: "All games",
description: "All games in the library",
example: [Examples.Game]
})
),
},
},
description: "Game details"
description: "All games in the library"
},
400: ErrorResponses[400],
404: ErrorResponses[404],

6
packages/functions/src/api/index.ts
View File

@@ -1,11 +1,11 @@
import "zod-openapi/extend";
import { Hono } from "hono";
import { cors } from "hono/cors";
import { GameApi } from "./game";
import { SteamApi } from "./steam";
import { auth } from "./utils/auth";
import { FriendApi } from "./friend";
import { logger } from "hono/logger";
import { type Env, Hono } from "hono";
import { Realtime } from "./realtime";
import { AccountApi } from "./account";
import { openAPISpecs } from "hono-openapi";
@@ -98,8 +98,8 @@ export default {
port: 3001,
idleTimeout: 255,
webSocketHandler: Realtime.webSocketHandler,
fetch: (req: Request) =>
app.fetch(req, undefined, {
fetch: (req: Request,env: Env) =>
app.fetch(req, env, {
waitUntil: (fn) => fn,
passThroughOnException: () => { },
}),

15
packages/functions/src/api/realtime/index.ts
View File

@@ -1,6 +1,11 @@
import { setup } from "actor-core";
import chatRoom from "./actor-core";
import { createRouter } from "@actor-core/bun";
import {
FileSystemGlobalState,
FileSystemActorDriver,
FileSystemManagerDriver,
} from "@actor-core/file-system";
export namespace Realtime {
const app = setup({
@@ -8,7 +13,15 @@ export namespace Realtime {
basePath: "/realtime"
});
const realtimeRouter = createRouter(app);
const fsState = new FileSystemGlobalState("/tmp");
const realtimeRouter = createRouter(app, {
topology: "standalone",
drivers: {
manager: new FileSystemManagerDriver(app, fsState),
actor: new FileSystemActorDriver(fsState),
}
});
export const route = realtimeRouter.router;
export const webSocketHandler = realtimeRouter.webSocketHandler;

129
packages/functions/src/api/steam.ts
View File

@@ -1,17 +1,24 @@
import { z } from "zod";
import { Hono } from "hono";
import crypto from 'crypto';
import { Resource } from "sst";
import { streamSSE } from "hono/streaming";
import { Actor } from "@nestri/core/actor";
import SteamCommunity from "steamcommunity";
import { describeRoute } from "hono-openapi";
import { Steam } from "@nestri/core/steam/index";
import { Team } from "@nestri/core/team/index";
import { Examples } from "@nestri/core/examples";
import { Steam } from "@nestri/core/steam/index";
import { Member } from "@nestri/core/member/index";
import { Client } from "@nestri/core/client/index";
import { Library } from "@nestri/core/library/index";
import { chunkArray } from "@nestri/core/utils/helper";
import { ErrorResponses, validator, Result } from "./utils";
import { Credentials } from "@nestri/core/credentials/index";
import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
import { LoginSession, EAuthTokenPlatformType } from "steam-session";
import type CSteamUser from "steamcommunity/classes/CSteamUser";
const sqs = new SQSClient({});
export namespace SteamApi {
export const route = new Hono()
@@ -157,15 +164,7 @@ export namespace SteamApi {
const community = new SteamCommunity();
community.setCookies(cookies);
const user = await new Promise((res, rej) => {
community.getSteamUser(session.steamID, async (error, user) => {
if (!error) {
res(user)
} else {
rej(error)
}
})
}) as CSteamUser
const user = await Client.getUserInfo({ steamID, cookies })
const wasAdded =
await Steam.create({
@@ -187,28 +186,58 @@ export namespace SteamApi {
})
// Does not matter if the user is already there or has just been created, just store the credentials
await Credentials.create({ refreshToken, id: steamID, username })
await Credentials.create({ refreshToken, steamID, username })
let teamID: string | undefined
if (wasAdded) {
const rawFirst = (user.name ?? username).trim().split(/\s+/)[0] ?? username;
const firstName = rawFirst
.charAt(0) // first character
.toUpperCase() // make it uppercase
+ rawFirst
.slice(1) // rest of the string
.toLowerCase();
if (!!wasAdded) {
// create a team
const teamID = await Team.create({
teamID = await Team.create({
slug: username,
name: `${user.name.split(" ")[0]}'s Team`,
name: firstName,
ownerID: currentUser.userID,
})
// Add us as a member
await Actor.provide(
"system",
{ teamID },
async () => {
async () =>
await Member.create({
role: "adult",
userID: currentUser.userID,
steamID
})
})
)
} else {
// Update the owner of the Steam account
await Steam.updateOwner({ userID: currentUser.userID, steamID })
const t = await Actor.provide(
"user",
currentUser,
async () => {
// Get the team associated with this username
const team = await Team.fromSlug(username);
// This should never happen
if (!team) throw Error(`Is Nestri okay???, we could not find the team with this slug ${username}`)
teamID = team.id
return team.id
}
)
console.log("t",t)
console.log("teamID",teamID)
}
await stream.writeSSE({
@@ -216,13 +245,71 @@ export namespace SteamApi {
data: JSON.stringify({ username })
})
//TODO: Get game library
// Get game library in the background
c.executionCtx.waitUntil((async () => {
const games = await Client.getUserLibrary(accessToken);
await stream.close()
// Get a batch of 5 games each
const apps = games?.response?.apps || [];
if (apps.length === 0) {
console.info("[SteamApi] Is Steam okay? No games returned for user:", { steamID });
return
}
resolve()
const chunkedGames = chunkArray(apps, 5);
// Get the batches to the queue
const processQueue = chunkedGames.map(async (chunk) => {
const myGames = chunk.map(i => {
return {
appID: i.appid,
totalPlaytime: i.rt_playtime,
isFamilyShareable: i.exclude_reason === 0,
lastPlayed: new Date(i.rt_last_played * 1000),
timeAcquired: new Date(i.rt_time_acquired * 1000),
isFamilyShared: !i.owner_steamids.includes(steamID) && i.exclude_reason === 0,
}
})
if (teamID) {
const deduplicationId = crypto
.createHash('md5')
.update(`${teamID}_${chunk.map(g => g.appid).join(',')}`)
.digest('hex');
await Actor.provide(
"member",
{
teamID,
steamID,
userID: currentUser.userID
},
async () => {
const payload = await Library.Events.Queue.create(myGames);
await sqs.send(
new SendMessageCommand({
MessageGroupId: teamID,
QueueUrl: Resource.LibraryQueue.url,
MessageBody: JSON.stringify(payload),
MessageDeduplicationId: deduplicationId,
})
)
}
)
}
})
const settled = await Promise.allSettled(processQueue)
settled
.filter(r => r.status === "rejected")
.forEach(r => console.error("[LibraryQueue] enqueue failed:", (r as PromiseRejectedResult).reason));
})())
await stream.close();
resolve();
})
})
})
}

91
packages/functions/src/api/team.ts Normal file
View File

@@ -0,0 +1,91 @@
import { z } from "zod"
import { Hono } from "hono";
import { describeRoute } from "hono-openapi";
import { Team } from "@nestri/core/team/index";
import { Examples } from "@nestri/core/examples";
import { ErrorResponses, Result, validator } from "./utils";
import { ErrorCodes, VisibleError } from "@nestri/core/error";
export namespace TeamApi {
export const route = new Hono()
.get("/",
describeRoute({
tags: ["Team"],
summary: "List user teams",
description: "List the current user's team details",
responses: {
200: {
content: {
"application/json": {
schema: Result(
Team.Info.array().openapi({
description: "All team information",
example: [Examples.Team]
})
),
},
},
description: "All team details"
},
400: ErrorResponses[400],
404: ErrorResponses[404],
429: ErrorResponses[429],
}
}),
async (c) =>
c.json({
data: await Team.list()
})
)
.get("/:slug",
describeRoute({
tags: ["Team"],
summary: "Get team by slug",
description: "Get the current user's team details, by its slug",
responses: {
200: {
content: {
"application/json": {
schema: Result(
Team.Info.openapi({
description: "Team details",
example: Examples.Team
})
),
},
},
description: "Team details"
},
400: ErrorResponses[400],
404: ErrorResponses[404],
429: ErrorResponses[429],
}
}),
validator(
"param",
z.object({
slug: z.string().openapi({
description: "SLug of the team to get",
example: Examples.Team.slug,
}),
}),
),
async (c) => {
const teamSlug = c.req.valid("param").slug
const team = await Team.fromSlug(teamSlug)
if (!team) {
throw new VisibleError(
"not_found",
ErrorCodes.NotFound.RESOURCE_NOT_FOUND,
`Team ${teamSlug} not found`
)
}
return c.json({
data: team
})
}
)
}

10
packages/functions/src/auth/index.ts
View File

@@ -1,10 +1,10 @@
import { Resource } from "sst"
import { Resource } from "sst";
import { type Env } from "hono";
import { PasswordUI, Select } from "./ui";
import { logger } from "hono/logger";
import { subjects } from "../subjects"
import { subjects } from "../subjects";
import { PasswordUI, Select } from "./ui";
import { issuer } from "@openauthjs/openauth";
import { User } from "@nestri/core/user/index"
import { User } from "@nestri/core/user/index";
import { Email } from "@nestri/core/email/index";
import { patchLogger } from "../utils/patch-logger";
import { handleDiscord, handleGithub } from "./utils";
@@ -26,7 +26,7 @@ const app = issuer({
logo: "https://nestri.io/logo.webp",
favicon: "https://nestri.io/seo/favicon.ico",
background: {
light: "#f5f5f5 ",
light: "#F5F5F5",
dark: "#171717"
},
radius: "lg",

68
packages/functions/src/event/event.ts
View File

@@ -1,68 +0,0 @@
import SteamID from "steamid"
import { bus } from "sst/aws/bus";
import SteamCommunity from "steamcommunity";
import { User } from "@nestri/core/user/index";
import { Steam } from "@nestri/core/steam/index";
import { Friend } from "@nestri/core/friend/index";
import { Credentials } from "@nestri/core/credentials/index";
import { EAuthTokenPlatformType, LoginSession } from "steam-session";
export const handler = bus.subscriber(
[Credentials.Events.New],
async (event) => {
console.log(event.type, event.properties, event.metadata);
switch (event.type) {
case "new_credentials.added": {
const input = event.properties
const credentials = await Credentials.getByID(input.steamID)
if (credentials) {
const session = new LoginSession(EAuthTokenPlatformType.MobileApp);
session.refreshToken = credentials.refreshToken;
const cookies = await session.getWebCookies()
const community = new SteamCommunity()
community.setCookies(cookies);
//FIXME: use a promise as promises inside callbacks are not awaited
community.getFriendsList((error, allFriends) => {
if (!error) {
const friends = Object.entries(allFriends);
for (const [id, nonce] of friends) {
const friendID = new SteamID(id);
community.getSteamUser(friendID, async (error, user) => {
if (!error) {
const wasAdded =
await Steam.create({
id: friendID.toString(),
name: user.name,
realName: user.realName,
avatarHash: user.avatarHash,
steamMemberSince: user.memberSince,
profileUrl: user.customURL?.trim() || null,
limitations: {
isLimited: user.isLimitedAccount,
isVacBanned: user.vacBanned,
tradeBanState: user.tradeBanState.toLowerCase() as any,
privacyState: user.privacyState as any,
visibilityState: Number(user.visibilityState)
}
})
if (!wasAdded) {
console.log(`steam user ${friendID.toString()} already exists`)
}
await Friend.add({ friendSteamID: friendID.toString(), steamID: input.steamID })
}
})
}
}
});
}
break;
}
}
},
);

117
packages/functions/src/events/index.ts Normal file
View File

@@ -0,0 +1,117 @@
import { Resource } from "sst";
import { bus } from "sst/aws/bus";
import { Steam } from "@nestri/core/steam/index";
import { Client } from "@nestri/core/client/index";
import { Images } from "@nestri/core/images/index";
import { Friend } from "@nestri/core/friend/index";
import { BaseGame } from "@nestri/core/base-game/index";
import { Credentials } from "@nestri/core/credentials/index";
import { EAuthTokenPlatformType, LoginSession } from "steam-session";
import { PutObjectCommand, S3Client, HeadObjectCommand } from "@aws-sdk/client-s3";
const s3 = new S3Client({});
export const handler = bus.subscriber(
[Credentials.Events.New, BaseGame.Events.New],
async (event) => {
console.log(event.type, event.properties, event.metadata);
switch (event.type) {
case "new_credentials.added": {
const input = event.properties
const credentials = await Credentials.fromSteamID(input.steamID)
if (credentials) {
const session = new LoginSession(EAuthTokenPlatformType.MobileApp);
session.refreshToken = credentials.refreshToken;
const cookies = await session.getWebCookies();
const friends = await Client.getFriendsList(cookies);
const putFriends = friends.map(async (user) => {
const wasAdded =
await Steam.create({
id: user.steamID.toString(),
name: user.name,
realName: user.realName,
avatarHash: user.avatarHash,
steamMemberSince: user.memberSince,
profileUrl: user.customURL?.trim() || null,
limitations: {
isLimited: user.isLimitedAccount,
isVacBanned: user.vacBanned,
tradeBanState: user.tradeBanState.toLowerCase() as any,
privacyState: user.privacyState as any,
visibilityState: Number(user.visibilityState)
}
})
if (!wasAdded) {
console.log(`Steam user ${user.steamID.toString()} already exists`)
}
await Friend.add({ friendSteamID: user.steamID.toString(), steamID: input.steamID })
})
const settled = await Promise.allSettled(putFriends);
settled
.filter(result => result.status === 'rejected')
.forEach(result => console.warn('[putFriends] failed:', (result as PromiseRejectedResult).reason))
}
break;
}
case "new_game.added": {
const input = event.properties
// Get images and save to s3
const images = await Client.getImages(input.appID);
(await Promise.allSettled(
images.map(async (image) => {
// Put the images into the db
await Images.create({
type: image.type,
imageHash: image.hash,
baseGameID: input.appID,
position: image.position,
fileSize: image.fileSize,
sourceUrl: image.sourceUrl,
dimensions: image.dimensions,
extractedColor: image.averageColor,
});
try {
//Check whether the image already exists
await s3.send(
new HeadObjectCommand({
Bucket: Resource.Storage.name,
Key: `images/${image.hash}`,
})
);
} catch (e) {
// Save to s3 because it doesn't already exist
await s3.send(
new PutObjectCommand({
Bucket: Resource.Storage.name,
Key: `images/${image.hash}`,
Body: image.buffer,
...(image.format && { ContentType: `image/${image.format}` }),
Metadata: {
type: image.type,
appID: input.appID,
}
})
)
}
})
))
.filter(i => i.status === "rejected")
.forEach(r => console.warn("[createImages] failed:", (r as PromiseRejectedResult).reason));
break;
}
}
},
);

87
packages/functions/src/queues/library.ts Normal file
View File

@@ -0,0 +1,87 @@
import { SQSHandler } from "aws-lambda";
import { Actor } from "@nestri/core/actor";
import { Game } from "@nestri/core/game/index";
import { Utils } from "@nestri/core/client/utils";
import { Client } from "@nestri/core/client/index";
import { Library } from "@nestri/core/library/index";
import { BaseGame } from "@nestri/core/base-game/index";
import { Categories } from "@nestri/core/categories/index";
export const handler: SQSHandler = async (event) => {
for (const record of event.Records) {
const parsed = JSON.parse(
record.body,
) as typeof Library.Events.Queue.$payload;
await Actor.provide(
parsed.metadata.actor.type,
parsed.metadata.actor.properties,
async () => {
const processGames = parsed.properties.map(async (game) => {
// First check whether the base_game exists, if not get it
const appID = game.appID.toString()
const exists = await BaseGame.fromID(appID)
if (!exists) {
const appInfo = await Client.getAppInfo(appID);
const tags = appInfo.tags;
await BaseGame.create({
id: appID,
name: appInfo.name,
size: appInfo.size,
score: appInfo.score,
slug: appInfo.slug,
description: appInfo.description,
releaseDate: appInfo.releaseDate,
primaryGenre: appInfo.primaryGenre,
compatibility: appInfo.compatibility,
controllerSupport: appInfo.controllerSupport,
})
if (game.isFamilyShareable) {
tags.push(Utils.createTag("Family Share"))
}
const allCategories = [...tags, ...appInfo.genres, ...appInfo.publishers, ...appInfo.developers]
const uniqueCategories = Array.from(
new Map(allCategories.map(c => [`${c.type}:${c.slug}`, c])).values()
);
const settled = await Promise.allSettled(
uniqueCategories.map(async (cat) => {
//Use a single db transaction to get or set the category
await Categories.create({
type: cat.type, slug: cat.slug, name: cat.name
})
// Use a single db transaction to get or create the game
await Game.create({ baseGameID: appID, categorySlug: cat.slug, categoryType: cat.type })
})
)
settled
.filter(r => r.status === "rejected")
.forEach(r => console.warn("[uniqueCategories] failed:", (r as PromiseRejectedResult).reason));
}
// Add to user's library
await Library.add({
baseGameID: appID,
lastPlayed: game.lastPlayed,
timeAcquired: game.timeAcquired,
totalPlaytime: game.totalPlaytime,
isFamilyShared: game.isFamilyShared,
})
})
const settled = await Promise.allSettled(processGames)
settled
.filter(r => r.status === "rejected")
.forEach(r => console.warn("[processGames] failed:", (r as PromiseRejectedResult).reason));
}
)
}
}