github-mirrors/netris-nestri
1
0
Fork 0
mirror of https://github.com/nestriness/nestri.git synced 2025-12-12 16:55:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

⭐ feat(core): Implement Steam library sync with metadata extraction and image processing (#278)

Browse Source 
## Description
<!-- Briefly describe the purpose and scope of your changes -->


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Added AWS queue infrastructure and SQS handler for processing Steam
game libraries and images.
- Introduced event-driven handling for new credentials and game
additions, including image uploads to S3.
- Added client functions to fetch Steam user libraries, friends lists,
app info, and related images.
- Added new database columns and schema updates to track game
acquisition, playtime, and family sharing.
  - Added utility function for chunking arrays.
- Added new event notifications for library queue processing and game
creation.
  - Added new lookup functions for categories and teams by slug.
- Introduced a new Team API with endpoints to list and fetch teams by
slug.
  - Added a new Steam library page displaying game images.

- **Enhancements**
  - Improved game creation with event notifications and upsert logic.
  - Enhanced category and team retrieval with new lookup functions.
  - Renamed and refined image categories for clearer classification.
  - Expanded dependencies for image processing and AWS SDK integration.
- Improved image processing utilities with caching, ranking, and
metadata extraction.
  - Refined Steam client utilities for concurrency and error handling.

- **Bug Fixes**
- Fixed event publishing timing and removed deprecated credential
retrieval methods.

- **Chores**
- Updated infrastructure configurations with increased timeouts, memory,
and resource linking.
- Added new dependencies for image processing, caching, and AWS SDK
clients.
  - Refined internal code structure and imports for clarity.
  - Removed Steam provider and related UI components from the frontend.
- Disabled authentication providers and Steam-related routes in the
frontend.
  - Updated API fetch handler to accept environment bindings.

- **Refactor**
- Simplified query result handling and renamed functions for better
clarity.
- Removed outdated event handler in favor of consolidated event
subscriber.
- Consolidated and simplified database relationships and permission
queries.

- **Tests**
  - No explicit test changes included in this release.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Wanjohi
2025-05-17 00:51:18 +03:00
committed by GitHub
parent cc2065299d
commit e1a903a7c9
82 changed files with 7819 additions and 1002 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
packages/core/src/base-game/index.ts
View File

@@ -1,9 +1,12 @@
import { z } from "zod";
import { fn } from "../utils";
import { Resource } from "sst";
import { bus } from "sst/aws/bus";
import { Common } from "../common";
import { Examples } from "../examples";
import { eq, isNull, or, and } from "drizzle-orm";
import { createTransaction } from "../drizzle/transaction";
import { createEvent } from "../event";
import { eq, isNull, and } from "drizzle-orm";
import { afterTx, createTransaction, useTransaction } from "../drizzle/transaction";
import { CompatibilityEnum, baseGamesTable, Size, ControllerEnum } from "./base-game.sql";
export namespace BaseGame {
@@ -56,32 +59,53 @@ export namespace BaseGame {
export type Info = z.infer<typeof Info>;
export const Events = {
New: createEvent(
"new_game.added",
z.object({
appID: Info.shape.id,
}),
),
};
export const create = fn(
Info,
(input) =>
createTransaction(async (tx) => {
const results = await tx
await tx
.insert(baseGamesTable)
.values(input)
.onConflictDoUpdate({
target: baseGamesTable.id,
set: {
timeDeleted: null
}
})
await afterTx(async () => {
await bus.publish(Resource.Bus, Events.New, { appID: input.id })
})
return input.id
})
)
export const fromID = fn(
Info.shape.id,
(id) =>
useTransaction(async (tx) =>
tx
.select()
.from(baseGamesTable)
.where(
and(
or(
eq(baseGamesTable.slug, input.slug),
eq(baseGamesTable.id, input.id),
),
eq(baseGamesTable.id, id),
isNull(baseGamesTable.timeDeleted)
)
)
.execute()
if (results.length > 0) return null
await tx
.insert(baseGamesTable)
.values(input)
return input.id
})
.limit(1)
.then(rows => rows.map(serialize).at(0))
)
)
export function serialize(

47
packages/core/src/categories/index.ts
View File

@@ -3,7 +3,7 @@ import { fn } from "../utils";
import { Examples } from "../examples";
import { createSelectSchema } from "drizzle-zod";
import { categoriesTable } from "./categories.sql";
import { createTransaction } from "../drizzle/transaction";
import { createTransaction, useTransaction } from "../drizzle/transaction";
import { eq, isNull, and } from "drizzle-orm";
export namespace Categories {
@@ -52,20 +52,21 @@ export namespace Categories {
InputInfo,
(input) =>
createTransaction(async (tx) => {
const results =
await tx
.select()
.from(categoriesTable)
.where(
and(
eq(categoriesTable.slug, input.slug),
eq(categoriesTable.type, input.type),
isNull(categoriesTable.timeDeleted)
)
const result = await tx
.select()
.from(categoriesTable)
.where(
and(
eq(categoriesTable.slug, input.slug),
eq(categoriesTable.type, input.type),
isNull(categoriesTable.timeDeleted)
)
.execute()
)
.limit(1)
.execute()
.then(rows => rows.at(0))
if (results.length > 0) return null
if (result) return result.slug
await tx
.insert(categoriesTable)
@@ -79,6 +80,26 @@ export namespace Categories {
})
)
export const get = fn(
InputInfo.pick({ slug: true, type: true }),
(input) =>
useTransaction((tx) =>
tx
.select()
.from(categoriesTable)
.where(
and(
eq(categoriesTable.slug, input.slug),
eq(categoriesTable.type, input.type),
isNull(categoriesTable.timeDeleted)
)
)
.limit(1)
.execute()
.then(rows => serialize(rows))
)
)
export function serialize(
input: typeof categoriesTable.$inferSelect[],
): z.infer<typeof Info> {

240
packages/core/src/client/index.ts Normal file
View File

@@ -0,0 +1,240 @@
import type {
AppInfo,
GameTagsResponse,
SteamApiResponse,
GameDetailsResponse,
SteamAppDataResponse,
ImageInfo,
ImageType,
Shot
} from "./types";
import { z } from "zod";
import pLimit from 'p-limit';
import SteamID from "steamid";
import { fn } from "../utils";
import { Utils } from "./utils";
import SteamCommunity from "steamcommunity";
import { Credentials } from "../credentials";
import type CSteamUser from "steamcommunity/classes/CSteamUser";
const requestLimit = pLimit(10); // max concurrent requests
export namespace Client {
export const getUserLibrary = fn(
Credentials.Info.shape.accessToken,
async (accessToken) =>
await Utils.fetchApi<SteamApiResponse>(`https://api.steampowered.com/IFamilyGroupsService/GetSharedLibraryApps/v1/?access_token=${accessToken}&family_groupid=0&include_excluded=true&include_free=true&include_non_games=false&include_own=true`)
)
export const getFriendsList = fn(
Credentials.Info.shape.cookies,
async (cookies): Promise<CSteamUser[]> => {
const community = new SteamCommunity();
community.setCookies(cookies);
const allFriends = await new Promise<Record<string, any>>((resolve, reject) => {
community.getFriendsList((err, friends) => {
if (err) {
return reject(new Error(`Could not get friends list: ${err.message}`));
}
resolve(friends);
});
});
const friendIds = Object.keys(allFriends);
const userPromises: Promise<CSteamUser>[] = friendIds.map(id =>
requestLimit(() => new Promise<CSteamUser>((resolve, reject) => {
const sid = new SteamID(id);
community.getSteamUser(sid, (err, user) => {
if (err) {
return reject(new Error(`Could not get steam user info for ${id}: ${err.message}`));
}
resolve(user);
});
}))
);
const settled = await Promise.allSettled(userPromises)
settled
.filter(r => r.status === "rejected")
.forEach(r => console.warn("[getFriendsList] failed:", (r as PromiseRejectedResult).reason));
return settled.filter(s => s.status === "fulfilled").map(r => (r as PromiseFulfilledResult<CSteamUser>).value);
}
);
export const getUserInfo = fn(
Credentials.Info.pick({ cookies: true, steamID: true }),
async (input) =>
new Promise((resolve, reject) => {
const community = new SteamCommunity()
community.setCookies(input.cookies);
const steamID = new SteamID(input.steamID);
community.getSteamUser(steamID, async (err, user) => {
if (err) {
reject(`Could not get steam user info: ${err.message}`)
} else {
resolve(user)
}
})
}) as Promise<CSteamUser>
)
export const getAppInfo = fn(
z.string(),
async (appid) => {
const [infoData, tagsData, details] = await Promise.all([
Utils.fetchApi<SteamAppDataResponse>(`https://api.steamcmd.net/v1/info/${appid}`),
Utils.fetchApi<GameTagsResponse>("https://store.steampowered.com/actions/ajaxgetstoretags"),
Utils.fetchApi<GameDetailsResponse>(
`https://store.steampowered.com/apphover/${appid}?full=1&review_score_preference=1&pagev6=true&json=1`
),
]);
const tags = tagsData.tags;
const game = infoData.data[appid];
// Guard against an empty string - When there are no genres, Steam returns an empty string
const genres = details.strGenres ? Utils.parseGenres(details.strGenres) : [];
const controllerTag = game.common.controller_support ?
Utils.createTag(`${Utils.capitalise(game.common.controller_support)} Controller Support`) :
Utils.createTag(`Unknown Controller Support`)
const compatibilityTag = Utils.createTag(`${Utils.capitalise(Utils.compatibilityType(game.common.steam_deck_compatibility?.category))} Compatibility`)
const controller = (game.common.controller_support === "partial" || game.common.controller_support === "full") ? game.common.controller_support : "unknown";
const appInfo: AppInfo = {
genres,
gameid: game.appid,
name: game.common.name.trim(),
size: Utils.getPublicDepotSizes(game.depots!),
slug: Utils.createSlug(game.common.name.trim()),
description: Utils.cleanDescription(details.strDescription),
controllerSupport: controller,
releaseDate: new Date(Number(game.common.steam_release_date) * 1000),
primaryGenre: (!!game?.common.genres && !!details.strGenres) ? Utils.getPrimaryGenre(
genres,
game.common.genres!,
game.common.primary_genre!
) : null,
developers: game.common.associations ?
Array.from(
Utils.getAssociationsByTypeWithSlug(
game.common.associations!,
"developer"
)
) : [],
publishers: game.common.associations ?
Array.from(
Utils.getAssociationsByTypeWithSlug(
game.common.associations!,
"publisher"
)
) : [],
compatibility: Utils.compatibilityType(game.common.steam_deck_compatibility?.category),
tags: [
...(game?.common.store_tags ?
Utils.mapGameTags(
tags,
game.common.store_tags!,
) : []),
controllerTag,
compatibilityTag
],
score: Utils.getRating(
details.ReviewSummary.cRecommendationsPositive,
details.ReviewSummary.cRecommendationsNegative
),
};
return appInfo
}
)
export const getImages = fn(
z.string(),
async (appid) => {
const [appData, details] = await Promise.all([
Utils.fetchApi<SteamAppDataResponse>(`https://api.steamcmd.net/v1/info/${appid}`),
Utils.fetchApi<GameDetailsResponse>(
`https://store.steampowered.com/apphover/${appid}?full=1&review_score_preference=1&pagev6=true&json=1`
),
]);
const game = appData.data[appid]?.common;
if (!game) throw new Error('Game info missing');
// 2. Prepare URLs
const screenshotUrls = Utils.getScreenshotUrls(details.rgScreenshots || []);
const assetUrls = Utils.getAssetUrls(game.library_assets_full, appid, game.header_image.english);
const iconUrl = `https://cdn.cloudflare.steamstatic.com/steamcommunity/public/images/apps/${appid}/${game.icon}.jpg`;
//2.5 Get the backdrop buffer and use it to get the best screenshot
const baselineBuffer = await Utils.fetchBuffer(assetUrls.backdrop);
// 3. Download screenshot buffers in parallel
const shots: Shot[] = await Promise.all(
screenshotUrls.map(async url => ({ url, buffer: await Utils.fetchBuffer(url) }))
);
// 4. Score screenshots (or pick single)
const scores =
shots.length === 1
? [{ url: shots[0].url, score: 0 }]
: (await Utils.rankScreenshots(baselineBuffer, shots, {
threshold: 0.08,
}))
// Build url->rank map
const rankMap = new Map<string, number>();
scores.forEach((s, i) => rankMap.set(s.url, i));
// 5. Create tasks for all images
const tasks: Array<Promise<ImageInfo>> = [];
// 5a. Screenshots and heroArt metadata (top 4)
for (const { url, buffer } of shots) {
const rank = rankMap.get(url);
if (rank === undefined || rank >= 4) continue;
const type: ImageType = rank === 0 ? 'heroArt' : 'screenshot';
tasks.push(
Utils.getImageMetadata(buffer).then(meta => ({ ...meta, sourceUrl: url, position: type == "screenshot" ? rank - 1 : rank, type } as ImageInfo))
);
}
// 5b. Asset images
for (const [type, url] of Object.entries({ ...assetUrls, icon: iconUrl })) {
if (!url || type === "backdrop") continue;
tasks.push(
Utils.fetchBuffer(url)
.then(buf => Utils.getImageMetadata(buf))
.then(meta => ({ ...meta, position: 0, sourceUrl: url, type: type as ImageType } as ImageInfo))
);
}
// 5c. Backdrop
tasks.push(
Utils.getImageMetadata(baselineBuffer)
.then(meta => ({ ...meta, position: 0, sourceUrl: assetUrls.backdrop, type: "backdrop" as const } as ImageInfo))
)
// 5d. Box art
tasks.push(
Utils.createBoxArtBuffer(game.library_assets_full, appid)
.then(buf => Utils.getImageMetadata(buf))
.then(meta => ({ ...meta, position: 0, sourceUrl: null, type: 'boxArt' as const }) as ImageInfo)
);
const settled = await Promise.allSettled(tasks)
settled
.filter(r => r.status === "rejected")
.forEach(r => console.warn("[getImages] failed:", (r as PromiseRejectedResult).reason));
// 6. Await all and return
return settled.filter(s => s.status === "fulfilled").map(r => (r as PromiseFulfilledResult<ImageInfo>).value)
}
)
}

344
packages/core/src/client/types.ts Normal file
View File

@@ -0,0 +1,344 @@
export interface SteamApp {
/** Steam application ID */
appid: number;
/** Array of Steam IDs that own this app */
owner_steamids: string[];
/** Name of the game/application */
name: string;
/** Filename of the game's capsule image */
capsule_filename: string;
/** Hash value for the game's icon */
img_icon_hash: string;
/** Reason code for exclusion (0 indicates no exclusion) */
exclude_reason: number;
/** Unix timestamp when the app was acquired */
rt_time_acquired: number;
/** Unix timestamp when the app was last played */
rt_last_played: number;
/** Total playtime in seconds */
rt_playtime: number;
/** Type identifier for the app (1 = game) */
app_type: number;
/** Array of content descriptor IDs */
content_descriptors?: number[];
}
export interface SteamApiResponse {
response: {
apps: SteamApp[];
owner_steamid: string;
};
}
export interface SteamAppDataResponse {
data: Record<string, SteamAppEntry>;
status: string;
}
export interface SteamAppEntry {
_change_number: number;
_missing_token: boolean;
_sha: string;
_size: number;
appid: string;
common: CommonData;
config: AppConfig;
depots: AppDepots;
extended: AppExtended;
ufs: UFSData;
}
export interface CommonData {
associations: Record<string, { name: string; type: string }>;
category: Record<string, string>;
clienticon: string;
clienttga: string;
community_hub_visible: string;
community_visible_stats: string;
content_descriptors: Record<string, string>;
controller_support?: string;
controllertagwizard: string;
gameid: string;
genres: Record<string, string>;
header_image: Record<string, string>;
icon: string;
languages: Record<string, string>;
library_assets: LibraryAssets;
library_assets_full: LibraryAssetsFull;
metacritic_fullurl: string;
metacritic_name: string;
metacritic_score: string;
name: string;
name_localized: Partial<Record<LanguageCode, string>>;
osarch: string;
osextended: string;
oslist: string;
primary_genre: string;
releasestate: string;
review_percentage: string;
review_score: string;
small_capsule: Record<string, string>;
steam_deck_compatibility: SteamDeckCompatibility;
steam_release_date: string;
store_asset_mtime: string;
store_tags: Record<string, string>;
supported_languages: Record<
string,
{
full_audio?: string;
subtitles?: string;
supported?: string;
}
>;
type: string;
}
export interface LibraryAssets {
library_capsule: string;
library_header: string;
library_hero: string;
library_logo: string;
logo_position: LogoPosition;
}
export interface LogoPosition {
height_pct: string;
pinned_position: string;
width_pct: string;
}
export interface LibraryAssetsFull {
library_capsule: ImageSet;
library_header: ImageSet;
library_hero: ImageSet;
library_logo: ImageSet & { logo_position: LogoPosition };
[key: string]: any
}
export interface ImageSet {
image: Record<string, string>;
image2x?: Record<string, string>;
}
export interface SteamDeckCompatibility {
category: string;
configuration: Record<string, string>;
test_timestamp: string;
tested_build_id: string;
tests: Record<string, { display: string; token: string }>;
}
export interface AppConfig {
installdir: string;
launch: Record<
string,
{
executable: string;
type: string;
arguments?: string;
description?: string;
description_loc?: Record<string, string>;
config?: {
betakey: string;
};
}
>;
steamcontrollertemplateindex: string;
steamdecktouchscreen: string;
}
export interface AppDepots {
branches: AppDepotBranches;
privatebranches: Record<string, AppDepotBranches>;
[depotId: string]: DepotEntry
| AppDepotBranches
| Record<string, AppDepotBranches>;
}
export interface DepotEntry {
manifests: {
public: {
download: string;
gid: string;
size: string;
};
};
}
export interface AppDepotBranches {
[branchName: string]: {
buildid: string;
timeupdated: string;
};
}
export interface AppExtended {
additional_dependencies: Array<{
dest_os: string;
h264: string;
src_os: string;
}>;
developer: string;
dlcavailableonstore: string;
homepage: string;
listofdlc: string;
publisher: string;
}
export interface UFSData {
maxnumfiles: string;
quota: string;
savefiles: Array<{
path: string;
pattern: string;
recursive: string;
root: string;
}>;
}
export type LanguageCode =
| "english"
| "french"
| "german"
| "italian"
| "japanese"
| "koreana"
| "polish"
| "russian"
| "schinese"
| "tchinese"
| "brazilian"
| "spanish";
export interface Screenshot {
appid: number;
id: number;
filename: string;
all_ages: string;
normalized_name: string;
}
export interface Category {
strDisplayName: string;
}
export interface ReviewSummary {
strReviewSummary: string;
cReviews: number;
cRecommendationsPositive: number;
cRecommendationsNegative: number;
nReviewScore: number;
}
export interface GameDetailsResponse {
strReleaseDate: string;
strDescription: string;
rgScreenshots: Screenshot[];
rgCategories: Category[];
strGenres?: string;
strFullDescription: string;
strMicroTrailerURL: string;
ReviewSummary: ReviewSummary;
}
// Define the TypeScript interfaces
export interface Tag {
tagid: number;
name: string;
}
export interface TagWithSlug {
name: string;
slug: string;
type: string;
}
export interface StoreTags {
[key: string]: string; // Index signature for numeric string keys to tag ID strings
}
export interface GameTagsResponse {
tags: Tag[];
success: number;
rwgrsn: number;
}
export type GenreType = {
type: 'genre';
name: string;
slug: string;
};
export interface AppInfo {
name: string;
slug: string;
score: number;
gameid: string;
releaseDate: Date;
description: string;
compatibility: "low" | "mid" | "high" | "unknown";
controllerSupport: "partial" | "full" | "unknown";
primaryGenre: string | null;
size: { downloadSize: number; sizeOnDisk: number };
tags: Array<{ name: string; slug: string; type: "tag" }>;
genres: Array<{ type: "genre"; name: string; slug: string }>;
developers: Array<{ name: string; slug: string; type: "developer" }>;
publishers: Array<{ name: string; slug: string; type: "publisher" }>;
}
export type ImageType =
| 'screenshot'
| 'boxArt'
| 'banner'
| 'backdrop'
| 'icon'
| 'logo'
| 'poster'
| 'heroArt';
export interface ImageInfo {
type: ImageType;
position: number;
hash: string;
sourceUrl: string | null;
format?: string;
averageColor: { hex: string; isDark: boolean };
dimensions: { width: number; height: number };
fileSize: number;
buffer: Buffer;
}
export interface CompareOpts {
/** Pixelmatch color threshold (0–1). Default: 0.1 */
threshold?: number;
/** If true, return an image buffer of the diff map. Default: false */
diffOutput?: boolean;
}
export interface CompareResult {
diffRatio: number;
/** Present only if `diffOutput: true` */
diffBuffer?: Buffer;
}
export interface Shot {
url: string;
buffer: Buffer;
}
export interface RankedShot {
url: string;
score: number;
}

422
packages/core/src/client/utils.ts Normal file
View File

@@ -0,0 +1,422 @@
import type {
Tag,
StoreTags,
AppDepots,
GenreType,
LibraryAssetsFull,
DepotEntry,
CompareOpts,
CompareResult,
RankedShot,
Shot,
} from "./types";
import crypto from 'crypto';
import pLimit from 'p-limit';
import { PNG } from 'pngjs';
import pixelmatch from 'pixelmatch';
import { LRUCache } from 'lru-cache';
import sanitizeHtml from 'sanitize-html';
import { Agent as HttpAgent } from 'http';
import { Agent as HttpsAgent } from 'https';
import sharp, { type Metadata } from 'sharp';
import AbortController from 'abort-controller';
import fetch, { RequestInit } from 'node-fetch';
import { FastAverageColor } from 'fast-average-color';
const fac = new FastAverageColor()
// --- Configuration ---
const httpAgent = new HttpAgent({ keepAlive: true, maxSockets: 50 });
const httpsAgent = new HttpsAgent({ keepAlive: true, maxSockets: 50 });
const downloadCache = new LRUCache<string, Buffer>({
max: 100,
ttl: 1000 * 60 * 30, // 30-minute expiry
allowStale: false,
});
const downloadLimit = pLimit(10); // max concurrent downloads
const compareCache = new LRUCache<string, CompareResult>({
max: 50,
ttl: 1000 * 60 * 10, // 10-minute expiry
});
export namespace Utils {
export async function fetchBuffer(url: string, retries = 3): Promise<Buffer> {
if (downloadCache.has(url)) {
return downloadCache.get(url)!;
}
let lastError: Error | null = null;
for (let attempt = 0; attempt < retries; attempt++) {
try {
const controller = new AbortController();
const id = setTimeout(() => controller.abort(), 15_000);
const res = await fetch(url, {
signal: controller.signal,
agent: (_parsed) => _parsed.protocol === 'http:' ? httpAgent : httpsAgent
} as RequestInit);
clearTimeout(id);
if (!res.ok) throw new Error(`Failed to fetch ${url}: ${res.status}`);
const buf = Buffer.from(await res.arrayBuffer());
downloadCache.set(url, buf);
return buf;
} catch (error: any) {
lastError = error as Error;
console.warn(`Attempt ${attempt + 1} failed for ${url}: ${error.message}`);
if (attempt < retries - 1) {
await new Promise(resolve => setTimeout(resolve, 1000 * Math.pow(2, attempt)));
}
}
}
throw lastError || new Error(`Failed to fetch ${url} after ${retries} attempts`);
}
export async function getImageMetadata(buffer: Buffer) {
const hash = crypto.createHash('sha256').update(buffer).digest('hex');
const { width, height, format, size: fileSize } = await sharp(buffer).metadata();
if (!width || !height) throw new Error('Invalid dimensions');
const slice = await sharp(buffer)
.resize({ width: Math.min(width, 256) }) // cheap shrink
.ensureAlpha()
.raw()
.toBuffer();
const pixelArray = new Uint8Array(slice.buffer);
const { hex, isDark } = fac.prepareResult(fac.getColorFromArray4(pixelArray, { mode: "precision" }));
return { hash, format, averageColor: { hex, isDark }, dimensions: { width, height }, fileSize, buffer };
}
// --- Optimized Box Art creation ---
export async function createBoxArtBuffer(
assets: LibraryAssetsFull,
appid: number | string,
logoPercent = 0.9
): Promise<Buffer> {
const base = `https://shared.cloudflare.steamstatic.com/store_item_assets/steam/apps/${appid}`;
const pick = (key: string) => {
const set = assets[key];
const path = set?.image2x?.english || set?.image?.english;
if (!path) throw new Error(`Missing asset for ${key}`);
return `${base}/${path}`;
};
const [bgBuf, logoBuf] = await Promise.all([
downloadLimit(() =>
fetchBuffer(pick('library_hero'))
.catch(error => {
console.error(`Failed to download hero image for ${appid}:`, error);
throw new Error(`Failed to create box art: hero image unavailable`);
}),
),
downloadLimit(() => fetchBuffer(pick('library_logo'))
.catch(error => {
console.error(`Failed to download logo image for ${appid}:`, error);
throw new Error(`Failed to create box art: logo image unavailable`);
}),
),
]);
const bgImage = sharp(bgBuf);
const meta = await bgImage.metadata();
if (!meta.width || !meta.height) throw new Error('Invalid background dimensions');
const size = Math.min(meta.width, meta.height);
const left = Math.floor((meta.width - size) / 2);
const top = Math.floor((meta.height - size) / 2);
const squareBg = bgImage.extract({ left, top, width: size, height: size });
// Resize logo
const logoTarget = Math.floor(size * logoPercent);
const logoResized = await sharp(logoBuf).resize({ width: logoTarget }).toBuffer();
const logoMeta = await sharp(logoResized).metadata();
if (!logoMeta.width || !logoMeta.height) throw new Error('Invalid logo dimensions');
const logoLeft = Math.floor((size - logoMeta.width) / 2);
const logoTop = Math.floor((size - logoMeta.height) / 2);
return await squareBg
.composite([{ input: logoResized, left: logoLeft, top: logoTop }])
.jpeg({ quality: 100 })
.toBuffer();
}
/**
* Fetch JSON from the given URL, with Steam-like headers
*/
export async function fetchApi<T>(url: string, retries = 3): Promise<T> {
let lastError: Error | null = null;
for (let attempt = 0; attempt < retries; attempt++) {
try {
const response = await fetch(url, {
agent: (_parsed) => _parsed.protocol === 'http:' ? httpAgent : httpsAgent,
method: "GET",
headers: {
"User-Agent": "Steam 1291812 / iPhone",
"Accept-Language": "en-us",
},
} as RequestInit);
if (!response.ok) {
throw new Error(`API error: ${response.status} ${response.statusText}`);
}
return (await response.json()) as T;
} catch (error: any) {
lastError = error as Error;
// Only retry on network errors or 5xx status codes
if (error.message.includes('API error: 5') || !error.message.includes('API error')) {
console.warn(`Attempt ${attempt + 1} failed for ${url}: ${error.message}`);
await new Promise(resolve => setTimeout(resolve, 1000 * Math.pow(2, attempt)));
continue;
}
throw error;
}
}
throw lastError || new Error(`Failed to fetch ${url} after ${retries} attempts`);
}
/**
* Generate a slug from a name
*/
export function createSlug(name: string): string {
return name
.toLowerCase()
.replace(/[^\w\s -]/g, "")
.replace(/\s+/g, "-")
.replace(/-+/g, "-")
.trim();
}
/**
* Compare a candidate screenshot against a UI-free baseline to find how much UI/HUD remains.
*
* @param baselineBuffer - PNG/JPEG buffer of the clean background.
* @param candidateBuffer - PNG/JPEG buffer of the screenshot to test.
* @param opts - Options.
* @returns Promise resolving to diff ratio (and optional diff image).
*/
export async function compareWithBaseline(
baselineBuffer: Buffer,
candidateBuffer: Buffer,
opts: CompareOpts = {}
): Promise<CompareResult> {
// Generate cache key from buffer hashes
const baseHash = crypto.createHash('md5').update(baselineBuffer).digest('hex');
const candHash = crypto.createHash('md5').update(candidateBuffer).digest('hex');
const optsKey = JSON.stringify(opts);
const cacheKey = `${baseHash}:${candHash}:${optsKey}`;
// Check cache
if (compareCache.has(cacheKey)) {
return compareCache.get(cacheKey)!;
}
const { threshold = 0.1, diffOutput = false } = opts;
// Get dimensions of baseline
const baseMeta: Metadata = await sharp(baselineBuffer).metadata();
if (!baseMeta.width || !baseMeta.height) {
throw new Error('Invalid baseline dimensions');
}
// Produce PNG buffers of same size
const [pngBaseBuf, pngCandBuf] = await Promise.all([
sharp(baselineBuffer).png().toBuffer(),
sharp(candidateBuffer)
.resize(baseMeta.width, baseMeta.height)
.png()
.toBuffer(),
]);
const imgBase = PNG.sync.read(pngBaseBuf);
const imgCand = PNG.sync.read(pngCandBuf);
const diffImg = new PNG({ width: baseMeta.width, height: baseMeta.height });
const numDiff = pixelmatch(
imgBase.data,
imgCand.data,
diffImg.data,
baseMeta.width,
baseMeta.height,
{ threshold }
);
const total = baseMeta.width * baseMeta.height;
const diffRatio = numDiff / total;
const result: CompareResult = { diffRatio };
if (diffOutput) {
result.diffBuffer = PNG.sync.write(diffImg);
}
compareCache.set(cacheKey, result);
return result;
}
/**
* Given a baseline buffer and an array of screenshots, returns them sorted
* ascending by diffRatio (least UI first).
*/
export async function rankScreenshots(
baselineBuffer: Buffer,
shots: Shot[],
opts: CompareOpts = {}
): Promise<RankedShot[]> {
// Process up to 5 comparisons in parallel
const compareLimit = pLimit(5);
// Run all comparisons with limited concurrency
const results = await Promise.all(
shots.map(shot =>
compareLimit(async () => {
const { diffRatio } = await compareWithBaseline(
baselineBuffer,
shot.buffer,
opts
);
return { url: shot.url, score: diffRatio };
})
)
);
return results.sort((a, b) => a.score - b.score);
}
// --- Helpers for URLs ---
export function getScreenshotUrls(screenshots: { appid: number; filename: string }[]): string[] {
return screenshots.map(s => `https://shared.cloudflare.steamstatic.com/store_item_assets/steam/apps/${s.appid}/${s.filename}`);
}
export function getAssetUrls(assets: LibraryAssetsFull, appid: number | string, header: string) {
const base = `https://shared.cloudflare.steamstatic.com/store_item_assets/steam/apps/${appid}`;
return {
logo: `${base}/${assets.library_logo?.image2x?.english || assets.library_logo?.image?.english}`,
backdrop: `${base}/${assets.library_hero?.image2x?.english || assets.library_hero?.image?.english}`,
poster: `${base}/${assets.library_capsule?.image2x?.english || assets.library_capsule?.image?.english}`,
banner: `${base}/${assets.library_header?.image2x?.english || assets.library_header?.image?.english || header}`,
};
}
/**
* Compute a 0–5 score from positive/negative votes using a Wilson score confidence interval.
* This formula adjusts the raw ratio based on the total number of votes to account for
* statistical confidence. With few votes, the score regresses toward 2.5 (neutral).
*
* Compute a 0–5 score from positive/negative votes
*/
export function getRating(positive: number, negative: number): number {
const total = positive + negative;
if (!total) return 0;
const avg = positive / total;
// Apply Wilson score confidence adjustment and scale to 0-5 range
const score = avg - (avg - 0.5) * Math.pow(2, -Math.log10(total + 1));
return Math.round(score * 5 * 10) / 10;
}
export function getAssociationsByTypeWithSlug<
T extends "developer" | "publisher"
>(
associations: Record<string, { name: string; type: string }>,
type: T
): Array<{ name: string; slug: string; type: T }> {
return Object.values(associations)
.filter((a) => a.type === type)
.map((a) => ({ name: a.name.trim(), slug: createSlug(a.name.trim()), type }));
}
export function compatibilityType(type?: string): "low" | "mid" | "high" | "unknown" {
switch (type) {
case "1":
return "low";
case "2":
return "mid";
case "3":
return "high";
default:
return "unknown";
}
}
export function mapGameTags<
T extends string = "tag"
>(
available: Tag[],
storeTags: StoreTags,
): Array<{ name: string; slug: string; type: T }> {
const tagMap = new Map<number, Tag>(available.map((t) => [t.tagid, t]));
const result: Array<{ name: string; slug: string; type: T }> = Object.values(storeTags)
.map((id) => tagMap.get(Number(id)))
.filter((t): t is Tag => Boolean(t))
.map((t) => ({ name: t.name.trim(), slug: createSlug(t.name), type: 'tag' as T }));
return result;
}
/**
* Create a tag object with name, slug, and type
* @typeparam T Literal type of the `type` field (defaults to 'tag')
*/
export function createTag<
T extends string = 'tag'
>(
name: string,
type?: T
): { name: string; slug: string; type: T } {
const tagType = (type ?? 'tag') as T;
return {
name: name.trim(),
slug: createSlug(name),
type: tagType,
};
}
export function capitalise(name: string) {
return name
.charAt(0) // first character
.toUpperCase() // make it uppercase
+ name
.slice(1) // rest of the string
.toLowerCase();
}
export function getPublicDepotSizes(depots: AppDepots) {
const sum = { download: 0, size: 0 };
for (const key in depots) {
if (key === 'branches' || key === 'privatebranches') continue;
const entry = depots[key] as DepotEntry;
if ('manifests' in entry && entry.manifests.public) {
sum.download += Number(entry.manifests.public.download);
sum.size += Number(entry.manifests.public.size);
}
}
return { downloadSize: sum.download, sizeOnDisk: sum.size };
}
export function parseGenres(str: string): GenreType[] {
return str.split(',')
.map((g) => g.trim())
.filter(Boolean)
.map((g) => ({ type: 'genre', name: g.trim(), slug: createSlug(g) }));
}
export function getPrimaryGenre(
genres: GenreType[],
map: Record<string, string>,
primaryId: string
): string | null {
const idx = Object.keys(map).find((k) => map[k] === primaryId);
return idx !== undefined ? genres[Number(idx)]?.name : null;
}
export function cleanDescription(input: string): string {
const cleaned = sanitizeHtml(input, {
allowedTags: [], // no tags allowed
allowedAttributes: {}, // no attributes anywhere
textFilter: (text) => text.replace(/\s+/g, ' '), // collapse runs of whitespace
});
return cleaned.trim()
}
}

15
packages/core/src/credentials/credentials.sql.ts
View File

@@ -1,14 +1,14 @@
import { steamTable } from "../steam/steam.sql";
import { pgTable, varchar } from "drizzle-orm/pg-core";
import { encryptedText, timestamps, utc } from "../drizzle/types";
import { pgTable, primaryKey, varchar } from "drizzle-orm/pg-core";
import { encryptedText, ulid, timestamps, utc } from "../drizzle/types";
export const steamCredentialsTable = pgTable(
"steam_account_credentials",
{
...timestamps,
id: varchar("steam_id", { length: 255 })
id: ulid("id").notNull(),
steamID: varchar("steam_id", { length: 255 })
.notNull()
.primaryKey()
.references(() => steamTable.id, {
onDelete: "cascade"
}),
@@ -16,5 +16,10 @@ export const steamCredentialsTable = pgTable(
.notNull(),
expiry: utc("expiry").notNull(),
username: varchar("username", { length: 255 }).notNull(),
}
},
(table) => [
primaryKey({
columns: [table.steamID, table.id]
})
]
)

50
packages/core/src/credentials/index.ts
View File

@@ -1,8 +1,8 @@
import { z } from "zod";
import { createID, fn } from "../utils";
import { Resource } from "sst";
import { bus } from "sst/aws/bus";
import { createEvent } from "../event";
import { createID, fn } from "../utils";
import { eq, and, isNull, gt } from "drizzle-orm";
import { createSelectSchema } from "drizzle-zod";
import { steamCredentialsTable } from "./credentials.sql";
@@ -22,39 +22,41 @@ export namespace Credentials {
New: createEvent(
"new_credentials.added",
z.object({
steamID: Info.shape.id,
steamID: Info.shape.steamID,
}),
),
};
export const create = fn(
Info
.omit({ accessToken: true, cookies: true, expiry: true }),
.omit({ accessToken: true, cookies: true, expiry: true })
.partial({ id: true }),
(input) => {
const part = input.refreshToken.split('.')[1] as string
const payload = JSON.parse(Buffer.from(part, 'base64').toString());
return createTransaction(async (tx) => {
const id = input.id
const id = input.id ?? createID("credentials")
await tx
.insert(steamCredentialsTable)
.values({
id,
steamID: input.steamID,
username: input.username,
refreshToken: input.refreshToken,
expiry: new Date(payload.exp * 1000),
})
// await afterTx(async () =>
// await bus.publish(Resource.Bus, Events.New, { steamID: input.id })
// );
await afterTx(async () =>
await bus.publish(Resource.Bus, Events.New, { steamID: input.steamID })
);
return id
})
});
export const getByID = fn(
Info.shape.id,
(id) =>
export const fromSteamID = fn(
Info.shape.steamID,
(steamID) =>
useTransaction(async (tx) => {
const now = new Date()
@@ -63,7 +65,7 @@ export namespace Credentials {
.from(steamCredentialsTable)
.where(
and(
eq(steamCredentialsTable.id, id),
eq(steamCredentialsTable.steamID, steamID),
isNull(steamCredentialsTable.timeDeleted),
gt(steamCredentialsTable.expiry, now)
)
@@ -76,31 +78,6 @@ export namespace Credentials {
return serialize(credential);
})
);
// export const getBySteamID = fn(
// Info.shape.steamID,
// (steamID) =>
// useTransaction(async (tx) => {
// const now = new Date()
// const credential = await tx
// .select()
// .from(steamCredentialsTable)
// .where(
// and(
// eq(steamCredentialsTable.steamID, steamID),
// isNull(steamCredentialsTable.timeDeleted),
// gt(steamCredentialsTable.expiry, now)
// )
// )
// .execute()
// .then(rows => rows.at(0));
// if (!credential) return null;
// return serialize(credential);
// })
// );
export function serialize(
input: typeof steamCredentialsTable.$inferSelect,
@@ -108,6 +85,7 @@ export namespace Credentials {
return {
id: input.id,
expiry: input.expiry,
steamID: input.steamID,
username: input.username,
refreshToken: input.refreshToken,
};

4
packages/core/src/examples.ts
View File

@@ -251,9 +251,9 @@ export namespace Examples {
screenshots: CommonImg,
boxArts: CommonImg,
posters: CommonImg,
banners: CommonImg,
heroArts: CommonImg,
superHeroArts: CommonImg,
backgrounds: CommonImg,
backdrops: CommonImg,
logos: CommonImg,
icons: CommonImg,
}

10
packages/core/src/game/index.ts
View File

@@ -8,8 +8,8 @@ import { Categories } from "../categories";
import { eq, and, isNull } from "drizzle-orm";
import { createSelectSchema } from "drizzle-zod";
import { imagesTable } from "../images/images.sql";
import { groupBy, map, pipe, uniqueBy, values } from "remeda";
import { baseGamesTable } from "../base-game/base-game.sql";
import { groupBy, map, pipe, uniqueBy, values } from "remeda";
import { categoriesTable } from "../categories/categories.sql";
import { createTransaction, useTransaction } from "../drizzle/transaction";
@@ -22,7 +22,7 @@ export namespace Game {
example: Examples.Game
})
export type Info = z.infer<typeof Info>
export type Info = z.infer<typeof Info>;
export const InputInfo = createSelectSchema(gamesTable)
.omit({ timeCreated: true, timeDeleted: true, timeUpdated: true })
@@ -31,7 +31,7 @@ export namespace Game {
InputInfo,
(input) =>
createTransaction(async (tx) => {
const results =
const result =
await tx
.select()
.from(gamesTable)
@@ -43,9 +43,11 @@ export namespace Game {
isNull(gamesTable.timeDeleted)
)
)
.limit(1)
.execute()
.then(rows => rows.at(0))
if (results.length > 0) return null
if (result) return result.baseGameID
await tx
.insert(gamesTable)

10
packages/core/src/images/images.sql.ts
View File

@@ -1,9 +1,9 @@
import { z } from "zod";
import { timestamps } from "../drizzle/types";
import { baseGamesTable } from "../base-game/base-game.sql";
import { index, integer, json, pgEnum, pgTable, primaryKey, text, varchar } from "drizzle-orm/pg-core";
import { z } from "zod";
export const ImageTypeEnum = pgEnum("image_type", ["heroArt", "icon", "logo", "superHeroArt", "poster", "boxArt", "screenshot","background"])
export const ImageTypeEnum = pgEnum("image_type", ["heroArt", "icon", "logo", "banner", "poster", "boxArt", "screenshot", "backdrop"])
export const ImageDimensions = z.object({
width: z.number().int(),
@@ -11,8 +11,8 @@ export const ImageDimensions = z.object({
})
export const ImageColor = z.object({
hex: z.string(),
isDark: z.boolean()
hex: z.string(),
isDark: z.boolean()
})
export type ImageColor = z.infer<typeof ImageColor>;
@@ -30,7 +30,7 @@ export const imagesTable = pgTable(
.references(() => baseGamesTable.id, {
onDelete: "cascade"
}),
sourceUrl: text("source_url").notNull(),
sourceUrl: text("source_url"), // The BoxArt is source Url will always be null;
position: integer("position").notNull().default(0),
fileSize: integer("file_size").notNull(),
dimensions: json("dimensions").$type<ImageDimensions>().notNull(),

20
packages/core/src/images/index.ts
View File

@@ -38,17 +38,17 @@ export namespace Images {
description: "Vertical 2:3 aspect ratio promotional artwork, similar to movie posters",
example: Examples.Images.posters
}),
heroArts: Image.array().openapi({
banners: Image.array().openapi({
description: "Horizontal promotional artwork optimized for header displays and banners",
example: Examples.Images.banners
}),
heroArts: Image.array().openapi({
description: "High-resolution, wide-format artwork designed for featured content and main entries",
example: Examples.Images.heroArts
}),
superHeroArts: Image.array().openapi({
description: "High-resolution, wide-format artwork designed for featured content and main entries",
example: Examples.Images.superHeroArts
}),
backgrounds: Image.array().openapi({
description: "Full-width background images optimized for page layouts and decorative purposes",
example: Examples.Images.backgrounds
backdrops: Image.array().openapi({
description: "Full-width backdrop images optimized for page layouts and decorative purposes",
example: Examples.Images.backdrops
}),
logos: Image.array().openapi({
description: "Official game logo artwork, typically with transparent backgrounds for flexible placement",
@@ -107,10 +107,10 @@ export namespace Images {
}, {
screenshots: [],
boxArts: [],
superHeroArts: [],
banners: [],
heroArts: [],
posters: [],
backgrounds: [],
backdrops: [],
icons: [],
logos: [],
})

43
packages/core/src/library/index.ts
View File

@@ -2,6 +2,7 @@ import { z } from "zod";
import { fn } from "../utils";
import { Game } from "../game";
import { Actor } from "../actor";
import { createEvent } from "../event";
import { gamesTable } from "../game/game.sql";
import { createSelectSchema } from "drizzle-zod";
import { steamLibraryTable } from "./library.sql";
@@ -17,31 +18,61 @@ export namespace Library {
export type Info = z.infer<typeof Info>;
export const Events = {
Queue: createEvent(
"library.queue",
z.object({
appID: z.number(),
lastPlayed: z.date(),
timeAcquired: z.date(),
totalPlaytime: z.number(),
isFamilyShared: z.boolean(),
isFamilyShareable: z.boolean(),
}).array(),
),
};
export const add = fn(
Info,
Info.partial({ ownerID: true }),
async (input) =>
createTransaction(async (tx) => {
const results =
const ownerSteamID = input.ownerID ?? Actor.steamID()
const result =
await tx
.select()
.from(steamLibraryTable)
.where(
and(
eq(steamLibraryTable.baseGameID, input.baseGameID),
eq(steamLibraryTable.ownerID, input.ownerID),
eq(steamLibraryTable.ownerID, ownerSteamID),
isNull(steamLibraryTable.timeDeleted)
)
)
.limit(1)
.execute()
.then(rows => rows.at(0))
if (results.length > 0) return null
if (result) return result.baseGameID
await tx
.insert(steamLibraryTable)
.values(input)
.values({
ownerID: ownerSteamID,
baseGameID: input.baseGameID,
lastPlayed: input.lastPlayed,
totalPlaytime: input.totalPlaytime,
timeAcquired: input.timeAcquired,
isFamilyShared: input.isFamilyShared
})
.onConflictDoUpdate({
target: [steamLibraryTable.ownerID, steamLibraryTable.baseGameID],
set: { timeDeleted: null }
set: {
timeDeleted: null,
lastPlayed: input.lastPlayed,
timeAcquired: input.timeAcquired,
totalPlaytime: input.totalPlaytime,
isFamilyShared: input.isFamilyShared
}
})
})

9
packages/core/src/library/library.sql.ts
View File

@@ -1,9 +1,8 @@
import { timestamps, } from "../drizzle/types";
import { timestamps, utc, } from "../drizzle/types";
import { steamTable } from "../steam/steam.sql";
import { baseGamesTable } from "../base-game/base-game.sql";
import { index, pgTable, primaryKey, varchar, } from "drizzle-orm/pg-core";
import { boolean, index, integer, pgTable, primaryKey, varchar, } from "drizzle-orm/pg-core";
//TODO: Add playtime here
export const steamLibraryTable = pgTable(
"game_libraries",
{
@@ -18,6 +17,10 @@ export const steamLibraryTable = pgTable(
.references(() => steamTable.id, {
onDelete: "cascade"
}),
timeAcquired: utc("time_acquired").notNull(),
lastPlayed: utc("last_played").notNull(),
totalPlaytime: integer("total_playtime").notNull(),
isFamilyShared: boolean("is_family_shared").notNull()
},
(table) => [
primaryKey({

30
packages/core/src/team/index.ts
View File

@@ -10,7 +10,6 @@ import { createID, fn, Invite } from "../utils";
import { memberTable } from "../member/member.sql";
import { groupBy, pipe, values, map } from "remeda";
import { createTransaction, useTransaction, type Transaction } from "../drizzle/transaction";
import { VisibleError } from "../error";
export namespace Team {
export const Info = z
@@ -144,6 +143,28 @@ export namespace Team {
.then((rows) => serialize(rows))
)
export const fromSlug = fn(
Info.shape.slug,
(slug) =>
useTransaction((tx) =>
tx
.select()
.from(teamTable)
.innerJoin(memberTable, eq(memberTable.teamID, teamTable.id))
.innerJoin(steamTable, eq(memberTable.steamID, steamTable.id))
.where(
and(
eq(memberTable.userID, Actor.userID()),
isNull(memberTable.timeDeleted),
isNull(steamTable.timeDeleted),
isNull(teamTable.timeDeleted),
eq(teamTable.slug, slug),
)
)
.then((rows) => serialize(rows).at(0))
)
)
export function serialize(
input: { teams: typeof teamTable.$inferSelect; steam_accounts: typeof steamTable.$inferSelect | null }[]
): z.infer<typeof Info>[] {
@@ -158,10 +179,9 @@ export namespace Team {
ownerID: group[0].teams.ownerID,
maxMembers: group[0].teams.maxMembers,
inviteCode: group[0].teams.inviteCode,
members:
!group[0].steam_accounts ?
[] :
group.map((item) => Steam.serialize(item.steam_accounts!))
members: group.map(i => i.steam_accounts)
.filter((c): c is typeof steamTable.$inferSelect => Boolean(c))
.map((item) => Steam.serialize(item))
})),
)
}

10
packages/core/src/utils/helper.ts Normal file
View File

@@ -0,0 +1,10 @@
export function chunkArray<T>(arr: T[], chunkSize: number): T[][] {
if (chunkSize <= 0) {
throw new Error("chunkSize must be a positive integer");
}
const chunks: T[][] = [];
for (let i = 0; i < arr.length; i += chunkSize) {
chunks.push(arr.slice(i, i + chunkSize));
}
return chunks;
}

3
packages/core/src/utils/index.ts
View File

@@ -2,4 +2,5 @@ export * from "./fn"
export * from "./log"
export * from "./id"
export * from "./invite"
export * from "./token"
export * from "./token"
export * from "./helper"