mirror of
https://github.com/nestriness/nestri.git
synced 2025-12-12 08:45:38 +02:00
⭐ feat: Upgrade to asynchronous event bus with retry queue and backoff strategy (#290)
## Description <!-- Briefly describe the purpose and scope of your changes --> <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Introduced a retry and dead-letter queue system for more robust event processing. - Added a retry handler for processing failed Lambda invocations with exponential backoff. - Enhanced event handling to support retry logic and improved error management. - **Refactor** - Replaced SQS-based library event processing with an event bus-based approach. - Updated event names and structure for improved clarity and consistency. - Removed legacy library queue and related infrastructure. - **Chores** - Updated dependencies to include the AWS Lambda client. - Cleaned up unused code and removed deprecated event handling logic. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Wanjohi
GitHub
@@ -9,12 +9,14 @@ import { Images } from "@nestri/core/images/index";
|
||||
import { Library } from "@nestri/core/library/index";
|
||||
import { BaseGame } from "@nestri/core/base-game/index";
|
||||
import { Categories } from "@nestri/core/categories/index";
|
||||
import { ImageTypeEnum } from "@nestri/core/images/images.sql";
|
||||
import { PutObjectCommand, S3Client, HeadObjectCommand } from "@aws-sdk/client-s3";
|
||||
|
||||
const s3 = new S3Client({});
|
||||
|
||||
export const handler = bus.subscriber(
|
||||
[
|
||||
Library.Events.Add,
|
||||
BaseGame.Events.New,
|
||||
Steam.Events.Updated,
|
||||
Steam.Events.Created,
|
||||
@@ -114,125 +116,154 @@ export const handler = bus.subscriber(
|
||||
|
||||
const images = await Client.createHeroArt(input);
|
||||
|
||||
const settled =
|
||||
await Promise.allSettled(
|
||||
images.map(async (image) => {
|
||||
await Images.create({
|
||||
type: image.type,
|
||||
imageHash: image.hash,
|
||||
baseGameID: input.appID,
|
||||
position: image.position,
|
||||
fileSize: image.fileSize,
|
||||
sourceUrl: image.sourceUrl,
|
||||
dimensions: image.dimensions,
|
||||
extractedColor: image.averageColor,
|
||||
});
|
||||
await Promise.all(
|
||||
images.map(async (image) => {
|
||||
await Images.create({
|
||||
type: image.type,
|
||||
imageHash: image.hash,
|
||||
baseGameID: input.appID,
|
||||
position: image.position,
|
||||
fileSize: image.fileSize,
|
||||
sourceUrl: image.sourceUrl,
|
||||
dimensions: image.dimensions,
|
||||
extractedColor: image.averageColor,
|
||||
});
|
||||
|
||||
try {
|
||||
//Check whether the image already exists
|
||||
await s3.send(
|
||||
new HeadObjectCommand({
|
||||
Bucket: Resource.Storage.name,
|
||||
Key: `images/${image.hash}`,
|
||||
})
|
||||
);
|
||||
try {
|
||||
//Check whether the image already exists
|
||||
await s3.send(
|
||||
new HeadObjectCommand({
|
||||
Bucket: Resource.Storage.name,
|
||||
Key: `images/${image.hash}`,
|
||||
})
|
||||
);
|
||||
|
||||
} catch (e) {
|
||||
// Save to s3 because it doesn't already exist
|
||||
await s3.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: Resource.Storage.name,
|
||||
Key: `images/${image.hash}`,
|
||||
Body: image.buffer,
|
||||
...(image.format && { ContentType: `image/${image.format}` }),
|
||||
Metadata: {
|
||||
type: image.type,
|
||||
appID: input.appID,
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
settled
|
||||
.filter(r => r.status === "rejected")
|
||||
.forEach(r => console.warn("[processHeroArt] failed:", (r as PromiseRejectedResult).reason));
|
||||
} catch (e) {
|
||||
// Save to s3 because it doesn't already exist
|
||||
await s3.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: Resource.Storage.name,
|
||||
Key: `images/${image.hash}`,
|
||||
Body: image.buffer,
|
||||
...(image.format && { ContentType: `image/${image.format}` }),
|
||||
Metadata: {
|
||||
type: image.type,
|
||||
appID: input.appID,
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
break;
|
||||
}
|
||||
// case "steam_account.updated":
|
||||
// case "steam_account.created": {
|
||||
// //Get user library and commit it to the db
|
||||
// const steamID = event.properties.steamID;
|
||||
case "library.add": {
|
||||
|
||||
// await Actor.provide(
|
||||
// event.metadata.actor.type,
|
||||
// event.metadata.actor.properties,
|
||||
// async () => {
|
||||
// //Get user library
|
||||
// const gameLibrary = await Client.getUserLibrary(steamID);
|
||||
await Actor.provide(
|
||||
event.metadata.actor.type,
|
||||
event.metadata.actor.properties,
|
||||
async () => {
|
||||
const game = event.properties
|
||||
// First check whether the base_game exists, if not get it
|
||||
const appID = game.appID.toString();
|
||||
const exists = await BaseGame.fromID(appID);
|
||||
|
||||
// const myLibrary = new Map(gameLibrary.response.games.map(g => [g.appid, g]))
|
||||
if (!exists) {
|
||||
const appInfo = await Client.getAppInfo(appID);
|
||||
|
||||
// const queryLib = await Promise.allSettled(
|
||||
// gameLibrary.response.games.map(async (game) => {
|
||||
// return await Client.getAppInfo(game.appid.toString())
|
||||
// })
|
||||
// )
|
||||
await BaseGame.create({
|
||||
id: appID,
|
||||
name: appInfo.name,
|
||||
size: appInfo.size,
|
||||
slug: appInfo.slug,
|
||||
links: appInfo.links,
|
||||
score: appInfo.score,
|
||||
description: appInfo.description,
|
||||
releaseDate: appInfo.releaseDate,
|
||||
primaryGenre: appInfo.primaryGenre,
|
||||
compatibility: appInfo.compatibility,
|
||||
controllerSupport: appInfo.controllerSupport,
|
||||
})
|
||||
|
||||
// queryLib
|
||||
// .filter(i => i.status === "rejected")
|
||||
// .forEach(e => console.warn(`[getAppInfo]: Failed to get game metadata: ${e.reason}`))
|
||||
const allCategories = [...appInfo.tags, ...appInfo.genres, ...appInfo.publishers, ...appInfo.developers, ...appInfo.categories, ...appInfo.franchises]
|
||||
|
||||
// const gameInfo = queryLib.filter(i => i.status === "fulfilled").map(f => f.value)
|
||||
const uniqueCategories = Array.from(
|
||||
new Map(allCategories.map(c => [`${c.type}:${c.slug}`, c])).values()
|
||||
);
|
||||
|
||||
// const queryGames = gameInfo.map(async (game) => {
|
||||
// await BaseGame.create(game);
|
||||
await Promise.all(
|
||||
uniqueCategories.map(async (cat) => {
|
||||
//Create category if it doesn't exist
|
||||
await Categories.create({
|
||||
type: cat.type, slug: cat.slug, name: cat.name
|
||||
})
|
||||
|
||||
// const allCategories = [...game.tags, ...game.genres, ...game.publishers, ...game.developers];
|
||||
//Create game if it doesn't exist
|
||||
await Game.create({ baseGameID: appID, categorySlug: cat.slug, categoryType: cat.type })
|
||||
})
|
||||
)
|
||||
|
||||
// const uniqueCategories = Array.from(
|
||||
// new Map(allCategories.map(c => [`${c.type}:${c.slug}`, c])).values()
|
||||
// );
|
||||
const imageUrls = appInfo.images
|
||||
|
||||
// const gameSettled = await Promise.allSettled(
|
||||
// uniqueCategories.map(async (cat) => {
|
||||
// //Use a single db transaction to get or set the category
|
||||
// await Categories.create({
|
||||
// type: cat.type, slug: cat.slug, name: cat.name
|
||||
// })
|
||||
await Promise.all(
|
||||
ImageTypeEnum.enumValues.map(async (type) => {
|
||||
switch (type) {
|
||||
case "backdrop": {
|
||||
await bus.publish(Resource.Bus, BaseGame.Events.New, { appID, type: "backdrop", url: imageUrls.backdrop })
|
||||
break;
|
||||
}
|
||||
case "banner": {
|
||||
await bus.publish(Resource.Bus, BaseGame.Events.New, { appID, type: "banner", url: imageUrls.banner })
|
||||
break;
|
||||
}
|
||||
case "icon": {
|
||||
await bus.publish(Resource.Bus, BaseGame.Events.New, { appID, type: "icon", url: imageUrls.icon })
|
||||
break;
|
||||
}
|
||||
case "logo": {
|
||||
await bus.publish(Resource.Bus, BaseGame.Events.New, { appID, type: "logo", url: imageUrls.logo })
|
||||
break;
|
||||
}
|
||||
case "poster": {
|
||||
await bus.publish(
|
||||
Resource.Bus,
|
||||
BaseGame.Events.New,
|
||||
{ appID, type: "poster", url: imageUrls.poster }
|
||||
)
|
||||
break;
|
||||
}
|
||||
case "heroArt": {
|
||||
await bus.publish(
|
||||
Resource.Bus,
|
||||
BaseGame.Events.NewHeroArt,
|
||||
{ appID, backdropUrl: imageUrls.backdrop, screenshots: imageUrls.screenshots }
|
||||
)
|
||||
break;
|
||||
}
|
||||
case "boxArt": {
|
||||
await bus.publish(
|
||||
Resource.Bus,
|
||||
BaseGame.Events.NewBoxArt,
|
||||
{ appID, logoUrl: imageUrls.logo, backgroundUrl: imageUrls.backdrop }
|
||||
)
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// // Use a single db transaction to get or create the game
|
||||
// await Game.create({ baseGameID: game.id, categorySlug: cat.slug, categoryType: cat.type })
|
||||
// })
|
||||
// )
|
||||
// Add to user's library
|
||||
await Library.add({
|
||||
baseGameID: appID,
|
||||
lastPlayed: game.lastPlayed ? new Date(game.lastPlayed) : null,
|
||||
totalPlaytime: game.totalPlaytime,
|
||||
})
|
||||
})
|
||||
|
||||
// gameSettled
|
||||
// .filter(r => r.status === "rejected")
|
||||
// .forEach(r => console.warn("[uniqueCategories] failed:", (r as PromiseRejectedResult).reason));
|
||||
|
||||
// const currentGameInLibrary = myLibrary.get(parseInt(game.id))
|
||||
// if (currentGameInLibrary) {
|
||||
// await Library.add({
|
||||
// baseGameID: game.id,
|
||||
// lastPlayed: currentGameInLibrary.rtime_last_played ? new Date(currentGameInLibrary.rtime_last_played * 1000) : null,
|
||||
// totalPlaytime: currentGameInLibrary.playtime_forever,
|
||||
// })
|
||||
// } else {
|
||||
// throw new Error(`Game is not in library, but was found in app info:${game.id}`)
|
||||
// }
|
||||
// })
|
||||
|
||||
// const settled = await Promise.allSettled(queryGames);
|
||||
|
||||
// settled
|
||||
// .filter(i => i.status === "rejected")
|
||||
// .forEach(e => console.warn(`[gameCreate]: Failed to create game: ${e.reason}`))
|
||||
// })
|
||||
|
||||
// break;
|
||||
// }
|
||||
break;
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
Reference in New Issue
Block a user