From a206dee8301e5a8723daab819f175d7ab01f9029 Mon Sep 17 00:00:00 2001 From: mat Date: Sun, 15 May 2022 22:37:22 -0500 Subject: add very basic auctionprices endpoint --- src/cleaners/skyblock/endedAuctions.ts | 41 ++++++++++++++ src/cleaners/skyblock/inventory.ts | 17 +++--- src/database.ts | 47 ++++++++++++++++ src/hypixel.ts | 98 +++++++++++++++++++++++++++++++++- src/index.ts | 16 +++++- 5 files changed, 206 insertions(+), 13 deletions(-) create mode 100644 src/cleaners/skyblock/endedAuctions.ts diff --git a/src/cleaners/skyblock/endedAuctions.ts b/src/cleaners/skyblock/endedAuctions.ts new file mode 100644 index 0000000..0dfa0ab --- /dev/null +++ b/src/cleaners/skyblock/endedAuctions.ts @@ -0,0 +1,41 @@ +import typedHypixelApi from 'typed-hypixel-api' +import { cleanInventory, headIdFromBase64, Item } from './inventory.js' +import { cleanItemId } from './itemId.js' + + +interface Auction { + id: string + sellerUuid: string + sellerProfileUuid: string + buyerUuid: string + timestamp: number + coins: number + bin: boolean + item: Item +} + +export interface EndedAuctions { + lastUpdated: number + auctions: Auction[] +} + +export async function cleanEndedAuctions(data: typedHypixelApi.SkyBlockRecentlyEndedAuctionsResponse): Promise { + const auctions: Auction[] = [] + for (const auction of data.auctions) { + auctions.push({ + id: auction.auction_id, + sellerUuid: auction.seller, + sellerProfileUuid: auction.seller_profile, + buyerUuid: auction.buyer, + timestamp: auction.timestamp, + coins: auction.price, + bin: auction.bin, + item: (await cleanInventory(auction.item_bytes))[0] + }) + } + + return { + lastUpdated: data.lastUpdated, + auctions + } +} \ No newline at end of file diff --git a/src/cleaners/skyblock/inventory.ts b/src/cleaners/skyblock/inventory.ts index c5ab952..df30be3 100644 --- a/src/cleaners/skyblock/inventory.ts +++ b/src/cleaners/skyblock/inventory.ts @@ -6,7 +6,7 @@ function base64decode(base64: string): Buffer { return Buffer.from(base64, 'base64') } -interface Item { +export interface Item { id: string count: number vanillaId: string @@ -79,15 +79,12 @@ function cleanItems(rawItems): Inventory { return rawItems.map(cleanItem) } -export function cleanInventory(encodedNbt: string): Promise { - return new Promise(resolve => { - const base64Data = base64decode(encodedNbt) - nbt.parse(base64Data, false, (err, value) => { - const simplifiedNbt = nbt.simplify(value) - // do some basic cleaning on the items and return - resolve(cleanItems(simplifiedNbt.i)) - }) - }) +export async function cleanInventory(encodedNbt: string): Promise { + const base64Data = base64decode(encodedNbt) + const value: any = await new Promise((resolve, reject) => nbt.parse(base64Data, false, (err, value) => { if (err) reject(err); else resolve(value) })) + const simplifiedNbt = nbt.simplify(value) + // do some basic cleaning on the items and return + return cleanItems(simplifiedNbt.i) } export const INVENTORIES = { diff --git a/src/database.ts b/src/database.ts index 04b3825..8f63d97 100644 --- a/src/database.ts +++ b/src/database.ts @@ -17,6 +17,7 @@ import { debug } from './index.js' import Queue from 'queue-promise' import { RANK_COLORS } from './cleaners/rank.js' import { cleanItemId } from './cleaners/skyblock/itemId.js' +import { periodicallyFetchRecentlyEndedAuctions } from './hypixel.js' // don't update the user for 3 minutes const recentlyUpdated = new NodeCache({ @@ -116,10 +117,30 @@ export interface AccountSchema { customization?: AccountCustomization } +export interface SimpleAuctionSchema { + /** The UUID of the auction so we can look it up later. */ + id: string + coins: number + /** + * The timestamp as **seconds** since epoch. It's in seconds instead of ms + * since we don't need to be super exact and so it's shorter. + */ + ts: number + /** Whether the auction was bought or simply expired. */ + success: boolean + bin: boolean +} +export interface ItemAuctionsSchema { + /** The id of the item */ + _id: string + auctions: SimpleAuctionSchema[] +} + let memberLeaderboardsCollection: Collection let profileLeaderboardsCollection: Collection let sessionsCollection: Collection let accountsCollection: Collection +let itemAuctionsCollection: Collection const leaderboardInfos: { [leaderboardName: string]: string } = { @@ -142,6 +163,10 @@ async function connect(): Promise { profileLeaderboardsCollection = database.collection('profile-leaderboards') sessionsCollection = database.collection('sessions') accountsCollection = database.collection('accounts') + itemAuctionsCollection = database.collection('item-auctions') + + periodicallyFetchRecentlyEndedAuctions() + console.log('Connected to database :)') } @@ -1064,6 +1089,28 @@ export async function updateAccount(discordId: string, schema: AccountSchema) { }, { $set: schema }, { upsert: true }) } +/** Fetch all the Item Auctions for the item ids in the given array. */ +export async function fetchItemsAuctions(itemIds: string[]): Promise { + const auctions = await itemAuctionsCollection?.find({ + _id: { $in: itemIds } + }).toArray() + return auctions +} + + +/** Fetch all the Item Auctions for the item ids in the given array. */ +export async function fetchPaginatedItemsAuctions(skip: number, limit: number): Promise { + const auctions = await itemAuctionsCollection?.find({}).skip(skip).limit(limit).toArray() + return auctions +} + +export async function updateItemAuction(auction: ItemAuctionsSchema) { + await itemAuctionsCollection?.updateOne({ + _id: auction._id, + }, { $set: auction }, { upsert: true }) +} + + export async function fetchServerStatus() { return await database.admin().serverStatus() } diff --git a/src/hypixel.ts b/src/hypixel.ts index d7a5887..060cff4 100644 --- a/src/hypixel.ts +++ b/src/hypixel.ts @@ -13,9 +13,13 @@ import { AccountCustomization, AccountSchema, fetchAccount, + fetchItemsAuctions, + ItemAuctionsSchema, queueUpdateDatabaseMember, queueUpdateDatabaseProfile, - removeDeletedProfilesFromLeaderboards + removeDeletedProfilesFromLeaderboards, + SimpleAuctionSchema, + updateItemAuction } from './database.js' import { cleanElectionResponse, ElectionData } from './cleaners/skyblock/election.js' import { cleanItemListResponse, ItemListData } from './cleaners/skyblock/itemList.js' @@ -27,6 +31,7 @@ import typedHypixelApi from 'typed-hypixel-api' import * as cached from './hypixelCached.js' import { debug } from './index.js' import { WithId } from 'mongodb' +import { cleanEndedAuctions } from './cleaners/skyblock/endedAuctions.js' export type Included = 'profiles' | 'player' | 'stats' | 'inventories' | undefined @@ -57,8 +62,9 @@ const cleanResponseFunctions = { 'player': (data, options) => cleanPlayerResponse(data.player), 'skyblock/profile': (data: typedHypixelApi.SkyBlockProfileResponse, options) => cleanSkyblockProfileResponse(data.profile, options), 'skyblock/profiles': (data, options) => cleanSkyblockProfilesResponse(data.profiles), + 'skyblock/auctions_ended': (data, options) => cleanEndedAuctions(data), 'resources/skyblock/election': (data, options) => cleanElectionResponse(data), - 'resources/skyblock/items': (data, options) => cleanItemListResponse(data) + 'resources/skyblock/items': (data, options) => cleanItemListResponse(data), } as const @@ -354,3 +360,91 @@ export async function fetchItemList() { return itemList } +// this function is called from database.ts so it starts when we connect to the database +// it should only ever be called once! +export async function periodicallyFetchRecentlyEndedAuctions() { + let previousAuctionIds = new Set() + + while (true) { + const endedAuctions = await sendCleanApiRequest( + 'skyblock/auctions_ended', + {} + ) + + + let newAuctionUuids: Set = new Set() + let newAuctionItemIds: Set = new Set() + + for (const auction of endedAuctions.auctions) { + if (previousAuctionIds.has(auction.id)) continue + newAuctionUuids.add(auction.id) + newAuctionItemIds.add(auction.item.id) + } + let updatedDatabaseAuctionItems: Map = new Map() + + const itemsAuctions = await fetchItemsAuctions(Array.from(newAuctionItemIds)) + for (const itemAuctions of itemsAuctions) { + updatedDatabaseAuctionItems[itemAuctions._id] = itemAuctions + } + + for (const auction of endedAuctions.auctions) { + if (previousAuctionIds.has(auction.id)) continue + + let auctions: SimpleAuctionSchema[] + if (!updatedDatabaseAuctionItems.has(auction.item.id)) { + updatedDatabaseAuctionItems.set(auction.item.id, { + _id: auction.item.id, + auctions: [], + }) + auctions = [] + } else { + auctions = updatedDatabaseAuctionItems.get(auction.item.id)!.auctions + } + + const simpleAuction: SimpleAuctionSchema = { + success: true, + coins: auction.coins, + id: auction.id, + ts: Math.floor(auction.timestamp / 1000), + bin: auction.bin, + } + // make sure the auction isn't already in there + if (auctions.findIndex((a) => a.id === simpleAuction.id) !== null) { + auctions.push(simpleAuction) + // keep only the last 100 items + if (auctions.length > 100) + auctions = auctions.slice(auctions.length - 100) + } + + updatedDatabaseAuctionItems.set(auction.item.id, { + _id: auction.item.id, + auctions, + }) + } + + // we use a promise pool to set all the things fast but not overload the database + let tasks = Array.from(updatedDatabaseAuctionItems.values()).map(t => updateItemAuction(t)) + async function doTasks() { + let hasTask = true + while (hasTask) { + const task = tasks.pop() + if (task) { + await task + } else + hasTask = false + } + } + // Promise.all 5 cycles + await Promise.all(Array(5).fill(0).map(_ => doTasks())) + + previousAuctionIds = newAuctionUuids + + let endedAgo = Date.now() - endedAuctions.lastUpdated + // +10 seconds just so we're sure we'll get the update + let refetchIn = 60 * 1000 - endedAgo + 10000 + + await new Promise(resolve => setTimeout(resolve, refetchIn)) + } +} + + diff --git a/src/index.ts b/src/index.ts index 1d75830..59e7761 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ -import { createSession, fetchAccountFromDiscord, fetchAllLeaderboardsCategorized, fetchLeaderboard, fetchMemberLeaderboardSpots, fetchSession, finishedCachingRawLeaderboards, leaderboardUpdateMemberQueue, leaderboardUpdateProfileQueue, updateAccount, deleteSession } from './database.js' +import { createSession, fetchAccountFromDiscord, fetchAllLeaderboardsCategorized, fetchLeaderboard, fetchMemberLeaderboardSpots, fetchSession, finishedCachingRawLeaderboards, leaderboardUpdateMemberQueue, leaderboardUpdateProfileQueue, updateAccount, deleteSession, fetchPaginatedItemsAuctions } from './database.js' import { fetchElection, fetchItemList, fetchMemberProfile, fetchUser } from './hypixel.js' import rateLimit from 'express-rate-limit' import * as constants from './constants.js' @@ -164,6 +164,20 @@ app.get('/items', async (req, res) => { } }) + +app.get('/auctionprices', async (req, res) => { + try { + res + .setHeader('Cache-Control', 'public, max-age=600') + .json( + await fetchPaginatedItemsAuctions(0, 100) + ) + } catch (err) { + console.error(err) + res.json({ ok: false }) + } +}) + app.post('/accounts/createsession', async (req, res) => { try { const { code } = req.body -- cgit