add very basic auctionprices endpoint

5 files changed, 206 insertions, 13 deletions

diff --git a/src/cleaners/skyblock/endedAuctions.ts b/src/cleaners/skyblock/endedAuctions.ts
new file mode 100644
index 0000000..0dfa0ab
--- /dev/null
+++ b/src/cleaners/skyblock/endedAuctions.ts
@@ -0,0 +1,41 @@
+import typedHypixelApi from 'typed-hypixel-api'
+import { cleanInventory, headIdFromBase64, Item } from './inventory.js'
+import { cleanItemId } from './itemId.js'
+
+
+interface Auction {
+    id: string
+    sellerUuid: string
+    sellerProfileUuid: string
+    buyerUuid: string
+    timestamp: number
+    coins: number
+    bin: boolean
+    item: Item
+}
+
+export interface EndedAuctions {
+    lastUpdated: number
+    auctions: Auction[]
+}
+
+export async function cleanEndedAuctions(data: typedHypixelApi.SkyBlockRecentlyEndedAuctionsResponse): Promise<EndedAuctions> {
+    const auctions: Auction[] = []
+    for (const auction of data.auctions) {
+        auctions.push({
+            id: auction.auction_id,
+            sellerUuid: auction.seller,
+            sellerProfileUuid: auction.seller_profile,
+            buyerUuid: auction.buyer,
+            timestamp: auction.timestamp,
+            coins: auction.price,
+            bin: auction.bin,
+            item: (await cleanInventory(auction.item_bytes))[0]
+        })
+    }
+
+    return {
+        lastUpdated: data.lastUpdated,
+        auctions
+    }
+}
\ No newline at end of file
diff --git a/src/cleaners/skyblock/inventory.ts b/src/cleaners/skyblock/inventory.ts
index c5ab952..df30be3 100644
--- a/src/cleaners/skyblock/inventory.ts
+++ b/src/cleaners/skyblock/inventory.ts
@@ -6,7 +6,7 @@ function base64decode(base64: string): Buffer {
 	return Buffer.from(base64, 'base64')
 }
 
-interface Item {
+export interface Item {
 	id: string
 	count: number
 	vanillaId: string
@@ -79,15 +79,12 @@ function cleanItems(rawItems): Inventory {
 	return rawItems.map(cleanItem)
 }
 
-export function cleanInventory(encodedNbt: string): Promise<Inventory> {
-	return new Promise(resolve => {
-		const base64Data = base64decode(encodedNbt)
-		nbt.parse(base64Data, false, (err, value) => {
-			const simplifiedNbt = nbt.simplify(value)
-			// do some basic cleaning on the items and return
-			resolve(cleanItems(simplifiedNbt.i))
-		})
-	})
+export async function cleanInventory(encodedNbt: string): Promise<Inventory> {
+	const base64Data = base64decode(encodedNbt)
+	const value: any = await new Promise((resolve, reject) => nbt.parse(base64Data, false, (err, value) => { if (err) reject(err); else resolve(value) }))
+	const simplifiedNbt = nbt.simplify(value)
+	// do some basic cleaning on the items and return
+	return cleanItems(simplifiedNbt.i)
 }
 
 export const INVENTORIES = {
diff --git a/src/database.ts b/src/database.ts
index 04b3825..8f63d97 100644
--- a/src/database.ts
+++ b/src/database.ts
@@ -17,6 +17,7 @@ import { debug } from './index.js'
 import Queue from 'queue-promise'
 import { RANK_COLORS } from './cleaners/rank.js'
 import { cleanItemId } from './cleaners/skyblock/itemId.js'
+import { periodicallyFetchRecentlyEndedAuctions } from './hypixel.js'
 
 // don't update the user for 3 minutes
 const recentlyUpdated = new NodeCache({
@@ -116,10 +117,30 @@ export interface AccountSchema {
 	customization?: AccountCustomization
 }
 
+export interface SimpleAuctionSchema {
+	/** The UUID of the auction so we can look it up later. */
+	id: string
+	coins: number
+	/**
+	 * The timestamp as **seconds** since epoch. It's in seconds instead of ms
+	 * since we don't need to be super exact and so it's shorter.
+	 */
+	ts: number
+	/** Whether the auction was bought or simply expired. */
+	success: boolean
+	bin: boolean
+}
+export interface ItemAuctionsSchema {
+	/** The id of the item */
+	_id: string
+	auctions: SimpleAuctionSchema[]
+}
+
 let memberLeaderboardsCollection: Collection<DatabaseMemberLeaderboardItem>
 let profileLeaderboardsCollection: Collection<DatabaseProfileLeaderboardItem>
 let sessionsCollection: Collection<SessionSchema>
 let accountsCollection: Collection<AccountSchema>
+let itemAuctionsCollection: Collection<ItemAuctionsSchema>
 
 
 const leaderboardInfos: { [leaderboardName: string]: string } = {
@@ -142,6 +163,10 @@ async function connect(): Promise<void> {
 	profileLeaderboardsCollection = database.collection('profile-leaderboards')
 	sessionsCollection = database.collection('sessions')
 	accountsCollection = database.collection('accounts')
+	itemAuctionsCollection = database.collection('item-auctions')
+
+	periodicallyFetchRecentlyEndedAuctions()
+
 	console.log('Connected to database :)')
 }
 
@@ -1064,6 +1089,28 @@ export async function updateAccount(discordId: string, schema: AccountSchema) {
 	}, { $set: schema }, { upsert: true })
 }
 
+/** Fetch all the Item Auctions for the item ids in the given array. */
+export async function fetchItemsAuctions(itemIds: string[]): Promise<ItemAuctionsSchema[]> {
+	const auctions = await itemAuctionsCollection?.find({
+		_id: { $in: itemIds }
+	}).toArray()
+	return auctions
+}
+
+
+/** Fetch all the Item Auctions for the item ids in the given array. */
+export async function fetchPaginatedItemsAuctions(skip: number, limit: number): Promise<ItemAuctionsSchema[]> {
+	const auctions = await itemAuctionsCollection?.find({}).skip(skip).limit(limit).toArray()
+	return auctions
+}
+
+export async function updateItemAuction(auction: ItemAuctionsSchema) {
+	await itemAuctionsCollection?.updateOne({
+		_id: auction._id,
+	}, { $set: auction }, { upsert: true })
+}
+
+
 export async function fetchServerStatus() {
 	return await database.admin().serverStatus()
 }
diff --git a/src/hypixel.ts b/src/hypixel.ts
index d7a5887..060cff4 100644
--- a/src/hypixel.ts
+++ b/src/hypixel.ts
@@ -13,9 +13,13 @@ import {
 	AccountCustomization,
 	AccountSchema,
 	fetchAccount,
+	fetchItemsAuctions,
+	ItemAuctionsSchema,
 	queueUpdateDatabaseMember,
 	queueUpdateDatabaseProfile,
-	removeDeletedProfilesFromLeaderboards
+	removeDeletedProfilesFromLeaderboards,
+	SimpleAuctionSchema,
+	updateItemAuction
 } from './database.js'
 import { cleanElectionResponse, ElectionData } from './cleaners/skyblock/election.js'
 import { cleanItemListResponse, ItemListData } from './cleaners/skyblock/itemList.js'
@@ -27,6 +31,7 @@ import typedHypixelApi from 'typed-hypixel-api'
 import * as cached from './hypixelCached.js'
 import { debug } from './index.js'
 import { WithId } from 'mongodb'
+import { cleanEndedAuctions } from './cleaners/skyblock/endedAuctions.js'
 
 export type Included = 'profiles' | 'player' | 'stats' | 'inventories' | undefined
 
@@ -57,8 +62,9 @@ const cleanResponseFunctions = {
 	'player': (data, options) => cleanPlayerResponse(data.player),
 	'skyblock/profile': (data: typedHypixelApi.SkyBlockProfileResponse, options) => cleanSkyblockProfileResponse(data.profile, options),
 	'skyblock/profiles': (data, options) => cleanSkyblockProfilesResponse(data.profiles),
+	'skyblock/auctions_ended': (data, options) => cleanEndedAuctions(data),
 	'resources/skyblock/election': (data, options) => cleanElectionResponse(data),
-	'resources/skyblock/items': (data, options) => cleanItemListResponse(data)
+	'resources/skyblock/items': (data, options) => cleanItemListResponse(data),
 } as const
 
 
@@ -354,3 +360,91 @@ export async function fetchItemList() {
 	return itemList
 }
 
+// this function is called from database.ts so it starts when we connect to the database
+// it should only ever be called once!
+export async function periodicallyFetchRecentlyEndedAuctions() {
+	let previousAuctionIds = new Set()
+
+	while (true) {
+		const endedAuctions = await sendCleanApiRequest(
+			'skyblock/auctions_ended',
+			{}
+		)
+
+
+		let newAuctionUuids: Set<string> = new Set()
+		let newAuctionItemIds: Set<string> = new Set()
+
+		for (const auction of endedAuctions.auctions) {
+			if (previousAuctionIds.has(auction.id)) continue
+			newAuctionUuids.add(auction.id)
+			newAuctionItemIds.add(auction.item.id)
+		}
+		let updatedDatabaseAuctionItems: Map<string, ItemAuctionsSchema> = new Map()
+
+		const itemsAuctions = await fetchItemsAuctions(Array.from(newAuctionItemIds))
+		for (const itemAuctions of itemsAuctions) {
+			updatedDatabaseAuctionItems[itemAuctions._id] = itemAuctions
+		}
+
+		for (const auction of endedAuctions.auctions) {
+			if (previousAuctionIds.has(auction.id)) continue
+
+			let auctions: SimpleAuctionSchema[]
+			if (!updatedDatabaseAuctionItems.has(auction.item.id)) {
+				updatedDatabaseAuctionItems.set(auction.item.id, {
+					_id: auction.item.id,
+					auctions: [],
+				})
+				auctions = []
+			} else {
+				auctions = updatedDatabaseAuctionItems.get(auction.item.id)!.auctions
+			}
+
+			const simpleAuction: SimpleAuctionSchema = {
+				success: true,
+				coins: auction.coins,
+				id: auction.id,
+				ts: Math.floor(auction.timestamp / 1000),
+				bin: auction.bin,
+			}
+			// make sure the auction isn't already in there
+			if (auctions.findIndex((a) => a.id === simpleAuction.id) !== null) {
+				auctions.push(simpleAuction)
+				// keep only the last 100 items
+				if (auctions.length > 100)
+					auctions = auctions.slice(auctions.length - 100)
+			}
+
+			updatedDatabaseAuctionItems.set(auction.item.id, {
+				_id: auction.item.id,
+				auctions,
+			})
+		}
+
+		// we use a promise pool to set all the things fast but not overload the database
+		let tasks = Array.from(updatedDatabaseAuctionItems.values()).map(t => updateItemAuction(t))
+		async function doTasks() {
+			let hasTask = true
+			while (hasTask) {
+				const task = tasks.pop()
+				if (task) {
+					await task
+				} else
+					hasTask = false
+			}
+		}
+		// Promise.all 5 cycles
+		await Promise.all(Array(5).fill(0).map(_ => doTasks()))
+
+		previousAuctionIds = newAuctionUuids
+
+		let endedAgo = Date.now() - endedAuctions.lastUpdated
+		// +10 seconds just so we're sure we'll get the update
+		let refetchIn = 60 * 1000 - endedAgo + 10000
+
+		await new Promise(resolve => setTimeout(resolve, refetchIn))
+	}
+}
+
+
diff --git a/src/index.ts b/src/index.ts
index 1d75830..59e7761 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,4 +1,4 @@
-import { createSession, fetchAccountFromDiscord, fetchAllLeaderboardsCategorized, fetchLeaderboard, fetchMemberLeaderboardSpots, fetchSession, finishedCachingRawLeaderboards, leaderboardUpdateMemberQueue, leaderboardUpdateProfileQueue, updateAccount, deleteSession } from './database.js'
+import { createSession, fetchAccountFromDiscord, fetchAllLeaderboardsCategorized, fetchLeaderboard, fetchMemberLeaderboardSpots, fetchSession, finishedCachingRawLeaderboards, leaderboardUpdateMemberQueue, leaderboardUpdateProfileQueue, updateAccount, deleteSession, fetchPaginatedItemsAuctions } from './database.js'
 import { fetchElection, fetchItemList, fetchMemberProfile, fetchUser } from './hypixel.js'
 import rateLimit from 'express-rate-limit'
 import * as constants from './constants.js'
@@ -164,6 +164,20 @@ app.get('/items', async (req, res) => {
 	}
 })
 
+
+app.get('/auctionprices', async (req, res) => {
+	try {
+		res
+			.setHeader('Cache-Control', 'public, max-age=600')
+			.json(
+				await fetchPaginatedItemsAuctions(0, 100)
+			)
+	} catch (err) {
+		console.error(err)
+		res.json({ ok: false })
+	}
+})
+
 app.post('/accounts/createsession', async (req, res) => {
 	try {
 		const { code } = req.body