optimize leaderboards

2 files changed, 78 insertions, 17 deletions

diff --git a/src/database.ts b/src/database.ts
index 29760ef..466e2f1 100644
--- a/src/database.ts
+++ b/src/database.ts
@@ -12,6 +12,7 @@ import { shuffle, sleep } from './util'
 import { CleanFullProfile } from './cleaners/skyblock/profile'
 import { categorizeStat } from './cleaners/skyblock/stats'
 import Queue from 'queue-promise'
+import { debug } from '.'
 
 // don't update the user for 3 minutes
 const recentlyUpdated = new NodeCache({
@@ -34,10 +35,14 @@ interface LeaderboardItem {
 const cachedRawLeaderboards: Map<string, DatabaseLeaderboardItem[]> = new Map()
 
 const leaderboardMax = 100
-const reversedStats = [
+const reversedLeaderboards = [
 	'first_join',
 	'_best_time', '_best_time_2'
 ]
+const leaderboardUnits = {
+	time: ['_best_time', '_best_time_2'],
+	date: ['first_join']
+}
 
 let client: MongoClient
 let database: Db
@@ -131,19 +136,34 @@ export async function fetchAllMemberLeaderboardAttributes(): Promise<string[]> {
 }
 
 function isLeaderboardReversed(name: string): boolean {
-	for (const statMatch of reversedStats) {
-		let trailingEnd = statMatch[0] === '_'
-		let trailingStart = statMatch.substr(-1) === '_'
+	for (const leaderboardMatch of reversedLeaderboards) {
+		let trailingEnd = leaderboardMatch[0] === '_'
+		let trailingStart = leaderboardMatch.substr(-1) === '_'
 		if (
-			(trailingStart && name.startsWith(statMatch))
-			|| (trailingEnd && name.endsWith(statMatch))
-			|| (name == statMatch)
+			(trailingStart && name.startsWith(leaderboardMatch))
+			|| (trailingEnd && name.endsWith(leaderboardMatch))
+			|| (name == leaderboardMatch)
 		)
 			return true
 	}
 	return false
 }
 
+function getLeaderboardUnit(name: string): string {
+	for (const [ unitName, leaderboardMatchers ] of Object.entries(leaderboardUnits)) {
+		for (const leaderboardMatch of leaderboardMatchers) {
+			let trailingEnd = leaderboardMatch[0] === '_'
+			let trailingStart = leaderboardMatch.substr(-1) === '_'
+			if (
+				(trailingStart && name.startsWith(leaderboardMatch))
+				|| (trailingEnd && name.endsWith(leaderboardMatch))
+				|| (name == leaderboardMatch)
+			)
+				return unitName
+		}
+	}
+}
+
 async function fetchMemberLeaderboardRaw(name: string): Promise<DatabaseLeaderboardItem[]> {
 	if (cachedRawLeaderboards.has(name))
 		return cachedRawLeaderboards.get(name)
@@ -163,7 +183,7 @@ export async function fetchMemberLeaderboard(name: string) {
 	const leaderboardRaw = await fetchMemberLeaderboardRaw(name)
 	const fetchLeaderboardPlayer = async(item: DatabaseLeaderboardItem): Promise<LeaderboardItem> => {
 		return {
-			player: await cached.fetchPlayer(item.uuid),
+			player: await cached.fetchBasicPlayer(item.uuid),
 			value: item.stats[name]
 		}
 	}
@@ -172,7 +192,11 @@ export async function fetchMemberLeaderboard(name: string) {
 		promises.push(fetchLeaderboardPlayer(item))
 	}
 	const leaderboard = await Promise.all(promises)
-	return leaderboard
+	return {
+		name: name,
+		unit: getLeaderboardUnit(name) ?? null,
+		list: leaderboard
+	}
 }
 
 async function getMemberLeaderboardRequirement(name: string): Promise<number> {
@@ -192,14 +216,12 @@ async function getApplicableAttributes(member): Promise<{ [key: string]: number
 		const requirement = await getMemberLeaderboardRequirement(attributeName)
 		if (!requirement || attributeValue > requirement)
 			applicableAttributes[attributeName] = attributeValue
-		console.log(attributeName)
 	}
 	return applicableAttributes
 }
 
 /** Update the member's leaderboard data on the server if applicable */
 export async function updateDatabaseMember(member: CleanMember, profile: CleanFullProfile) {
-	console.log('updating', member.uuid)
 	if (!client) return // the db client hasn't been initialized
 	// the member's been updated too recently, just return
 	if (recentlyUpdated.get(profile.uuid + member.uuid))
@@ -243,7 +265,6 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu
 			.slice(0, 100)
 		cachedRawLeaderboards.set(attributeName, newRawLeaderboard)
 	}
-	console.log('updated', member.uuid)
 }
 
 const queue = new Queue({
@@ -265,7 +286,7 @@ async function removeBadMemberLeaderboardAttributes() {
 	// shuffle so if the application is restarting many times itll still be useful
 	for (const leaderboard of shuffle(leaderboards)) {
 		// wait 10 seconds so it doesnt use as much ram
-		await sleep(100000)
+		await sleep(10 * 1000)
 
 		const unsetValue = {}
 		unsetValue[leaderboard] = ''
@@ -283,7 +304,26 @@ async function removeBadMemberLeaderboardAttributes() {
 	}
 }
 
+/** Fetch all the leaderboards, used for caching. Don't call this often! */
+async function fetchAllLeaderboards() {
+	const leaderboards = await fetchAllMemberLeaderboardAttributes()
+	// shuffle so if the application is restarting many times itll still be useful
+	if (debug) console.log('Caching leaderboards!')
+	for (const leaderboard of shuffle(leaderboards)) {
+		// wait 2 seconds so it doesnt use as much ram
+		await sleep(2 * 1000)
+
+		await fetchMemberLeaderboard(leaderboard)
+	}
+	if (debug) console.log('Finished caching leaderboards!')
+}
 
-connect()
-	.then(removeBadMemberLeaderboardAttributes)
 
+connect().then(() => {
+	// when it connects, cache the leaderboards and remove bad members
+	removeBadMemberLeaderboardAttributes()
+	// cache leaderboards on startup so its faster later on
+	fetchAllLeaderboards()
+	// cache leaderboard players again every hour
+	setInterval(fetchAllLeaderboards, 4 * 60 * 60 * 1000)
+})
diff --git a/src/hypixelCached.ts b/src/hypixelCached.ts
index 4063069..5338229 100644
--- a/src/hypixelCached.ts
+++ b/src/hypixelCached.ts
@@ -29,6 +29,13 @@ const playerCache = new NodeCache({
 	useClones: true,
 })
 
+// cache "basic players" (players without profiles) for 4 hours
+const basicPlayerCache = new NodeCache({
+	stdTTL: 60 * 60 * 4,
+	checkperiod: 60 * 10,
+	useClones: true
+})
+
 const profileCache = new NodeCache({
 	stdTTL: 30,
 	checkperiod: 10,
@@ -130,9 +137,8 @@ export async function usernameFromUser(user: string): Promise<string> {
 export async function fetchPlayer(user: string): Promise<CleanPlayer> {
 	const playerUuid = await uuidFromUser(user)
 
-	if (playerCache.has(playerUuid)) {
+	if (playerCache.has(playerUuid))
 		return playerCache.get(playerUuid)
-	}
 
 	const cleanPlayer: CleanPlayer = await hypixel.sendCleanApiRequest({
 		path: 'player',
@@ -144,10 +150,25 @@ export async function fetchPlayer(user: string): Promise<CleanPlayer> {
 	// clone in case it gets modified somehow later
 	playerCache.set(playerUuid, cleanPlayer)
 	usernameCache.set(playerUuid, cleanPlayer.username)
+	
+	const cleanBasicPlayer = Object.assign({}, cleanPlayer)
+	delete cleanBasicPlayer.profiles
+	basicPlayerCache.set(playerUuid, cleanBasicPlayer)
 
 	return cleanPlayer
 }
 
+/** Fetch a player without their profiles. This is heavily cached. */
+export async function fetchBasicPlayer(user: string): Promise<CleanPlayer> {
+	const playerUuid = await uuidFromUser(user)
+
+	if (basicPlayerCache.has(playerUuid))
+		return basicPlayerCache.get(playerUuid)
+	
+	const player = await fetchPlayer(playerUuid)
+	delete player.profiles
+	return player
+}
 
 export async function fetchSkyblockProfiles(playerUuid: string): Promise<CleanProfile[]> {
 	if (profilesCache.has(playerUuid)) {