optimize leaderboards

4 files changed, 153 insertions, 35 deletions

diff --git a/build/database.js b/build/database.js
index d28d934..05a082f 100644
--- a/build/database.js
+++ b/build/database.js
@@ -33,6 +33,7 @@ const node_cache_1 = __importDefault(require("node-cache"));
 const util_1 = require("./util");
 const stats_1 = require("./cleaners/skyblock/stats");
 const queue_promise_1 = __importDefault(require("queue-promise"));
+const _1 = require(".");
 // don't update the user for 3 minutes
 const recentlyUpdated = new node_cache_1.default({
     stdTTL: 60 * 3,
@@ -41,10 +42,14 @@ const recentlyUpdated = new node_cache_1.default({
 });
 const cachedRawLeaderboards = new Map();
 const leaderboardMax = 100;
-const reversedStats = [
+const reversedLeaderboards = [
     'first_join',
     '_best_time', '_best_time_2'
 ];
+const leaderboardUnits = {
+    time: ['_best_time', '_best_time_2'],
+    date: ['first_join']
+};
 let client;
 let database;
 let memberLeaderboardsCollection;
@@ -121,16 +126,28 @@ async function fetchAllMemberLeaderboardAttributes() {
 }
 exports.fetchAllMemberLeaderboardAttributes = fetchAllMemberLeaderboardAttributes;
 function isLeaderboardReversed(name) {
-    for (const statMatch of reversedStats) {
-        let trailingEnd = statMatch[0] === '_';
-        let trailingStart = statMatch.substr(-1) === '_';
-        if ((trailingStart && name.startsWith(statMatch))
-            || (trailingEnd && name.endsWith(statMatch))
-            || (name == statMatch))
+    for (const leaderboardMatch of reversedLeaderboards) {
+        let trailingEnd = leaderboardMatch[0] === '_';
+        let trailingStart = leaderboardMatch.substr(-1) === '_';
+        if ((trailingStart && name.startsWith(leaderboardMatch))
+            || (trailingEnd && name.endsWith(leaderboardMatch))
+            || (name == leaderboardMatch))
             return true;
     }
     return false;
 }
+function getLeaderboardUnit(name) {
+    for (const [unitName, leaderboardMatchers] of Object.entries(leaderboardUnits)) {
+        for (const leaderboardMatch of leaderboardMatchers) {
+            let trailingEnd = leaderboardMatch[0] === '_';
+            let trailingStart = leaderboardMatch.substr(-1) === '_';
+            if ((trailingStart && name.startsWith(leaderboardMatch))
+                || (trailingEnd && name.endsWith(leaderboardMatch))
+                || (name == leaderboardMatch))
+                return unitName;
+        }
+    }
+}
 async function fetchMemberLeaderboardRaw(name) {
     if (cachedRawLeaderboards.has(name))
         return cachedRawLeaderboards.get(name);
@@ -144,10 +161,11 @@ async function fetchMemberLeaderboardRaw(name) {
     return leaderboardRaw;
 }
 async function fetchMemberLeaderboard(name) {
+    var _a;
     const leaderboardRaw = await fetchMemberLeaderboardRaw(name);
     const fetchLeaderboardPlayer = async (item) => {
         return {
-            player: await cached.fetchPlayer(item.uuid),
+            player: await cached.fetchBasicPlayer(item.uuid),
             value: item.stats[name]
         };
     };
@@ -156,7 +174,11 @@ async function fetchMemberLeaderboard(name) {
         promises.push(fetchLeaderboardPlayer(item));
     }
     const leaderboard = await Promise.all(promises);
-    return leaderboard;
+    return {
+        name: name,
+        unit: (_a = getLeaderboardUnit(name)) !== null && _a !== void 0 ? _a : null,
+        list: leaderboard
+    };
 }
 exports.fetchMemberLeaderboard = fetchMemberLeaderboard;
 async function getMemberLeaderboardRequirement(name) {
@@ -175,13 +197,11 @@ async function getApplicableAttributes(member) {
         const requirement = await getMemberLeaderboardRequirement(attributeName);
         if (!requirement || attributeValue > requirement)
             applicableAttributes[attributeName] = attributeValue;
-        console.log(attributeName);
     }
     return applicableAttributes;
 }
 /** Update the member's leaderboard data on the server if applicable */
 async function updateDatabaseMember(member, profile) {
-    console.log('updating', member.uuid);
     if (!client)
         return; // the db client hasn't been initialized
     // the member's been updated too recently, just return
@@ -218,7 +238,6 @@ async function updateDatabaseMember(member, profile) {
             .slice(0, 100);
         cachedRawLeaderboards.set(attributeName, newRawLeaderboard);
     }
-    console.log('updated', member.uuid);
 }
 exports.updateDatabaseMember = updateDatabaseMember;
 const queue = new queue_promise_1.default({
@@ -238,7 +257,7 @@ async function removeBadMemberLeaderboardAttributes() {
     // shuffle so if the application is restarting many times itll still be useful
     for (const leaderboard of util_1.shuffle(leaderboards)) {
         // wait 10 seconds so it doesnt use as much ram
-        await util_1.sleep(100000);
+        await util_1.sleep(10 * 1000);
         const unsetValue = {};
         unsetValue[leaderboard] = '';
         const filter = {};
@@ -251,5 +270,25 @@ async function removeBadMemberLeaderboardAttributes() {
         }
     }
 }
-connect()
-    .then(removeBadMemberLeaderboardAttributes);
+/** Fetch all the leaderboards, used for caching. Don't call this often! */
+async function fetchAllLeaderboards() {
+    const leaderboards = await fetchAllMemberLeaderboardAttributes();
+    // shuffle so if the application is restarting many times itll still be useful
+    if (_1.debug)
+        console.log('Caching leaderboards!');
+    for (const leaderboard of util_1.shuffle(leaderboards)) {
+        // wait 2 seconds so it doesnt use as much ram
+        await util_1.sleep(2 * 1000);
+        await fetchMemberLeaderboard(leaderboard);
+    }
+    if (_1.debug)
+        console.log('Finished caching leaderboards!');
+}
+connect().then(() => {
+    // when it connects, cache the leaderboards and remove bad members
+    removeBadMemberLeaderboardAttributes();
+    // cache leaderboards on startup so its faster later on
+    fetchAllLeaderboards();
+    // cache leaderboard players again every hour
+    setInterval(fetchAllLeaderboards, 4 * 60 * 60 * 1000);
+});
diff --git a/build/hypixelCached.js b/build/hypixelCached.js
index ccb0ca6..87f4a9e 100644
--- a/build/hypixelCached.js
+++ b/build/hypixelCached.js
@@ -25,7 +25,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.fetchProfileName = exports.fetchProfile = exports.fetchProfileUuid = exports.fetchSkyblockProfiles = exports.fetchPlayer = exports.usernameFromUser = exports.uuidFromUser = void 0;
+exports.fetchProfileName = exports.fetchProfile = exports.fetchProfileUuid = exports.fetchSkyblockProfiles = exports.fetchBasicPlayer = exports.fetchPlayer = exports.usernameFromUser = exports.uuidFromUser = void 0;
 const node_cache_1 = __importDefault(require("node-cache"));
 const mojang = __importStar(require("./mojang"));
 const hypixel = __importStar(require("./hypixel"));
@@ -47,6 +47,12 @@ const playerCache = new node_cache_1.default({
     checkperiod: 10,
     useClones: true,
 });
+// cache "basic players" (players without profiles) for 4 hours
+const basicPlayerCache = new node_cache_1.default({
+    stdTTL: 60 * 60 * 4,
+    checkperiod: 60 * 10,
+    useClones: true
+});
 const profileCache = new node_cache_1.default({
     stdTTL: 30,
     checkperiod: 10,
@@ -137,9 +143,8 @@ async function usernameFromUser(user) {
 exports.usernameFromUser = usernameFromUser;
 async function fetchPlayer(user) {
     const playerUuid = await uuidFromUser(user);
-    if (playerCache.has(playerUuid)) {
+    if (playerCache.has(playerUuid))
         return playerCache.get(playerUuid);
-    }
     const cleanPlayer = await hypixel.sendCleanApiRequest({
         path: 'player',
         args: { uuid: playerUuid }
@@ -149,9 +154,22 @@ async function fetchPlayer(user) {
     // clone in case it gets modified somehow later
     playerCache.set(playerUuid, cleanPlayer);
     usernameCache.set(playerUuid, cleanPlayer.username);
+    const cleanBasicPlayer = Object.assign({}, cleanPlayer);
+    delete cleanBasicPlayer.profiles;
+    basicPlayerCache.set(playerUuid, cleanBasicPlayer);
     return cleanPlayer;
 }
 exports.fetchPlayer = fetchPlayer;
+/** Fetch a player without their profiles. This is heavily cached. */
+async function fetchBasicPlayer(user) {
+    const playerUuid = await uuidFromUser(user);
+    if (basicPlayerCache.has(playerUuid))
+        return basicPlayerCache.get(playerUuid);
+    const player = await fetchPlayer(playerUuid);
+    delete player.profiles;
+    return player;
+}
+exports.fetchBasicPlayer = fetchBasicPlayer;
 async function fetchSkyblockProfiles(playerUuid) {
     if (profilesCache.has(playerUuid)) {
         if (_1.debug)
diff --git a/src/database.ts b/src/database.ts
index 29760ef..466e2f1 100644
--- a/src/database.ts
+++ b/src/database.ts
@@ -12,6 +12,7 @@ import { shuffle, sleep } from './util'
 import { CleanFullProfile } from './cleaners/skyblock/profile'
 import { categorizeStat } from './cleaners/skyblock/stats'
 import Queue from 'queue-promise'
+import { debug } from '.'
 
 // don't update the user for 3 minutes
 const recentlyUpdated = new NodeCache({
@@ -34,10 +35,14 @@ interface LeaderboardItem {
 const cachedRawLeaderboards: Map<string, DatabaseLeaderboardItem[]> = new Map()
 
 const leaderboardMax = 100
-const reversedStats = [
+const reversedLeaderboards = [
 	'first_join',
 	'_best_time', '_best_time_2'
 ]
+const leaderboardUnits = {
+	time: ['_best_time', '_best_time_2'],
+	date: ['first_join']
+}
 
 let client: MongoClient
 let database: Db
@@ -131,19 +136,34 @@ export async function fetchAllMemberLeaderboardAttributes(): Promise<string[]> {
 }
 
 function isLeaderboardReversed(name: string): boolean {
-	for (const statMatch of reversedStats) {
-		let trailingEnd = statMatch[0] === '_'
-		let trailingStart = statMatch.substr(-1) === '_'
+	for (const leaderboardMatch of reversedLeaderboards) {
+		let trailingEnd = leaderboardMatch[0] === '_'
+		let trailingStart = leaderboardMatch.substr(-1) === '_'
 		if (
-			(trailingStart && name.startsWith(statMatch))
-			|| (trailingEnd && name.endsWith(statMatch))
-			|| (name == statMatch)
+			(trailingStart && name.startsWith(leaderboardMatch))
+			|| (trailingEnd && name.endsWith(leaderboardMatch))
+			|| (name == leaderboardMatch)
 		)
 			return true
 	}
 	return false
 }
 
+function getLeaderboardUnit(name: string): string {
+	for (const [ unitName, leaderboardMatchers ] of Object.entries(leaderboardUnits)) {
+		for (const leaderboardMatch of leaderboardMatchers) {
+			let trailingEnd = leaderboardMatch[0] === '_'
+			let trailingStart = leaderboardMatch.substr(-1) === '_'
+			if (
+				(trailingStart && name.startsWith(leaderboardMatch))
+				|| (trailingEnd && name.endsWith(leaderboardMatch))
+				|| (name == leaderboardMatch)
+			)
+				return unitName
+		}
+	}
+}
+
 async function fetchMemberLeaderboardRaw(name: string): Promise<DatabaseLeaderboardItem[]> {
 	if (cachedRawLeaderboards.has(name))
 		return cachedRawLeaderboards.get(name)
@@ -163,7 +183,7 @@ export async function fetchMemberLeaderboard(name: string) {
 	const leaderboardRaw = await fetchMemberLeaderboardRaw(name)
 	const fetchLeaderboardPlayer = async(item: DatabaseLeaderboardItem): Promise<LeaderboardItem> => {
 		return {
-			player: await cached.fetchPlayer(item.uuid),
+			player: await cached.fetchBasicPlayer(item.uuid),
 			value: item.stats[name]
 		}
 	}
@@ -172,7 +192,11 @@ export async function fetchMemberLeaderboard(name: string) {
 		promises.push(fetchLeaderboardPlayer(item))
 	}
 	const leaderboard = await Promise.all(promises)
-	return leaderboard
+	return {
+		name: name,
+		unit: getLeaderboardUnit(name) ?? null,
+		list: leaderboard
+	}
 }
 
 async function getMemberLeaderboardRequirement(name: string): Promise<number> {
@@ -192,14 +216,12 @@ async function getApplicableAttributes(member): Promise<{ [key: string]: number
 		const requirement = await getMemberLeaderboardRequirement(attributeName)
 		if (!requirement || attributeValue > requirement)
 			applicableAttributes[attributeName] = attributeValue
-		console.log(attributeName)
 	}
 	return applicableAttributes
 }
 
 /** Update the member's leaderboard data on the server if applicable */
 export async function updateDatabaseMember(member: CleanMember, profile: CleanFullProfile) {
-	console.log('updating', member.uuid)
 	if (!client) return // the db client hasn't been initialized
 	// the member's been updated too recently, just return
 	if (recentlyUpdated.get(profile.uuid + member.uuid))
@@ -243,7 +265,6 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu
 			.slice(0, 100)
 		cachedRawLeaderboards.set(attributeName, newRawLeaderboard)
 	}
-	console.log('updated', member.uuid)
 }
 
 const queue = new Queue({
@@ -265,7 +286,7 @@ async function removeBadMemberLeaderboardAttributes() {
 	// shuffle so if the application is restarting many times itll still be useful
 	for (const leaderboard of shuffle(leaderboards)) {
 		// wait 10 seconds so it doesnt use as much ram
-		await sleep(100000)
+		await sleep(10 * 1000)
 
 		const unsetValue = {}
 		unsetValue[leaderboard] = ''
@@ -283,7 +304,26 @@ async function removeBadMemberLeaderboardAttributes() {
 	}
 }
 
+/** Fetch all the leaderboards, used for caching. Don't call this often! */
+async function fetchAllLeaderboards() {
+	const leaderboards = await fetchAllMemberLeaderboardAttributes()
+	// shuffle so if the application is restarting many times itll still be useful
+	if (debug) console.log('Caching leaderboards!')
+	for (const leaderboard of shuffle(leaderboards)) {
+		// wait 2 seconds so it doesnt use as much ram
+		await sleep(2 * 1000)
+
+		await fetchMemberLeaderboard(leaderboard)
+	}
+	if (debug) console.log('Finished caching leaderboards!')
+}
 
-connect()
-	.then(removeBadMemberLeaderboardAttributes)
 
+connect().then(() => {
+	// when it connects, cache the leaderboards and remove bad members
+	removeBadMemberLeaderboardAttributes()
+	// cache leaderboards on startup so its faster later on
+	fetchAllLeaderboards()
+	// cache leaderboard players again every hour
+	setInterval(fetchAllLeaderboards, 4 * 60 * 60 * 1000)
+})
diff --git a/src/hypixelCached.ts b/src/hypixelCached.ts
index 4063069..5338229 100644
--- a/src/hypixelCached.ts
+++ b/src/hypixelCached.ts
@@ -29,6 +29,13 @@ const playerCache = new NodeCache({
 	useClones: true,
 })
 
+// cache "basic players" (players without profiles) for 4 hours
+const basicPlayerCache = new NodeCache({
+	stdTTL: 60 * 60 * 4,
+	checkperiod: 60 * 10,
+	useClones: true
+})
+
 const profileCache = new NodeCache({
 	stdTTL: 30,
 	checkperiod: 10,
@@ -130,9 +137,8 @@ export async function usernameFromUser(user: string): Promise<string> {
 export async function fetchPlayer(user: string): Promise<CleanPlayer> {
 	const playerUuid = await uuidFromUser(user)
 
-	if (playerCache.has(playerUuid)) {
+	if (playerCache.has(playerUuid))
 		return playerCache.get(playerUuid)
-	}
 
 	const cleanPlayer: CleanPlayer = await hypixel.sendCleanApiRequest({
 		path: 'player',
@@ -144,10 +150,25 @@ export async function fetchPlayer(user: string): Promise<CleanPlayer> {
 	// clone in case it gets modified somehow later
 	playerCache.set(playerUuid, cleanPlayer)
 	usernameCache.set(playerUuid, cleanPlayer.username)
+	
+	const cleanBasicPlayer = Object.assign({}, cleanPlayer)
+	delete cleanBasicPlayer.profiles
+	basicPlayerCache.set(playerUuid, cleanBasicPlayer)
 
 	return cleanPlayer
 }
 
+/** Fetch a player without their profiles. This is heavily cached. */
+export async function fetchBasicPlayer(user: string): Promise<CleanPlayer> {
+	const playerUuid = await uuidFromUser(user)
+
+	if (basicPlayerCache.has(playerUuid))
+		return basicPlayerCache.get(playerUuid)
+	
+	const player = await fetchPlayer(playerUuid)
+	delete player.profiles
+	return player
+}
 
 export async function fetchSkyblockProfiles(playerUuid: string): Promise<CleanProfile[]> {
 	if (profilesCache.has(playerUuid)) {