Total leaderboards leaderboard working (#6)

* add leaderboards leaderboard

* slightly optimize fetchPlayer to not unnecessarily fetch the same player twice

* Compiled TS into JS

* fix errors with fetch

* fix random bugs that made people disappear from leaderboards

7 files changed, 136 insertions, 82 deletions

diff --git a/build/cleaners/skyblock/stats.js b/build/cleaners/skyblock/stats.js
index 0eb94c7..482771c 100644
--- a/build/cleaners/skyblock/stats.js
+++ b/build/cleaners/skyblock/stats.js
@@ -60,7 +60,8 @@ exports.categorizeStat = categorizeStat;
 exports.statUnits = {
     time: ['_best_time', '_best_time_2'],
     date: ['first_join'],
-    coins: ['purse']
+    coins: ['purse'],
+    leaderboards: ['leaderboards_count']
 };
 function getStatUnit(name) {
     for (const [unitName, statMatchers] of Object.entries(exports.statUnits)) {
diff --git a/build/database.js b/build/database.js
index e889f1e..b76b85c 100644
--- a/build/database.js
+++ b/build/database.js
@@ -155,6 +155,7 @@ async function fetchAllMemberLeaderboardAttributes() {
         'first_join',
         'purse',
         'visited_zones',
+        'leaderboards_count'
     ];
 }
 exports.fetchAllMemberLeaderboardAttributes = fetchAllMemberLeaderboardAttributes;
@@ -249,6 +250,13 @@ async function getApplicableAttributes(member) {
             applicableAttributes[leaderboard] = attributeValue;
         }
     }
+    let leaderboardsCount = Object.keys(applicableAttributes).length;
+    const leaderboardsCountRequirement = await getMemberLeaderboardRequirement('leaderboards_count');
+    if ((leaderboardsCountRequirement === null)
+        || (leaderboardsCount > leaderboardsCountRequirement)) {
+        // add 1 extra because this attribute also counts :)
+        applicableAttributes['leaderboards_count'] = leaderboardsCount;
+    }
     return applicableAttributes;
 }
 /** Update the member's leaderboard data on the server if applicable */
@@ -273,7 +281,7 @@ async function updateDatabaseMember(member, profile) {
         console.log('done constants..');
     const leaderboardAttributes = await getApplicableAttributes(member);
     if (_1.debug)
-        console.log('done getApplicableAttributes..', leaderboardAttributes);
+        console.log('done getApplicableAttributes..', leaderboardAttributes, member.username, profile.name);
     await memberLeaderboardsCollection.updateOne({
         uuid: member.uuid,
         profile: profile.uuid
@@ -288,7 +296,7 @@ async function updateDatabaseMember(member, profile) {
         const leaderboardReverse = isLeaderboardReversed(attributeName);
         const newRawLeaderboard = existingRawLeaderboard
             // remove the player from the leaderboard, if they're there
-            .filter(value => value.uuid !== member.uuid)
+            .filter(value => value.uuid !== member.uuid || value.profile !== profile.uuid)
             .concat([{
                 last_updated: new Date(),
                 stats: leaderboardAttributes,
@@ -354,7 +362,8 @@ connect().then(() => {
     // when it connects, cache the leaderboards and remove bad members
     removeBadMemberLeaderboardAttributes();
     // cache leaderboards on startup so its faster later on
-    fetchAllLeaderboards(true);
+    // fetchAllLeaderboards(true)
+    fetchAllLeaderboards(false);
     // cache leaderboard players again every 4 hours
     setInterval(fetchAllLeaderboards, 4 * 60 * 60 * 1000);
 });
diff --git a/build/hypixelCached.js b/build/hypixelCached.js
index a667ec4..8a35482 100644
--- a/build/hypixelCached.js
+++ b/build/hypixelCached.js
@@ -145,14 +145,24 @@ async function usernameFromUser(user) {
     return username;
 }
 exports.usernameFromUser = usernameFromUser;
+let fetchingPlayers = new Set();
 async function fetchPlayer(user) {
     const playerUuid = await uuidFromUser(user);
     if (playerCache.has(playerUuid))
         return playerCache.get(playerUuid);
+    // if it's already in the process of fetching, check every 100ms until it's not fetching the player anymore and fetch it again, since it'll be cached now
+    if (fetchingPlayers.has(playerUuid)) {
+        while (fetchingPlayers.has(playerUuid)) {
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+        return await fetchPlayer(user);
+    }
+    fetchingPlayers.add(playerUuid);
     const cleanPlayer = await hypixel.sendCleanApiRequest({
         path: 'player',
         args: { uuid: playerUuid }
     });
+    fetchingPlayers.delete(playerUuid);
     if (!cleanPlayer)
         return;
     // clone in case it gets modified somehow later
diff --git a/src/cleaners/skyblock/stats.ts b/src/cleaners/skyblock/stats.ts
index 218e034..31a894c 100644
--- a/src/cleaners/skyblock/stats.ts
+++ b/src/cleaners/skyblock/stats.ts
@@ -1,16 +1,16 @@
 const statCategories: { [ key: string ]: string[] | null } = { // sorted in order of importance
-    'deaths': ['deaths_', 'deaths'],
-    'kills': ['kills_', 'kills'],
-    'fishing': ['items_fished_', 'items_fished', 'shredder_'],
-    'auctions': ['auctions_'],
-    'races': ['_best_time', '_best_time_2'],
-    'mythos': ['mythos_burrows_', 'mythos_kills'],
+	'deaths': ['deaths_', 'deaths'],
+	'kills': ['kills_', 'kills'],
+	'fishing': ['items_fished_', 'items_fished', 'shredder_'],
+	'auctions': ['auctions_'],
+	'races': ['_best_time', '_best_time_2'],
+	'mythos': ['mythos_burrows_', 'mythos_kills'],
 
-    'collection': ['collection_'],
-    'skills': ['skill_'],
-    'slayer': ['slayer_'],
+	'collection': ['collection_'],
+	'skills': ['skill_'],
+	'slayer': ['slayer_'],
 
-    'misc': null // everything else goes here
+	'misc': null // everything else goes here
 }
 
 export interface StatCategory {
@@ -19,59 +19,60 @@ export interface StatCategory {
 }
 
 export function categorizeStat(statNameRaw: string): StatCategory {
-    // 'deaths_void'
-    for (const statCategory in statCategories) {
-        // 'deaths'
-        const statCategoryMatchers = statCategories[statCategory]
-        if (statCategoryMatchers == null) {
-            // If it's null, just go with this. Can only ever be 'misc'
-            return {
-                category: statCategory,
-                name: statNameRaw
-            }
-        }
-        for (const categoryMatch of statCategoryMatchers) {
-            // ['deaths_']
-            let trailingEnd = categoryMatch[0] === '_'
-            let trailingStart = categoryMatch.substr(-1) === '_'
-            if (trailingStart && statNameRaw.startsWith(categoryMatch)) {
-                return {
-                    category: statCategory,
-                    name: statNameRaw.substr(categoryMatch.length)
-                }
-            } else if (trailingEnd && statNameRaw.endsWith(categoryMatch)) {
-                return {
-                    category: statCategory,
-                    name: statNameRaw.substr(0, statNameRaw.length - categoryMatch.length)
-                }
-            } else if (statNameRaw == categoryMatch) {
-                // if it matches exactly, we don't know the name. will be defaulted to category later on
-                return {
-                    category: statCategory,
-                    name: null
-                }
-            }
-        }
-    }
-    // this should never happen, as it'll default to misc and return if nothing is found
-    return {
-        category: null,
-        name: statNameRaw
-    }
+	// 'deaths_void'
+	for (const statCategory in statCategories) {
+		// 'deaths'
+		const statCategoryMatchers = statCategories[statCategory]
+		if (statCategoryMatchers == null) {
+			// If it's null, just go with this. Can only ever be 'misc'
+			return {
+				category: statCategory,
+				name: statNameRaw
+			}
+		}
+		for (const categoryMatch of statCategoryMatchers) {
+			// ['deaths_']
+			let trailingEnd = categoryMatch[0] === '_'
+			let trailingStart = categoryMatch.substr(-1) === '_'
+			if (trailingStart && statNameRaw.startsWith(categoryMatch)) {
+				return {
+					category: statCategory,
+					name: statNameRaw.substr(categoryMatch.length)
+				}
+			} else if (trailingEnd && statNameRaw.endsWith(categoryMatch)) {
+				return {
+					category: statCategory,
+					name: statNameRaw.substr(0, statNameRaw.length - categoryMatch.length)
+				}
+			} else if (statNameRaw == categoryMatch) {
+				// if it matches exactly, we don't know the name. will be defaulted to category later on
+				return {
+					category: statCategory,
+					name: null
+				}
+			}
+		}
+	}
+	// this should never happen, as it'll default to misc and return if nothing is found
+	return {
+		category: null,
+		name: statNameRaw
+	}
 }
 
 export const statUnits = {
 	time: ['_best_time', '_best_time_2'],
 	date: ['first_join'],
-	coins: ['purse']
+	coins: ['purse'],
+	leaderboards: ['leaderboards_count']
 }
 
 export interface StatItem {
-    rawName: string
-    value: number
-    categorizedName: string
-    category: string
-    unit: string
+	rawName: string
+	value: number
+	categorizedName: string
+	category: string
+	unit: string
 }
 
 export function getStatUnit(name: string): string {
@@ -91,22 +92,22 @@ export function getStatUnit(name: string): string {
 
 
 export function cleanProfileStats(data: any): StatItem[] {
-    // TODO: add type for statsRaw (probably in hypixelApi.ts since its coming from there)
-    const stats: StatItem[] = []
+	// TODO: add type for statsRaw (probably in hypixelApi.ts since its coming from there)
+	const stats: StatItem[] = []
 
-    const rawStats = data?.stats ?? {}
+	const rawStats = data?.stats ?? {}
 
-    for (const statNameRaw in rawStats) {
-        const statValue = rawStats[statNameRaw]
-        let { category: statCategory, name: statName } = categorizeStat(statNameRaw)
-        stats.push({
-            categorizedName: statName ?? 'total',
-            value: statValue,
-            rawName: statNameRaw,
-            category: statCategory,
-            unit: getStatUnit(statNameRaw) ?? null
-        })
-    }
+	for (const statNameRaw in rawStats) {
+		const statValue = rawStats[statNameRaw]
+		let { category: statCategory, name: statName } = categorizeStat(statNameRaw)
+		stats.push({
+			categorizedName: statName ?? 'total',
+			value: statValue,
+			rawName: statNameRaw,
+			category: statCategory,
+			unit: getStatUnit(statNameRaw) ?? null
+		})
+	}
 
-    return stats
+	return stats
 }
diff --git a/src/database.ts b/src/database.ts
index 08a8114..afdaa0c 100644
--- a/src/database.ts
+++ b/src/database.ts
@@ -3,7 +3,7 @@
 */
 
 import { categorizeStat, getStatUnit } from './cleaners/skyblock/stats'
-import { CleanFullProfile } from './cleaners/skyblock/profile'
+import { CleanFullProfile, CleanProfile } from './cleaners/skyblock/profile'
 import { CleanMember } from './cleaners/skyblock/member'
 import { Collection, Db, MongoClient } from 'mongodb'
 import { CleanPlayer } from './cleaners/player'
@@ -119,7 +119,7 @@ function getMemberLeaderboardAttributes(member: CleanMember): StringNumber {
 
 
 export async function fetchAllLeaderboardsCategorized(): Promise<{ [ category: string ]: string[] }> {
-	const memberLeaderboardAttributes = await fetchAllMemberLeaderboardAttributes()
+	const memberLeaderboardAttributes: string[] = await fetchAllMemberLeaderboardAttributes()
 	const categorizedLeaderboards: { [ category: string ]: string[] } = {}
 	for (const leaderboard of memberLeaderboardAttributes) {
 		const { category } = categorizeStat(leaderboard)
@@ -175,6 +175,7 @@ export async function fetchAllMemberLeaderboardAttributes(): Promise<string[]> {
 		'first_join',
 		'purse',
 		'visited_zones',
+		'leaderboards_count'
 	]
 }
 
@@ -221,6 +222,7 @@ interface Leaderboard {
 /** Fetch a leaderboard that ranks members, as opposed to profiles */
 export async function fetchMemberLeaderboard(name: string): Promise<Leaderboard> {
 	const leaderboardRaw = await fetchMemberLeaderboardRaw(name)
+
 	const fetchLeaderboardPlayer = async(item: DatabaseLeaderboardItem): Promise<LeaderboardItem> => {
 		return {
 			player: await cached.fetchBasicPlayer(item.uuid),
@@ -254,6 +256,7 @@ export async function fetchMemberLeaderboardSpots(player: string, profile: strin
 	for (const leaderboardName in applicableAttributes) {
 		const leaderboard = await fetchMemberLeaderboardRaw(leaderboardName)
 		const leaderboardPositionIndex = leaderboard.findIndex(i => i.uuid === fullMember.uuid && i.profile === fullProfile.uuid)
+
 		memberLeaderboardSpots.push({
 			name: leaderboardName,
 			positionIndex: leaderboardPositionIndex,
@@ -279,6 +282,7 @@ async function getMemberLeaderboardRequirement(name: string): Promise<number> {
 async function getApplicableAttributes(member: CleanMember): Promise<StringNumber> {
 	const leaderboardAttributes = getMemberLeaderboardAttributes(member)
 	const applicableAttributes = {}
+
 	for (const [ leaderboard, attributeValue ] of Object.entries(leaderboardAttributes)) {
 		const requirement = await getMemberLeaderboardRequirement(leaderboard)
 		const leaderboardReversed = isLeaderboardReversed(leaderboard)
@@ -289,6 +293,20 @@ async function getApplicableAttributes(member: CleanMember): Promise<StringNumbe
 			applicableAttributes[leaderboard] = attributeValue
 		}
 	}
+
+
+	let leaderboardsCount: number = Object.keys(applicableAttributes).length
+	const leaderboardsCountRequirement: number = await getMemberLeaderboardRequirement('leaderboards_count')
+
+	if (
+		(leaderboardsCountRequirement === null)
+		|| (leaderboardsCount > leaderboardsCountRequirement)
+	) {
+		// add 1 extra because this attribute also counts :)
+		applicableAttributes['leaderboards_count'] = leaderboardsCount
+	}
+
+
 	return applicableAttributes
 }
 
@@ -314,7 +332,7 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu
 
 	const leaderboardAttributes = await getApplicableAttributes(member)
 
-	if (debug) console.log('done getApplicableAttributes..', leaderboardAttributes)
+	if (debug) console.log('done getApplicableAttributes..', leaderboardAttributes, member.username, profile.name)
 
 	await memberLeaderboardsCollection.updateOne(
 		{
@@ -335,7 +353,7 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu
 		const leaderboardReverse = isLeaderboardReversed(attributeName)
 		const newRawLeaderboard = existingRawLeaderboard
 			// remove the player from the leaderboard, if they're there
-			.filter(value => value.uuid !== member.uuid)
+			.filter(value => value.uuid !== member.uuid || value.profile !== profile.uuid)
 			.concat([{
 				last_updated: new Date(),
 				stats: leaderboardAttributes,
@@ -365,8 +383,9 @@ export async function queueUpdateDatabaseMember(member: CleanMember, profile: Cl
  * Remove leaderboard attributes for members that wouldn't actually be on the leaderboard. This saves a lot of storage space
  */
 async function removeBadMemberLeaderboardAttributes(): Promise<void> {
-	const leaderboards = await fetchAllMemberLeaderboardAttributes()
+	const leaderboards: string[] = await fetchAllMemberLeaderboardAttributes()
 	// shuffle so if the application is restarting many times itll still be useful
+
 	for (const leaderboard of shuffle(leaderboards)) {
 		// wait 10 seconds so it doesnt use as much ram
 		await sleep(10 * 1000)
@@ -391,7 +410,7 @@ async function removeBadMemberLeaderboardAttributes(): Promise<void> {
 
 /** Fetch all the leaderboards, used for caching. Don't call this often! */
 async function fetchAllLeaderboards(fast?: boolean): Promise<void> {
-	const leaderboards = await fetchAllMemberLeaderboardAttributes()
+	const leaderboards: string[] = await fetchAllMemberLeaderboardAttributes()
 
 	// shuffle so if the application is restarting many times itll still be useful
 	if (debug) console.log('Caching leaderboards!')
@@ -410,7 +429,8 @@ connect().then(() => {
 	// when it connects, cache the leaderboards and remove bad members
 	removeBadMemberLeaderboardAttributes()
 	// cache leaderboards on startup so its faster later on
-	fetchAllLeaderboards(true)
+	// fetchAllLeaderboards(true)
+	fetchAllLeaderboards(false)
 	// cache leaderboard players again every 4 hours
 	setInterval(fetchAllLeaderboards, 4 * 60 * 60 * 1000)
 })
diff --git a/src/hypixelApi.ts b/src/hypixelApi.ts
index 8541089..9840c79 100644
--- a/src/hypixelApi.ts
+++ b/src/hypixelApi.ts
@@ -125,7 +125,6 @@ export interface HypixelPlayer {
 	socialMedia?: HypixelPlayerSocialMedia
 }
 
-
 /** Send an HTTP request to the Hypixel API */
 export async function sendApiRequest({ path, key, args }): Promise<HypixelResponse> {
 	// Send a raw http request to api.hypixel.net, and return the parsed json
diff --git a/src/hypixelCached.ts b/src/hypixelCached.ts
index c347a1a..3dc89a3 100644
--- a/src/hypixelCached.ts
+++ b/src/hypixelCached.ts
@@ -143,18 +143,32 @@ export async function usernameFromUser(user: string): Promise<string> {
 	return username
 }
 
+let fetchingPlayers: Set<string> = new Set()
 
 export async function fetchPlayer(user: string): Promise<CleanPlayer> {
 	const playerUuid = await uuidFromUser(user)
 
+
 	if (playerCache.has(playerUuid))
 		return playerCache.get(playerUuid)
 
+	// if it's already in the process of fetching, check every 100ms until it's not fetching the player anymore and fetch it again, since it'll be cached now
+	if (fetchingPlayers.has(playerUuid)) {
+		while (fetchingPlayers.has(playerUuid)) {
+			await new Promise(resolve => setTimeout(resolve, 100))
+		}
+		return await fetchPlayer(user)
+	}
+
+	fetchingPlayers.add(playerUuid)
+
 	const cleanPlayer: CleanPlayer = await hypixel.sendCleanApiRequest({
 		path: 'player',
 		args: { uuid: playerUuid }
 	})
 
+	fetchingPlayers.delete(playerUuid)
+
 	if (!cleanPlayer) return
 
 	// clone in case it gets modified somehow later