fix leaderboards not loading sometimes, hopefully

7 files changed, 76 insertions, 91 deletions

diff --git a/package-lock.json b/package-lock.json
index 504c5c9..87603aa 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -265,6 +265,15 @@
         "node": ">=6"
       }
     },
+    "node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/ansi-styles": {
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
@@ -517,15 +526,6 @@
         "wrap-ansi": "^7.0.0"
       }
     },
-    "node_modules/cliui/node_modules/ansi-regex": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
-      "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
-      "dev": true,
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/cliui/node_modules/is-fullwidth-code-point": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -2082,15 +2082,6 @@
         "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
       }
     },
-    "node_modules/wrap-ansi/node_modules/ansi-regex": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
-      "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
-      "dev": true,
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/wrap-ansi/node_modules/is-fullwidth-code-point": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -2188,15 +2179,6 @@
         "node": ">=10"
       }
     },
-    "node_modules/yargs/node_modules/ansi-regex": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
-      "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
-      "dev": true,
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/yargs/node_modules/is-fullwidth-code-point": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -2457,6 +2439,12 @@
       "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==",
       "dev": true
     },
+    "ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "dev": true
+    },
     "ansi-styles": {
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
@@ -2633,12 +2621,6 @@
         "wrap-ansi": "^7.0.0"
       },
       "dependencies": {
-        "ansi-regex": {
-          "version": "5.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
-          "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
-          "dev": true
-        },
         "is-fullwidth-code-point": {
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -3292,7 +3274,7 @@
     "node-fetch": {
       "version": "git+ssh://git@github.com/tekwiz/node-fetch.git#be5ef867e0bfe78985ecb809777b84185533f7cf",
       "integrity": "sha512-HRfWbb6K5g+IF62gQFAPzZDaD6c6EKHg9zcn/cJ6yq8TzO65HzShXafYRdCnQji6hEPsfUnh4dMEiMV6+dMXFg==",
-      "from": "node-fetch@github:tekwiz/node-fetch#be5ef867e0bfe78985ecb809777b84185533f7cf",
+      "from": "node-fetch@github:tekwiz/node-fetch#fix/chunked-encoding-keepalive",
       "requires": {
         "data-uri-to-buffer": "^4.0.0",
         "fetch-blob": "^3.1.3",
@@ -3371,7 +3353,7 @@
     "prismarine-nbt": {
       "version": "git+ssh://git@github.com/PrismarineJS/prismarine-nbt.git#02e32042d357cbc179f1ac9c9c4566b828655600",
       "integrity": "sha512-8VHErjmjuL/JplgDfLhrMoT7QwYO6QYCbMMSSCoXXbyZe/1U9V58CLMe9aoy0bAoJDqo+ZrjpNx0kwb2T2umIA==",
-      "from": "prismarine-nbt@github:PrismarineJS/prismarine-nbt#02e32042d357cbc179f1ac9c9c4566b828655600",
+      "from": "prismarine-nbt@github:PrismarineJS/prismarine-nbt",
       "requires": {
         "protodef": "^1.9.0"
       }
@@ -3750,12 +3732,6 @@
         "strip-ansi": "^6.0.0"
       },
       "dependencies": {
-        "ansi-regex": {
-          "version": "5.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
-          "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
-          "dev": true
-        },
         "is-fullwidth-code-point": {
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -3816,12 +3792,6 @@
         "yargs-parser": "^20.2.2"
       },
       "dependencies": {
-        "ansi-regex": {
-          "version": "5.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
-          "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
-          "dev": true
-        },
         "is-fullwidth-code-point": {
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
diff --git a/src/constants.ts b/src/constants.ts
index 3cc77d2..27f413b 100644
--- a/src/constants.ts
+++ b/src/constants.ts
@@ -8,6 +8,7 @@ import * as constants from './constants.js'
 import * as nodeFetch from 'node-fetch'
 import NodeCache from 'node-cache'
 import { debug } from './index.js'
+import { sleep } from './util.js'
 import Queue from 'queue-promise'
 import fetch from 'node-fetch'
 import { Agent } from 'https'
@@ -51,7 +52,7 @@ async function fetchGithubApi(method: string, route: string, headers?: any, json
 		return data
 	} catch {
 		// if there's an error, wait a second and try again
-		await new Promise((resolve) => setTimeout(resolve, 1000))
+		await sleep(1000)
 		return await fetchGithubApi(method, route, headers, json)
 	}
 }
diff --git a/src/database.ts b/src/database.ts
index 571ec5e..e195eed 100644
--- a/src/database.ts
+++ b/src/database.ts
@@ -313,21 +313,28 @@ async function fetchMemberLeaderboardRaw(name: string): Promise<memberRawLeaderb
 	sortQuery[`stats.${name}`] = isLeaderboardReversed(name) ? 1 : -1
 
 	fetchingRawLeaderboardNames.add(name)
-	const leaderboardRaw: memberRawLeaderboardItem[] = (await memberLeaderboardsCollection
-		.find(query)
-		.sort(sortQuery)
-		.limit(leaderboardMax)
-		.toArray())
-		.map((i: DatabaseMemberLeaderboardItem): memberRawLeaderboardItem => {
-			return {
-				profile: i.profile,
-				uuid: i.uuid,
-				value: i.stats[name]
-			}
-		})
-	fetchingRawLeaderboardNames.delete(name)
-	cachedRawLeaderboards.set(name, leaderboardRaw)
-	return leaderboardRaw
+
+	try {
+		const leaderboardRaw: memberRawLeaderboardItem[] = (await memberLeaderboardsCollection
+			.find(query)
+			.sort(sortQuery)
+			.limit(leaderboardMax)
+			.toArray())
+			.map((i: DatabaseMemberLeaderboardItem): memberRawLeaderboardItem => {
+				return {
+					profile: i.profile,
+					uuid: i.uuid,
+					value: i.stats[name]
+				}
+			})
+		fetchingRawLeaderboardNames.delete(name)
+		cachedRawLeaderboards.set(name, leaderboardRaw)
+		return leaderboardRaw
+	} catch (e) {
+		// if it fails while fetching, remove it from fetchingRawLeaderboardNames
+		fetchingRawLeaderboardNames.delete(name)
+		throw e
+	}
 }
 
 async function fetchProfileLeaderboardRaw(name: string): Promise<profileRawLeaderboardItem[]> {
@@ -351,22 +358,28 @@ async function fetchProfileLeaderboardRaw(name: string): Promise<profileRawLeade
 	sortQuery[`stats.${name}`] = isLeaderboardReversed(name) ? 1 : -1
 
 	fetchingRawLeaderboardNames.add(name)
-	const leaderboardRaw: profileRawLeaderboardItem[] = (await profileLeaderboardsCollection
-		.find(query)
-		.sort(sortQuery)
-		.limit(leaderboardMax)
-		.toArray())
-		.map((i: DatabaseProfileLeaderboardItem): profileRawLeaderboardItem => {
-			return {
-				players: i.players,
-				uuid: i.uuid,
-				value: i.stats[name]
-			}
-		})
-	fetchingRawLeaderboardNames.delete(name)
-
-	cachedRawLeaderboards.set(name, leaderboardRaw)
-	return leaderboardRaw
+	try {
+		const leaderboardRaw: profileRawLeaderboardItem[] = (await profileLeaderboardsCollection
+			.find(query)
+			.sort(sortQuery)
+			.limit(leaderboardMax)
+			.toArray())
+			.map((i: DatabaseProfileLeaderboardItem): profileRawLeaderboardItem => {
+				return {
+					players: i.players,
+					uuid: i.uuid,
+					value: i.stats[name]
+				}
+			})
+		fetchingRawLeaderboardNames.delete(name)
+
+		cachedRawLeaderboards.set(name, leaderboardRaw)
+		return leaderboardRaw
+	} catch (e) {
+		// if it fails while fetching, remove it from fetchingRawLeaderboardNames
+		fetchingRawLeaderboardNames.delete(name)
+		throw e
+	}
 }
 
 interface MemberLeaderboard {
diff --git a/src/hypixel.ts b/src/hypixel.ts
index dd9d91d..745e036 100644
--- a/src/hypixel.ts
+++ b/src/hypixel.ts
@@ -22,6 +22,7 @@ import { cleanSkyblockProfilesResponse } from './cleaners/skyblock/profiles.js'
 import { CleanPlayer, cleanPlayerResponse } from './cleaners/player.js'
 import * as cached from './hypixelCached.js'
 import { debug } from './index.js'
+import { sleep } from './util.js'
 import { WithId } from 'mongodb'
 
 export type Included = 'profiles' | 'player' | 'stats' | 'inventories' | undefined
@@ -48,7 +49,7 @@ export async function sendCleanApiRequest({ path, args }, included?: Included[],
 	const rawResponse = await sendApiRequest({ path, key, args })
 	if (rawResponse.throttled) {
 		// if it's throttled, wait a second and try again
-		await new Promise(resolve => setTimeout(resolve, 1000))
+		await sleep(1000)
 		return await sendCleanApiRequest({ path, args }, included, options)
 	}
 	// clean the response
diff --git a/src/hypixelApi.ts b/src/hypixelApi.ts
index 7b6a4a8..13866f9 100644
--- a/src/hypixelApi.ts
+++ b/src/hypixelApi.ts
@@ -1,9 +1,9 @@
 /**
  * Fetch the raw Hypixel API
  */
-import fetch from 'node-fetch'
+import { jsonToQuery, shuffle, sleep } from './util.js'
 import * as nodeFetch from 'node-fetch'
-import { jsonToQuery, shuffle } from './util.js'
+import fetch from 'node-fetch'
 import { Agent } from 'https'
 
 if (!process.env.hypixel_keys)
@@ -167,13 +167,13 @@ export let sendApiRequest = async function sendApiRequest({ path, key, args }):
 		fetchJsonParsed = await fetchResponse.json()
 	} catch {
 		// if there's an error, wait a second and try again
-		await new Promise((resolve) => setTimeout(resolve, 1000))
+		await sleep(1000)
 		return await sendApiRequest({ path, key, args })
 	}
 
 	// bruh
 	if (fetchJsonParsed.cause === 'This endpoint is currently disabled') {
-		await new Promise((resolve) => setTimeout(resolve, 30000))
+		await sleep(30000)
 		return await sendApiRequest({ path, key, args })
 	}
 
@@ -204,7 +204,7 @@ export let sendApiRequest = async function sendApiRequest({ path, key, args }):
 		if (apiKeyUsage[key])
 			apiKeyUsage[key].remaining = 0
 		// if it's throttled, wait 10 seconds and try again
-		await new Promise((resolve) => setTimeout(resolve, 10000))
+		await sleep(10000)
 		return await sendApiRequest({ path, key, args })
 	}
 	return fetchJsonParsed
diff --git a/src/hypixelCached.ts b/src/hypixelCached.ts
index d5b6628..b49a190 100644
--- a/src/hypixelCached.ts
+++ b/src/hypixelCached.ts
@@ -3,8 +3,8 @@
  */
 
 import { CleanProfile, CleanFullProfile, CleanBasicProfile } from './cleaners/skyblock/profile.js'
+import { isUuid, sleep, undashUuid } from './util.js'
 import { CleanPlayer } from './cleaners/player.js'
-import { isUuid, undashUuid } from './util.js'
 import * as hypixel from './hypixel.js'
 import * as mojang from './mojang.js'
 import NodeCache from 'node-cache'
@@ -166,7 +166,7 @@ export async function fetchPlayer(user: string): Promise<CleanPlayer | null> {
 	// if it's already in the process of fetching, check every 100ms until it's not fetching the player anymore and fetch it again, since it'll be cached now
 	if (fetchingPlayers.has(playerUuid)) {
 		while (fetchingPlayers.has(playerUuid)) {
-			await new Promise(resolve => setTimeout(resolve, 100))
+			await sleep(100)
 		}
 		return await fetchPlayer(user)
 	}
diff --git a/src/mojang.ts b/src/mojang.ts
index 9b224d6..0ad22ab 100644
--- a/src/mojang.ts
+++ b/src/mojang.ts
@@ -2,7 +2,7 @@
  * Fetch the Mojang username API through api.ashcon.app
  */
 
-import { isUuid, undashUuid } from './util.js'
+import { isUuid, sleep, undashUuid } from './util.js'
 import * as nodeFetch from 'node-fetch'
 import fetch from 'node-fetch'
 import { Agent } from 'https'
@@ -32,7 +32,7 @@ export let profileFromUuid = async function profileFromUuid(uuid: string): Promi
 		)
 	} catch {
 		// if there's an error, wait a second and try again
-		await new Promise((resolve) => setTimeout(resolve, 1000))
+		await sleep(1000)
 		return await profileFromUuid(uuid)
 	}
 
@@ -68,7 +68,7 @@ export let profileFromUsername = async function profileFromUsername(username: st
 		)
 	} catch {
 		// if there's an error, wait a second and try again
-		await new Promise((resolve) => setTimeout(resolve, 1000))
+		await sleep(1000)
 		return await profileFromUsername(username)
 	}
 
@@ -100,7 +100,7 @@ export async function profileFromUsernameAlternative(username: string): Promise<
 		)
 	} catch {
 		// if there's an error, wait a second and try again
-		await new Promise((resolve) => setTimeout(resolve, 1000))
+		await sleep(1000)
 		return await profileFromUsernameAlternative(username)
 	}