Slayer leaderboard (#4)

* add slayers to constants

* Compiled TS into JS

* change units to also show up when only changing one constant item

* Compiled TS into JS

* add slayer leaderboards

* remove debugging stuff

6 files changed, 152 insertions, 41 deletions

diff --git a/src/cleaners/skyblock/slayers.ts b/src/cleaners/skyblock/slayers.ts
index dd3881b..2026496 100644
--- a/src/cleaners/skyblock/slayers.ts
+++ b/src/cleaners/skyblock/slayers.ts
@@ -1,5 +1,4 @@
-
-const slayerLevels = 4 // number of slayer levels, this might be 5 soon
+export const slayerLevels = 4 // number of slayer levels, this might be 5 soon
 
 const SLAYER_NAMES = {
 	spider: 'tarantula',
@@ -17,12 +16,15 @@ interface SlayerTier {
 
 export interface Slayer {
 	name: SlayerName
+	raw_name: string
 	xp: number
+	kills: number
 	tiers: SlayerTier[]
 }
 
 export interface SlayerData {
 	xp: number
+	kills: number
 	bosses: Slayer[]
 }
 
@@ -31,27 +33,34 @@ export function cleanSlayers(data: any): SlayerData {
 
 	const slayersDataRaw = data?.slayer_bosses
 
-	let totalXp = 0
+	let totalXp: number = 0
+	let totalKills: number = 0
 
 	for (const slayerNameRaw in slayersDataRaw) {
 		const slayerDataRaw = slayersDataRaw[slayerNameRaw]
 
 		// convert name provided by api (spider) to the real name (tarantula)
-		const slayerName: SlayerName = SLAYER_NAMES[slayerDataRaw]
+		const slayerName: SlayerName = SLAYER_NAMES[slayerNameRaw]
 
-		const slayerXp: number = slayerDataRaw.xp
+		const slayerXp: number = slayerDataRaw.xp ?? 0
+		let slayerKills: number = 0
 		const slayerTiers: SlayerTier[] = []
+
 		for (const slayerDataKey in slayerDataRaw) {
 			// if a key starts with boss_kills_tier_ (boss_kills_tier_1), get the last number
 			if (slayerDataKey.startsWith('boss_kills_tier_')) {
 				const slayerTierRaw = parseInt(slayerDataKey.substr('boss_kills_tier_'.length))
-				const slayerTierKills = slayerDataRaw[slayerDataKey]
+				const slayerTierKills = slayerDataRaw[slayerDataKey] ?? 0
 				// add 1 since hypixel is using 0 indexed tiers
 				const slayerTier = slayerTierRaw + 1
 				slayerTiers.push({
 					kills: slayerTierKills,
 					tier: slayerTier
 				})
+
+				// count up the total number of kills for this slayer
+				if (slayerTierKills)
+					slayerKills += slayerTierKills
 			}
 		}
 
@@ -64,15 +73,25 @@ export function cleanSlayers(data: any): SlayerData {
 
 		const slayer: Slayer = {
 			name: slayerName,
+			raw_name: slayerNameRaw,
 			tiers: slayerTiers,
-			xp: slayerXp
+			xp: slayerXp ?? 0,
+			kills: slayerKills
 		}
+
 		slayers.push(slayer)
-		// add the xp from this slayer to the total xp
-		totalXp += slayerXp
+
+		// add the xp and kills from this slayer to the total xp
+		if (slayerXp)
+			totalXp += slayerXp
+		if (slayerKills)
+			totalKills += slayerKills
 	}
+
 	return {
 		xp: totalXp,
+		kills: totalKills,
 		bosses: slayers
 	}
 }
+
diff --git a/src/cleaners/skyblock/stats.ts b/src/cleaners/skyblock/stats.ts
index efb79bd..218e034 100644
--- a/src/cleaners/skyblock/stats.ts
+++ b/src/cleaners/skyblock/stats.ts
@@ -8,6 +8,7 @@ const statCategories: { [ key: string ]: string[] | null } = { // sorted in orde
 
     'collection': ['collection_'],
     'skills': ['skill_'],
+    'slayer': ['slayer_'],
 
     'misc': null // everything else goes here
 }
diff --git a/src/constants.ts b/src/constants.ts
index 760dcb1..5f20147 100644
--- a/src/constants.ts
+++ b/src/constants.ts
@@ -30,17 +30,23 @@ const queue = new Queue({
  * @param json The JSON body, only applicable for some types of methods
  */
 async function fetchGithubApi(method: string, route: string, headers?: any, json?: any): Promise<nodeFetch.Response> {
-	return await fetch(
-		githubApiBase + route,
-		{
-			agent: () => httpsAgent,
-			body: json ? JSON.stringify(json) : null,
-			method,
-			headers: Object.assign({
-				'Authorization': `token ${process.env.github_token}`
-			}, headers),
-		}
-	)
+	try {
+		return await fetch(
+			githubApiBase + route,
+			{
+				agent: () => httpsAgent,
+				body: json ? JSON.stringify(json) : null,
+				method,
+				headers: Object.assign({
+					'Authorization': `token ${process.env.github_token}`
+				}, headers),
+			}
+		)
+	} catch {
+		// if there's an error, wait a second and try again
+		await new Promise((resolve) => setTimeout(resolve, 1000))
+		return await fetchGithubApi(method, route, headers, json)
+	}
 }
 
 interface GithubFile {
@@ -73,6 +79,7 @@ async function fetchFile(path: string): Promise<GithubFile> {
 		},
 	)
 	const data = await r.json()
+
 	const file = {
 		path: data.path,
 		content: Buffer.from(data.content, data.encoding).toString(),
@@ -119,7 +126,7 @@ async function fetchJSONConstant(filename: string): Promise<string[]> {
 }
 
 /** Add stats to skyblock-constants. This has caching so it's fine to call many times */
-export async function addJSONConstants(filename: string, addingValues: string[], units: string='stats'): Promise<void> {
+export async function addJSONConstants(filename: string, addingValues: string[], unit: string='stat'): Promise<void> {
 	if (addingValues.length === 0) return // no stats provided, just return
 
 	queue.enqueue(async() => {
@@ -143,7 +150,7 @@ export async function addJSONConstants(filename: string, addingValues: string[],
 		// there's not actually any new stats, just return
 		if (newStats.length === 0) return
 
-		const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new ${units}` : `Add '${newStats[0]}'`
+		const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new ${unit}s` : `Add '${newStats[0]}' ${unit}`
 		await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2))
 	})
 }
@@ -156,7 +163,7 @@ export async function fetchStats(): Promise<string[]> {
 
 /** Add stats to skyblock-constants. This has caching so it's fine to call many times */
 export async function addStats(addingStats: string[]): Promise<void> {
-	await addJSONConstants('stats.json', addingStats, 'stats')
+	await addJSONConstants('stats.json', addingStats, 'stat')
 }
 
 /** Fetch all the known SkyBlock collections as an array of strings */
@@ -166,7 +173,7 @@ export async function fetchCollections(): Promise<string[]> {
 
 /** Add collections to skyblock-constants. This has caching so it's fine to call many times */
 export async function addCollections(addingCollections: string[]): Promise<void> {
-	await addJSONConstants('collections.json', addingCollections, 'collections')
+	await addJSONConstants('collections.json', addingCollections, 'collection')
 }
 
 /** Fetch all the known SkyBlock collections as an array of strings */
@@ -176,7 +183,7 @@ export async function fetchSkills(): Promise<string[]> {
 
 /** Add skills to skyblock-constants. This has caching so it's fine to call many times */
 export async function addSkills(addingSkills: string[]): Promise<void> {
-	await addJSONConstants('skills.json', addingSkills, 'skills')
+	await addJSONConstants('skills.json', addingSkills, 'skill')
 }
 
 /** Fetch all the known SkyBlock collections as an array of strings */
@@ -186,5 +193,16 @@ export async function fetchZones(): Promise<string[]> {
 
 /** Add skills to skyblock-constants. This has caching so it's fine to call many times */
 export async function addZones(addingZones: string[]): Promise<void> {
-	await addJSONConstants('zones.json', addingZones, 'zones')
+	await addJSONConstants('zones.json', addingZones, 'zone')
+}
+
+
+/** Fetch all the known SkyBlock slayer names as an array of strings */
+export async function fetchSlayers(): Promise<string[]> {
+	return await fetchJSONConstant('slayers.json')
+}
+
+/** Add skills to skyblock-constants. This has caching so it's fine to call many times */
+export async function addSlayers(addingSlayers: string[]): Promise<void> {
+	await addJSONConstants('slayers.json', addingSlayers, 'slayer')
 }
diff --git a/src/database.ts b/src/database.ts
index 9e81f5b..3f44b39 100644
--- a/src/database.ts
+++ b/src/database.ts
@@ -13,6 +13,7 @@ import { shuffle, sleep } from './util'
 import NodeCache from 'node-cache'
 import Queue from 'queue-promise'
 import { debug } from '.'
+import { slayerLevels } from './cleaners/skyblock/slayers'
 
 // don't update the user for 3 minutes
 const recentlyUpdated = new NodeCache({
@@ -77,6 +78,23 @@ function getMemberSkillAttributes(member: CleanMember): StringNumber {
 	return skillAttributes
 }
 
+function getMemberSlayerAttributes(member: CleanMember): StringNumber {
+	const slayerAttributes: StringNumber = {
+		slayer_total_xp: member.slayers.xp,
+		slayer_total_kills: member.slayers.kills,
+	}
+
+	for (const slayer of member.slayers.bosses) {
+		slayerAttributes[`slayer_${slayer.name}_total_xp`] = slayer.xp
+		slayerAttributes[`slayer_${slayer.name}_total_kills`] = slayer.kills
+		for (const tier of slayer.tiers) {
+			slayerAttributes[`slayer_${slayer.name}_${tier.tier}_kills`] = tier.kills
+		}
+	}
+
+	return slayerAttributes
+}
+
 function getMemberLeaderboardAttributes(member: CleanMember): StringNumber {
 	// if you want to add a new leaderboard for member attributes, add it here (and getAllLeaderboardAttributes)
 	return {
@@ -89,6 +107,9 @@ function getMemberLeaderboardAttributes(member: CleanMember): StringNumber {
 		// skill leaderboards
 		...getMemberSkillAttributes(member),
 
+		// slayer leaderboards
+		...getMemberSlayerAttributes(member),
+
 		fairy_souls: member.fairy_souls.total,
 		first_join: member.first_join,
 		purse: member.purse,
@@ -115,6 +136,25 @@ export async function fetchAllLeaderboardsCategorized(): Promise<{ [ category: s
 	return categorizedLeaderboards
 }
 
+/** Fetch the raw names for the slayer leaderboards */
+export async function fetchSlayerLeaderboards(): Promise<string[]> {
+	const rawSlayerNames = await constants.fetchSlayers()
+	let leaderboardNames: string[] = [
+		'slayer_total_xp',
+		'slayer_total_kills'
+	]
+
+	// we use the raw names (zombie, spider, wolf) instead of the clean names (revenant, tarantula, sven) because the raw names are guaranteed to never change
+	for (const slayerNameRaw of rawSlayerNames) {
+		leaderboardNames.push(`slayer_${slayerNameRaw}_total_xp`)
+		leaderboardNames.push(`slayer_${slayerNameRaw}_total_kills`)
+		for (let slayerTier = 1; slayerTier <= slayerLevels; slayerTier ++) {
+			leaderboardNames.push(`slayer_${slayerNameRaw}_${slayerTier}_kills`)
+		}
+	}
+
+	return leaderboardNames
+}
 
 /** Fetch the names of all the leaderboards */
 export async function fetchAllMemberLeaderboardAttributes(): Promise<string[]> {
@@ -128,6 +168,9 @@ export async function fetchAllMemberLeaderboardAttributes(): Promise<string[]> {
 		// skill leaderboards
 		...(await constants.fetchSkills()).map(value => `skill_${value}`),
 
+		// slayer leaderboards
+		...await fetchSlayerLeaderboards(),
+
 		'fairy_souls',
 		'first_join',
 		'purse',
@@ -265,6 +308,7 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu
 	await constants.addCollections(member.collections.map(coll => coll.name))
 	await constants.addSkills(member.skills.map(skill => skill.name))
 	await constants.addZones(member.visited_zones.map(zone => zone.name))
+	await constants.addSlayers(member.slayers.bosses.map(s => s.raw_name))
 
 	if (debug) console.log('done constants..')
 
@@ -306,14 +350,14 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu
 	if (debug) console.log('added member to leaderboards', member.username, leaderboardAttributes)
 }
 
-const queue = new Queue({
+const leaderboardUpdateQueue = new Queue({
 	concurrent: 1,
 	interval: 500
 })
 
 /** Queue an update for the member's leaderboard data on the server if applicable */
 export async function queueUpdateDatabaseMember(member: CleanMember, profile: CleanFullProfile): Promise<void> {
-	queue.enqueue(async() => await updateDatabaseMember(member, profile))
+	leaderboardUpdateQueue.enqueue(async() => await updateDatabaseMember(member, profile))
 }
 
 
diff --git a/src/hypixelApi.ts b/src/hypixelApi.ts
index 5c4bb2b..8541089 100644
--- a/src/hypixelApi.ts
+++ b/src/hypixelApi.ts
@@ -2,6 +2,7 @@
  * Fetch the raw Hypixel API
  */
 import fetch from 'node-fetch'
+import * as nodeFetch from 'node-fetch'
 import { jsonToQuery, shuffle } from './util'
 import { Agent } from 'https'
 
@@ -136,10 +137,18 @@ export async function sendApiRequest({ path, key, args }): Promise<HypixelRespon
 	// Construct a url from the base api url, path, and arguments
 	const fetchUrl = baseHypixelAPI + '/' + path + '?' + jsonToQuery(args)
 
-	const fetchResponse = await fetch(
-		fetchUrl,
-		{ agent: () => httpsAgent }
-	)
+	let fetchResponse: nodeFetch.Response
+
+	try {
+		fetchResponse = await fetch(
+			fetchUrl,
+			{ agent: () => httpsAgent }
+		)
+	} catch {
+		// if there's an error, wait a second and try again
+		await new Promise((resolve) => setTimeout(resolve, 1000))
+		return await sendApiRequest({ path, key, args })
+	}
 
 	if (fetchResponse.headers['ratelimit-limit'])
 		// remember how many uses it has
diff --git a/src/mojang.ts b/src/mojang.ts
index ce5b0bb..829de7f 100644
--- a/src/mojang.ts
+++ b/src/mojang.ts
@@ -3,6 +3,7 @@
  */
 
 import fetch from 'node-fetch'
+import * as nodeFetch from 'node-fetch'
 import { Agent } from 'https'
 import { isUuid, undashUuid } from './util'
 
@@ -22,11 +23,20 @@ interface MojangApiResponse {
  * Get mojang api data from the session server
  */
 export async function profileFromUuid(uuid: string): Promise<MojangApiResponse> {
-	const fetchResponse = await fetch(
-		// using mojang directly is faster than ashcon lol, also mojang removed the ratelimits from here
-		`https://sessionserver.mojang.com/session/minecraft/profile/${undashUuid(uuid)}`,
-		{ agent: () => httpsAgent }
-	)
+	let fetchResponse: nodeFetch.Response
+
+	try {
+		fetchResponse = await fetch(
+			// using mojang directly is faster than ashcon lol, also mojang removed the ratelimits from here
+			`https://sessionserver.mojang.com/session/minecraft/profile/${undashUuid(uuid)}`,
+			{ agent: () => httpsAgent }
+		)
+	} catch {
+		// if there's an error, wait a second and try again
+		await new Promise((resolve) => setTimeout(resolve, 1000))
+		return await profileFromUuid(uuid)
+	}
+
 	let data
 	try {
 		data = await fetchResponse.json()
@@ -43,10 +53,20 @@ export async function profileFromUuid(uuid: string): Promise<MojangApiResponse>
 
 export async function profileFromUsername(username: string): Promise<MojangApiResponse> {
 	// since we don't care about anything other than the uuid, we can use /uuid/ instead of /user/
-	const fetchResponse = await fetch(
-		`https://api.mojang.com/users/profiles/minecraft/${username}`,
-		{ agent: () => httpsAgent }
-	)
+
+	let fetchResponse: nodeFetch.Response
+
+	try {
+		fetchResponse = await fetch(
+			`https://api.mojang.com/users/profiles/minecraft/${username}`,
+			{ agent: () => httpsAgent }
+		)
+	} catch {
+		// if there's an error, wait a second and try again
+		await new Promise((resolve) => setTimeout(resolve, 1000))
+		return await profileFromUsername(username)
+	}
+
 	let data
 	try {
 		data = await fetchResponse.json()