Show minions that haven't been unlocked in response (#20)

* add minions that haven't been unlocked yet to response, and slightly optimize constants

* remove most console.logs with console.debugs

* mock constants

7 files changed, 138 insertions, 89 deletions

diff --git a/src/cleaners/skyblock/member.ts b/src/cleaners/skyblock/member.ts
index 5d8d4f9..cfa9a71 100644
--- a/src/cleaners/skyblock/member.ts
+++ b/src/cleaners/skyblock/member.ts
@@ -67,7 +67,7 @@ export async function cleanSkyBlockProfileMemberResponse(member, included: Inclu
 		// this is used for leaderboards
 		rawHypixelStats: member.stats ?? {},
 
-		minions: cleanMinions(member),
+		minions: await cleanMinions(member),
 		fairy_souls: cleanFairySouls(member),
 		inventories: inventoriesIncluded ? await cleanInventories(member) : undefined,
 		objectives: cleanObjectives(member),
diff --git a/src/cleaners/skyblock/minions.ts b/src/cleaners/skyblock/minions.ts
index 06e7752..21f7b66 100644
--- a/src/cleaners/skyblock/minions.ts
+++ b/src/cleaners/skyblock/minions.ts
@@ -1,4 +1,5 @@
 import { maxMinion } from '../../hypixel'
+import * as constants from '../../constants'
 
 export interface CleanMinion {
     name: string,
@@ -10,9 +11,11 @@ export interface CleanMinion {
  * Clean the minions provided by Hypixel
  * @param minionsRaw The minion data provided by the Hypixel API
  */
-export function cleanMinions(data: any): CleanMinion[] {
+export async function cleanMinions(member: any): Promise<CleanMinion[]> {
     const minions: CleanMinion[] = []
-    for (const minionRaw of data?.crafted_generators ?? []) {
+    const processedMinionNames: Set<string> = new Set()
+
+    for (const minionRaw of member?.crafted_generators ?? []) {
         // do some regex magic to get the minion name and level
         // examples of potential minion names: CLAY_11, PIG_1, MAGMA_CUBE_4
         const minionName = minionRaw.split(/_\d/)[0].toLowerCase()
@@ -32,8 +35,29 @@ export function cleanMinions(data: any): CleanMinion[] {
 
         // set the minion at that level to true
         matchingMinion.levels[minionLevel - 1] = true
+        processedMinionNames.add(minionName)
+    }
+
+    const allMinionNames = new Set(await constants.fetchMinions())
+
+    for (const minionName of processedMinionNames) {
+        if (!allMinionNames.has(minionName)) {
+            constants.addMinions(Array.from(processedMinionNames))
+            break
+        }
     }
-    return minions
+
+    for (const minionName of allMinionNames) {
+        if (!processedMinionNames.has(minionName)) {
+            processedMinionNames.add(minionName)
+            minions.push({
+                name: minionName,
+                levels: new Array(maxMinion).fill(false)
+            })
+        }
+    }
+
+    return minions.sort((a, b) => a.name > b.name ? 1 : (a.name < b.name ? -1 : 0))
 }
 
 /**
diff --git a/src/constants.ts b/src/constants.ts
index 5f20147..1652b6b 100644
--- a/src/constants.ts
+++ b/src/constants.ts
@@ -2,11 +2,15 @@
  * Fetch and edit constants from the skyblock-constants repo
  */
 
-import fetch from 'node-fetch'
+// we have to do this so we can mock the function from the tests properly
+import * as constants from './constants'
+
 import * as nodeFetch from 'node-fetch'
-import { Agent } from 'https'
 import NodeCache from 'node-cache'
 import Queue from 'queue-promise'
+import fetch from 'node-fetch'
+import { Agent } from 'https'
+import { debug } from '.'
 
 const httpsAgent = new Agent({
 	keepAlive: true
@@ -16,10 +20,10 @@ const githubApiBase = 'https://api.github.com'
 const owner = 'skyblockstats'
 const repo = 'skyblock-constants'
 
-// we use a queue for editing so it doesnt hit the github ratelimit as much
+// we use a queue for editing so it always utilizes the cache if possible, and to avoid hitting the github rateimit
 const queue = new Queue({
 	concurrent: 1,
-	interval: 500
+	interval: 10
 })
 
 /**
@@ -31,6 +35,7 @@ const queue = new Queue({
  */
 async function fetchGithubApi(method: string, route: string, headers?: any, json?: any): Promise<nodeFetch.Response> {
 	try {
+		if (debug) console.debug('fetching github api', method, route)
 		return await fetch(
 			githubApiBase + route,
 			{
@@ -67,26 +72,30 @@ const fileCache = new NodeCache({
  * Fetch a file from skyblock-constants
  * @param path The file path, for example stats.json
  */
-async function fetchFile(path: string): Promise<GithubFile> {
-	if (fileCache.has(path))
-		return fileCache.get(path)
-
-	const r = await fetchGithubApi(
-		'GET',
-		`/repos/${owner}/${repo}/contents/${path}`,
-		{
-			'Accept': 'application/vnd.github.v3+json',
-		},
-	)
-	const data = await r.json()
-
-	const file = {
-		path: data.path,
-		content: Buffer.from(data.content, data.encoding).toString(),
-		sha: data.sha
-	}
-	fileCache.set(path, file)
-	return file
+function fetchFile(path: string): Promise<GithubFile> {
+	return new Promise(resolve => {
+		queue.enqueue(async() => {
+			if (fileCache.has(path))
+				return resolve(fileCache.get(path))
+
+			const r = await fetchGithubApi(
+				'GET',
+				`/repos/${owner}/${repo}/contents/${path}`,
+				{
+					'Accept': 'application/vnd.github.v3+json',
+				},
+			)
+			const data = await r.json()
+
+			const file = {
+				path: data.path,
+				content: Buffer.from(data.content, data.encoding).toString(),
+				sha: data.sha
+			}
+			fileCache.set(path, file)
+			resolve(file)
+		})
+	})
 }
 
 /**
@@ -115,7 +124,8 @@ async function editFile(file: GithubFile, message: string, newContent: string):
 	})
 }
 
-async function fetchJSONConstant(filename: string): Promise<string[]> {
+export async function fetchJSONConstant(filename: string): Promise<string[]> {
+	console.log('actually fetchJSONConstant')
 	const file = await fetchFile(filename)
 	try {
 		return JSON.parse(file.content)
@@ -129,80 +139,94 @@ async function fetchJSONConstant(filename: string): Promise<string[]> {
 export async function addJSONConstants(filename: string, addingValues: string[], unit: string='stat'): Promise<void> {
 	if (addingValues.length === 0) return // no stats provided, just return
 
-	queue.enqueue(async() => {
-		const file = await fetchFile(filename)
-		if (!file.path)
-			return
-		let oldStats: string[]
-		try {
-			oldStats = JSON.parse(file.content)
-		} catch {
-			// invalid json, set it as an empty array
-			oldStats = []
-		}
-		const updatedStats = oldStats
-			.concat(addingValues)
-			// remove duplicates
-			.filter((value, index, array) => array.indexOf(value) === index)
-			.sort((a, b) => a.localeCompare(b))
-		const newStats = updatedStats.filter(value => !oldStats.includes(value))
+	let file: GithubFile = await fetchFile(filename)
+	if (!file.path)
+		return
+	let oldStats: string[]
+	try {
+		oldStats = JSON.parse(file.content)
+	} catch {
+		// invalid json, set it as an empty array
+		oldStats = []
+	}
+	const updatedStats = oldStats
+		.concat(addingValues)
+		// remove duplicates
+		.filter((value, index, array) => array.indexOf(value) === index)
+		.sort((a, b) => a.localeCompare(b))
+	const newStats = updatedStats.filter(value => !oldStats.includes(value))
 
-		// there's not actually any new stats, just return
-		if (newStats.length === 0) return
+	// there's not actually any new stats, just return
+	if (newStats.length === 0) return
 
-		const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new ${unit}s` : `Add '${newStats[0]}' ${unit}`
+	const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new ${unit}s` : `Add '${newStats[0]}' ${unit}`
+	try {
 		await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2))
-	})
+	} catch {
+		// the file probably changed or something, try again
+		file = await fetchFile(filename)
+		await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2))
+	}
 }
 
 
 /** Fetch all the known SkyBlock stats as an array of strings */
 export async function fetchStats(): Promise<string[]> {
-	return await fetchJSONConstant('stats.json')
+	return await constants.fetchJSONConstant('stats.json')
 }
 
 /** Add stats to skyblock-constants. This has caching so it's fine to call many times */
 export async function addStats(addingStats: string[]): Promise<void> {
-	await addJSONConstants('stats.json', addingStats, 'stat')
+	await constants.addJSONConstants('stats.json', addingStats, 'stat')
 }
 
 /** Fetch all the known SkyBlock collections as an array of strings */
 export async function fetchCollections(): Promise<string[]> {
-	return await fetchJSONConstant('collections.json')
+	return await constants.fetchJSONConstant('collections.json')
 }
 
 /** Add collections to skyblock-constants. This has caching so it's fine to call many times */
 export async function addCollections(addingCollections: string[]): Promise<void> {
-	await addJSONConstants('collections.json', addingCollections, 'collection')
+	await constants.addJSONConstants('collections.json', addingCollections, 'collection')
 }
 
 /** Fetch all the known SkyBlock collections as an array of strings */
 export async function fetchSkills(): Promise<string[]> {
-	return await fetchJSONConstant('skills.json')
+	return await constants.fetchJSONConstant('skills.json')
 }
 
 /** Add skills to skyblock-constants. This has caching so it's fine to call many times */
 export async function addSkills(addingSkills: string[]): Promise<void> {
-	await addJSONConstants('skills.json', addingSkills, 'skill')
+	await constants.addJSONConstants('skills.json', addingSkills, 'skill')
 }
 
 /** Fetch all the known SkyBlock collections as an array of strings */
 export async function fetchZones(): Promise<string[]> {
-	return await fetchJSONConstant('zones.json')
+	return await constants.fetchJSONConstant('zones.json')
 }
 
 /** Add skills to skyblock-constants. This has caching so it's fine to call many times */
 export async function addZones(addingZones: string[]): Promise<void> {
-	await addJSONConstants('zones.json', addingZones, 'zone')
+	await constants.addJSONConstants('zones.json', addingZones, 'zone')
 }
 
 
 /** Fetch all the known SkyBlock slayer names as an array of strings */
 export async function fetchSlayers(): Promise<string[]> {
-	return await fetchJSONConstant('slayers.json')
+	return await constants.fetchJSONConstant('slayers.json')
 }
 
 /** Add skills to skyblock-constants. This has caching so it's fine to call many times */
 export async function addSlayers(addingSlayers: string[]): Promise<void> {
-	await addJSONConstants('slayers.json', addingSlayers, 'slayer')
+	await constants.addJSONConstants('slayers.json', addingSlayers, 'slayer')
+}
+
+/** Fetch all the known SkyBlock slayer names as an array of strings */
+export async function fetchMinions(): Promise<string[]> {
+	return await constants.fetchJSONConstant('minions.json')
+}
+
+/** Add skills to skyblock-constants. This has caching so it's fine to call many times */
+export async function addMinions(addingMinions: string[]): Promise<void> {
+	await constants.addJSONConstants('minions.json', addingMinions, 'minion')
 }
diff --git a/src/database.ts b/src/database.ts
index 8aa334a..4c6dadf 100644
--- a/src/database.ts
+++ b/src/database.ts
@@ -442,15 +442,15 @@ async function getApplicableProfileLeaderboardAttributes(profile: CleanFullProfi
 
 /** Update the member's leaderboard data on the server if applicable */
 export async function updateDatabaseMember(member: CleanMember, profile: CleanFullProfile): Promise<void> {
-	if (debug) console.log('updateDatabaseMember', member.username)
 	if (!client) return // the db client hasn't been initialized
+	if (debug) console.debug('updateDatabaseMember', member.username)
 	// the member's been updated too recently, just return
 	if (recentlyUpdated.get(profile.uuid + member.uuid))
 		return
 	// store the member in recentlyUpdated so it cant update for 3 more minutes
 	recentlyUpdated.set(profile.uuid + member.uuid, true)
 
-	if (debug) console.log('adding member to leaderboards', member.username)
+	if (debug) console.debug('adding member to leaderboards', member.username)
 
 	await constants.addStats(Object.keys(member.rawHypixelStats))
 	await constants.addCollections(member.collections.map(coll => coll.name))
@@ -458,11 +458,11 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu
 	await constants.addZones(member.visited_zones.map(zone => zone.name))
 	await constants.addSlayers(member.slayers.bosses.map(s => s.raw_name))
 
-	if (debug) console.log('done constants..')
+	if (debug) console.debug('done constants..')
 
 	const leaderboardAttributes = await getApplicableMemberLeaderboardAttributes(member)
 
-	if (debug) console.log('done getApplicableMemberLeaderboardAttributes..', leaderboardAttributes, member.username, profile.name)
+	if (debug) console.debug('done getApplicableMemberLeaderboardAttributes..', leaderboardAttributes, member.username, profile.name)
 
 	await memberLeaderboardsCollection.updateOne(
 		{
@@ -495,7 +495,7 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu
 		cachedRawLeaderboards.set(attributeName, newRawLeaderboard)
 	}
 
-	if (debug) console.log('added member to leaderboards', member.username, leaderboardAttributes)
+	if (debug) console.debug('added member to leaderboards', member.username, leaderboardAttributes)
 }
 
 /**
@@ -503,8 +503,8 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu
  * This will not also update the members, you have to call updateDatabaseMember separately for that
  */
 export async function updateDatabaseProfile(profile: CleanFullProfile): Promise<void> {
-	if (debug) console.log('updateDatabaseProfile', profile.name)
 	if (!client) return // the db client hasn't been initialized
+	if (debug) console.debug('updateDatabaseProfile', profile.name)
 
 	// the profile's been updated too recently, just return
 	if (recentlyUpdated.get(profile.uuid + 'profile'))
@@ -512,11 +512,11 @@ export async function updateDatabaseProfile(profile: CleanFullProfile): Promise<
 	// store the profile in recentlyUpdated so it cant update for 3 more minutes
 	recentlyUpdated.set(profile.uuid + 'profile', true)
 
-	if (debug) console.log('adding profile to leaderboards', profile.name)
+	if (debug) console.debug('adding profile to leaderboards', profile.name)
 
 	const leaderboardAttributes = await getApplicableProfileLeaderboardAttributes(profile)
 
-	if (debug) console.log('done getApplicableProfileLeaderboardAttributes..', leaderboardAttributes, profile.name)
+	if (debug) console.debug('done getApplicableProfileLeaderboardAttributes..', leaderboardAttributes, profile.name)
 
 	await profileLeaderboardsCollection.updateOne(
 		{
@@ -550,7 +550,7 @@ export async function updateDatabaseProfile(profile: CleanFullProfile): Promise<
 		cachedRawLeaderboards.set(attributeName, newRawLeaderboard)
 	}
 
-	if (debug) console.log('added profile to leaderboards', profile.name, leaderboardAttributes)
+	if (debug) console.debug('added profile to leaderboards', profile.name, leaderboardAttributes)
 }
 
 const leaderboardUpdateMemberQueue = new Queue({
@@ -607,7 +607,7 @@ async function fetchAllLeaderboards(fast?: boolean): Promise<void> {
 	const leaderboards: string[] = await fetchAllMemberLeaderboardAttributes()
 
 	// shuffle so if the application is restarting many times itll still be useful
-	if (debug) console.log('Caching leaderboards!')
+	if (debug) console.debug('Caching leaderboards!')
 	for (const leaderboard of shuffle(leaderboards)) {
 		if (!fast)
 			// wait 2 seconds so it doesnt use as much ram
@@ -615,11 +615,11 @@ async function fetchAllLeaderboards(fast?: boolean): Promise<void> {
 
 		await fetchMemberLeaderboard(leaderboard)
 	}
-	if (debug) console.log('Finished caching leaderboards!')
+	if (debug) console.debug('Finished caching leaderboards!')
 }
 
 // make sure it's not in a test
-if (typeof global.it !== 'function') {
+if (!globalThis.isTest) {
 	connect().then(() => {
 		// when it connects, cache the leaderboards and remove bad members
 		removeBadMemberLeaderboardAttributes()
diff --git a/src/hypixel.ts b/src/hypixel.ts
index 599a7e1..464f3dd 100644
--- a/src/hypixel.ts
+++ b/src/hypixel.ts
@@ -83,7 +83,7 @@ export async function fetchUser({ user, uuid, username }: UserAny, included: Inc
 	}
 	if (!uuid) {
 		// the user doesn't exist.
-		if (debug) console.log('error:', user, 'doesnt exist')
+		if (debug) console.debug('error:', user, 'doesnt exist')
 		return null
 	}
 
diff --git a/src/hypixelCached.ts b/src/hypixelCached.ts
index a5a5909..f7b96ef 100644
--- a/src/hypixelCached.ts
+++ b/src/hypixelCached.ts
@@ -105,7 +105,7 @@ export async function uuidFromUser(user: string): Promise<string> {
 			return uuid
 	}
 
-	if (debug) console.log('Cache miss: uuidFromUser', user)
+	if (debug) console.debug('Cache miss: uuidFromUser', user)
 
 	// set it as waitForCacheSet (a promise) in case uuidFromUser gets called while its fetching mojang
 	usernameCache.set(undashUuid(user), waitForCacheSet(usernameCache, user, user))
@@ -132,11 +132,11 @@ export async function uuidFromUser(user: string): Promise<string> {
  */
 export async function usernameFromUser(user: string): Promise<string> {
 	if (usernameCache.has(undashUuid(user))) {
-		if (debug) console.log('Cache hit! usernameFromUser', user)
+		if (debug) console.debug('Cache hit! usernameFromUser', user)
 		return usernameCache.get(undashUuid(user))
 	}
 
-	if (debug) console.log('Cache miss: usernameFromUser', user)
+	if (debug) console.debug('Cache miss: usernameFromUser', user)
 
 	let { uuid, username } = await mojang.profileFromUser(user)
 	uuid = undashUuid(uuid)
@@ -191,7 +191,7 @@ export async function fetchBasicPlayer(user: string): Promise<CleanPlayer> {
 		return basicPlayerCache.get(playerUuid)
 	
 	const player = await fetchPlayer(playerUuid)
-	if (!player) console.log('no player? this should never happen', user)
+	if (!player) console.debug('no player? this should never happen', user)
 
 	delete player.profiles
 	return player
@@ -199,11 +199,11 @@ export async function fetchBasicPlayer(user: string): Promise<CleanPlayer> {
 
 export async function fetchSkyblockProfiles(playerUuid: string): Promise<CleanProfile[]> {
 	if (profilesCache.has(playerUuid)) {
-		if (debug) console.log('Cache hit! fetchSkyblockProfiles', playerUuid)
+		if (debug) console.debug('Cache hit! fetchSkyblockProfiles', playerUuid)
 		return profilesCache.get(playerUuid)
 	}
 
-	if (debug) console.log('Cache miss: fetchSkyblockProfiles', playerUuid)
+	if (debug) console.debug('Cache miss: fetchSkyblockProfiles', playerUuid)
 
 	const profiles: CleanProfile[] = await hypixel.fetchMemberProfilesUncached(playerUuid)
 
@@ -240,11 +240,11 @@ async function fetchBasicProfiles(user: string): Promise<CleanBasicProfile[]> {
 	if (!playerUuid) return // invalid player, just return
 
 	if (basicProfilesCache.has(playerUuid)) {
-		if (debug) console.log('Cache hit! fetchBasicProfiles', playerUuid)
+		if (debug) console.debug('Cache hit! fetchBasicProfiles', playerUuid)
 		return basicProfilesCache.get(playerUuid)
 	}
 
-	if (debug) console.log('Cache miss: fetchBasicProfiles', user)
+	if (debug) console.debug('Cache miss: fetchBasicProfiles', user)
 
 	const player = await fetchPlayer(playerUuid)
 	const profiles = player.profiles
@@ -265,11 +265,11 @@ async function fetchBasicProfiles(user: string): Promise<CleanBasicProfile[]> {
 export async function fetchProfileUuid(user: string, profile: string): Promise<string> {
 	// if a profile wasn't provided, return
 	if (!profile) {
-		if (debug) console.log('no profile provided?', user, profile)
+		if (debug) console.debug('no profile provided?', user, profile)
 		return null
 	}
 
-	if (debug) console.log('Cache miss: fetchProfileUuid', user)
+	if (debug) console.debug('Cache miss: fetchProfileUuid', user)
 
 	const profiles = await fetchBasicProfiles(user)
 	if (!profiles) return // user probably doesnt exist
@@ -295,11 +295,11 @@ export async function fetchProfile(user: string, profile: string): Promise<Clean
 
 	if (profileCache.has(profileUuid)) {
 		// we have the profile cached, return it :)
-		if (debug) console.log('Cache hit! fetchProfile', profileUuid)
+		if (debug) console.debug('Cache hit! fetchProfile', profileUuid)
 		return profileCache.get(profileUuid)
 	}
 
-	if (debug) console.log('Cache miss: fetchProfile', user, profile)
+	if (debug) console.debug('Cache miss: fetchProfile', user, profile)
 
 	const profileName = await fetchProfileName(user, profile)
 
@@ -320,7 +320,7 @@ export async function fetchProfile(user: string, profile: string): Promise<Clean
 export async function fetchBasicProfileFromUuid(profileUuid: string): Promise<CleanProfile> {
 	if (profileCache.has(profileUuid)) {
 		// we have the profile cached, return it :)
-		if (debug) console.log('Cache hit! fetchBasicProfileFromUuid', profileUuid)
+		if (debug) console.debug('Cache hit! fetchBasicProfileFromUuid', profileUuid)
 		const profile: CleanFullProfile = profileCache.get(profileUuid)
 		return {
 			uuid: profile.uuid,
@@ -351,11 +351,11 @@ export async function fetchProfileName(user: string, profile: string): Promise<s
 
 	if (profileNameCache.has(`${playerUuid}.${profileUuid}`)) {
 		// Return the profile name if it's cached
-		if (debug) console.log('Cache hit! fetchProfileName', profileUuid)
+		if (debug) console.debug('Cache hit! fetchProfileName', profileUuid)
 		return profileNameCache.get(`${playerUuid}.${profileUuid}`)
 	}
 
-	if (debug) console.log('Cache miss: fetchProfileName', user, profile)
+	if (debug) console.debug('Cache miss: fetchProfileName', user, profile)
 
 	const basicProfiles = await fetchBasicProfiles(playerUuid)
 	let profileName
diff --git a/src/index.ts b/src/index.ts
index 06bdc5f..d25efc9 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -7,6 +7,7 @@ const app = express()
 
 export const debug = false
 
+
 // 200 requests over 5 minutes
 const limiter = rateLimit({
 	windowMs: 60 * 1000 * 5,
@@ -69,5 +70,5 @@ app.get('/leaderboards', async(req, res) => {
 
 
 // only run the server if it's not doing tests
-if (typeof global.it !== 'function')
+if (!globalThis.isTest)
 	app.listen(8080, () => console.log('App started :)'))