add skills and zones as constants

3 files changed, 78 insertions, 65 deletions

diff --git a/src/constants.ts b/src/constants.ts
index 610d1a8..1c27e14 100644
--- a/src/constants.ts
+++ b/src/constants.ts
@@ -5,6 +5,7 @@
 import fetch from 'node-fetch'
 import { Agent } from 'https'
 import NodeCache from 'node-cache'
+import Queue from 'queue-promise'
 
 const httpsAgent = new Agent({
 	keepAlive: true
@@ -14,6 +15,12 @@ const githubApiBase = 'https://api.github.com'
 const owner = 'skyblockstats'
 const repo = 'skyblock-constants'
 
+// we use a queue for editing so it doesnt hit the github ratelimit as much
+const queue = new Queue({
+	concurrent: 1,
+	interval: 500
+})
+
 /**
  * Send a request to the GitHub API
  * @param method The HTTP method, for example GET, PUT, POST, etc
@@ -62,7 +69,6 @@ async function fetchFile(path: string): Promise<GithubFile> {
 		`/repos/${owner}/${repo}/contents/${path}`,
 		{
 			'Accept': 'application/vnd.github.v3+json',
-			'Authorization': undefined
 		},
 	)
 	const data = await r.json()
@@ -101,9 +107,8 @@ async function editFile(file: GithubFile, message: string, newContent: string) {
 	})
 }
 
-/** Fetch all the known SkyBlock stats as an array of strings */
-export async function fetchStats(): Promise<string[]> {
-	const file = await fetchFile('stats.json')
+async function fetchJSONConstant(filename: string): Promise<string[]> {
+	const file = await fetchFile(filename)
 	try {
 		return JSON.parse(file.content)
 	} catch {
@@ -112,71 +117,73 @@ export async function fetchStats(): Promise<string[]> {
 	}
 }
 
-/** Fetch all the known SkyBlock collections as an array of strings */
-export async function fetchCollections(): Promise<string[]> {
-	const file = await fetchFile('collections.json')
-	try {
-		return JSON.parse(file.content)
-	} catch {
-		// probably invalid json, return an empty array
-		return []
-	}
+/** Add stats to skyblock-constants. This has caching so it's fine to call many times */
+export async function addJSONConstants(filename: string, addingValues: string[], units: string='stats') {
+	if (addingValues.length === 0) return // no stats provided, just return
+
+	queue.enqueue(async() => {
+		const file = await fetchFile(filename)
+		if (!file.path)
+			return
+		let oldStats: string[]
+		try {
+			oldStats = JSON.parse(file.content)
+		} catch {
+			// invalid json, set it as an empty array
+			oldStats = []
+		}
+		const updatedStats = oldStats
+			.concat(addingValues)
+			// remove duplicates
+			.filter((value, index, array) => array.indexOf(value) === index)
+			.sort((a, b) => a.localeCompare(b))
+		const newStats = updatedStats.filter(value => !oldStats.includes(value))
+
+		// there's not actually any new stats, just return
+		if (newStats.length === 0) return
+
+		const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new ${units}` : `Add '${newStats[0]}'`
+		await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2))
+	})
 }
 
-/** Add stats to skyblock-constants. This has caching so it's fine to call many times */
-export async function addStats(addingStats: string[]) {
-	if (addingStats.length === 0) return // no stats provided, just return
 
-	const file = await fetchFile('stats.json')
-	if (!file.path)
-		return
-	let oldStats: string[]
-	try {
-		oldStats = JSON.parse(file.content)
-	} catch {
-		// invalid json, set it as an empty array
-		oldStats = []
-	}
-	const updatedStats = oldStats
-		.concat(addingStats)
-		// remove duplicates
-		.filter((value, index, array) => array.indexOf(value) === index)
-		.sort((a, b) => a.localeCompare(b))
-	const newStats = updatedStats.filter(value => !oldStats.includes(value))
-
-	// there's not actually any new stats, just return
-	if (newStats.length === 0) return
+/** Fetch all the known SkyBlock stats as an array of strings */
+export async function fetchStats(): Promise<string[]> {
+	return await fetchJSONConstant('stats.json')
+}
 
-	const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new stats` : `Add '${newStats[0]}'`
+/** Add stats to skyblock-constants. This has caching so it's fine to call many times */
+export async function addStats(addingStats: string[]) {
+	await addJSONConstants('stats.json', addingStats, 'stats')
+}
 
-	await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2))
+/** Fetch all the known SkyBlock collections as an array of strings */
+export async function fetchCollections(): Promise<string[]> {
+	return await fetchJSONConstant('collections.json')
 }
 
-/** Add stats to skyblock-constants. This has caching so it's fine to call many times */
+/** Add collections to skyblock-constants. This has caching so it's fine to call many times */
 export async function addCollections(addingCollections: string[]) {
-	if (addingCollections.length === 0) return // no stats provided, just return
+	await addJSONConstants('collections.json', addingCollections, 'collections')
+}
 
-	const file = await fetchFile('collections.json')
-	if (!file.path)
-		return
-	let oldCollections: string[]
-	try {
-		oldCollections = JSON.parse(file.content)
-	} catch {
-		// invalid json, set it as an empty array
-		oldCollections = []
-	}
-	const updatedCollections = oldCollections
-		.concat(addingCollections)
-		// remove duplicates
-		.filter((value, index, array) => array.indexOf(value) === index)
-		.sort((a, b) => a.localeCompare(b))
-	const newCollections = updatedCollections.filter(value => !oldCollections.includes(value))
+/** Fetch all the known SkyBlock collections as an array of strings */
+export async function fetchSkills(): Promise<string[]> {
+	return await fetchJSONConstant('skills.json')
+}
 
-	// there's not actually any new stats, just return
-	if (newCollections.length === 0) return
+/** Add skills to skyblock-constants. This has caching so it's fine to call many times */
+export async function addSkills(addingSkills: string[]) {
+	await addJSONConstants('skills.json', addingSkills, 'skills')
+}
 
-	const commitMessage = newCollections.length >= 2 ? `Add ${newCollections.length} new collections` : `Add '${newCollections[0]}'`
+/** Fetch all the known SkyBlock collections as an array of strings */
+export async function fetchZones(): Promise<string[]> {
+	return await fetchJSONConstant('zones.json')
+}
 
-	await editFile(file, commitMessage, JSON.stringify(updatedCollections, null, 2))
+/** Add skills to skyblock-constants. This has caching so it's fine to call many times */
+export async function addZones(addingZones: string[]) {
+	await addJSONConstants('zones.json', addingZones, 'zones')
 }
diff --git a/src/database.ts b/src/database.ts
index 2e48dbb..ce95578 100644
--- a/src/database.ts
+++ b/src/database.ts
@@ -190,7 +190,10 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu
 	recentlyUpdated.set(profile.uuid + member.uuid, true)
 
 	await constants.addStats(Object.keys(member.rawHypixelStats))
-	await constants.addCollections(member.collections.map(value => value.name))
+	await constants.addCollections(member.collections.map(coll => coll.name))
+	await constants.addSkills(member.skills.map(skill => skill.name))
+	await constants.addZones(member.visited_zones.map(zone => zone.name))
+
 	const leaderboardAttributes = await getApplicableAttributes(member)
 
 	await memberLeaderboardsCollection.updateOne(
diff --git a/src/hypixelCached.ts b/src/hypixelCached.ts
index 4ddb07c..b407a69 100644
--- a/src/hypixelCached.ts
+++ b/src/hypixelCached.ts
@@ -20,13 +20,13 @@ const usernameCache = new NodeCache({
 const basicProfilesCache = new NodeCache({
 	stdTTL: 60 * 10,
 	checkperiod: 60,
-	useClones: false,
+	useClones: true,
 })
 
 const playerCache = new NodeCache({
 	stdTTL: 60,
 	checkperiod: 10,
-	useClones: false,
+	useClones: true,
 })
 
 const profileCache = new NodeCache({
@@ -64,6 +64,10 @@ function waitForSet(cache: NodeCache, key?: string, value?: string): Promise<any
  * @param user A user can be either a uuid or a username 
  */
 export async function uuidFromUser(user: string): Promise<string> {
+	// if the user is 32 characters long, it has to be a uuid
+	if (undashUuid(user).length === 32)
+		return undashUuid(user)
+
 	if (usernameCache.has(undashUuid(user))) {
 		// check if the uuid is a key
 		const username: any = usernameCache.get(undashUuid(user))
@@ -132,9 +136,8 @@ export async function fetchPlayer(user: string): Promise<CleanPlayer> {
 	if (!cleanPlayer) return
 
 	// clone in case it gets modified somehow later
-	const cleanPlayerClone = Object.assign({}, cleanPlayer)
-	playerCache.set(playerUuid, cleanPlayerClone)
-	usernameCache.set(playerUuid, cleanPlayerClone.username)
+	playerCache.set(playerUuid, cleanPlayer)
+	usernameCache.set(playerUuid, cleanPlayer.username)
 
 	return cleanPlayer
 }