Add leaderboards

10 files changed, 604 insertions, 175 deletions

diff --git a/src/cleaners/rank.ts b/src/cleaners/rank.ts
index d565502..0a3a4a7 100644
--- a/src/cleaners/rank.ts
+++ b/src/cleaners/rank.ts
@@ -22,48 +22,50 @@ export interface CleanRank {
 
 /** Response cleaning (reformatting to be nicer) */
 export function cleanRank({
-    packageRank,
-    newPackageRank,
-    monthlyPackageRank,
-    rankPlusColor,
-    rank,
-    prefix
+	packageRank,
+	newPackageRank,
+	monthlyPackageRank,
+	rankPlusColor,
+	rank,
+	prefix
 }: HypixelPlayer): CleanRank {
-    let name
-    let color
-    let colored
-    if (prefix) { // derive values from prefix
-        colored = prefix
-        color = minecraftColorCodes[colored.match(/§./)[0][1]]
-        name = colored.replace(/§./g, '').replace(/[\[\]]/g, '')
-    } else {
-        name = monthlyPackageRank
-                || rank
-                || newPackageRank?.replace('_PLUS', '+')
-                || packageRank?.replace('_PLUS', '+')
+	let name
+	let color
+	let colored
+	if (prefix) { // derive values from prefix
+		colored = prefix
+		color = minecraftColorCodes[colored.match(/§./)[0][1]]
+		name = colored.replace(/§./g, '').replace(/[\[\]]/g, '')
+	} else {
+		if (monthlyPackageRank !== 'NONE')
+			name = monthlyPackageRank
+		else
+			name = rank
+				|| newPackageRank?.replace('_PLUS', '+')
+				|| packageRank?.replace('_PLUS', '+')
 
-        // MVP++ is called Superstar for some reason
-        if (name === 'SUPERSTAR') name = 'MVP++'
-        // YouTube rank is called YouTuber, change this to the proper name
-        else if (name === 'YOUTUBER') name = 'YOUTUBE'
-        else if (name === undefined) name = 'NONE'
+		// MVP++ is called Superstar for some reason
+		if (name === 'SUPERSTAR') name = 'MVP++'
+		// YouTube rank is called YouTuber, change this to the proper name
+		else if (name === 'YOUTUBER') name = 'YOUTUBE'
+		else if (name === undefined) name = 'NONE'
 
-        const plusColor = rankPlusColor ? colorCodeFromName(rankPlusColor) : null
-        color = minecraftColorCodes[rankColors[name]]
-        const rankColorPrefix = rankColors[name] ? '§' + rankColors[name] : ''
-        const nameWithoutPlus = name.split('+')[0]
-        const plusesInName = '+'.repeat(name.split('+').length - 1)
-        if (plusColor && plusesInName.length >= 1)
-            colored = `${rankColorPrefix}[${nameWithoutPlus}§${plusColor}${plusesInName}${rankColorPrefix}]`
-        else if (name !== 'NONE')
-            colored = `${rankColorPrefix}[${name}]`
-        else
-            // nons don't have a prefix
-            colored = `${rankColorPrefix}`
-    }
-    return {
-        name,
-        color,
-        colored
-    }
+		const plusColor = rankPlusColor ? colorCodeFromName(rankPlusColor) : null
+		color = minecraftColorCodes[rankColors[name]]
+		const rankColorPrefix = rankColors[name] ? '§' + rankColors[name] : ''
+		const nameWithoutPlus = name.split('+')[0]
+		const plusesInName = '+'.repeat(name.split('+').length - 1)
+		if (plusColor && plusesInName.length >= 1)
+			colored = `${rankColorPrefix}[${nameWithoutPlus}§${plusColor}${plusesInName}${rankColorPrefix}]`
+		else if (name !== 'NONE')
+			colored = `${rankColorPrefix}[${name}]`
+		else
+			// nons don't have a prefix
+			colored = `${rankColorPrefix}`
+	}
+	return {
+		name,
+		color,
+		colored
+	}
 }
diff --git a/src/cleaners/skyblock/member.ts b/src/cleaners/skyblock/member.ts
index 154ff22..a4ca053 100644
--- a/src/cleaners/skyblock/member.ts
+++ b/src/cleaners/skyblock/member.ts
@@ -5,7 +5,7 @@ import { cleanObjectives, Objective } from './objectives'
 import { CleanMinion, cleanMinions } from './minions'
 import { cleanSkills, Skill } from './skills'
 import * as cached from '../../hypixelCached'
-import { CleanFullProfile } from './profile'
+import { CleanFullProfile, CleanFullProfileBasicMembers } from './profile'
 import { Included } from '../../hypixel'
 import { CleanPlayer } from '../player'
 import { Bank } from './bank'
@@ -25,6 +25,7 @@ export interface CleanBasicMember {
 export interface CleanMember extends CleanBasicMember {
     purse: number
     stats: CleanProfileStats
+    rawHypixelStats?: { [ key: string ]: number }
     minions: CleanMinion[]
 	fairy_souls: FairySouls
     inventories: Inventories
@@ -61,6 +62,10 @@ export async function cleanSkyBlockProfileMemberResponse(member, included: Inclu
         purse: member.coin_purse,
 
         stats: cleanProfileStats(member),
+
+        // this is used for leaderboards
+        rawHypixelStats: member.stats ?? {},
+
         minions: cleanMinions(member),
         fairy_souls: cleanFairySouls(member),
         inventories: inventoriesIncluded ? await cleanInventories(member) : undefined,
@@ -83,5 +88,5 @@ export interface CleanMemberProfilePlayer extends CleanPlayer {
 
 export interface CleanMemberProfile {
     member: CleanMemberProfilePlayer
-    profile: CleanFullProfile
+    profile: CleanFullProfileBasicMembers
 }
diff --git a/src/cleaners/skyblock/profile.ts b/src/cleaners/skyblock/profile.ts
index 2b092a1..6e98f8f 100644
--- a/src/cleaners/skyblock/profile.ts
+++ b/src/cleaners/skyblock/profile.ts
@@ -8,7 +8,14 @@ export interface CleanProfile extends CleanBasicProfile {
 }
 
 export interface CleanFullProfile extends CleanProfile {
-    members: (CleanMember|CleanBasicMember)[]
+    members: CleanMember[]
+    bank: Bank
+    minions: CleanMinion[]
+	minion_count: number
+}
+
+export interface CleanFullProfileBasicMembers extends CleanProfile {
+    members: CleanBasicMember[]
     bank: Bank
     minions: CleanMinion[]
 	minion_count: number
@@ -38,19 +45,21 @@ export async function cleanSkyblockProfileResponseLighter(data): Promise<CleanPr
 /**
  * This function is somewhat costly and shouldn't be called often. Use cleanSkyblockProfileResponseLighter if you don't need all the data
  */
-export async function cleanSkyblockProfileResponse(data: any, { mainMemberUuid }: ApiOptions): Promise<CleanFullProfile> {
-    const cleanedMembers: CleanMember[] = []
-
+export async function cleanSkyblockProfileResponse(data: any, options?: ApiOptions): Promise<CleanFullProfile> {
+    // We use Promise.all so it can fetch all the users at once instead of waiting for the previous promise to complete
+    const promises: Promise<CleanMember>[] = []
+    
     for (const memberUUID in data.members) {
         const memberRaw = data.members[memberUUID]
         memberRaw.uuid = memberUUID
-        const member: CleanMember = await cleanSkyBlockProfileMemberResponse(
+        promises.push(cleanSkyBlockProfileMemberResponse(
             memberRaw,
-            ['stats', mainMemberUuid === memberUUID ? 'inventories' : undefined]
-        )
-        cleanedMembers.push(member)
+            ['stats', options?.mainMemberUuid === memberUUID ? 'inventories' : undefined]
+        ))
     }
 
+    const cleanedMembers: CleanMember[] = await Promise.all(promises)
+
     const memberMinions: CleanMinion[][] = []
 
     for (const member of cleanedMembers) {
diff --git a/src/cleaners/skyblock/profiles.ts b/src/cleaners/skyblock/profiles.ts
index ea290f6..c9f5628 100644
--- a/src/cleaners/skyblock/profiles.ts
+++ b/src/cleaners/skyblock/profiles.ts
@@ -1,5 +1,5 @@
 import { HypixelPlayerStatsSkyBlockProfiles } from "../../hypixelApi"
-import { CleanBasicProfile, CleanProfile, cleanSkyblockProfileResponseLighter } from "./profile"
+import { CleanBasicProfile, CleanProfile, cleanSkyblockProfileResponse, cleanSkyblockProfileResponseLighter } from "./profile"
 
 export function cleanPlayerSkyblockProfiles(rawProfiles: HypixelPlayerStatsSkyBlockProfiles): CleanBasicProfile[] {
     let profiles: CleanBasicProfile[] = []
@@ -16,7 +16,8 @@ export function cleanPlayerSkyblockProfiles(rawProfiles: HypixelPlayerStatsSkyBl
 export async function cleanSkyblockProfilesResponse(data: any[]): Promise<CleanProfile[]> {
     const cleanedProfiles: CleanProfile[] = []
     for (const profile of data ?? []) {
-        let cleanedProfile = await cleanSkyblockProfileResponseLighter(profile)
+        // let cleanedProfile = await cleanSkyblockProfileResponseLighter(profile)
+        let cleanedProfile = await cleanSkyblockProfileResponse(profile)
         cleanedProfiles.push(cleanedProfile)
     }
     return cleanedProfiles
diff --git a/src/constants.ts b/src/constants.ts
new file mode 100644
index 0000000..c7581ae
--- /dev/null
+++ b/src/constants.ts
@@ -0,0 +1,172 @@
+/**
+ * Fetch and edit constants from the skyblock-constants repo
+ */
+
+import fetch from 'node-fetch'
+import { Agent } from 'https'
+import NodeCache from 'node-cache'
+
+const httpsAgent = new Agent({
+	keepAlive: true
+})
+
+const githubApiBase = 'https://api.github.com'
+const owner = 'skyblockstats'
+const repo = 'skyblock-constants'
+
+/**
+ * Send a request to the GitHub API
+ * @param method The HTTP method, for example GET, PUT, POST, etc
+ * @param route The route to send the request to
+ * @param headers The extra headers
+ * @param json The JSON body, only applicable for some types of methods
+ */
+async function fetchGithubApi(method: string, route: string, headers?: any, json?: any) {
+	return await fetch(
+		githubApiBase + route,
+		{
+			agent: () => httpsAgent,
+			body: json ? JSON.stringify(json) : null,
+			method,
+			headers: Object.assign({
+				'Authorization': `token ${process.env.github_token}`
+			}, headers),
+		}
+	)
+}
+
+interface GithubFile {
+	path: string
+	content: string
+	sha: string
+}
+
+// cache files for a day
+const fileCache = new NodeCache({
+	stdTTL: 60 * 60 * 24,
+	checkperiod: 60,
+	useClones: false,
+})
+
+
+/**
+ * Fetch a file from skyblock-constants
+ * @param path The file path, for example stats.json
+ */
+async function fetchFile(path: string): Promise<GithubFile> {
+	if (fileCache.has(path))
+		return fileCache.get(path)
+
+	const r = await fetchGithubApi(
+		'GET',
+		`/repos/${owner}/${repo}/contents/${path}`,
+		{ 'Accept': 'application/vnd.github.v3+json' },
+	)
+	const data = await r.json()
+	return {
+		path: data.path,
+		content: Buffer.from(data.content, data.encoding).toString(),
+		sha: data.sha
+	}
+}
+
+/**
+ * Edit a file on skyblock-constants
+ * @param file The GithubFile you got from fetchFile
+ * @param message The commit message
+ * @param newContent The new content in the file
+ */
+async function editFile(file: GithubFile, message: string, newContent: string) {
+	fileCache.set(file.path, newContent)
+	await fetchGithubApi(
+		'PUT',
+		`/repos/${owner}/${repo}/contents/${file.path}`,
+		{ 'Content-Type': 'application/json' },
+		{
+			message: message,
+			content: Buffer.from(newContent).toString('base64'),
+			sha: file.sha,
+			branch: 'main'
+		}
+	)
+}
+
+/** Fetch all the known SkyBlock stats as an array of strings */
+export async function fetchStats(): Promise<string[]> {
+	const file = await fetchFile('stats.json')
+	try {
+		return JSON.parse(file.content)
+	} catch {
+		// probably invalid json, return an empty array
+		return []
+	}
+}
+
+/** Fetch all the known SkyBlock collections as an array of strings */
+export async function fetchCollections(): Promise<string[]> {
+	const file = await fetchFile('collections.json')
+	try {
+		return JSON.parse(file.content)
+	} catch {
+		// probably invalid json, return an empty array
+		return []
+	}
+}
+
+/** Add stats to skyblock-constants. This has caching so it's fine to call many times */
+export async function addStats(addingStats: string[]) {
+	if (addingStats.length === 0) return // no stats provided, just return
+
+	const file = await fetchFile('stats.json')
+	if (!file.path)
+		return
+	let oldStats: string[]
+	try {
+		oldStats = JSON.parse(file.content)
+	} catch {
+		// invalid json, set it as an empty array
+		oldStats = []
+	}
+	const updatedStats = oldStats
+		.concat(addingStats)
+		// remove duplicates
+		.filter((value, index, array) => array.indexOf(value) === index)
+		.sort((a, b) => a.localeCompare(b))
+	const newStats = updatedStats.filter(value => !oldStats.includes(value))
+
+	// there's not actually any new stats, just return
+	if (newStats.length === 0) return
+
+	const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new stats` : `Add '${newStats[0]}'`
+
+	await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2))
+}
+
+/** Add stats to skyblock-constants. This has caching so it's fine to call many times */
+export async function addCollections(addingCollections: string[]) {
+	if (addingCollections.length === 0) return // no stats provided, just return
+
+	const file = await fetchFile('collections.json')
+	if (!file.path)
+		return
+	let oldCollections: string[]
+	try {
+		oldCollections = JSON.parse(file.content)
+	} catch {
+		// invalid json, set it as an empty array
+		oldCollections = []
+	}
+	const updatedCollections = oldCollections
+		.concat(addingCollections)
+		// remove duplicates
+		.filter((value, index, array) => array.indexOf(value) === index)
+		.sort((a, b) => a.localeCompare(b))
+	const newCollections = updatedCollections.filter(value => !oldCollections.includes(value))
+
+	// there's not actually any new stats, just return
+	if (newCollections.length === 0) return
+
+	const commitMessage = newCollections.length >= 2 ? `Add ${newCollections.length} new collections` : `Add '${newCollections[0]}'`
+
+	await editFile(file, commitMessage, JSON.stringify(updatedCollections, null, 2))
+}
diff --git a/src/database.ts b/src/database.ts
new file mode 100644
index 0000000..a1178bc
--- /dev/null
+++ b/src/database.ts
@@ -0,0 +1,173 @@
+/**
+ * Store data about members for leaderboards
+*/
+
+import * as constants from './constants'
+import * as cached from './hypixelCached'
+import { Collection, Db, FilterQuery, MongoClient } from 'mongodb'
+import NodeCache from 'node-cache'
+import { CleanMember } from './cleaners/skyblock/member'
+
+// don't update the user for 3 minutes
+const recentlyUpdated = new NodeCache({
+	stdTTL: 60 * 3,
+	checkperiod: 60,
+	useClones: false,
+})
+
+interface LeaderboardItem {
+	uuid: string
+	stats: any
+	last_updated: Date
+}
+
+const cachedLeaderboards: Map<string, any> = new Map()
+
+
+let client: MongoClient
+let database: Db
+let memberLeaderboardsCollection: Collection<LeaderboardItem>
+
+async function connect() {
+	if (!process.env.db_uri)
+		return console.warn('Warning: db_uri was not found in .env. Features that utilize the database such as leaderboards won\'t work.')
+	if (!process.env.db_name)
+		return console.warn('Warning: db_name was not found in .env. Features that utilize the database such as leaderboards won\'t work.')
+	client = await MongoClient.connect(process.env.db_uri, { useNewUrlParser: true, useUnifiedTopology: true })
+	database = client.db(process.env.db_name)
+	memberLeaderboardsCollection = database.collection('member-leaderboards')
+}
+
+
+function getMemberCollectionAttributes(member: CleanMember) {
+	const collectionAttributes = {}
+	for (const collection of member.collections) {
+		const collectionLeaderboardName = `collection_${collection.name}`
+		collectionAttributes[collectionLeaderboardName] = collection.xp
+	}
+	return collectionAttributes
+}
+
+function getMemberLeaderboardAttributes(member: CleanMember) {
+	// if you want to add a new leaderboard for member attributes, add it here (and getAllLeaderboardAttributes)
+	return {
+		// we use the raw stat names rather than the clean stats in case hypixel adds a new stat and it takes a while for us to clean it
+		...member.rawHypixelStats,
+
+		// collection leaderboards
+		...getMemberCollectionAttributes(member),
+
+		fairy_souls: member.fairy_souls.total,
+		first_join: member.first_join,
+		purse: member.purse,
+		visited_zones: member.visited_zones.length,
+	}
+}
+
+/** Fetch the names of all the leaderboards */
+async function fetchAllMemberLeaderboardAttributes(): Promise<string[]> {
+	return [
+		// we use the raw stat names rather than the clean stats in case hypixel adds a new stat and it takes a while for us to clean it
+		...await constants.fetchStats(),
+
+		// collection leaderboards
+		...(await constants.fetchCollections()).map(value => `collection_${value}`),
+
+		'fairy_souls',
+		'first_join',
+		'purse',
+		'visited_zones',
+	]
+}
+
+export async function fetchMemberLeaderboard(name: string) {
+	if (cachedLeaderboards.has(name))
+		return cachedLeaderboards.get(name)
+	// typescript forces us to make a new variable and set it this way because it gives an error otherwise
+	const query: FilterQuery<any> = {}
+	query[`stats.${name}`] = { '$exists': true }
+
+	const sortQuery: any = {}
+	sortQuery[`stats.${name}`] = -1
+
+
+	const leaderboardRaw = await memberLeaderboardsCollection.find(query).sort(sortQuery).limit(100).toArray()
+	const fetchLeaderboardPlayer = async(item: LeaderboardItem) => {
+		return {
+			player: await cached.fetchPlayer(item.uuid),
+			value: item.stats[name]
+		}
+	}
+	const promises = []
+	for (const item of leaderboardRaw) {
+		promises.push(fetchLeaderboardPlayer(item))
+	}
+	const leaderboard = await Promise.all(promises)
+	cachedLeaderboards.set(name, leaderboard)
+	return leaderboard
+}
+
+async function getMemberLeaderboardRequirement(name: string): Promise<LeaderboardItem> {
+	const leaderboard = await fetchMemberLeaderboard(name)
+	// if there's more than 100 items, return the 100th. if there's less, return null
+	if (leaderboard.length >= 100)
+		return leaderboard[99].value
+	else
+		return null
+}
+
+/** Update the member's leaderboard data on the server if applicable */
+export async function updateDatabaseMember(member: CleanMember) {
+	if (!client) return // the db client hasn't been initialized
+	// the member's been updated too recently, just return
+	if (recentlyUpdated.get(member.uuid))
+		return
+	// store the member in recentlyUpdated so it cant update for 3 more minutes
+	recentlyUpdated.set(member.uuid, true)
+
+	await constants.addStats(Object.keys(member.rawHypixelStats))
+	await constants.addCollections(member.collections.map(value => value.name))
+
+	const leaderboardAttributes = getMemberLeaderboardAttributes(member)
+
+	await memberLeaderboardsCollection.updateOne({
+		uuid: member.uuid
+	}, {
+		'$set': {
+			'stats': leaderboardAttributes,
+			'last_updated': new Date()
+		}
+	}, {
+		upsert: true
+	})
+}
+
+
+/**
+ * Remove leaderboard attributes for members that wouldn't actually be on the leaderboard. This saves a lot of storage space
+ */
+async function removeBadMemberLeaderboardAttributes() {
+	const leaderboards = await fetchAllMemberLeaderboardAttributes()
+	for (const leaderboard of leaderboards) {
+		// wait 10 seconds so it doesnt use as much ram
+		await new Promise(resolve => setTimeout(resolve, 10000))
+
+		const unsetValue = {}
+		unsetValue[leaderboard] = ''
+		const filter = {}
+		const requirement = await getMemberLeaderboardRequirement(leaderboard)
+		if (requirement !== null) {
+			filter[`stats.${leaderboard}`] = {
+				'$lt': requirement
+			}
+			await memberLeaderboardsCollection.updateMany(
+				filter,
+				{ '$unset': unsetValue }
+			)
+		}
+	}
+}
+
+
+connect()
+	.then(removeBadMemberLeaderboardAttributes)
\ No newline at end of file
diff --git a/src/hypixel.ts b/src/hypixel.ts
index 83ad419..3b8a952 100644
--- a/src/hypixel.ts
+++ b/src/hypixel.ts
@@ -6,9 +6,10 @@ import { CleanPlayer, cleanPlayerResponse } from './cleaners/player'
 import { chooseApiKey, HypixelResponse, sendApiRequest } from './hypixelApi'
 import * as cached from './hypixelCached'
 import { CleanBasicMember, CleanMemberProfile } from './cleaners/skyblock/member'
-import { cleanSkyblockProfileResponse, CleanProfile, CleanBasicProfile } from './cleaners/skyblock/profile'
+import { cleanSkyblockProfileResponse, CleanProfile, CleanBasicProfile, CleanFullProfile, CleanFullProfileBasicMembers } from './cleaners/skyblock/profile'
 import { cleanSkyblockProfilesResponse } from './cleaners/skyblock/profiles'
 import { debug } from '.'
+import { updateDatabaseMember } from './database'
 
 export type Included = 'profiles' | 'player' | 'stats' | 'inventories'
 
@@ -23,46 +24,47 @@ export const maxMinion = 11
  */ 
 
 export interface ApiOptions {
-    mainMemberUuid?: string
+	mainMemberUuid?: string
 }
 
+/** Sends an API request to Hypixel and cleans it up. */
 export async function sendCleanApiRequest({ path, args }, included?: Included[], options?: ApiOptions) {
-    const key = await chooseApiKey()
-    const rawResponse = await sendApiRequest({ path, key, args })
-    if (rawResponse.throttled) {
+	const key = await chooseApiKey()
+	const rawResponse = await sendApiRequest({ path, key, args })
+	if (rawResponse.throttled) {
 		// if it's throttled, wait a second and try again
 		await new Promise(resolve => setTimeout(resolve, 1000))
-        return await sendCleanApiRequest({ path, args }, included, options)
-    }
+		return await sendCleanApiRequest({ path, args }, included, options)
+	}
 
-    // clean the response
-    return await cleanResponse({ path, data: rawResponse }, options ?? {})
+	// clean the response
+	return await cleanResponse({ path, data: rawResponse }, options ?? {})
 }
 
 
 
 async function cleanResponse({ path, data }: { path: string, data: HypixelResponse }, options: ApiOptions) {
-    // Cleans up an api response
-    switch (path) {
-        case 'player': return await cleanPlayerResponse(data.player)
-        case 'skyblock/profile': return await cleanSkyblockProfileResponse(data.profile, options)
-        case 'skyblock/profiles': return await cleanSkyblockProfilesResponse(data.profiles)
-    }
+	// Cleans up an api response
+	switch (path) {
+		case 'player': return await cleanPlayerResponse(data.player)
+		case 'skyblock/profile': return await cleanSkyblockProfileResponse(data.profile, options)
+		case 'skyblock/profiles': return await cleanSkyblockProfilesResponse(data.profiles)
+	}
 }
 
 /* ----------------------------- */
 
 export interface UserAny {
-    user?: string
-    uuid?: string
-    username?: string
+	user?: string
+	uuid?: string
+	username?: string
 }
 
 export interface CleanUser {
-    player: CleanPlayer
-    profiles?: CleanProfile[]
-    activeProfile?: string
-    online?: boolean
+	player: CleanPlayer
+	profiles?: CleanProfile[]
+	activeProfile?: string
+	online?: boolean
 }
 
 
@@ -73,52 +75,52 @@ export interface CleanUser {
  * used inclusions: player, profiles
  */
 export async function fetchUser({ user, uuid, username }: UserAny, included: Included[]=['player']): Promise<CleanUser> {
-    if (!uuid) {
-        // If the uuid isn't provided, get it
-        uuid = await cached.uuidFromUser(user || username)
-    }
-    if (!uuid) {
-        // the user doesn't exist.
-        if (debug) console.log('error:', user, 'doesnt exist')
-        return null
-    }
-
-    const includePlayers = included.includes('player')
-    const includeProfiles = included.includes('profiles')
-
-    let profilesData: CleanProfile[]
-    let basicProfilesData: CleanBasicProfile[]
-    let playerData: CleanPlayer
-
-    if (includePlayers) {
-        playerData = await cached.fetchPlayer(uuid)
-        // if not including profiles, include lightweight profiles just in case
-        if (!includeProfiles)
-            basicProfilesData = playerData.profiles
-        playerData.profiles = undefined
-    }
-    if (includeProfiles) {
-        profilesData = await cached.fetchSkyblockProfiles(uuid)
-    }
-
-    let activeProfile: CleanProfile = null
-    let lastOnline: number = 0
-
-    if (includeProfiles) {
-        for (const profile of profilesData) {
-            const member = profile.members.find(member => member.uuid === uuid)
-            if (member.last_save > lastOnline) {
-                lastOnline = member.last_save
-                activeProfile = profile
-            }
-        }
-    }
-    return {
-        player: playerData ?? null,
-        profiles: profilesData ?? basicProfilesData,
-        activeProfile: includeProfiles ? activeProfile?.uuid : undefined,
-        online: includeProfiles ? lastOnline > (Date.now() - saveInterval): undefined
-    }
+	if (!uuid) {
+		// If the uuid isn't provided, get it
+		uuid = await cached.uuidFromUser(user || username)
+	}
+	if (!uuid) {
+		// the user doesn't exist.
+		if (debug) console.log('error:', user, 'doesnt exist')
+		return null
+	}
+
+	const includePlayers = included.includes('player')
+	const includeProfiles = included.includes('profiles')
+
+	let profilesData: CleanProfile[]
+	let basicProfilesData: CleanBasicProfile[]
+	let playerData: CleanPlayer
+
+	if (includePlayers) {
+		playerData = await cached.fetchPlayer(uuid)
+		// if not including profiles, include lightweight profiles just in case
+		if (!includeProfiles)
+			basicProfilesData = playerData.profiles
+		playerData.profiles = undefined
+	}
+	if (includeProfiles) {
+		profilesData = await cached.fetchSkyblockProfiles(uuid)
+	}
+
+	let activeProfile: CleanProfile = null
+	let lastOnline: number = 0
+
+	if (includeProfiles) {
+		for (const profile of profilesData) {
+			const member = profile.members.find(member => member.uuid === uuid)
+			if (member.last_save > lastOnline) {
+				lastOnline = member.last_save
+				activeProfile = profile
+			}
+		}
+	}
+	return {
+		player: playerData ?? null,
+		profiles: profilesData ?? basicProfilesData,
+		activeProfile: includeProfiles ? activeProfile?.uuid : undefined,
+		online: includeProfiles ? lastOnline > (Date.now() - saveInterval): undefined
+	}
 }
 
 /**
@@ -128,40 +130,78 @@ export async function fetchUser({ user, uuid, username }: UserAny, included: Inc
  * @param profile A profile name or profile uuid
  */
 export async function fetchMemberProfile(user: string, profile: string): Promise<CleanMemberProfile> {
-    const playerUuid = await cached.uuidFromUser(user)
-    const profileUuid = await cached.fetchProfileUuid(user, profile)
+	const playerUuid = await cached.uuidFromUser(user)
+	const profileUuid = await cached.fetchProfileUuid(user, profile)
 
-    // if the profile doesn't have an id, just return
-    if (!profileUuid) return null
+	// if the profile doesn't have an id, just return
+	if (!profileUuid) return null
 
-    const player = await cached.fetchPlayer(playerUuid)
+	const player = await cached.fetchPlayer(playerUuid)
 
-    const cleanProfile = await cached.fetchProfile(playerUuid, profileUuid)
+	const cleanProfile = await cached.fetchProfile(playerUuid, profileUuid) as CleanFullProfileBasicMembers
 
-    const member = cleanProfile.members.find(m => m.uuid === playerUuid)
+	const member = cleanProfile.members.find(m => m.uuid === playerUuid)
 
-    // remove unnecessary member data
-    const simpleMembers: CleanBasicMember[] = cleanProfile.members.map(m => {
-        return {
-            uuid: m.uuid,
-            username: m.username,
-            first_join: m.first_join,
-            last_save: m.last_save,
-            rank: m.rank
-        }
-    })
+	// remove unnecessary member data
+	const simpleMembers: CleanBasicMember[] = cleanProfile.members.map(m => {
+		return {
+			uuid: m.uuid,
+			username: m.username,
+			first_join: m.first_join,
+			last_save: m.last_save,
+			rank: m.rank
+		}
+	})
 
-    cleanProfile.members = simpleMembers
+	cleanProfile.members = simpleMembers
 
-    return {
-        member: {
+	return {
+		member: {
 			// the profile name is in member rather than profile since they sometimes differ for each member
-            profileName: cleanProfile.name,
+			profileName: cleanProfile.name,
 			// add all the member data
-            ...member,
-            // add all other data relating to the hypixel player, such as username, rank, etc
-            ...player
-        },
-        profile: cleanProfile
-    }
+			...member,
+			// add all other data relating to the hypixel player, such as username, rank, etc
+			...player
+		},
+		profile: cleanProfile
+	}
 }
+
+/**
+ * Fetches the Hypixel API to get a CleanFullProfile. This doesn't do any caching and you should use hypixelCached.fetchProfile instead
+ * @param playerUuid The UUID of the Minecraft player
+ * @param profileUuid The UUID of the Hypixel SkyBlock profile
+ */
+export async function fetchMemberProfileUncached(playerUuid: string, profileUuid: string): Promise<CleanFullProfile> {
+	const profile: CleanFullProfile = await sendCleanApiRequest(
+		{
+			path: 'skyblock/profile',
+			args: { profile: profileUuid }
+		},
+		null,
+		{ mainMemberUuid: playerUuid }
+	)
+	for (const member of profile.members)
+		updateDatabaseMember(member)
+	return profile
+}
+
+
+export async function fetchMemberProfilesUncached(playerUuid: string): Promise<CleanFullProfile[]> {
+	const profiles: CleanFullProfile[] = await sendCleanApiRequest({
+		path: 'skyblock/profiles',
+		args: {
+			uuid: playerUuid
+		}},
+		null,
+		{
+			// only the inventories for the main player are generated, this is for optimization purposes
+			mainMemberUuid: playerUuid
+		}
+	)
+	for (const profile of profiles)
+		for (const member of profile.members)
+			updateDatabaseMember(member)
+	return profiles
+}
\ No newline at end of file
diff --git a/src/hypixelCached.ts b/src/hypixelCached.ts
index 1b8b617..83360a9 100644
--- a/src/hypixelCached.ts
+++ b/src/hypixelCached.ts
@@ -2,7 +2,7 @@
  * Fetch the clean and cached Hypixel API
  */
 
-import NodeCache from 'node-cache'
+import NodeCache, { EventEmitter, Key } from 'node-cache'
 import * as mojang from './mojang'
 import * as hypixel from './hypixel'
 import { CleanPlayer } from './cleaners/player'
@@ -10,8 +10,6 @@ import { undashUuid } from './util'
 import { CleanProfile, CleanFullProfile, CleanBasicProfile } from './cleaners/skyblock/profile'
 import { debug } from '.'
 
-
-
 // cache usernames for 4 hours
 const usernameCache = new NodeCache({
 	stdTTL: 60 * 60 * 4,
@@ -49,28 +47,54 @@ const profileNameCache = new NodeCache({
 	useClones: false,
 })
 
+function waitForSet(cache: NodeCache, key?: string, value?: string): Promise<any> {
+	return new Promise((resolve, reject) => {
+		const listener = (setKey, setValue) => {
+			if (setKey === key || (value && setValue === value)) {
+				cache.removeListener('set', listener)
+				return resolve({ key, value })
+			}
+		}
+		cache.on('set', listener)
+	})
+}
+
 /**
  * Fetch the uuid from a user
  * @param user A user can be either a uuid or a username 
  */
 export async function uuidFromUser(user: string): Promise<string> {
-	if (usernameCache.has(undashUuid(user)))
+	if (usernameCache.has(undashUuid(user))) {
 		// check if the uuid is a key
-		return undashUuid(user)
+		const username: any = usernameCache.get(undashUuid(user))
+		// if it has .then, then that means its a waitForSet promise. This is done to prevent requests made while it is already requesting
+		if (username.then) {
+			return (await username()).key
+		} else
+			return undashUuid(user)
+	}
 
 	// check if the username is a value
 	const uuidToUsername: {[ key: string ]: string} = usernameCache.mget(usernameCache.keys())
 	for (const [ uuid, username ] of Object.entries(uuidToUsername)) {
-		if (user.toLowerCase() === username.toLowerCase())
+		if (username.toLowerCase && user.toLowerCase() === username.toLowerCase())
 			return uuid
 	}
 
+	if (debug) console.log('Cache miss: uuidFromUser', user)
+
+	// set it as waitForSet (a promise) in case uuidFromUser gets called while its fetching mojang
+	usernameCache.set(undashUuid(user), waitForSet(usernameCache, user, user))
+	
 	// not cached, actually fetch mojang api now
 	let { uuid, username } = await mojang.mojangDataFromUser(user)
 	if (!uuid) return
 
 	// remove dashes from the uuid so its more normal
 	uuid = undashUuid(uuid)
+
+	if (user !== uuid) usernameCache.del(user)
+
 	usernameCache.set(uuid, username)
 	return uuid
 }
@@ -85,6 +109,8 @@ export async function usernameFromUser(user: string): Promise<string> {
 		return usernameCache.get(undashUuid(user))
 	}
 
+	if (debug) console.log('Cache miss: usernameFromUser', user)
+
 	let { uuid, username } = await mojang.mojangDataFromUser(user)
 	uuid = undashUuid(uuid)
 	usernameCache.set(uuid, username)
@@ -96,7 +122,6 @@ export async function fetchPlayer(user: string): Promise<CleanPlayer> {
 	const playerUuid = await uuidFromUser(user)
 
 	if (playerCache.has(playerUuid)) {
-		if (debug) console.log('Cache hit! fetchPlayer', playerUuid)
 		return playerCache.get(playerUuid)
 	}
 
@@ -120,17 +145,9 @@ export async function fetchSkyblockProfiles(playerUuid: string): Promise<CleanPr
 		return profilesCache.get(playerUuid)
 	}
 
-	const profiles: CleanFullProfile[] = await hypixel.sendCleanApiRequest({
-		path: 'skyblock/profiles',
-		args: {
-			uuid: playerUuid
-		}},
-		null,
-		{
-			// only the inventories for the main player are generated, this is for optimization purposes
-			mainMemberUuid: playerUuid
-		}
-	)
+	if (debug) console.log('Cache miss: fetchSkyblockProfiles', playerUuid)
+
+	const profiles: CleanProfile[] = await hypixel.fetchMemberProfilesUncached(playerUuid)
 
 	const basicProfiles: CleanProfile[] = []
 
@@ -165,6 +182,9 @@ async function fetchBasicProfiles(user: string): Promise<CleanBasicProfile[]> {
 		if (debug) console.log('Cache hit! fetchBasicProfiles', playerUuid)
 		return basicProfilesCache.get(playerUuid)
 	}
+
+	if (debug) console.log('Cache miss: fetchBasicProfiles', user)
+
 	const player = await fetchPlayer(playerUuid)
 	const profiles = player.profiles
 	basicProfilesCache.set(playerUuid, profiles)
@@ -188,6 +208,8 @@ export async function fetchProfileUuid(user: string, profile: string) {
 		return null
 	}
 
+	if (debug) console.log('Cache miss: fetchProfileUuid', user)
+
 	const profiles = await fetchBasicProfiles(user)
 
 	const profileUuid = undashUuid(profile)
@@ -215,16 +237,11 @@ export async function fetchProfile(user: string, profile: string): Promise<Clean
 		return profileCache.get(profileUuid)
 	}
 
+	if (debug) console.log('Cache miss: fetchProfile', user, profile)
+
 	const profileName = await fetchProfileName(user, profile)
 
-	const cleanProfile: CleanFullProfile = await hypixel.sendCleanApiRequest(
-		{
-			path: 'skyblock/profile',
-			args: { profile: profileUuid }
-		},
-		null,
-		{ mainMemberUuid: playerUuid }
-	)
+	const cleanProfile: CleanFullProfile = await hypixel.fetchMemberProfileUncached(playerUuid, profileUuid)
 
 	// we know the name from fetchProfileName, so set it here
 	cleanProfile.name = profileName
@@ -250,6 +267,8 @@ export async function fetchProfileName(user: string, profile: string): Promise<s
 		return profileNameCache.get(`${playerUuid}.${profileUuid}`)
 	}
 
+	if (debug) console.log('Cache miss: fetchProfileName', user, profile)
+
 	const basicProfiles = await fetchBasicProfiles(playerUuid)
 	let profileName
 	for (const basicProfile of basicProfiles)
diff --git a/src/index.ts b/src/index.ts
index adeab57..0c33930 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,5 +1,6 @@
 import { fetchMemberProfile, fetchUser } from './hypixel'
 import express from 'express'
+import { fetchMemberLeaderboard } from './database'
 
 const app = express()
 
@@ -33,4 +34,11 @@ app.get('/player/:user/:profile', async(req, res) => {
 	)
 })
 
+app.get('/leaderboard/:name', async(req, res) => {
+	res.json(
+		await fetchMemberLeaderboard(req.params.name)
+	)
+})
+
+
 app.listen(8080, () => console.log('App started :)'))
diff --git a/src/util.ts b/src/util.ts
index e9fa145..2ff55a8 100644
--- a/src/util.ts
+++ b/src/util.ts
@@ -3,7 +3,7 @@
  */
 
 export function undashUuid(uuid: string): string {
-	return uuid.replace(/-/g, '')	
+	return uuid.replace(/-/g, '').toLowerCase()
 }