diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/cleaners/rank.ts | 84 | ||||
| -rw-r--r-- | src/cleaners/skyblock/member.ts | 9 | ||||
| -rw-r--r-- | src/cleaners/skyblock/profile.ts | 25 | ||||
| -rw-r--r-- | src/cleaners/skyblock/profiles.ts | 5 | ||||
| -rw-r--r-- | src/constants.ts | 172 | ||||
| -rw-r--r-- | src/database.ts | 173 | ||||
| -rw-r--r-- | src/hypixel.ts | 230 | ||||
| -rw-r--r-- | src/hypixelCached.ts | 71 | ||||
| -rw-r--r-- | src/index.ts | 8 | ||||
| -rw-r--r-- | src/util.ts | 2 |
10 files changed, 604 insertions, 175 deletions
diff --git a/src/cleaners/rank.ts b/src/cleaners/rank.ts index d565502..0a3a4a7 100644 --- a/src/cleaners/rank.ts +++ b/src/cleaners/rank.ts @@ -22,48 +22,50 @@ export interface CleanRank { /** Response cleaning (reformatting to be nicer) */ export function cleanRank({ - packageRank, - newPackageRank, - monthlyPackageRank, - rankPlusColor, - rank, - prefix + packageRank, + newPackageRank, + monthlyPackageRank, + rankPlusColor, + rank, + prefix }: HypixelPlayer): CleanRank { - let name - let color - let colored - if (prefix) { // derive values from prefix - colored = prefix - color = minecraftColorCodes[colored.match(/§./)[0][1]] - name = colored.replace(/§./g, '').replace(/[\[\]]/g, '') - } else { - name = monthlyPackageRank - || rank - || newPackageRank?.replace('_PLUS', '+') - || packageRank?.replace('_PLUS', '+') + let name + let color + let colored + if (prefix) { // derive values from prefix + colored = prefix + color = minecraftColorCodes[colored.match(/§./)[0][1]] + name = colored.replace(/§./g, '').replace(/[\[\]]/g, '') + } else { + if (monthlyPackageRank !== 'NONE') + name = monthlyPackageRank + else + name = rank + || newPackageRank?.replace('_PLUS', '+') + || packageRank?.replace('_PLUS', '+') - // MVP++ is called Superstar for some reason - if (name === 'SUPERSTAR') name = 'MVP++' - // YouTube rank is called YouTuber, change this to the proper name - else if (name === 'YOUTUBER') name = 'YOUTUBE' - else if (name === undefined) name = 'NONE' + // MVP++ is called Superstar for some reason + if (name === 'SUPERSTAR') name = 'MVP++' + // YouTube rank is called YouTuber, change this to the proper name + else if (name === 'YOUTUBER') name = 'YOUTUBE' + else if (name === undefined) name = 'NONE' - const plusColor = rankPlusColor ? colorCodeFromName(rankPlusColor) : null - color = minecraftColorCodes[rankColors[name]] - const rankColorPrefix = rankColors[name] ? '§' + rankColors[name] : '' - const nameWithoutPlus = name.split('+')[0] - const plusesInName = '+'.repeat(name.split('+').length - 1) - if (plusColor && plusesInName.length >= 1) - colored = `${rankColorPrefix}[${nameWithoutPlus}§${plusColor}${plusesInName}${rankColorPrefix}]` - else if (name !== 'NONE') - colored = `${rankColorPrefix}[${name}]` - else - // nons don't have a prefix - colored = `${rankColorPrefix}` - } - return { - name, - color, - colored - } + const plusColor = rankPlusColor ? colorCodeFromName(rankPlusColor) : null + color = minecraftColorCodes[rankColors[name]] + const rankColorPrefix = rankColors[name] ? '§' + rankColors[name] : '' + const nameWithoutPlus = name.split('+')[0] + const plusesInName = '+'.repeat(name.split('+').length - 1) + if (plusColor && plusesInName.length >= 1) + colored = `${rankColorPrefix}[${nameWithoutPlus}§${plusColor}${plusesInName}${rankColorPrefix}]` + else if (name !== 'NONE') + colored = `${rankColorPrefix}[${name}]` + else + // nons don't have a prefix + colored = `${rankColorPrefix}` + } + return { + name, + color, + colored + } } diff --git a/src/cleaners/skyblock/member.ts b/src/cleaners/skyblock/member.ts index 154ff22..a4ca053 100644 --- a/src/cleaners/skyblock/member.ts +++ b/src/cleaners/skyblock/member.ts @@ -5,7 +5,7 @@ import { cleanObjectives, Objective } from './objectives' import { CleanMinion, cleanMinions } from './minions' import { cleanSkills, Skill } from './skills' import * as cached from '../../hypixelCached' -import { CleanFullProfile } from './profile' +import { CleanFullProfile, CleanFullProfileBasicMembers } from './profile' import { Included } from '../../hypixel' import { CleanPlayer } from '../player' import { Bank } from './bank' @@ -25,6 +25,7 @@ export interface CleanBasicMember { export interface CleanMember extends CleanBasicMember { purse: number stats: CleanProfileStats + rawHypixelStats?: { [ key: string ]: number } minions: CleanMinion[] fairy_souls: FairySouls inventories: Inventories @@ -61,6 +62,10 @@ export async function cleanSkyBlockProfileMemberResponse(member, included: Inclu purse: member.coin_purse, stats: cleanProfileStats(member), + + // this is used for leaderboards + rawHypixelStats: member.stats ?? {}, + minions: cleanMinions(member), fairy_souls: cleanFairySouls(member), inventories: inventoriesIncluded ? await cleanInventories(member) : undefined, @@ -83,5 +88,5 @@ export interface CleanMemberProfilePlayer extends CleanPlayer { export interface CleanMemberProfile { member: CleanMemberProfilePlayer - profile: CleanFullProfile + profile: CleanFullProfileBasicMembers } diff --git a/src/cleaners/skyblock/profile.ts b/src/cleaners/skyblock/profile.ts index 2b092a1..6e98f8f 100644 --- a/src/cleaners/skyblock/profile.ts +++ b/src/cleaners/skyblock/profile.ts @@ -8,7 +8,14 @@ export interface CleanProfile extends CleanBasicProfile { } export interface CleanFullProfile extends CleanProfile { - members: (CleanMember|CleanBasicMember)[] + members: CleanMember[] + bank: Bank + minions: CleanMinion[] + minion_count: number +} + +export interface CleanFullProfileBasicMembers extends CleanProfile { + members: CleanBasicMember[] bank: Bank minions: CleanMinion[] minion_count: number @@ -38,19 +45,21 @@ export async function cleanSkyblockProfileResponseLighter(data): Promise<CleanPr /** * This function is somewhat costly and shouldn't be called often. Use cleanSkyblockProfileResponseLighter if you don't need all the data */ -export async function cleanSkyblockProfileResponse(data: any, { mainMemberUuid }: ApiOptions): Promise<CleanFullProfile> { - const cleanedMembers: CleanMember[] = [] - +export async function cleanSkyblockProfileResponse(data: any, options?: ApiOptions): Promise<CleanFullProfile> { + // We use Promise.all so it can fetch all the users at once instead of waiting for the previous promise to complete + const promises: Promise<CleanMember>[] = [] + for (const memberUUID in data.members) { const memberRaw = data.members[memberUUID] memberRaw.uuid = memberUUID - const member: CleanMember = await cleanSkyBlockProfileMemberResponse( + promises.push(cleanSkyBlockProfileMemberResponse( memberRaw, - ['stats', mainMemberUuid === memberUUID ? 'inventories' : undefined] - ) - cleanedMembers.push(member) + ['stats', options?.mainMemberUuid === memberUUID ? 'inventories' : undefined] + )) } + const cleanedMembers: CleanMember[] = await Promise.all(promises) + const memberMinions: CleanMinion[][] = [] for (const member of cleanedMembers) { diff --git a/src/cleaners/skyblock/profiles.ts b/src/cleaners/skyblock/profiles.ts index ea290f6..c9f5628 100644 --- a/src/cleaners/skyblock/profiles.ts +++ b/src/cleaners/skyblock/profiles.ts @@ -1,5 +1,5 @@ import { HypixelPlayerStatsSkyBlockProfiles } from "../../hypixelApi" -import { CleanBasicProfile, CleanProfile, cleanSkyblockProfileResponseLighter } from "./profile" +import { CleanBasicProfile, CleanProfile, cleanSkyblockProfileResponse, cleanSkyblockProfileResponseLighter } from "./profile" export function cleanPlayerSkyblockProfiles(rawProfiles: HypixelPlayerStatsSkyBlockProfiles): CleanBasicProfile[] { let profiles: CleanBasicProfile[] = [] @@ -16,7 +16,8 @@ export function cleanPlayerSkyblockProfiles(rawProfiles: HypixelPlayerStatsSkyBl export async function cleanSkyblockProfilesResponse(data: any[]): Promise<CleanProfile[]> { const cleanedProfiles: CleanProfile[] = [] for (const profile of data ?? []) { - let cleanedProfile = await cleanSkyblockProfileResponseLighter(profile) + // let cleanedProfile = await cleanSkyblockProfileResponseLighter(profile) + let cleanedProfile = await cleanSkyblockProfileResponse(profile) cleanedProfiles.push(cleanedProfile) } return cleanedProfiles diff --git a/src/constants.ts b/src/constants.ts new file mode 100644 index 0000000..c7581ae --- /dev/null +++ b/src/constants.ts @@ -0,0 +1,172 @@ +/** + * Fetch and edit constants from the skyblock-constants repo + */ + +import fetch from 'node-fetch' +import { Agent } from 'https' +import NodeCache from 'node-cache' + +const httpsAgent = new Agent({ + keepAlive: true +}) + +const githubApiBase = 'https://api.github.com' +const owner = 'skyblockstats' +const repo = 'skyblock-constants' + +/** + * Send a request to the GitHub API + * @param method The HTTP method, for example GET, PUT, POST, etc + * @param route The route to send the request to + * @param headers The extra headers + * @param json The JSON body, only applicable for some types of methods + */ +async function fetchGithubApi(method: string, route: string, headers?: any, json?: any) { + return await fetch( + githubApiBase + route, + { + agent: () => httpsAgent, + body: json ? JSON.stringify(json) : null, + method, + headers: Object.assign({ + 'Authorization': `token ${process.env.github_token}` + }, headers), + } + ) +} + +interface GithubFile { + path: string + content: string + sha: string +} + +// cache files for a day +const fileCache = new NodeCache({ + stdTTL: 60 * 60 * 24, + checkperiod: 60, + useClones: false, +}) + + +/** + * Fetch a file from skyblock-constants + * @param path The file path, for example stats.json + */ +async function fetchFile(path: string): Promise<GithubFile> { + if (fileCache.has(path)) + return fileCache.get(path) + + const r = await fetchGithubApi( + 'GET', + `/repos/${owner}/${repo}/contents/${path}`, + { 'Accept': 'application/vnd.github.v3+json' }, + ) + const data = await r.json() + return { + path: data.path, + content: Buffer.from(data.content, data.encoding).toString(), + sha: data.sha + } +} + +/** + * Edit a file on skyblock-constants + * @param file The GithubFile you got from fetchFile + * @param message The commit message + * @param newContent The new content in the file + */ +async function editFile(file: GithubFile, message: string, newContent: string) { + fileCache.set(file.path, newContent) + await fetchGithubApi( + 'PUT', + `/repos/${owner}/${repo}/contents/${file.path}`, + { 'Content-Type': 'application/json' }, + { + message: message, + content: Buffer.from(newContent).toString('base64'), + sha: file.sha, + branch: 'main' + } + ) +} + +/** Fetch all the known SkyBlock stats as an array of strings */ +export async function fetchStats(): Promise<string[]> { + const file = await fetchFile('stats.json') + try { + return JSON.parse(file.content) + } catch { + // probably invalid json, return an empty array + return [] + } +} + +/** Fetch all the known SkyBlock collections as an array of strings */ +export async function fetchCollections(): Promise<string[]> { + const file = await fetchFile('collections.json') + try { + return JSON.parse(file.content) + } catch { + // probably invalid json, return an empty array + return [] + } +} + +/** Add stats to skyblock-constants. This has caching so it's fine to call many times */ +export async function addStats(addingStats: string[]) { + if (addingStats.length === 0) return // no stats provided, just return + + const file = await fetchFile('stats.json') + if (!file.path) + return + let oldStats: string[] + try { + oldStats = JSON.parse(file.content) + } catch { + // invalid json, set it as an empty array + oldStats = [] + } + const updatedStats = oldStats + .concat(addingStats) + // remove duplicates + .filter((value, index, array) => array.indexOf(value) === index) + .sort((a, b) => a.localeCompare(b)) + const newStats = updatedStats.filter(value => !oldStats.includes(value)) + + // there's not actually any new stats, just return + if (newStats.length === 0) return + + const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new stats` : `Add '${newStats[0]}'` + + await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2)) +} + +/** Add stats to skyblock-constants. This has caching so it's fine to call many times */ +export async function addCollections(addingCollections: string[]) { + if (addingCollections.length === 0) return // no stats provided, just return + + const file = await fetchFile('collections.json') + if (!file.path) + return + let oldCollections: string[] + try { + oldCollections = JSON.parse(file.content) + } catch { + // invalid json, set it as an empty array + oldCollections = [] + } + const updatedCollections = oldCollections + .concat(addingCollections) + // remove duplicates + .filter((value, index, array) => array.indexOf(value) === index) + .sort((a, b) => a.localeCompare(b)) + const newCollections = updatedCollections.filter(value => !oldCollections.includes(value)) + + // there's not actually any new stats, just return + if (newCollections.length === 0) return + + const commitMessage = newCollections.length >= 2 ? `Add ${newCollections.length} new collections` : `Add '${newCollections[0]}'` + + await editFile(file, commitMessage, JSON.stringify(updatedCollections, null, 2)) +} diff --git a/src/database.ts b/src/database.ts new file mode 100644 index 0000000..a1178bc --- /dev/null +++ b/src/database.ts @@ -0,0 +1,173 @@ +/** + * Store data about members for leaderboards +*/ + +import * as constants from './constants' +import * as cached from './hypixelCached' +import { Collection, Db, FilterQuery, MongoClient } from 'mongodb' +import NodeCache from 'node-cache' +import { CleanMember } from './cleaners/skyblock/member' + +// don't update the user for 3 minutes +const recentlyUpdated = new NodeCache({ + stdTTL: 60 * 3, + checkperiod: 60, + useClones: false, +}) + +interface LeaderboardItem { + uuid: string + stats: any + last_updated: Date +} + +const cachedLeaderboards: Map<string, any> = new Map() + + +let client: MongoClient +let database: Db +let memberLeaderboardsCollection: Collection<LeaderboardItem> + +async function connect() { + if (!process.env.db_uri) + return console.warn('Warning: db_uri was not found in .env. Features that utilize the database such as leaderboards won\'t work.') + if (!process.env.db_name) + return console.warn('Warning: db_name was not found in .env. Features that utilize the database such as leaderboards won\'t work.') + client = await MongoClient.connect(process.env.db_uri, { useNewUrlParser: true, useUnifiedTopology: true }) + database = client.db(process.env.db_name) + memberLeaderboardsCollection = database.collection('member-leaderboards') +} + + +function getMemberCollectionAttributes(member: CleanMember) { + const collectionAttributes = {} + for (const collection of member.collections) { + const collectionLeaderboardName = `collection_${collection.name}` + collectionAttributes[collectionLeaderboardName] = collection.xp + } + return collectionAttributes +} + +function getMemberLeaderboardAttributes(member: CleanMember) { + // if you want to add a new leaderboard for member attributes, add it here (and getAllLeaderboardAttributes) + return { + // we use the raw stat names rather than the clean stats in case hypixel adds a new stat and it takes a while for us to clean it + ...member.rawHypixelStats, + + // collection leaderboards + ...getMemberCollectionAttributes(member), + + fairy_souls: member.fairy_souls.total, + first_join: member.first_join, + purse: member.purse, + visited_zones: member.visited_zones.length, + } +} + +/** Fetch the names of all the leaderboards */ +async function fetchAllMemberLeaderboardAttributes(): Promise<string[]> { + return [ + // we use the raw stat names rather than the clean stats in case hypixel adds a new stat and it takes a while for us to clean it + ...await constants.fetchStats(), + + // collection leaderboards + ...(await constants.fetchCollections()).map(value => `collection_${value}`), + + 'fairy_souls', + 'first_join', + 'purse', + 'visited_zones', + ] +} + +export async function fetchMemberLeaderboard(name: string) { + if (cachedLeaderboards.has(name)) + return cachedLeaderboards.get(name) + // typescript forces us to make a new variable and set it this way because it gives an error otherwise + const query: FilterQuery<any> = {} + query[`stats.${name}`] = { '$exists': true } + + const sortQuery: any = {} + sortQuery[`stats.${name}`] = -1 + + + const leaderboardRaw = await memberLeaderboardsCollection.find(query).sort(sortQuery).limit(100).toArray() + const fetchLeaderboardPlayer = async(item: LeaderboardItem) => { + return { + player: await cached.fetchPlayer(item.uuid), + value: item.stats[name] + } + } + const promises = [] + for (const item of leaderboardRaw) { + promises.push(fetchLeaderboardPlayer(item)) + } + const leaderboard = await Promise.all(promises) + cachedLeaderboards.set(name, leaderboard) + return leaderboard +} + +async function getMemberLeaderboardRequirement(name: string): Promise<LeaderboardItem> { + const leaderboard = await fetchMemberLeaderboard(name) + // if there's more than 100 items, return the 100th. if there's less, return null + if (leaderboard.length >= 100) + return leaderboard[99].value + else + return null +} + +/** Update the member's leaderboard data on the server if applicable */ +export async function updateDatabaseMember(member: CleanMember) { + if (!client) return // the db client hasn't been initialized + // the member's been updated too recently, just return + if (recentlyUpdated.get(member.uuid)) + return + // store the member in recentlyUpdated so it cant update for 3 more minutes + recentlyUpdated.set(member.uuid, true) + + await constants.addStats(Object.keys(member.rawHypixelStats)) + await constants.addCollections(member.collections.map(value => value.name)) + + const leaderboardAttributes = getMemberLeaderboardAttributes(member) + + await memberLeaderboardsCollection.updateOne({ + uuid: member.uuid + }, { + '$set': { + 'stats': leaderboardAttributes, + 'last_updated': new Date() + } + }, { + upsert: true + }) +} + + +/** + * Remove leaderboard attributes for members that wouldn't actually be on the leaderboard. This saves a lot of storage space + */ +async function removeBadMemberLeaderboardAttributes() { + const leaderboards = await fetchAllMemberLeaderboardAttributes() + for (const leaderboard of leaderboards) { + // wait 10 seconds so it doesnt use as much ram + await new Promise(resolve => setTimeout(resolve, 10000)) + + const unsetValue = {} + unsetValue[leaderboard] = '' + const filter = {} + const requirement = await getMemberLeaderboardRequirement(leaderboard) + if (requirement !== null) { + filter[`stats.${leaderboard}`] = { + '$lt': requirement + } + await memberLeaderboardsCollection.updateMany( + filter, + { '$unset': unsetValue } + ) + } + } +} + + +connect() + .then(removeBadMemberLeaderboardAttributes)
\ No newline at end of file diff --git a/src/hypixel.ts b/src/hypixel.ts index 83ad419..3b8a952 100644 --- a/src/hypixel.ts +++ b/src/hypixel.ts @@ -6,9 +6,10 @@ import { CleanPlayer, cleanPlayerResponse } from './cleaners/player' import { chooseApiKey, HypixelResponse, sendApiRequest } from './hypixelApi' import * as cached from './hypixelCached' import { CleanBasicMember, CleanMemberProfile } from './cleaners/skyblock/member' -import { cleanSkyblockProfileResponse, CleanProfile, CleanBasicProfile } from './cleaners/skyblock/profile' +import { cleanSkyblockProfileResponse, CleanProfile, CleanBasicProfile, CleanFullProfile, CleanFullProfileBasicMembers } from './cleaners/skyblock/profile' import { cleanSkyblockProfilesResponse } from './cleaners/skyblock/profiles' import { debug } from '.' +import { updateDatabaseMember } from './database' export type Included = 'profiles' | 'player' | 'stats' | 'inventories' @@ -23,46 +24,47 @@ export const maxMinion = 11 */ export interface ApiOptions { - mainMemberUuid?: string + mainMemberUuid?: string } +/** Sends an API request to Hypixel and cleans it up. */ export async function sendCleanApiRequest({ path, args }, included?: Included[], options?: ApiOptions) { - const key = await chooseApiKey() - const rawResponse = await sendApiRequest({ path, key, args }) - if (rawResponse.throttled) { + const key = await chooseApiKey() + const rawResponse = await sendApiRequest({ path, key, args }) + if (rawResponse.throttled) { // if it's throttled, wait a second and try again await new Promise(resolve => setTimeout(resolve, 1000)) - return await sendCleanApiRequest({ path, args }, included, options) - } + return await sendCleanApiRequest({ path, args }, included, options) + } - // clean the response - return await cleanResponse({ path, data: rawResponse }, options ?? {}) + // clean the response + return await cleanResponse({ path, data: rawResponse }, options ?? {}) } async function cleanResponse({ path, data }: { path: string, data: HypixelResponse }, options: ApiOptions) { - // Cleans up an api response - switch (path) { - case 'player': return await cleanPlayerResponse(data.player) - case 'skyblock/profile': return await cleanSkyblockProfileResponse(data.profile, options) - case 'skyblock/profiles': return await cleanSkyblockProfilesResponse(data.profiles) - } + // Cleans up an api response + switch (path) { + case 'player': return await cleanPlayerResponse(data.player) + case 'skyblock/profile': return await cleanSkyblockProfileResponse(data.profile, options) + case 'skyblock/profiles': return await cleanSkyblockProfilesResponse(data.profiles) + } } /* ----------------------------- */ export interface UserAny { - user?: string - uuid?: string - username?: string + user?: string + uuid?: string + username?: string } export interface CleanUser { - player: CleanPlayer - profiles?: CleanProfile[] - activeProfile?: string - online?: boolean + player: CleanPlayer + profiles?: CleanProfile[] + activeProfile?: string + online?: boolean } @@ -73,52 +75,52 @@ export interface CleanUser { * used inclusions: player, profiles */ export async function fetchUser({ user, uuid, username }: UserAny, included: Included[]=['player']): Promise<CleanUser> { - if (!uuid) { - // If the uuid isn't provided, get it - uuid = await cached.uuidFromUser(user || username) - } - if (!uuid) { - // the user doesn't exist. - if (debug) console.log('error:', user, 'doesnt exist') - return null - } - - const includePlayers = included.includes('player') - const includeProfiles = included.includes('profiles') - - let profilesData: CleanProfile[] - let basicProfilesData: CleanBasicProfile[] - let playerData: CleanPlayer - - if (includePlayers) { - playerData = await cached.fetchPlayer(uuid) - // if not including profiles, include lightweight profiles just in case - if (!includeProfiles) - basicProfilesData = playerData.profiles - playerData.profiles = undefined - } - if (includeProfiles) { - profilesData = await cached.fetchSkyblockProfiles(uuid) - } - - let activeProfile: CleanProfile = null - let lastOnline: number = 0 - - if (includeProfiles) { - for (const profile of profilesData) { - const member = profile.members.find(member => member.uuid === uuid) - if (member.last_save > lastOnline) { - lastOnline = member.last_save - activeProfile = profile - } - } - } - return { - player: playerData ?? null, - profiles: profilesData ?? basicProfilesData, - activeProfile: includeProfiles ? activeProfile?.uuid : undefined, - online: includeProfiles ? lastOnline > (Date.now() - saveInterval): undefined - } + if (!uuid) { + // If the uuid isn't provided, get it + uuid = await cached.uuidFromUser(user || username) + } + if (!uuid) { + // the user doesn't exist. + if (debug) console.log('error:', user, 'doesnt exist') + return null + } + + const includePlayers = included.includes('player') + const includeProfiles = included.includes('profiles') + + let profilesData: CleanProfile[] + let basicProfilesData: CleanBasicProfile[] + let playerData: CleanPlayer + + if (includePlayers) { + playerData = await cached.fetchPlayer(uuid) + // if not including profiles, include lightweight profiles just in case + if (!includeProfiles) + basicProfilesData = playerData.profiles + playerData.profiles = undefined + } + if (includeProfiles) { + profilesData = await cached.fetchSkyblockProfiles(uuid) + } + + let activeProfile: CleanProfile = null + let lastOnline: number = 0 + + if (includeProfiles) { + for (const profile of profilesData) { + const member = profile.members.find(member => member.uuid === uuid) + if (member.last_save > lastOnline) { + lastOnline = member.last_save + activeProfile = profile + } + } + } + return { + player: playerData ?? null, + profiles: profilesData ?? basicProfilesData, + activeProfile: includeProfiles ? activeProfile?.uuid : undefined, + online: includeProfiles ? lastOnline > (Date.now() - saveInterval): undefined + } } /** @@ -128,40 +130,78 @@ export async function fetchUser({ user, uuid, username }: UserAny, included: Inc * @param profile A profile name or profile uuid */ export async function fetchMemberProfile(user: string, profile: string): Promise<CleanMemberProfile> { - const playerUuid = await cached.uuidFromUser(user) - const profileUuid = await cached.fetchProfileUuid(user, profile) + const playerUuid = await cached.uuidFromUser(user) + const profileUuid = await cached.fetchProfileUuid(user, profile) - // if the profile doesn't have an id, just return - if (!profileUuid) return null + // if the profile doesn't have an id, just return + if (!profileUuid) return null - const player = await cached.fetchPlayer(playerUuid) + const player = await cached.fetchPlayer(playerUuid) - const cleanProfile = await cached.fetchProfile(playerUuid, profileUuid) + const cleanProfile = await cached.fetchProfile(playerUuid, profileUuid) as CleanFullProfileBasicMembers - const member = cleanProfile.members.find(m => m.uuid === playerUuid) + const member = cleanProfile.members.find(m => m.uuid === playerUuid) - // remove unnecessary member data - const simpleMembers: CleanBasicMember[] = cleanProfile.members.map(m => { - return { - uuid: m.uuid, - username: m.username, - first_join: m.first_join, - last_save: m.last_save, - rank: m.rank - } - }) + // remove unnecessary member data + const simpleMembers: CleanBasicMember[] = cleanProfile.members.map(m => { + return { + uuid: m.uuid, + username: m.username, + first_join: m.first_join, + last_save: m.last_save, + rank: m.rank + } + }) - cleanProfile.members = simpleMembers + cleanProfile.members = simpleMembers - return { - member: { + return { + member: { // the profile name is in member rather than profile since they sometimes differ for each member - profileName: cleanProfile.name, + profileName: cleanProfile.name, // add all the member data - ...member, - // add all other data relating to the hypixel player, such as username, rank, etc - ...player - }, - profile: cleanProfile - } + ...member, + // add all other data relating to the hypixel player, such as username, rank, etc + ...player + }, + profile: cleanProfile + } } + +/** + * Fetches the Hypixel API to get a CleanFullProfile. This doesn't do any caching and you should use hypixelCached.fetchProfile instead + * @param playerUuid The UUID of the Minecraft player + * @param profileUuid The UUID of the Hypixel SkyBlock profile + */ +export async function fetchMemberProfileUncached(playerUuid: string, profileUuid: string): Promise<CleanFullProfile> { + const profile: CleanFullProfile = await sendCleanApiRequest( + { + path: 'skyblock/profile', + args: { profile: profileUuid } + }, + null, + { mainMemberUuid: playerUuid } + ) + for (const member of profile.members) + updateDatabaseMember(member) + return profile +} + + +export async function fetchMemberProfilesUncached(playerUuid: string): Promise<CleanFullProfile[]> { + const profiles: CleanFullProfile[] = await sendCleanApiRequest({ + path: 'skyblock/profiles', + args: { + uuid: playerUuid + }}, + null, + { + // only the inventories for the main player are generated, this is for optimization purposes + mainMemberUuid: playerUuid + } + ) + for (const profile of profiles) + for (const member of profile.members) + updateDatabaseMember(member) + return profiles +}
\ No newline at end of file diff --git a/src/hypixelCached.ts b/src/hypixelCached.ts index 1b8b617..83360a9 100644 --- a/src/hypixelCached.ts +++ b/src/hypixelCached.ts @@ -2,7 +2,7 @@ * Fetch the clean and cached Hypixel API */ -import NodeCache from 'node-cache' +import NodeCache, { EventEmitter, Key } from 'node-cache' import * as mojang from './mojang' import * as hypixel from './hypixel' import { CleanPlayer } from './cleaners/player' @@ -10,8 +10,6 @@ import { undashUuid } from './util' import { CleanProfile, CleanFullProfile, CleanBasicProfile } from './cleaners/skyblock/profile' import { debug } from '.' - - // cache usernames for 4 hours const usernameCache = new NodeCache({ stdTTL: 60 * 60 * 4, @@ -49,28 +47,54 @@ const profileNameCache = new NodeCache({ useClones: false, }) +function waitForSet(cache: NodeCache, key?: string, value?: string): Promise<any> { + return new Promise((resolve, reject) => { + const listener = (setKey, setValue) => { + if (setKey === key || (value && setValue === value)) { + cache.removeListener('set', listener) + return resolve({ key, value }) + } + } + cache.on('set', listener) + }) +} + /** * Fetch the uuid from a user * @param user A user can be either a uuid or a username */ export async function uuidFromUser(user: string): Promise<string> { - if (usernameCache.has(undashUuid(user))) + if (usernameCache.has(undashUuid(user))) { // check if the uuid is a key - return undashUuid(user) + const username: any = usernameCache.get(undashUuid(user)) + // if it has .then, then that means its a waitForSet promise. This is done to prevent requests made while it is already requesting + if (username.then) { + return (await username()).key + } else + return undashUuid(user) + } // check if the username is a value const uuidToUsername: {[ key: string ]: string} = usernameCache.mget(usernameCache.keys()) for (const [ uuid, username ] of Object.entries(uuidToUsername)) { - if (user.toLowerCase() === username.toLowerCase()) + if (username.toLowerCase && user.toLowerCase() === username.toLowerCase()) return uuid } + if (debug) console.log('Cache miss: uuidFromUser', user) + + // set it as waitForSet (a promise) in case uuidFromUser gets called while its fetching mojang + usernameCache.set(undashUuid(user), waitForSet(usernameCache, user, user)) + // not cached, actually fetch mojang api now let { uuid, username } = await mojang.mojangDataFromUser(user) if (!uuid) return // remove dashes from the uuid so its more normal uuid = undashUuid(uuid) + + if (user !== uuid) usernameCache.del(user) + usernameCache.set(uuid, username) return uuid } @@ -85,6 +109,8 @@ export async function usernameFromUser(user: string): Promise<string> { return usernameCache.get(undashUuid(user)) } + if (debug) console.log('Cache miss: usernameFromUser', user) + let { uuid, username } = await mojang.mojangDataFromUser(user) uuid = undashUuid(uuid) usernameCache.set(uuid, username) @@ -96,7 +122,6 @@ export async function fetchPlayer(user: string): Promise<CleanPlayer> { const playerUuid = await uuidFromUser(user) if (playerCache.has(playerUuid)) { - if (debug) console.log('Cache hit! fetchPlayer', playerUuid) return playerCache.get(playerUuid) } @@ -120,17 +145,9 @@ export async function fetchSkyblockProfiles(playerUuid: string): Promise<CleanPr return profilesCache.get(playerUuid) } - const profiles: CleanFullProfile[] = await hypixel.sendCleanApiRequest({ - path: 'skyblock/profiles', - args: { - uuid: playerUuid - }}, - null, - { - // only the inventories for the main player are generated, this is for optimization purposes - mainMemberUuid: playerUuid - } - ) + if (debug) console.log('Cache miss: fetchSkyblockProfiles', playerUuid) + + const profiles: CleanProfile[] = await hypixel.fetchMemberProfilesUncached(playerUuid) const basicProfiles: CleanProfile[] = [] @@ -165,6 +182,9 @@ async function fetchBasicProfiles(user: string): Promise<CleanBasicProfile[]> { if (debug) console.log('Cache hit! fetchBasicProfiles', playerUuid) return basicProfilesCache.get(playerUuid) } + + if (debug) console.log('Cache miss: fetchBasicProfiles', user) + const player = await fetchPlayer(playerUuid) const profiles = player.profiles basicProfilesCache.set(playerUuid, profiles) @@ -188,6 +208,8 @@ export async function fetchProfileUuid(user: string, profile: string) { return null } + if (debug) console.log('Cache miss: fetchProfileUuid', user) + const profiles = await fetchBasicProfiles(user) const profileUuid = undashUuid(profile) @@ -215,16 +237,11 @@ export async function fetchProfile(user: string, profile: string): Promise<Clean return profileCache.get(profileUuid) } + if (debug) console.log('Cache miss: fetchProfile', user, profile) + const profileName = await fetchProfileName(user, profile) - const cleanProfile: CleanFullProfile = await hypixel.sendCleanApiRequest( - { - path: 'skyblock/profile', - args: { profile: profileUuid } - }, - null, - { mainMemberUuid: playerUuid } - ) + const cleanProfile: CleanFullProfile = await hypixel.fetchMemberProfileUncached(playerUuid, profileUuid) // we know the name from fetchProfileName, so set it here cleanProfile.name = profileName @@ -250,6 +267,8 @@ export async function fetchProfileName(user: string, profile: string): Promise<s return profileNameCache.get(`${playerUuid}.${profileUuid}`) } + if (debug) console.log('Cache miss: fetchProfileName', user, profile) + const basicProfiles = await fetchBasicProfiles(playerUuid) let profileName for (const basicProfile of basicProfiles) diff --git a/src/index.ts b/src/index.ts index adeab57..0c33930 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,6 @@ import { fetchMemberProfile, fetchUser } from './hypixel' import express from 'express' +import { fetchMemberLeaderboard } from './database' const app = express() @@ -33,4 +34,11 @@ app.get('/player/:user/:profile', async(req, res) => { ) }) +app.get('/leaderboard/:name', async(req, res) => { + res.json( + await fetchMemberLeaderboard(req.params.name) + ) +}) + + app.listen(8080, () => console.log('App started :)')) diff --git a/src/util.ts b/src/util.ts index e9fa145..2ff55a8 100644 --- a/src/util.ts +++ b/src/util.ts @@ -3,7 +3,7 @@ */ export function undashUuid(uuid: string): string { - return uuid.replace(/-/g, '') + return uuid.replace(/-/g, '').toLowerCase() } |