From 52e38809212133ef673d11bfa96ba3bb43c3644c Mon Sep 17 00:00:00 2001 From: mat <27899617+mat-1@users.noreply.github.com> Date: Sat, 13 Feb 2021 14:13:42 -0600 Subject: move stuff into src folder --- cleaners/player.ts | 28 ---- cleaners/rank.ts | 66 --------- cleaners/skyblock/minions.ts | 70 ---------- cleaners/skyblock/stats.ts | 73 ---------- cleaners/socialmedia.ts | 14 -- hypixel.ts | 284 --------------------------------------- hypixelApi.ts | 154 --------------------- hypixelCached.ts | 240 --------------------------------- index.ts | 27 ---- mojang.ts | 59 -------- package.json | 3 +- src/cleaners/player.ts | 28 ++++ src/cleaners/rank.ts | 66 +++++++++ src/cleaners/skyblock/minions.ts | 70 ++++++++++ src/cleaners/skyblock/stats.ts | 73 ++++++++++ src/cleaners/socialmedia.ts | 14 ++ src/hypixel.ts | 284 +++++++++++++++++++++++++++++++++++++++ src/hypixelApi.ts | 154 +++++++++++++++++++++ src/hypixelCached.ts | 240 +++++++++++++++++++++++++++++++++ src/index.ts | 27 ++++ src/mojang.ts | 59 ++++++++ src/util.ts | 80 +++++++++++ tsconfig.json | 3 +- util.ts | 80 ----------- 24 files changed, 1099 insertions(+), 1097 deletions(-) delete mode 100644 cleaners/player.ts delete mode 100644 cleaners/rank.ts delete mode 100644 cleaners/skyblock/minions.ts delete mode 100644 cleaners/skyblock/stats.ts delete mode 100644 cleaners/socialmedia.ts delete mode 100644 hypixel.ts delete mode 100644 hypixelApi.ts delete mode 100644 hypixelCached.ts delete mode 100644 index.ts delete mode 100644 mojang.ts create mode 100644 src/cleaners/player.ts create mode 100644 src/cleaners/rank.ts create mode 100644 src/cleaners/skyblock/minions.ts create mode 100644 src/cleaners/skyblock/stats.ts create mode 100644 src/cleaners/socialmedia.ts create mode 100644 src/hypixel.ts create mode 100644 src/hypixelApi.ts create mode 100644 src/hypixelCached.ts create mode 100644 src/index.ts create mode 100644 src/mojang.ts create mode 100644 src/util.ts delete mode 100644 util.ts diff --git a/cleaners/player.ts b/cleaners/player.ts deleted file mode 100644 index c3afd6f..0000000 --- a/cleaners/player.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { CleanBasicProfile, cleanPlayerSkyblockProfiles, Included } from '../hypixel' -import { CleanSocialMedia, parseSocialMedia } from './socialmedia' -import { CleanRank, parseRank } from './rank' -import { HypixelPlayer } from '../hypixelApi' -import { undashUuid } from '../util' - -export interface CleanBasicPlayer { - uuid: string - username: string -} - -export interface CleanPlayer extends CleanBasicPlayer { - rank: CleanRank - socials: CleanSocialMedia - profiles?: CleanBasicProfile[] -} - -export async function cleanPlayerResponse(data: HypixelPlayer): Promise { - // Cleans up a 'player' api response - console.log('cleanPlayerResponse', data.stats.SkyBlock.profiles) - return { - uuid: undashUuid(data.uuid), - username: data.displayname, - rank: parseRank(data), - socials: parseSocialMedia(data.socialMedia), - profiles: cleanPlayerSkyblockProfiles(data.stats.SkyBlock.profiles) - } -} diff --git a/cleaners/rank.ts b/cleaners/rank.ts deleted file mode 100644 index 928373a..0000000 --- a/cleaners/rank.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { HypixelPlayer } from '../hypixelApi' -import { colorCodeFromName, minecraftColorCodes } from '../util' - -const rankColors: { [ name: string ]: string } = { - 'NONE': '7', - 'VIP': 'a', - 'VIP+': 'a', - 'MVP': 'b', - 'MVP+': 'b', - 'MVP++': '6', - 'YOUTUBE': 'c', - 'HELPER': '9', - 'MODERATOR': '2', - 'ADMIN': 'c' -} - -export interface CleanRank { - name: string, - color: string | null, - colored: string | null, -} - -/** Response cleaning (reformatting to be nicer) */ -export function parseRank({ - packageRank, - newPackageRank, - monthlyPackageRank, - rankPlusColor, - rank, - prefix -}: HypixelPlayer): CleanRank { - let name - let color - let colored - if (prefix) { // derive values from prefix - colored = prefix - color = minecraftColorCodes[colored.match(/§./)[0][1]] - name = colored.replace(/§./g, '').replace(/[\[\]]/g, '') - } else { - name = rank - || newPackageRank.replace('_PLUS', '+') - || packageRank.replace('_PLUS', '+') - || monthlyPackageRank - - // MVP++ is called Superstar for some reason - if (name === 'SUPERSTAR') name = 'MVP++' - // YouTube rank is called YouTuber, change this to the proper name - else if (name === 'YOUTUBER') name = 'YOUTUBE' - - const plusColor = colorCodeFromName(rankPlusColor) - color = minecraftColorCodes[rankColors[name]] - const rankColorPrefix = rankColors[name] ? '§' + rankColors[name] : '' - const nameWithoutPlus = name.split('+')[0] - const plusesInName = '+'.repeat(name.split('+').length - 1) - console.log(plusColor, nameWithoutPlus, plusesInName) - if (plusColor && plusesInName.length >= 1) - colored = `${rankColorPrefix}[${nameWithoutPlus}§${plusColor}${plusesInName}${rankColorPrefix}]` - else - colored = `${rankColorPrefix}[${name}]` - } - return { - name, - color, - colored - } -} diff --git a/cleaners/skyblock/minions.ts b/cleaners/skyblock/minions.ts deleted file mode 100644 index da69634..0000000 --- a/cleaners/skyblock/minions.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { maxMinion } from '../../hypixel' - -export interface CleanMinion { - name: string, - levels: boolean[] -} - - -/** - * Clean the minions provided by Hypixel - * @param minionsRaw The minion data provided by the Hypixel API - */ -export function cleanMinions(minionsRaw: string[]): CleanMinion[] { - const minions: CleanMinion[] = [] - for (const minionRaw of minionsRaw ?? []) { - // do some regex magic to get the minion name and level - // examples of potential minion names: CLAY_11, PIG_1, MAGMA_CUBE_4 - const minionName = minionRaw.split(/_\d/)[0].toLowerCase() - const minionLevel = parseInt(minionRaw.split(/\D*_/)[1]) - let matchingMinion = minions.find(m => m.name === minionName) - if (!matchingMinion) { - // if the minion doesnt already exist in the minions array, then create it - matchingMinion = { - name: minionName, - levels: new Array(maxMinion).fill(false) - } - minions.push(matchingMinion) - } - while (minionLevel > matchingMinion.levels.length) - // if hypixel increases the minion level, this will increase with it - matchingMinion.levels.push(false) - - // set the minion at that level to true - matchingMinion.levels[minionLevel - 1] = true - } - return minions -} - -/** - * Combine multiple arrays of minions into one, useful when getting the minions for members - * @param minions An array of arrays of minions - */ -export function combineMinionArrays(minions: CleanMinion[][]): CleanMinion[] { - const resultMinions: CleanMinion[] = [] - - for (const memberMinions of minions) { - for (const minion of memberMinions) { - // this is a reference, so we can directly modify the attributes for matchingMinionReference - // and they'll be in the resultMinions array - const matchingMinionReference = resultMinions.find(m => m.name === minion.name) - if (!matchingMinionReference) { - // if the minion name isn't already in the array, add it! - resultMinions.push(minion) - } else { - - // This should never happen, but in case the length of `minion.levels` is longer than - // `matchingMinionReference.levels`, then it should be extended to be equal length - while (matchingMinionReference.levels.length < minion.levels.length) - matchingMinionReference.levels.push(null) - - for (let i = 0; i < minion.levels.length; i++) { - if (minion.levels[i]) - matchingMinionReference.levels[i] = true - } - } - } - } - - return resultMinions -} \ No newline at end of file diff --git a/cleaners/skyblock/stats.ts b/cleaners/skyblock/stats.ts deleted file mode 100644 index 07d9133..0000000 --- a/cleaners/skyblock/stats.ts +++ /dev/null @@ -1,73 +0,0 @@ -const statCategories: { [ key: string ]: string[] | null } = { // sorted in order of importance - 'deaths': ['deaths_', 'deaths'], - 'kills': ['kills_', 'kills'], - 'fishing': ['items_fished_', 'items_fished'], - 'auctions': ['auctions_'], - 'races': ['_best_time'], - 'misc': null // everything else goes here -} - -interface statCategory { - category: string, - name: string -} - -function categorizeStat(statNameRaw: string): statCategory { - // 'deaths_void' - for (const statCategory in statCategories) { - // 'deaths' - const statCategoryMatchers = statCategories[statCategory] - if (statCategoryMatchers == null) { - // If it's null, just go with this. Can only ever be 'misc' - return { - category: statCategory, - name: statNameRaw - } - } - for (const categoryMatch of statCategoryMatchers) { - // ['deaths_'] - let trailingEnd = categoryMatch[0] == '_' - let trailingStart = categoryMatch.substr(-1) == '_' - if (trailingStart && statNameRaw.startsWith(categoryMatch)) { - return { - category: statCategory, - name: statNameRaw.substr(categoryMatch.length) - } - } else if (trailingEnd && statNameRaw.endsWith(categoryMatch)) { - return { - category: statCategory, - name: statNameRaw.substr(0, categoryMatch.length) - } - } else if (statNameRaw == categoryMatch) { - // if it matches exactly, we don't know the name. will be defaulted to category later on - return { - category: statCategory, - name: null - } - } - } - } - // this should never happen, as it'll default to misc and return if nothing is found - return { - category: null, - name: statNameRaw - } -} - -export interface CleanProfileStats { - [ category: string ]: { - [ stat: string ]: any - total?: any - } -} - -export function cleanProfileStats(statsRaw): CleanProfileStats { - // TODO: add type for statsRaw (probably in hypixelApi.ts since its coming from there) - const stats: CleanProfileStats = {} - for (let statNameRaw in statsRaw) { - let { category: statCategory, name: statName } = categorizeStat(statNameRaw) - if (!stats[statCategory]) stats[statCategory] = {} - stats[statCategory][statName || 'total'] = statsRaw[statNameRaw] - } - return stats -} diff --git a/cleaners/socialmedia.ts b/cleaners/socialmedia.ts deleted file mode 100644 index c1f9551..0000000 --- a/cleaners/socialmedia.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { HypixelPlayerSocialMedia } from "../hypixelApi"; - -export interface CleanSocialMedia { - discord: string | null - forums: string | null -} - -export function parseSocialMedia(socialMedia: HypixelPlayerSocialMedia): CleanSocialMedia { - return { - discord: socialMedia?.links?.DISCORD || null, - forums: socialMedia?.links?.HYPIXEL || null - } -} - diff --git a/hypixel.ts b/hypixel.ts deleted file mode 100644 index f17dc9e..0000000 --- a/hypixel.ts +++ /dev/null @@ -1,284 +0,0 @@ -import { CleanMinion, cleanMinions, combineMinionArrays } from './cleaners/skyblock/minions' -import { CleanProfileStats, cleanProfileStats } from './cleaners/skyblock/stats' -import { CleanPlayer, cleanPlayerResponse } from './cleaners/player' -import { chooseApiKey, HypixelPlayerStatsSkyBlockProfiles, HypixelResponse, sendApiRequest } from './hypixelApi' -import * as cached from './hypixelCached' - -export type Included = 'profiles' | 'player' | 'stats' - -// the interval at which the "last_save" parameter updates in the hypixel api, this is 3 minutes -export const saveInterval = 60 * 3 * 1000 - -// the highest level a minion can be -export const maxMinion = 11 - -/** - * Send a request to api.hypixel.net using a random key, clean it up to be more useable, and return it - */ -export async function sendCleanApiRequest({ path, args }, included?: Included[], cleaned=true) { - const key = await chooseApiKey() - const rawResponse = await sendApiRequest({ path, key, args }) - if (rawResponse.throttled) { - // if it's throttled, wait a second and try again - console.log('throttled :/') - await new Promise(resolve => setTimeout(resolve, 1000)) - return await sendCleanApiRequest({ path, args }, included, cleaned) - } - if (cleaned) { - // if it needs to clean the response, call cleanResponse - return await cleanResponse({ path, data: rawResponse }, included=included) - } else { - // this is provided in case the caller wants to do the cleaning itself - // used in skyblock/profile, as cleaning the entire profile would use too much cpu - return rawResponse - } -} - -export interface CleanBasicMember { - uuid: string - username: string - last_save: number - first_join: number -} - -interface CleanMember extends CleanBasicMember { - stats?: CleanProfileStats - minions?: CleanMinion[] -} - -async function cleanSkyBlockProfileMemberResponse(member, included: Included[] = null): Promise { - // Cleans up a member (from skyblock/profile) - // profiles.members[] - const statsIncluded = included == null || included.includes('stats') - return { - uuid: member.uuid, - username: await cached.usernameFromUser(member.uuid), - last_save: member.last_save, - first_join: member.first_join, - // last_death: ??? idk how this is formatted, - stats: statsIncluded ? cleanProfileStats(member.stats) : undefined, - minions: statsIncluded ? cleanMinions(member.crafted_generators) : undefined, - } -} - - -export interface CleanMemberProfilePlayer extends CleanPlayer { - // The profile name may be different for each player, so we put it here - profileName: string - first_join: number - last_save: number - bank?: { - balance: number - history: any[] - } -} - -export interface CleanMemberProfile { - member: CleanMemberProfilePlayer - profile: { - - } -} - -export interface CleanProfile extends CleanBasicProfile { - members?: CleanBasicMember[] -} - -export interface CleanFullProfile extends CleanProfile { - members: CleanMember[] - bank?: { - balance: number - history: any[] - } - minions: CleanMinion[] -} - -/** Return a `CleanProfile` instead of a `CleanFullProfile`, useful when we need to get members but don't want to waste much ram */ -async function cleanSkyblockProfileResponseLighter(data): Promise { - // We use Promise.all so it can fetch all the usernames at once instead of waiting for the previous promise to complete - const promises: Promise[] = [] - - for (const memberUUID in data.members) { - const memberRaw = data.members[memberUUID] - memberRaw.uuid = memberUUID - // we pass an empty array to make it not check stats - promises.push(cleanSkyBlockProfileMemberResponse(memberRaw, [])) - } - - const cleanedMembers: CleanMember[] = await Promise.all(promises) - - return { - uuid: data.profile_id, - name: data.cute_name, - members: cleanedMembers, - } -} - -/** This function is very costly and shouldn't be called often. Use cleanSkyblockProfileResponseLighter if you don't need all the data */ -async function cleanSkyblockProfileResponse(data: any): Promise { - const cleanedMembers: CleanMember[] = [] - - for (const memberUUID in data.members) { - const memberRaw = data.members[memberUUID] - memberRaw.uuid = memberUUID - const member: CleanMember = await cleanSkyBlockProfileMemberResponse(memberRaw, ['stats']) - cleanedMembers.push(member) - } - - const memberMinions: CleanMinion[][] = [] - - for (const member of cleanedMembers) { - memberMinions.push(member.minions) - } - const minions: CleanMinion[] = combineMinionArrays(memberMinions) - - // return more detailed info - return { - uuid: data.profile_id, - name: data.cute_name, - members: cleanedMembers, - bank: { - balance: data?.banking?.balance ?? 0, - - // TODO: make transactions good - history: data?.banking?.transactions ?? [] - }, - minions - } -} - -/** A basic profile that only includes the profile uuid and name */ -export interface CleanBasicProfile { - uuid: string - - // the name depends on the user, so its sometimes not included - name?: string -} - -export function cleanPlayerSkyblockProfiles(rawProfiles: HypixelPlayerStatsSkyBlockProfiles): CleanBasicProfile[] { - let profiles: CleanBasicProfile[] = [] - for (const profile of Object.values(rawProfiles)) { - profiles.push({ - uuid: profile.profile_id, - name: profile.cute_name - }) - } - console.log('cleanPlayerSkyblockProfiles', profiles) - return profiles -} - -/** Convert an array of raw profiles into clean profiles */ -async function cleanSkyblockProfilesResponse(data: any[]): Promise { - const cleanedProfiles: CleanProfile[] = [] - for (const profile of data) { - let cleanedProfile = await cleanSkyblockProfileResponseLighter(profile) - cleanedProfiles.push(cleanedProfile) - } - return cleanedProfiles -} - -async function cleanResponse({ path, data }: { path: string, data: HypixelResponse }, included?: Included[]) { - // Cleans up an api response - switch (path) { - case 'player': return await cleanPlayerResponse(data.player) - case 'skyblock/profile': return await cleanSkyblockProfileResponse(data.profile) - case 'skyblock/profiles': return await cleanSkyblockProfilesResponse(data.profiles) - } -} - -/* ----------------------------- */ - -export interface UserAny { - user?: string - uuid?: string - username?: string -} - -export interface CleanUser { - player: any - profiles?: any - activeProfile?: string - online?: boolean -} - - -/** - * Higher level function that requests the api for a user, and returns the cleaned response - * This is safe to fetch many times because the results are cached! - * @param included lets you choose what is returned, so there's less processing required on the backend - * used inclusions: player, profiles - */ -export async function fetchUser({ user, uuid, username }: UserAny, included: Included[]=['player']): Promise { - if (!uuid) { - // If the uuid isn't provided, get it - uuid = await cached.uuidFromUser(user || username) - } - - const includePlayers = included.includes('player') - const includeProfiles = included.includes('profiles') - - let profilesData: CleanProfile[] - let basicProfilesData: CleanBasicProfile[] - let playerData: CleanPlayer - - if (includePlayers) { - playerData = await cached.fetchPlayer(uuid) - // if not including profiles, include lightweight profiles just in case - if (!includeProfiles) - basicProfilesData = playerData.profiles - playerData.profiles = undefined - } - if (includeProfiles) { - profilesData = await cached.fetchSkyblockProfiles(uuid) - } - - let activeProfile: CleanProfile = null - let lastOnline: number = 0 - - if (includeProfiles) { - for (const profile of profilesData) { - const member = profile.members.find(member => member.uuid === uuid) - if (member.last_save > lastOnline) { - lastOnline = member.last_save - activeProfile = profile - } - } - } - return { - player: playerData ?? null, - profiles: profilesData ?? basicProfilesData, - activeProfile: includeProfiles ? activeProfile?.uuid : undefined, - online: includeProfiles ? lastOnline > (Date.now() - saveInterval): undefined - } -} - -/** - * Fetch a CleanMemberProfile from a user and string - * This is safe to use many times as the results are cached! - * @param user A username or uuid - * @param profile A profile name or profile uuid - */ -export async function fetchMemberProfile(user: string, profile: string): Promise { - const playerUuid = await cached.uuidFromUser(user) - const profileUuid = await cached.fetchProfileUuid(user, profile) - - const player = await cached.fetchPlayer(playerUuid) - - const cleanProfile = await cached.fetchProfile(playerUuid, profileUuid) - - const member = cleanProfile.members.find(m => m.uuid === playerUuid) - - return { - member: { - profileName: cleanProfile.name, - first_join: member.first_join, - last_save: member.last_save, - - // add all other data relating to the hypixel player, such as username, rank, etc - ...player - }, - profile: { - minions: cleanProfile.minions - } - } -} diff --git a/hypixelApi.ts b/hypixelApi.ts deleted file mode 100644 index e91d4f5..0000000 --- a/hypixelApi.ts +++ /dev/null @@ -1,154 +0,0 @@ -import fetch from 'node-fetch' -import { jsonToQuery, shuffle } from './util' -import { Agent } from 'https' -require('dotenv').config() - -// We need to create an agent to prevent memory leaks and to only do dns lookups once -const httpsAgent = new Agent({ - keepAlive: true -}) - -/* Lower level code related to the Hypixel api */ - -const apiKeys = process.env.keys.split(' ') - -interface KeyUsage { - remaining: number - limit: number - reset: number -} - -const apiKeyUsage: { [ key: string ]: KeyUsage } = {} - - -const baseHypixelAPI = 'https://api.hypixel.net' - -/** Choose the best current API key */ -export function chooseApiKey(): string { - // find the api key with the lowest amount of uses - let bestKeyUsage: KeyUsage = null - let bestKey: string = null - for (var key of shuffle(apiKeys)) { - const keyUsage = apiKeyUsage[key] - - // if the key has never been used before, use it - if (!keyUsage) return key - - // if the key has reset since the last use, set the remaining count to the default - if (Date.now() > keyUsage.reset) - keyUsage.remaining = keyUsage.limit - - // if this key has more uses remaining than the current known best one, save it - if (!bestKeyUsage || keyUsage.remaining > bestKeyUsage.remaining) { - bestKeyUsage = keyUsage - bestKey = key - } - } - return bestKey -} - -export interface HypixelResponse { - [key: string]: any | { - success: boolean - throttled?: boolean - } -} - - -export interface HypixelPlayerStatsSkyBlockProfiles { - [ uuid: string ]: { - profile_id: string - cute_name: string - } -} - -interface HypixelPlayerStatsSkyBlock { - profiles: HypixelPlayerStatsSkyBlockProfiles -} - -export interface HypixelPlayerSocialMedia { - YOUTUBE?: string - prompt: boolean - links: { - DISCORD?: string - HYPIXEL?: string - } -} - -export interface HypixelPlayer { - _id: string - achievementsOneTime: string[] - displayname: string - - firstLogin: number, - lastLogin: number, - lastLogout: number - - knownAliases: string[], - knownAliasesLower: string[] - - networkExp: number - playername: string - stats: { - SkyBlock: HypixelPlayerStatsSkyBlock - [ name: string ]: any - }, - timePlaying: number, - uuid: string, - achievements: { [ name: string ]: number }, - petConsumables: { [ name: string ]: number }, - vanityMeta: { - packages: string[] - }, - - language: string, - userLanguage?: string - - packageRank?: string - newPackageRank?: string - rankPlusColor?: string - monthlyPackageRank?: string - rank?: string - prefix?: string - - claimed_potato_talisman?: number - skyblock_free_cookie?: number - - socialMedia?: HypixelPlayerSocialMedia -} - - -/** Send an HTTP request to the Hypixel API */ -export async function sendApiRequest({ path, key, args }): Promise { - console.log('sending api request to', path, args) - // Send a raw http request to api.hypixel.net, and return the parsed json - - if (key) - // If there's an api key, add it to the arguments - args.key = key - - // Construct a url from the base api url, path, and arguments - const fetchUrl = baseHypixelAPI + '/' + path + '?' + jsonToQuery(args) - - const fetchResponse = await fetch( - fetchUrl, - { agent: () => httpsAgent } - ) - - if (fetchResponse.headers['ratelimit-limit']) - // remember how many uses it has - apiKeyUsage[key] = { - remaining: fetchResponse.headers['ratelimit-remaining'], - limit: fetchResponse.headers['ratelimit-limit'], - reset: Date.now() + parseInt(fetchResponse.headers['ratelimit-reset']) * 1000 - } - - const fetchJsonParsed = await fetchResponse.json() - if (fetchJsonParsed.throttle) { - apiKeyUsage[key].remaining = 0 - console.log('throttled :(') - return { throttled: true } - } - return fetchJsonParsed -} - diff --git a/hypixelCached.ts b/hypixelCached.ts deleted file mode 100644 index 5fe65a3..0000000 --- a/hypixelCached.ts +++ /dev/null @@ -1,240 +0,0 @@ -import NodeCache from 'node-cache' -import * as mojang from './mojang' -import * as hypixel from './hypixel' -import { CleanPlayer } from './cleaners/player' -import { CleanBasicProfile, CleanFullProfile, CleanProfile } from './hypixel' -import { undashUuid } from './util' - -/** -Hypixel... but with caching -All the caching in this project is done here! -*/ - - -// cache usernames for 4 hours -const usernameCache = new NodeCache({ - stdTTL: 60 * 60 * 4, - checkperiod: 60, - useClones: false, -}) - -const basicProfilesCache = new NodeCache({ - stdTTL: 60 * 10, - checkperiod: 60, - useClones: false, -}) - -const playerCache = new NodeCache({ - stdTTL: 60, - checkperiod: 10, - useClones: false, -}) - -const profileCache = new NodeCache({ - stdTTL: 30, - checkperiod: 10, - useClones: false, -}) - -const profilesCache = new NodeCache({ - stdTTL: 60 * 3, - checkperiod: 10, - useClones: false, -}) - -const profileNameCache = new NodeCache({ - stdTTL: 60 * 60, - checkperiod: 60, - useClones: false, -}) - -/** - * Fetch the uuid from a user - * @param user A user can be either a uuid or a username - */ -export async function uuidFromUser(user: string): Promise { - if (usernameCache.has(undashUuid(user))) - // check if the uuid is a key - return undashUuid(user) - - // check if the username is a value - const uuidToUsername: {[ key: string ]: string} = usernameCache.mget(usernameCache.keys()) - for (const [ uuid, username ] of Object.entries(uuidToUsername)) { - if (user.toLowerCase() === username.toLowerCase()) - return uuid - } - - // not cached, actually fetch mojang api now - let { uuid, username } = await mojang.mojangDataFromUser(user) - - // remove dashes from the uuid so its more normal - uuid = undashUuid(uuid) - usernameCache.set(uuid, username) - return uuid -} - -/** - * Fetch the username from a user - * @param user A user can be either a uuid or a username - */ -export async function usernameFromUser(user: string): Promise { - if (usernameCache.has(undashUuid(user))) { - return usernameCache.get(undashUuid(user)) - } - - let { uuid, username } = await mojang.mojangDataFromUser(user) - uuid = undashUuid(uuid) - usernameCache.set(uuid, username) - return username -} - - -export async function fetchPlayer(user: string): Promise { - const playerUuid = await uuidFromUser(user) - - if (playerCache.has(playerUuid)) { - console.log('cache hit! fetchPlayer', playerUuid) - return playerCache.get(playerUuid) - } - - const cleanPlayer: CleanPlayer = await hypixel.sendCleanApiRequest({ - path: 'player', - args: { uuid: playerUuid } - }) - playerCache.set(playerUuid, cleanPlayer) - return cleanPlayer -} - - -export async function fetchSkyblockProfiles(playerUuid: string): Promise { - if (profilesCache.has(playerUuid)) { - console.log('cache hit! fetchSkyblockProfiles', playerUuid) - return profilesCache.get(playerUuid) - } - - const profiles: CleanFullProfile[] = await hypixel.sendCleanApiRequest({ - path: 'skyblock/profiles', - args: { - uuid: playerUuid - }}, - ) - - const basicProfiles: CleanProfile[] = [] - - // create the basicProfiles array - for (const profile of profiles) { - const basicProfile: CleanProfile = { - name: profile.name, - uuid: profile.uuid, - members: profile.members.map(m => { - return { - uuid: m.uuid, - username: m.username, - first_join: m.first_join, - last_save: m.last_save - } - }) - } - basicProfiles.push(basicProfile) - } - - // cache the profiles - profilesCache.set(playerUuid, basicProfiles) - - return basicProfiles -} - -/** Fetch an array of `BasicProfile`s */ -async function fetchBasicProfiles(user: string): Promise { - const playerUuid = await uuidFromUser(user) - if (basicProfilesCache.has(playerUuid)) { - console.log('cache hit! fetchBasicProfiles') - return basicProfilesCache.get(playerUuid) - } - const player = await fetchPlayer(playerUuid) - const profiles = player.profiles - basicProfilesCache.set(playerUuid, profiles) - - console.log(player) - - // cache the profile names and uuids to profileNameCache because we can - for (const profile of profiles) - profileNameCache.set(`${playerUuid}.${profile.uuid}`, profile.name) - - return profiles -} - -/** - * Fetch a profile UUID from its name and user - * @param user A username or uuid - * @param profile A profile name or profile uuid - */ -export async function fetchProfileUuid(user: string, profile: string) { - const profiles = await fetchBasicProfiles(user) - - const profileUuid = undashUuid(profile) - - for (const p of profiles) { - if (p.name.toLowerCase() === profileUuid.toLowerCase()) - return undashUuid(p.uuid) - else if (undashUuid(p.uuid) === undashUuid(profileUuid)) - return undashUuid(p.uuid) - } -} - -/** - * Fetch an entire profile from the user and profile data - * @param user A username or uuid - * @param profile A profile name or profile uuid - */ -export async function fetchProfile(user: string, profile: string): Promise { - const profileUuid = await fetchProfileUuid(user, profile) - - if (profileCache.has(profileUuid)) { - console.log('cache hit! fetchProfile') - // we have the profile cached, return it :) - return profileCache.get(profileUuid) - } - - const profileName = await fetchProfileName(user, profile) - - const cleanProfile: CleanFullProfile = await hypixel.sendCleanApiRequest({ - path: 'skyblock/profile', - args: { - profile: profileUuid - } - }) - - // we know the name from fetchProfileName, so set it here - cleanProfile.name = profileName - - profileCache.set(profileUuid, cleanProfile) - - return cleanProfile -} - -/** - * Fetch the name of a profile from the user and profile uuid - * @param user A player uuid or username - * @param profile A profile uuid or name - */ -export async function fetchProfileName(user: string, profile: string): Promise { - // we're fetching the profile and player uuid again in case we were given a name, but it's cached so it's not much of a problem - const profileUuid = await fetchProfileUuid(user, profile) - const playerUuid = await uuidFromUser(user) - - if (profileNameCache.has(`${playerUuid}.${profileUuid}`)) { - // Return the profile name if it's cached - console.log('cache hit! fetchProfileName') - return profileNameCache.get(`${playerUuid}.${profileUuid}`) - } - - const basicProfiles = await fetchBasicProfiles(playerUuid) - let profileName - for (const basicProfile of basicProfiles) - if (basicProfile.uuid === playerUuid) - profileName = basicProfile.name - - profileNameCache.set(`${playerUuid}.${profileUuid}`, profileName) - return profileName -} \ No newline at end of file diff --git a/index.ts b/index.ts deleted file mode 100644 index af5095b..0000000 --- a/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -import express from 'express' -import { fetchMemberProfile, fetchUser } from './hypixel' -import { fetchProfile } from './hypixelCached' - -const app = express() - - -app.get('/', async(req, res) => { - res.json({ ok: true }) -}) - -app.get('/player/:user', async(req, res) => { - res.json( - await fetchUser( - { user: req.params.user }, - ['profiles', 'player'] - ) - ) -}) - -app.get('/player/:user/:profile', async(req, res) => { - res.json( - await fetchMemberProfile(req.params.user, req.params.profile) - ) -}) - -app.listen(8080, () => console.log('App started :)')) \ No newline at end of file diff --git a/mojang.ts b/mojang.ts deleted file mode 100644 index 746f674..0000000 --- a/mojang.ts +++ /dev/null @@ -1,59 +0,0 @@ -import fetch from 'node-fetch' -import { Agent } from 'https' - -// We need to create an agent to prevent memory leaks -const httpsAgent = new Agent({ - keepAlive: true -}) - -interface AshconHistoryItem { - username: string - changed_at?: string -} - -interface AshconTextures { - custom: boolean - slim: boolean - skin: { url: string, data: string } - raw: { value: string, signature: string } -} - -interface AshconResponse { - uuid: string - username: string - username_history: AshconHistoryItem[] - textures: AshconTextures - created_at?: string -} - -/** - * Get mojang api data from ashcon.app - */ -export async function mojangDataFromUser(user: string): Promise { - console.log('cache miss :( mojangDataFromUser', user) - const fetchResponse = await fetch( - 'https://api.ashcon.app/mojang/v2/user/' + user, - { agent: () => httpsAgent } - ) - return await fetchResponse.json() -} - -/** - * Fetch the uuid from a user - * @param user A user can be either a uuid or a username - */ -export async function uuidFromUser(user: string): Promise { - const fetchJSON = await mojangDataFromUser(user) - return fetchJSON.uuid.replace(/-/g, '') -} - -/** - * Fetch the username from a user - * @param user A user can be either a uuid or a username - */ -export async function usernameFromUser(user: string): Promise { - // get a minecraft uuid from a username, using ashcon.app's mojang api - const fetchJSON = await mojangDataFromUser(user) - return fetchJSON.username -} - diff --git a/package.json b/package.json index eedc492..4b544ef 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "description": "Wrapper for the Hypixel SkyBlock API", "main": "index.ts", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "compile": "" }, "repository": { "type": "git", diff --git a/src/cleaners/player.ts b/src/cleaners/player.ts new file mode 100644 index 0000000..c3afd6f --- /dev/null +++ b/src/cleaners/player.ts @@ -0,0 +1,28 @@ +import { CleanBasicProfile, cleanPlayerSkyblockProfiles, Included } from '../hypixel' +import { CleanSocialMedia, parseSocialMedia } from './socialmedia' +import { CleanRank, parseRank } from './rank' +import { HypixelPlayer } from '../hypixelApi' +import { undashUuid } from '../util' + +export interface CleanBasicPlayer { + uuid: string + username: string +} + +export interface CleanPlayer extends CleanBasicPlayer { + rank: CleanRank + socials: CleanSocialMedia + profiles?: CleanBasicProfile[] +} + +export async function cleanPlayerResponse(data: HypixelPlayer): Promise { + // Cleans up a 'player' api response + console.log('cleanPlayerResponse', data.stats.SkyBlock.profiles) + return { + uuid: undashUuid(data.uuid), + username: data.displayname, + rank: parseRank(data), + socials: parseSocialMedia(data.socialMedia), + profiles: cleanPlayerSkyblockProfiles(data.stats.SkyBlock.profiles) + } +} diff --git a/src/cleaners/rank.ts b/src/cleaners/rank.ts new file mode 100644 index 0000000..928373a --- /dev/null +++ b/src/cleaners/rank.ts @@ -0,0 +1,66 @@ +import { HypixelPlayer } from '../hypixelApi' +import { colorCodeFromName, minecraftColorCodes } from '../util' + +const rankColors: { [ name: string ]: string } = { + 'NONE': '7', + 'VIP': 'a', + 'VIP+': 'a', + 'MVP': 'b', + 'MVP+': 'b', + 'MVP++': '6', + 'YOUTUBE': 'c', + 'HELPER': '9', + 'MODERATOR': '2', + 'ADMIN': 'c' +} + +export interface CleanRank { + name: string, + color: string | null, + colored: string | null, +} + +/** Response cleaning (reformatting to be nicer) */ +export function parseRank({ + packageRank, + newPackageRank, + monthlyPackageRank, + rankPlusColor, + rank, + prefix +}: HypixelPlayer): CleanRank { + let name + let color + let colored + if (prefix) { // derive values from prefix + colored = prefix + color = minecraftColorCodes[colored.match(/§./)[0][1]] + name = colored.replace(/§./g, '').replace(/[\[\]]/g, '') + } else { + name = rank + || newPackageRank.replace('_PLUS', '+') + || packageRank.replace('_PLUS', '+') + || monthlyPackageRank + + // MVP++ is called Superstar for some reason + if (name === 'SUPERSTAR') name = 'MVP++' + // YouTube rank is called YouTuber, change this to the proper name + else if (name === 'YOUTUBER') name = 'YOUTUBE' + + const plusColor = colorCodeFromName(rankPlusColor) + color = minecraftColorCodes[rankColors[name]] + const rankColorPrefix = rankColors[name] ? '§' + rankColors[name] : '' + const nameWithoutPlus = name.split('+')[0] + const plusesInName = '+'.repeat(name.split('+').length - 1) + console.log(plusColor, nameWithoutPlus, plusesInName) + if (plusColor && plusesInName.length >= 1) + colored = `${rankColorPrefix}[${nameWithoutPlus}§${plusColor}${plusesInName}${rankColorPrefix}]` + else + colored = `${rankColorPrefix}[${name}]` + } + return { + name, + color, + colored + } +} diff --git a/src/cleaners/skyblock/minions.ts b/src/cleaners/skyblock/minions.ts new file mode 100644 index 0000000..da69634 --- /dev/null +++ b/src/cleaners/skyblock/minions.ts @@ -0,0 +1,70 @@ +import { maxMinion } from '../../hypixel' + +export interface CleanMinion { + name: string, + levels: boolean[] +} + + +/** + * Clean the minions provided by Hypixel + * @param minionsRaw The minion data provided by the Hypixel API + */ +export function cleanMinions(minionsRaw: string[]): CleanMinion[] { + const minions: CleanMinion[] = [] + for (const minionRaw of minionsRaw ?? []) { + // do some regex magic to get the minion name and level + // examples of potential minion names: CLAY_11, PIG_1, MAGMA_CUBE_4 + const minionName = minionRaw.split(/_\d/)[0].toLowerCase() + const minionLevel = parseInt(minionRaw.split(/\D*_/)[1]) + let matchingMinion = minions.find(m => m.name === minionName) + if (!matchingMinion) { + // if the minion doesnt already exist in the minions array, then create it + matchingMinion = { + name: minionName, + levels: new Array(maxMinion).fill(false) + } + minions.push(matchingMinion) + } + while (minionLevel > matchingMinion.levels.length) + // if hypixel increases the minion level, this will increase with it + matchingMinion.levels.push(false) + + // set the minion at that level to true + matchingMinion.levels[minionLevel - 1] = true + } + return minions +} + +/** + * Combine multiple arrays of minions into one, useful when getting the minions for members + * @param minions An array of arrays of minions + */ +export function combineMinionArrays(minions: CleanMinion[][]): CleanMinion[] { + const resultMinions: CleanMinion[] = [] + + for (const memberMinions of minions) { + for (const minion of memberMinions) { + // this is a reference, so we can directly modify the attributes for matchingMinionReference + // and they'll be in the resultMinions array + const matchingMinionReference = resultMinions.find(m => m.name === minion.name) + if (!matchingMinionReference) { + // if the minion name isn't already in the array, add it! + resultMinions.push(minion) + } else { + + // This should never happen, but in case the length of `minion.levels` is longer than + // `matchingMinionReference.levels`, then it should be extended to be equal length + while (matchingMinionReference.levels.length < minion.levels.length) + matchingMinionReference.levels.push(null) + + for (let i = 0; i < minion.levels.length; i++) { + if (minion.levels[i]) + matchingMinionReference.levels[i] = true + } + } + } + } + + return resultMinions +} \ No newline at end of file diff --git a/src/cleaners/skyblock/stats.ts b/src/cleaners/skyblock/stats.ts new file mode 100644 index 0000000..07d9133 --- /dev/null +++ b/src/cleaners/skyblock/stats.ts @@ -0,0 +1,73 @@ +const statCategories: { [ key: string ]: string[] | null } = { // sorted in order of importance + 'deaths': ['deaths_', 'deaths'], + 'kills': ['kills_', 'kills'], + 'fishing': ['items_fished_', 'items_fished'], + 'auctions': ['auctions_'], + 'races': ['_best_time'], + 'misc': null // everything else goes here +} + +interface statCategory { + category: string, + name: string +} + +function categorizeStat(statNameRaw: string): statCategory { + // 'deaths_void' + for (const statCategory in statCategories) { + // 'deaths' + const statCategoryMatchers = statCategories[statCategory] + if (statCategoryMatchers == null) { + // If it's null, just go with this. Can only ever be 'misc' + return { + category: statCategory, + name: statNameRaw + } + } + for (const categoryMatch of statCategoryMatchers) { + // ['deaths_'] + let trailingEnd = categoryMatch[0] == '_' + let trailingStart = categoryMatch.substr(-1) == '_' + if (trailingStart && statNameRaw.startsWith(categoryMatch)) { + return { + category: statCategory, + name: statNameRaw.substr(categoryMatch.length) + } + } else if (trailingEnd && statNameRaw.endsWith(categoryMatch)) { + return { + category: statCategory, + name: statNameRaw.substr(0, categoryMatch.length) + } + } else if (statNameRaw == categoryMatch) { + // if it matches exactly, we don't know the name. will be defaulted to category later on + return { + category: statCategory, + name: null + } + } + } + } + // this should never happen, as it'll default to misc and return if nothing is found + return { + category: null, + name: statNameRaw + } +} + +export interface CleanProfileStats { + [ category: string ]: { + [ stat: string ]: any + total?: any + } +} + +export function cleanProfileStats(statsRaw): CleanProfileStats { + // TODO: add type for statsRaw (probably in hypixelApi.ts since its coming from there) + const stats: CleanProfileStats = {} + for (let statNameRaw in statsRaw) { + let { category: statCategory, name: statName } = categorizeStat(statNameRaw) + if (!stats[statCategory]) stats[statCategory] = {} + stats[statCategory][statName || 'total'] = statsRaw[statNameRaw] + } + return stats +} diff --git a/src/cleaners/socialmedia.ts b/src/cleaners/socialmedia.ts new file mode 100644 index 0000000..c1f9551 --- /dev/null +++ b/src/cleaners/socialmedia.ts @@ -0,0 +1,14 @@ +import { HypixelPlayerSocialMedia } from "../hypixelApi"; + +export interface CleanSocialMedia { + discord: string | null + forums: string | null +} + +export function parseSocialMedia(socialMedia: HypixelPlayerSocialMedia): CleanSocialMedia { + return { + discord: socialMedia?.links?.DISCORD || null, + forums: socialMedia?.links?.HYPIXEL || null + } +} + diff --git a/src/hypixel.ts b/src/hypixel.ts new file mode 100644 index 0000000..f17dc9e --- /dev/null +++ b/src/hypixel.ts @@ -0,0 +1,284 @@ +import { CleanMinion, cleanMinions, combineMinionArrays } from './cleaners/skyblock/minions' +import { CleanProfileStats, cleanProfileStats } from './cleaners/skyblock/stats' +import { CleanPlayer, cleanPlayerResponse } from './cleaners/player' +import { chooseApiKey, HypixelPlayerStatsSkyBlockProfiles, HypixelResponse, sendApiRequest } from './hypixelApi' +import * as cached from './hypixelCached' + +export type Included = 'profiles' | 'player' | 'stats' + +// the interval at which the "last_save" parameter updates in the hypixel api, this is 3 minutes +export const saveInterval = 60 * 3 * 1000 + +// the highest level a minion can be +export const maxMinion = 11 + +/** + * Send a request to api.hypixel.net using a random key, clean it up to be more useable, and return it + */ +export async function sendCleanApiRequest({ path, args }, included?: Included[], cleaned=true) { + const key = await chooseApiKey() + const rawResponse = await sendApiRequest({ path, key, args }) + if (rawResponse.throttled) { + // if it's throttled, wait a second and try again + console.log('throttled :/') + await new Promise(resolve => setTimeout(resolve, 1000)) + return await sendCleanApiRequest({ path, args }, included, cleaned) + } + if (cleaned) { + // if it needs to clean the response, call cleanResponse + return await cleanResponse({ path, data: rawResponse }, included=included) + } else { + // this is provided in case the caller wants to do the cleaning itself + // used in skyblock/profile, as cleaning the entire profile would use too much cpu + return rawResponse + } +} + +export interface CleanBasicMember { + uuid: string + username: string + last_save: number + first_join: number +} + +interface CleanMember extends CleanBasicMember { + stats?: CleanProfileStats + minions?: CleanMinion[] +} + +async function cleanSkyBlockProfileMemberResponse(member, included: Included[] = null): Promise { + // Cleans up a member (from skyblock/profile) + // profiles.members[] + const statsIncluded = included == null || included.includes('stats') + return { + uuid: member.uuid, + username: await cached.usernameFromUser(member.uuid), + last_save: member.last_save, + first_join: member.first_join, + // last_death: ??? idk how this is formatted, + stats: statsIncluded ? cleanProfileStats(member.stats) : undefined, + minions: statsIncluded ? cleanMinions(member.crafted_generators) : undefined, + } +} + + +export interface CleanMemberProfilePlayer extends CleanPlayer { + // The profile name may be different for each player, so we put it here + profileName: string + first_join: number + last_save: number + bank?: { + balance: number + history: any[] + } +} + +export interface CleanMemberProfile { + member: CleanMemberProfilePlayer + profile: { + + } +} + +export interface CleanProfile extends CleanBasicProfile { + members?: CleanBasicMember[] +} + +export interface CleanFullProfile extends CleanProfile { + members: CleanMember[] + bank?: { + balance: number + history: any[] + } + minions: CleanMinion[] +} + +/** Return a `CleanProfile` instead of a `CleanFullProfile`, useful when we need to get members but don't want to waste much ram */ +async function cleanSkyblockProfileResponseLighter(data): Promise { + // We use Promise.all so it can fetch all the usernames at once instead of waiting for the previous promise to complete + const promises: Promise[] = [] + + for (const memberUUID in data.members) { + const memberRaw = data.members[memberUUID] + memberRaw.uuid = memberUUID + // we pass an empty array to make it not check stats + promises.push(cleanSkyBlockProfileMemberResponse(memberRaw, [])) + } + + const cleanedMembers: CleanMember[] = await Promise.all(promises) + + return { + uuid: data.profile_id, + name: data.cute_name, + members: cleanedMembers, + } +} + +/** This function is very costly and shouldn't be called often. Use cleanSkyblockProfileResponseLighter if you don't need all the data */ +async function cleanSkyblockProfileResponse(data: any): Promise { + const cleanedMembers: CleanMember[] = [] + + for (const memberUUID in data.members) { + const memberRaw = data.members[memberUUID] + memberRaw.uuid = memberUUID + const member: CleanMember = await cleanSkyBlockProfileMemberResponse(memberRaw, ['stats']) + cleanedMembers.push(member) + } + + const memberMinions: CleanMinion[][] = [] + + for (const member of cleanedMembers) { + memberMinions.push(member.minions) + } + const minions: CleanMinion[] = combineMinionArrays(memberMinions) + + // return more detailed info + return { + uuid: data.profile_id, + name: data.cute_name, + members: cleanedMembers, + bank: { + balance: data?.banking?.balance ?? 0, + + // TODO: make transactions good + history: data?.banking?.transactions ?? [] + }, + minions + } +} + +/** A basic profile that only includes the profile uuid and name */ +export interface CleanBasicProfile { + uuid: string + + // the name depends on the user, so its sometimes not included + name?: string +} + +export function cleanPlayerSkyblockProfiles(rawProfiles: HypixelPlayerStatsSkyBlockProfiles): CleanBasicProfile[] { + let profiles: CleanBasicProfile[] = [] + for (const profile of Object.values(rawProfiles)) { + profiles.push({ + uuid: profile.profile_id, + name: profile.cute_name + }) + } + console.log('cleanPlayerSkyblockProfiles', profiles) + return profiles +} + +/** Convert an array of raw profiles into clean profiles */ +async function cleanSkyblockProfilesResponse(data: any[]): Promise { + const cleanedProfiles: CleanProfile[] = [] + for (const profile of data) { + let cleanedProfile = await cleanSkyblockProfileResponseLighter(profile) + cleanedProfiles.push(cleanedProfile) + } + return cleanedProfiles +} + +async function cleanResponse({ path, data }: { path: string, data: HypixelResponse }, included?: Included[]) { + // Cleans up an api response + switch (path) { + case 'player': return await cleanPlayerResponse(data.player) + case 'skyblock/profile': return await cleanSkyblockProfileResponse(data.profile) + case 'skyblock/profiles': return await cleanSkyblockProfilesResponse(data.profiles) + } +} + +/* ----------------------------- */ + +export interface UserAny { + user?: string + uuid?: string + username?: string +} + +export interface CleanUser { + player: any + profiles?: any + activeProfile?: string + online?: boolean +} + + +/** + * Higher level function that requests the api for a user, and returns the cleaned response + * This is safe to fetch many times because the results are cached! + * @param included lets you choose what is returned, so there's less processing required on the backend + * used inclusions: player, profiles + */ +export async function fetchUser({ user, uuid, username }: UserAny, included: Included[]=['player']): Promise { + if (!uuid) { + // If the uuid isn't provided, get it + uuid = await cached.uuidFromUser(user || username) + } + + const includePlayers = included.includes('player') + const includeProfiles = included.includes('profiles') + + let profilesData: CleanProfile[] + let basicProfilesData: CleanBasicProfile[] + let playerData: CleanPlayer + + if (includePlayers) { + playerData = await cached.fetchPlayer(uuid) + // if not including profiles, include lightweight profiles just in case + if (!includeProfiles) + basicProfilesData = playerData.profiles + playerData.profiles = undefined + } + if (includeProfiles) { + profilesData = await cached.fetchSkyblockProfiles(uuid) + } + + let activeProfile: CleanProfile = null + let lastOnline: number = 0 + + if (includeProfiles) { + for (const profile of profilesData) { + const member = profile.members.find(member => member.uuid === uuid) + if (member.last_save > lastOnline) { + lastOnline = member.last_save + activeProfile = profile + } + } + } + return { + player: playerData ?? null, + profiles: profilesData ?? basicProfilesData, + activeProfile: includeProfiles ? activeProfile?.uuid : undefined, + online: includeProfiles ? lastOnline > (Date.now() - saveInterval): undefined + } +} + +/** + * Fetch a CleanMemberProfile from a user and string + * This is safe to use many times as the results are cached! + * @param user A username or uuid + * @param profile A profile name or profile uuid + */ +export async function fetchMemberProfile(user: string, profile: string): Promise { + const playerUuid = await cached.uuidFromUser(user) + const profileUuid = await cached.fetchProfileUuid(user, profile) + + const player = await cached.fetchPlayer(playerUuid) + + const cleanProfile = await cached.fetchProfile(playerUuid, profileUuid) + + const member = cleanProfile.members.find(m => m.uuid === playerUuid) + + return { + member: { + profileName: cleanProfile.name, + first_join: member.first_join, + last_save: member.last_save, + + // add all other data relating to the hypixel player, such as username, rank, etc + ...player + }, + profile: { + minions: cleanProfile.minions + } + } +} diff --git a/src/hypixelApi.ts b/src/hypixelApi.ts new file mode 100644 index 0000000..e91d4f5 --- /dev/null +++ b/src/hypixelApi.ts @@ -0,0 +1,154 @@ +import fetch from 'node-fetch' +import { jsonToQuery, shuffle } from './util' +import { Agent } from 'https' +require('dotenv').config() + +// We need to create an agent to prevent memory leaks and to only do dns lookups once +const httpsAgent = new Agent({ + keepAlive: true +}) + +/* Lower level code related to the Hypixel api */ + +const apiKeys = process.env.keys.split(' ') + +interface KeyUsage { + remaining: number + limit: number + reset: number +} + +const apiKeyUsage: { [ key: string ]: KeyUsage } = {} + + +const baseHypixelAPI = 'https://api.hypixel.net' + +/** Choose the best current API key */ +export function chooseApiKey(): string { + // find the api key with the lowest amount of uses + let bestKeyUsage: KeyUsage = null + let bestKey: string = null + for (var key of shuffle(apiKeys)) { + const keyUsage = apiKeyUsage[key] + + // if the key has never been used before, use it + if (!keyUsage) return key + + // if the key has reset since the last use, set the remaining count to the default + if (Date.now() > keyUsage.reset) + keyUsage.remaining = keyUsage.limit + + // if this key has more uses remaining than the current known best one, save it + if (!bestKeyUsage || keyUsage.remaining > bestKeyUsage.remaining) { + bestKeyUsage = keyUsage + bestKey = key + } + } + return bestKey +} + +export interface HypixelResponse { + [key: string]: any | { + success: boolean + throttled?: boolean + } +} + + +export interface HypixelPlayerStatsSkyBlockProfiles { + [ uuid: string ]: { + profile_id: string + cute_name: string + } +} + +interface HypixelPlayerStatsSkyBlock { + profiles: HypixelPlayerStatsSkyBlockProfiles +} + +export interface HypixelPlayerSocialMedia { + YOUTUBE?: string + prompt: boolean + links: { + DISCORD?: string + HYPIXEL?: string + } +} + +export interface HypixelPlayer { + _id: string + achievementsOneTime: string[] + displayname: string + + firstLogin: number, + lastLogin: number, + lastLogout: number + + knownAliases: string[], + knownAliasesLower: string[] + + networkExp: number + playername: string + stats: { + SkyBlock: HypixelPlayerStatsSkyBlock + [ name: string ]: any + }, + timePlaying: number, + uuid: string, + achievements: { [ name: string ]: number }, + petConsumables: { [ name: string ]: number }, + vanityMeta: { + packages: string[] + }, + + language: string, + userLanguage?: string + + packageRank?: string + newPackageRank?: string + rankPlusColor?: string + monthlyPackageRank?: string + rank?: string + prefix?: string + + claimed_potato_talisman?: number + skyblock_free_cookie?: number + + socialMedia?: HypixelPlayerSocialMedia +} + + +/** Send an HTTP request to the Hypixel API */ +export async function sendApiRequest({ path, key, args }): Promise { + console.log('sending api request to', path, args) + // Send a raw http request to api.hypixel.net, and return the parsed json + + if (key) + // If there's an api key, add it to the arguments + args.key = key + + // Construct a url from the base api url, path, and arguments + const fetchUrl = baseHypixelAPI + '/' + path + '?' + jsonToQuery(args) + + const fetchResponse = await fetch( + fetchUrl, + { agent: () => httpsAgent } + ) + + if (fetchResponse.headers['ratelimit-limit']) + // remember how many uses it has + apiKeyUsage[key] = { + remaining: fetchResponse.headers['ratelimit-remaining'], + limit: fetchResponse.headers['ratelimit-limit'], + reset: Date.now() + parseInt(fetchResponse.headers['ratelimit-reset']) * 1000 + } + + const fetchJsonParsed = await fetchResponse.json() + if (fetchJsonParsed.throttle) { + apiKeyUsage[key].remaining = 0 + console.log('throttled :(') + return { throttled: true } + } + return fetchJsonParsed +} + diff --git a/src/hypixelCached.ts b/src/hypixelCached.ts new file mode 100644 index 0000000..5fe65a3 --- /dev/null +++ b/src/hypixelCached.ts @@ -0,0 +1,240 @@ +import NodeCache from 'node-cache' +import * as mojang from './mojang' +import * as hypixel from './hypixel' +import { CleanPlayer } from './cleaners/player' +import { CleanBasicProfile, CleanFullProfile, CleanProfile } from './hypixel' +import { undashUuid } from './util' + +/** +Hypixel... but with caching +All the caching in this project is done here! +*/ + + +// cache usernames for 4 hours +const usernameCache = new NodeCache({ + stdTTL: 60 * 60 * 4, + checkperiod: 60, + useClones: false, +}) + +const basicProfilesCache = new NodeCache({ + stdTTL: 60 * 10, + checkperiod: 60, + useClones: false, +}) + +const playerCache = new NodeCache({ + stdTTL: 60, + checkperiod: 10, + useClones: false, +}) + +const profileCache = new NodeCache({ + stdTTL: 30, + checkperiod: 10, + useClones: false, +}) + +const profilesCache = new NodeCache({ + stdTTL: 60 * 3, + checkperiod: 10, + useClones: false, +}) + +const profileNameCache = new NodeCache({ + stdTTL: 60 * 60, + checkperiod: 60, + useClones: false, +}) + +/** + * Fetch the uuid from a user + * @param user A user can be either a uuid or a username + */ +export async function uuidFromUser(user: string): Promise { + if (usernameCache.has(undashUuid(user))) + // check if the uuid is a key + return undashUuid(user) + + // check if the username is a value + const uuidToUsername: {[ key: string ]: string} = usernameCache.mget(usernameCache.keys()) + for (const [ uuid, username ] of Object.entries(uuidToUsername)) { + if (user.toLowerCase() === username.toLowerCase()) + return uuid + } + + // not cached, actually fetch mojang api now + let { uuid, username } = await mojang.mojangDataFromUser(user) + + // remove dashes from the uuid so its more normal + uuid = undashUuid(uuid) + usernameCache.set(uuid, username) + return uuid +} + +/** + * Fetch the username from a user + * @param user A user can be either a uuid or a username + */ +export async function usernameFromUser(user: string): Promise { + if (usernameCache.has(undashUuid(user))) { + return usernameCache.get(undashUuid(user)) + } + + let { uuid, username } = await mojang.mojangDataFromUser(user) + uuid = undashUuid(uuid) + usernameCache.set(uuid, username) + return username +} + + +export async function fetchPlayer(user: string): Promise { + const playerUuid = await uuidFromUser(user) + + if (playerCache.has(playerUuid)) { + console.log('cache hit! fetchPlayer', playerUuid) + return playerCache.get(playerUuid) + } + + const cleanPlayer: CleanPlayer = await hypixel.sendCleanApiRequest({ + path: 'player', + args: { uuid: playerUuid } + }) + playerCache.set(playerUuid, cleanPlayer) + return cleanPlayer +} + + +export async function fetchSkyblockProfiles(playerUuid: string): Promise { + if (profilesCache.has(playerUuid)) { + console.log('cache hit! fetchSkyblockProfiles', playerUuid) + return profilesCache.get(playerUuid) + } + + const profiles: CleanFullProfile[] = await hypixel.sendCleanApiRequest({ + path: 'skyblock/profiles', + args: { + uuid: playerUuid + }}, + ) + + const basicProfiles: CleanProfile[] = [] + + // create the basicProfiles array + for (const profile of profiles) { + const basicProfile: CleanProfile = { + name: profile.name, + uuid: profile.uuid, + members: profile.members.map(m => { + return { + uuid: m.uuid, + username: m.username, + first_join: m.first_join, + last_save: m.last_save + } + }) + } + basicProfiles.push(basicProfile) + } + + // cache the profiles + profilesCache.set(playerUuid, basicProfiles) + + return basicProfiles +} + +/** Fetch an array of `BasicProfile`s */ +async function fetchBasicProfiles(user: string): Promise { + const playerUuid = await uuidFromUser(user) + if (basicProfilesCache.has(playerUuid)) { + console.log('cache hit! fetchBasicProfiles') + return basicProfilesCache.get(playerUuid) + } + const player = await fetchPlayer(playerUuid) + const profiles = player.profiles + basicProfilesCache.set(playerUuid, profiles) + + console.log(player) + + // cache the profile names and uuids to profileNameCache because we can + for (const profile of profiles) + profileNameCache.set(`${playerUuid}.${profile.uuid}`, profile.name) + + return profiles +} + +/** + * Fetch a profile UUID from its name and user + * @param user A username or uuid + * @param profile A profile name or profile uuid + */ +export async function fetchProfileUuid(user: string, profile: string) { + const profiles = await fetchBasicProfiles(user) + + const profileUuid = undashUuid(profile) + + for (const p of profiles) { + if (p.name.toLowerCase() === profileUuid.toLowerCase()) + return undashUuid(p.uuid) + else if (undashUuid(p.uuid) === undashUuid(profileUuid)) + return undashUuid(p.uuid) + } +} + +/** + * Fetch an entire profile from the user and profile data + * @param user A username or uuid + * @param profile A profile name or profile uuid + */ +export async function fetchProfile(user: string, profile: string): Promise { + const profileUuid = await fetchProfileUuid(user, profile) + + if (profileCache.has(profileUuid)) { + console.log('cache hit! fetchProfile') + // we have the profile cached, return it :) + return profileCache.get(profileUuid) + } + + const profileName = await fetchProfileName(user, profile) + + const cleanProfile: CleanFullProfile = await hypixel.sendCleanApiRequest({ + path: 'skyblock/profile', + args: { + profile: profileUuid + } + }) + + // we know the name from fetchProfileName, so set it here + cleanProfile.name = profileName + + profileCache.set(profileUuid, cleanProfile) + + return cleanProfile +} + +/** + * Fetch the name of a profile from the user and profile uuid + * @param user A player uuid or username + * @param profile A profile uuid or name + */ +export async function fetchProfileName(user: string, profile: string): Promise { + // we're fetching the profile and player uuid again in case we were given a name, but it's cached so it's not much of a problem + const profileUuid = await fetchProfileUuid(user, profile) + const playerUuid = await uuidFromUser(user) + + if (profileNameCache.has(`${playerUuid}.${profileUuid}`)) { + // Return the profile name if it's cached + console.log('cache hit! fetchProfileName') + return profileNameCache.get(`${playerUuid}.${profileUuid}`) + } + + const basicProfiles = await fetchBasicProfiles(playerUuid) + let profileName + for (const basicProfile of basicProfiles) + if (basicProfile.uuid === playerUuid) + profileName = basicProfile.name + + profileNameCache.set(`${playerUuid}.${profileUuid}`, profileName) + return profileName +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..af5095b --- /dev/null +++ b/src/index.ts @@ -0,0 +1,27 @@ +import express from 'express' +import { fetchMemberProfile, fetchUser } from './hypixel' +import { fetchProfile } from './hypixelCached' + +const app = express() + + +app.get('/', async(req, res) => { + res.json({ ok: true }) +}) + +app.get('/player/:user', async(req, res) => { + res.json( + await fetchUser( + { user: req.params.user }, + ['profiles', 'player'] + ) + ) +}) + +app.get('/player/:user/:profile', async(req, res) => { + res.json( + await fetchMemberProfile(req.params.user, req.params.profile) + ) +}) + +app.listen(8080, () => console.log('App started :)')) \ No newline at end of file diff --git a/src/mojang.ts b/src/mojang.ts new file mode 100644 index 0000000..746f674 --- /dev/null +++ b/src/mojang.ts @@ -0,0 +1,59 @@ +import fetch from 'node-fetch' +import { Agent } from 'https' + +// We need to create an agent to prevent memory leaks +const httpsAgent = new Agent({ + keepAlive: true +}) + +interface AshconHistoryItem { + username: string + changed_at?: string +} + +interface AshconTextures { + custom: boolean + slim: boolean + skin: { url: string, data: string } + raw: { value: string, signature: string } +} + +interface AshconResponse { + uuid: string + username: string + username_history: AshconHistoryItem[] + textures: AshconTextures + created_at?: string +} + +/** + * Get mojang api data from ashcon.app + */ +export async function mojangDataFromUser(user: string): Promise { + console.log('cache miss :( mojangDataFromUser', user) + const fetchResponse = await fetch( + 'https://api.ashcon.app/mojang/v2/user/' + user, + { agent: () => httpsAgent } + ) + return await fetchResponse.json() +} + +/** + * Fetch the uuid from a user + * @param user A user can be either a uuid or a username + */ +export async function uuidFromUser(user: string): Promise { + const fetchJSON = await mojangDataFromUser(user) + return fetchJSON.uuid.replace(/-/g, '') +} + +/** + * Fetch the username from a user + * @param user A user can be either a uuid or a username + */ +export async function usernameFromUser(user: string): Promise { + // get a minecraft uuid from a username, using ashcon.app's mojang api + const fetchJSON = await mojangDataFromUser(user) + return fetchJSON.username +} + diff --git a/src/util.ts b/src/util.ts new file mode 100644 index 0000000..80067ff --- /dev/null +++ b/src/util.ts @@ -0,0 +1,80 @@ +/* Utility functions (not related to Hypixel) */ + +export function undashUuid(uuid: string): string { + return uuid.replace(/-/g, '') +} + + + +export function queryToJson(queryString) { + var query = {}; + var pairs = (queryString[0] === '?' ? queryString.substr(1) : queryString).split('&'); + for (var i = 0; i < pairs.length; i++) { + var pair = pairs[i].split('='); + query[decodeURIComponent(pair[0])] = decodeURIComponent(pair[1] || ''); + } + return query; +} + +export function jsonToQuery(data) { + return Object.entries(data || {}).map(e => e.join('=')).join('&') +} + +export function shuffle(a) { + for (let i = a.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [a[i], a[j]] = [a[j], a[i]]; + } + return a; +} + + +export const minecraftColorCodes: { [ key: string ]: string } = { + '0': '#000000', + '1': '#0000be', + '2': '#00be00', + '3': '#00bebe', + '4': '#be0000', // red + '5': '#be00be', + '6': '#ffaa00', // gold + '7': '#bebebe', + '8': '#3f3f3f', + '9': '#3f3ffe', + 'a': '#3ffe3f', + 'b': '#3ffefe', + 'c': '#fe3f3f', // light red + 'd': '#fe3ffe', + 'e': '#fefe3f', + 'f': '#ffffff', + + 'black': '#000000', + 'dark_blue': '#0000be', + 'dark_green': '#00be00', + 'dark_aqua': '#00bebe', + 'dark_red': '#be0000', // red + 'dark_purple': '#be00be', + 'gold': '#ffaa00', // gold + 'gray': '#bebebe', + 'dark_gray': '#3f3f3f', + 'blue': '#3f3ffe', + 'green': '#3ffe3f', + 'aqua': '#3ffefe', + 'red': '#fe3f3f', // light red + 'light_purple': '#fe3ffe', + 'yellow': '#fefe3f', + 'white': '#ffffff', +} + +/** + * Converts a color name to the code + * For example: blue -> 9 + * @param colorName The name of the color (blue, red, aqua, etc) + */ +export function colorCodeFromName(colorName: string): string { + const hexColor = minecraftColorCodes[colorName.toLowerCase()] + for (const key in minecraftColorCodes) { + const value = minecraftColorCodes[key] + if (key.length === 1 && value === hexColor) + return key + } +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index ed4a062..e0bd8b9 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,8 @@ "module": "commonjs", "lib": ["esnext", "dom", "DOM.Iterable"], "target": "esnext", - "esModuleInterop": true + "esModuleInterop": true, + "outDir": "build" }, "include": ["**/*"], "exclude": ["node_modules"], diff --git a/util.ts b/util.ts deleted file mode 100644 index 80067ff..0000000 --- a/util.ts +++ /dev/null @@ -1,80 +0,0 @@ -/* Utility functions (not related to Hypixel) */ - -export function undashUuid(uuid: string): string { - return uuid.replace(/-/g, '') -} - - - -export function queryToJson(queryString) { - var query = {}; - var pairs = (queryString[0] === '?' ? queryString.substr(1) : queryString).split('&'); - for (var i = 0; i < pairs.length; i++) { - var pair = pairs[i].split('='); - query[decodeURIComponent(pair[0])] = decodeURIComponent(pair[1] || ''); - } - return query; -} - -export function jsonToQuery(data) { - return Object.entries(data || {}).map(e => e.join('=')).join('&') -} - -export function shuffle(a) { - for (let i = a.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)); - [a[i], a[j]] = [a[j], a[i]]; - } - return a; -} - - -export const minecraftColorCodes: { [ key: string ]: string } = { - '0': '#000000', - '1': '#0000be', - '2': '#00be00', - '3': '#00bebe', - '4': '#be0000', // red - '5': '#be00be', - '6': '#ffaa00', // gold - '7': '#bebebe', - '8': '#3f3f3f', - '9': '#3f3ffe', - 'a': '#3ffe3f', - 'b': '#3ffefe', - 'c': '#fe3f3f', // light red - 'd': '#fe3ffe', - 'e': '#fefe3f', - 'f': '#ffffff', - - 'black': '#000000', - 'dark_blue': '#0000be', - 'dark_green': '#00be00', - 'dark_aqua': '#00bebe', - 'dark_red': '#be0000', // red - 'dark_purple': '#be00be', - 'gold': '#ffaa00', // gold - 'gray': '#bebebe', - 'dark_gray': '#3f3f3f', - 'blue': '#3f3ffe', - 'green': '#3ffe3f', - 'aqua': '#3ffefe', - 'red': '#fe3f3f', // light red - 'light_purple': '#fe3ffe', - 'yellow': '#fefe3f', - 'white': '#ffffff', -} - -/** - * Converts a color name to the code - * For example: blue -> 9 - * @param colorName The name of the color (blue, red, aqua, etc) - */ -export function colorCodeFromName(colorName: string): string { - const hexColor = minecraftColorCodes[colorName.toLowerCase()] - for (const key in minecraftColorCodes) { - const value = minecraftColorCodes[key] - if (key.length === 1 && value === hexColor) - return key - } -} \ No newline at end of file -- cgit