diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/cleaners/skyblock/member.ts | 2 | ||||
| -rw-r--r-- | src/cleaners/skyblock/minions.ts | 30 | ||||
| -rw-r--r-- | src/constants.ts | 136 | ||||
| -rw-r--r-- | src/database.ts | 24 | ||||
| -rw-r--r-- | src/hypixel.ts | 2 | ||||
| -rw-r--r-- | src/hypixelCached.ts | 30 | ||||
| -rw-r--r-- | src/index.ts | 3 |
7 files changed, 138 insertions, 89 deletions
diff --git a/src/cleaners/skyblock/member.ts b/src/cleaners/skyblock/member.ts index 5d8d4f9..cfa9a71 100644 --- a/src/cleaners/skyblock/member.ts +++ b/src/cleaners/skyblock/member.ts @@ -67,7 +67,7 @@ export async function cleanSkyBlockProfileMemberResponse(member, included: Inclu // this is used for leaderboards rawHypixelStats: member.stats ?? {}, - minions: cleanMinions(member), + minions: await cleanMinions(member), fairy_souls: cleanFairySouls(member), inventories: inventoriesIncluded ? await cleanInventories(member) : undefined, objectives: cleanObjectives(member), diff --git a/src/cleaners/skyblock/minions.ts b/src/cleaners/skyblock/minions.ts index 06e7752..21f7b66 100644 --- a/src/cleaners/skyblock/minions.ts +++ b/src/cleaners/skyblock/minions.ts @@ -1,4 +1,5 @@ import { maxMinion } from '../../hypixel' +import * as constants from '../../constants' export interface CleanMinion { name: string, @@ -10,9 +11,11 @@ export interface CleanMinion { * Clean the minions provided by Hypixel * @param minionsRaw The minion data provided by the Hypixel API */ -export function cleanMinions(data: any): CleanMinion[] { +export async function cleanMinions(member: any): Promise<CleanMinion[]> { const minions: CleanMinion[] = [] - for (const minionRaw of data?.crafted_generators ?? []) { + const processedMinionNames: Set<string> = new Set() + + for (const minionRaw of member?.crafted_generators ?? []) { // do some regex magic to get the minion name and level // examples of potential minion names: CLAY_11, PIG_1, MAGMA_CUBE_4 const minionName = minionRaw.split(/_\d/)[0].toLowerCase() @@ -32,8 +35,29 @@ export function cleanMinions(data: any): CleanMinion[] { // set the minion at that level to true matchingMinion.levels[minionLevel - 1] = true + processedMinionNames.add(minionName) + } + + const allMinionNames = new Set(await constants.fetchMinions()) + + for (const minionName of processedMinionNames) { + if (!allMinionNames.has(minionName)) { + constants.addMinions(Array.from(processedMinionNames)) + break + } } - return minions + + for (const minionName of allMinionNames) { + if (!processedMinionNames.has(minionName)) { + processedMinionNames.add(minionName) + minions.push({ + name: minionName, + levels: new Array(maxMinion).fill(false) + }) + } + } + + return minions.sort((a, b) => a.name > b.name ? 1 : (a.name < b.name ? -1 : 0)) } /** diff --git a/src/constants.ts b/src/constants.ts index 5f20147..1652b6b 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -2,11 +2,15 @@ * Fetch and edit constants from the skyblock-constants repo */ -import fetch from 'node-fetch' +// we have to do this so we can mock the function from the tests properly +import * as constants from './constants' + import * as nodeFetch from 'node-fetch' -import { Agent } from 'https' import NodeCache from 'node-cache' import Queue from 'queue-promise' +import fetch from 'node-fetch' +import { Agent } from 'https' +import { debug } from '.' const httpsAgent = new Agent({ keepAlive: true @@ -16,10 +20,10 @@ const githubApiBase = 'https://api.github.com' const owner = 'skyblockstats' const repo = 'skyblock-constants' -// we use a queue for editing so it doesnt hit the github ratelimit as much +// we use a queue for editing so it always utilizes the cache if possible, and to avoid hitting the github rateimit const queue = new Queue({ concurrent: 1, - interval: 500 + interval: 10 }) /** @@ -31,6 +35,7 @@ const queue = new Queue({ */ async function fetchGithubApi(method: string, route: string, headers?: any, json?: any): Promise<nodeFetch.Response> { try { + if (debug) console.debug('fetching github api', method, route) return await fetch( githubApiBase + route, { @@ -67,26 +72,30 @@ const fileCache = new NodeCache({ * Fetch a file from skyblock-constants * @param path The file path, for example stats.json */ -async function fetchFile(path: string): Promise<GithubFile> { - if (fileCache.has(path)) - return fileCache.get(path) - - const r = await fetchGithubApi( - 'GET', - `/repos/${owner}/${repo}/contents/${path}`, - { - 'Accept': 'application/vnd.github.v3+json', - }, - ) - const data = await r.json() - - const file = { - path: data.path, - content: Buffer.from(data.content, data.encoding).toString(), - sha: data.sha - } - fileCache.set(path, file) - return file +function fetchFile(path: string): Promise<GithubFile> { + return new Promise(resolve => { + queue.enqueue(async() => { + if (fileCache.has(path)) + return resolve(fileCache.get(path)) + + const r = await fetchGithubApi( + 'GET', + `/repos/${owner}/${repo}/contents/${path}`, + { + 'Accept': 'application/vnd.github.v3+json', + }, + ) + const data = await r.json() + + const file = { + path: data.path, + content: Buffer.from(data.content, data.encoding).toString(), + sha: data.sha + } + fileCache.set(path, file) + resolve(file) + }) + }) } /** @@ -115,7 +124,8 @@ async function editFile(file: GithubFile, message: string, newContent: string): }) } -async function fetchJSONConstant(filename: string): Promise<string[]> { +export async function fetchJSONConstant(filename: string): Promise<string[]> { + console.log('actually fetchJSONConstant') const file = await fetchFile(filename) try { return JSON.parse(file.content) @@ -129,80 +139,94 @@ async function fetchJSONConstant(filename: string): Promise<string[]> { export async function addJSONConstants(filename: string, addingValues: string[], unit: string='stat'): Promise<void> { if (addingValues.length === 0) return // no stats provided, just return - queue.enqueue(async() => { - const file = await fetchFile(filename) - if (!file.path) - return - let oldStats: string[] - try { - oldStats = JSON.parse(file.content) - } catch { - // invalid json, set it as an empty array - oldStats = [] - } - const updatedStats = oldStats - .concat(addingValues) - // remove duplicates - .filter((value, index, array) => array.indexOf(value) === index) - .sort((a, b) => a.localeCompare(b)) - const newStats = updatedStats.filter(value => !oldStats.includes(value)) + let file: GithubFile = await fetchFile(filename) + if (!file.path) + return + let oldStats: string[] + try { + oldStats = JSON.parse(file.content) + } catch { + // invalid json, set it as an empty array + oldStats = [] + } + const updatedStats = oldStats + .concat(addingValues) + // remove duplicates + .filter((value, index, array) => array.indexOf(value) === index) + .sort((a, b) => a.localeCompare(b)) + const newStats = updatedStats.filter(value => !oldStats.includes(value)) - // there's not actually any new stats, just return - if (newStats.length === 0) return + // there's not actually any new stats, just return + if (newStats.length === 0) return - const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new ${unit}s` : `Add '${newStats[0]}' ${unit}` + const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new ${unit}s` : `Add '${newStats[0]}' ${unit}` + try { await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2)) - }) + } catch { + // the file probably changed or something, try again + file = await fetchFile(filename) + await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2)) + } } /** Fetch all the known SkyBlock stats as an array of strings */ export async function fetchStats(): Promise<string[]> { - return await fetchJSONConstant('stats.json') + return await constants.fetchJSONConstant('stats.json') } /** Add stats to skyblock-constants. This has caching so it's fine to call many times */ export async function addStats(addingStats: string[]): Promise<void> { - await addJSONConstants('stats.json', addingStats, 'stat') + await constants.addJSONConstants('stats.json', addingStats, 'stat') } /** Fetch all the known SkyBlock collections as an array of strings */ export async function fetchCollections(): Promise<string[]> { - return await fetchJSONConstant('collections.json') + return await constants.fetchJSONConstant('collections.json') } /** Add collections to skyblock-constants. This has caching so it's fine to call many times */ export async function addCollections(addingCollections: string[]): Promise<void> { - await addJSONConstants('collections.json', addingCollections, 'collection') + await constants.addJSONConstants('collections.json', addingCollections, 'collection') } /** Fetch all the known SkyBlock collections as an array of strings */ export async function fetchSkills(): Promise<string[]> { - return await fetchJSONConstant('skills.json') + return await constants.fetchJSONConstant('skills.json') } /** Add skills to skyblock-constants. This has caching so it's fine to call many times */ export async function addSkills(addingSkills: string[]): Promise<void> { - await addJSONConstants('skills.json', addingSkills, 'skill') + await constants.addJSONConstants('skills.json', addingSkills, 'skill') } /** Fetch all the known SkyBlock collections as an array of strings */ export async function fetchZones(): Promise<string[]> { - return await fetchJSONConstant('zones.json') + return await constants.fetchJSONConstant('zones.json') } /** Add skills to skyblock-constants. This has caching so it's fine to call many times */ export async function addZones(addingZones: string[]): Promise<void> { - await addJSONConstants('zones.json', addingZones, 'zone') + await constants.addJSONConstants('zones.json', addingZones, 'zone') } /** Fetch all the known SkyBlock slayer names as an array of strings */ export async function fetchSlayers(): Promise<string[]> { - return await fetchJSONConstant('slayers.json') + return await constants.fetchJSONConstant('slayers.json') } /** Add skills to skyblock-constants. This has caching so it's fine to call many times */ export async function addSlayers(addingSlayers: string[]): Promise<void> { - await addJSONConstants('slayers.json', addingSlayers, 'slayer') + await constants.addJSONConstants('slayers.json', addingSlayers, 'slayer') +} + +/** Fetch all the known SkyBlock slayer names as an array of strings */ +export async function fetchMinions(): Promise<string[]> { + return await constants.fetchJSONConstant('minions.json') +} + +/** Add skills to skyblock-constants. This has caching so it's fine to call many times */ +export async function addMinions(addingMinions: string[]): Promise<void> { + await constants.addJSONConstants('minions.json', addingMinions, 'minion') } diff --git a/src/database.ts b/src/database.ts index 8aa334a..4c6dadf 100644 --- a/src/database.ts +++ b/src/database.ts @@ -442,15 +442,15 @@ async function getApplicableProfileLeaderboardAttributes(profile: CleanFullProfi /** Update the member's leaderboard data on the server if applicable */ export async function updateDatabaseMember(member: CleanMember, profile: CleanFullProfile): Promise<void> { - if (debug) console.log('updateDatabaseMember', member.username) if (!client) return // the db client hasn't been initialized + if (debug) console.debug('updateDatabaseMember', member.username) // the member's been updated too recently, just return if (recentlyUpdated.get(profile.uuid + member.uuid)) return // store the member in recentlyUpdated so it cant update for 3 more minutes recentlyUpdated.set(profile.uuid + member.uuid, true) - if (debug) console.log('adding member to leaderboards', member.username) + if (debug) console.debug('adding member to leaderboards', member.username) await constants.addStats(Object.keys(member.rawHypixelStats)) await constants.addCollections(member.collections.map(coll => coll.name)) @@ -458,11 +458,11 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu await constants.addZones(member.visited_zones.map(zone => zone.name)) await constants.addSlayers(member.slayers.bosses.map(s => s.raw_name)) - if (debug) console.log('done constants..') + if (debug) console.debug('done constants..') const leaderboardAttributes = await getApplicableMemberLeaderboardAttributes(member) - if (debug) console.log('done getApplicableMemberLeaderboardAttributes..', leaderboardAttributes, member.username, profile.name) + if (debug) console.debug('done getApplicableMemberLeaderboardAttributes..', leaderboardAttributes, member.username, profile.name) await memberLeaderboardsCollection.updateOne( { @@ -495,7 +495,7 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu cachedRawLeaderboards.set(attributeName, newRawLeaderboard) } - if (debug) console.log('added member to leaderboards', member.username, leaderboardAttributes) + if (debug) console.debug('added member to leaderboards', member.username, leaderboardAttributes) } /** @@ -503,8 +503,8 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu * This will not also update the members, you have to call updateDatabaseMember separately for that */ export async function updateDatabaseProfile(profile: CleanFullProfile): Promise<void> { - if (debug) console.log('updateDatabaseProfile', profile.name) if (!client) return // the db client hasn't been initialized + if (debug) console.debug('updateDatabaseProfile', profile.name) // the profile's been updated too recently, just return if (recentlyUpdated.get(profile.uuid + 'profile')) @@ -512,11 +512,11 @@ export async function updateDatabaseProfile(profile: CleanFullProfile): Promise< // store the profile in recentlyUpdated so it cant update for 3 more minutes recentlyUpdated.set(profile.uuid + 'profile', true) - if (debug) console.log('adding profile to leaderboards', profile.name) + if (debug) console.debug('adding profile to leaderboards', profile.name) const leaderboardAttributes = await getApplicableProfileLeaderboardAttributes(profile) - if (debug) console.log('done getApplicableProfileLeaderboardAttributes..', leaderboardAttributes, profile.name) + if (debug) console.debug('done getApplicableProfileLeaderboardAttributes..', leaderboardAttributes, profile.name) await profileLeaderboardsCollection.updateOne( { @@ -550,7 +550,7 @@ export async function updateDatabaseProfile(profile: CleanFullProfile): Promise< cachedRawLeaderboards.set(attributeName, newRawLeaderboard) } - if (debug) console.log('added profile to leaderboards', profile.name, leaderboardAttributes) + if (debug) console.debug('added profile to leaderboards', profile.name, leaderboardAttributes) } const leaderboardUpdateMemberQueue = new Queue({ @@ -607,7 +607,7 @@ async function fetchAllLeaderboards(fast?: boolean): Promise<void> { const leaderboards: string[] = await fetchAllMemberLeaderboardAttributes() // shuffle so if the application is restarting many times itll still be useful - if (debug) console.log('Caching leaderboards!') + if (debug) console.debug('Caching leaderboards!') for (const leaderboard of shuffle(leaderboards)) { if (!fast) // wait 2 seconds so it doesnt use as much ram @@ -615,11 +615,11 @@ async function fetchAllLeaderboards(fast?: boolean): Promise<void> { await fetchMemberLeaderboard(leaderboard) } - if (debug) console.log('Finished caching leaderboards!') + if (debug) console.debug('Finished caching leaderboards!') } // make sure it's not in a test -if (typeof global.it !== 'function') { +if (!globalThis.isTest) { connect().then(() => { // when it connects, cache the leaderboards and remove bad members removeBadMemberLeaderboardAttributes() diff --git a/src/hypixel.ts b/src/hypixel.ts index 599a7e1..464f3dd 100644 --- a/src/hypixel.ts +++ b/src/hypixel.ts @@ -83,7 +83,7 @@ export async function fetchUser({ user, uuid, username }: UserAny, included: Inc } if (!uuid) { // the user doesn't exist. - if (debug) console.log('error:', user, 'doesnt exist') + if (debug) console.debug('error:', user, 'doesnt exist') return null } diff --git a/src/hypixelCached.ts b/src/hypixelCached.ts index a5a5909..f7b96ef 100644 --- a/src/hypixelCached.ts +++ b/src/hypixelCached.ts @@ -105,7 +105,7 @@ export async function uuidFromUser(user: string): Promise<string> { return uuid } - if (debug) console.log('Cache miss: uuidFromUser', user) + if (debug) console.debug('Cache miss: uuidFromUser', user) // set it as waitForCacheSet (a promise) in case uuidFromUser gets called while its fetching mojang usernameCache.set(undashUuid(user), waitForCacheSet(usernameCache, user, user)) @@ -132,11 +132,11 @@ export async function uuidFromUser(user: string): Promise<string> { */ export async function usernameFromUser(user: string): Promise<string> { if (usernameCache.has(undashUuid(user))) { - if (debug) console.log('Cache hit! usernameFromUser', user) + if (debug) console.debug('Cache hit! usernameFromUser', user) return usernameCache.get(undashUuid(user)) } - if (debug) console.log('Cache miss: usernameFromUser', user) + if (debug) console.debug('Cache miss: usernameFromUser', user) let { uuid, username } = await mojang.profileFromUser(user) uuid = undashUuid(uuid) @@ -191,7 +191,7 @@ export async function fetchBasicPlayer(user: string): Promise<CleanPlayer> { return basicPlayerCache.get(playerUuid) const player = await fetchPlayer(playerUuid) - if (!player) console.log('no player? this should never happen', user) + if (!player) console.debug('no player? this should never happen', user) delete player.profiles return player @@ -199,11 +199,11 @@ export async function fetchBasicPlayer(user: string): Promise<CleanPlayer> { export async function fetchSkyblockProfiles(playerUuid: string): Promise<CleanProfile[]> { if (profilesCache.has(playerUuid)) { - if (debug) console.log('Cache hit! fetchSkyblockProfiles', playerUuid) + if (debug) console.debug('Cache hit! fetchSkyblockProfiles', playerUuid) return profilesCache.get(playerUuid) } - if (debug) console.log('Cache miss: fetchSkyblockProfiles', playerUuid) + if (debug) console.debug('Cache miss: fetchSkyblockProfiles', playerUuid) const profiles: CleanProfile[] = await hypixel.fetchMemberProfilesUncached(playerUuid) @@ -240,11 +240,11 @@ async function fetchBasicProfiles(user: string): Promise<CleanBasicProfile[]> { if (!playerUuid) return // invalid player, just return if (basicProfilesCache.has(playerUuid)) { - if (debug) console.log('Cache hit! fetchBasicProfiles', playerUuid) + if (debug) console.debug('Cache hit! fetchBasicProfiles', playerUuid) return basicProfilesCache.get(playerUuid) } - if (debug) console.log('Cache miss: fetchBasicProfiles', user) + if (debug) console.debug('Cache miss: fetchBasicProfiles', user) const player = await fetchPlayer(playerUuid) const profiles = player.profiles @@ -265,11 +265,11 @@ async function fetchBasicProfiles(user: string): Promise<CleanBasicProfile[]> { export async function fetchProfileUuid(user: string, profile: string): Promise<string> { // if a profile wasn't provided, return if (!profile) { - if (debug) console.log('no profile provided?', user, profile) + if (debug) console.debug('no profile provided?', user, profile) return null } - if (debug) console.log('Cache miss: fetchProfileUuid', user) + if (debug) console.debug('Cache miss: fetchProfileUuid', user) const profiles = await fetchBasicProfiles(user) if (!profiles) return // user probably doesnt exist @@ -295,11 +295,11 @@ export async function fetchProfile(user: string, profile: string): Promise<Clean if (profileCache.has(profileUuid)) { // we have the profile cached, return it :) - if (debug) console.log('Cache hit! fetchProfile', profileUuid) + if (debug) console.debug('Cache hit! fetchProfile', profileUuid) return profileCache.get(profileUuid) } - if (debug) console.log('Cache miss: fetchProfile', user, profile) + if (debug) console.debug('Cache miss: fetchProfile', user, profile) const profileName = await fetchProfileName(user, profile) @@ -320,7 +320,7 @@ export async function fetchProfile(user: string, profile: string): Promise<Clean export async function fetchBasicProfileFromUuid(profileUuid: string): Promise<CleanProfile> { if (profileCache.has(profileUuid)) { // we have the profile cached, return it :) - if (debug) console.log('Cache hit! fetchBasicProfileFromUuid', profileUuid) + if (debug) console.debug('Cache hit! fetchBasicProfileFromUuid', profileUuid) const profile: CleanFullProfile = profileCache.get(profileUuid) return { uuid: profile.uuid, @@ -351,11 +351,11 @@ export async function fetchProfileName(user: string, profile: string): Promise<s if (profileNameCache.has(`${playerUuid}.${profileUuid}`)) { // Return the profile name if it's cached - if (debug) console.log('Cache hit! fetchProfileName', profileUuid) + if (debug) console.debug('Cache hit! fetchProfileName', profileUuid) return profileNameCache.get(`${playerUuid}.${profileUuid}`) } - if (debug) console.log('Cache miss: fetchProfileName', user, profile) + if (debug) console.debug('Cache miss: fetchProfileName', user, profile) const basicProfiles = await fetchBasicProfiles(playerUuid) let profileName diff --git a/src/index.ts b/src/index.ts index 06bdc5f..d25efc9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ const app = express() export const debug = false + // 200 requests over 5 minutes const limiter = rateLimit({ windowMs: 60 * 1000 * 5, @@ -69,5 +70,5 @@ app.get('/leaderboards', async(req, res) => { // only run the server if it's not doing tests -if (typeof global.it !== 'function') +if (!globalThis.isTest) app.listen(8080, () => console.log('App started :)')) |