/**
* Fetch the clean Hypixel API
*/
import {
cleanSkyblockProfileResponse,
CleanProfile,
CleanBasicProfile,
CleanFullProfile,
CleanFullProfileBasicMembers
} from './cleaners/skyblock/profile.js'
import {
AccountCustomization,
AccountSchema,
fetchAccount,
fetchItemsAuctions,
ItemAuctionsSchema,
queueUpdateDatabaseMember,
queueUpdateDatabaseProfile,
removeDeletedProfilesFromLeaderboards,
SimpleAuctionSchema,
updateItemAuction
} from './database.js'
import { cleanElectionResponse, ElectionData } from './cleaners/skyblock/election.js'
import { cleanItemListResponse, ItemListData } from './cleaners/skyblock/itemList.js'
import { CleanBasicMember, CleanMemberProfile } from './cleaners/skyblock/member.js'
import { cleanSkyblockProfilesResponse } from './cleaners/skyblock/profiles.js'
import { CleanPlayer, cleanPlayerResponse } from './cleaners/player.js'
import { chooseApiKey, sendApiRequest } from './hypixelApi.js'
import typedHypixelApi from 'typed-hypixel-api'
import * as cached from './hypixelCached.js'
import { debug } from './index.js'
import { WithId } from 'mongodb'
import { cleanEndedAuctions } from './cleaners/skyblock/endedAuctions.js'
export type Included = 'profiles' | 'player' | 'stats' | 'inventories' | undefined
// the interval at which the "last_save" parameter updates in the hypixel api, this is 3 minutes
export const saveInterval = 60 * 3 * 1000
/**
* Send a request to api.hypixel.net using a random key, clean it up to be more useable, and return it
*/
export interface ApiOptions {
mainMemberUuid?: string
/** Only get the most basic information, like uuids and names */
basic?: boolean
}
/** Sends an API request to Hypixel and cleans it up. */
export async function sendCleanApiRequest
(path: P, args: Omit, options?: ApiOptions): Promise>> {
const key = await chooseApiKey()
const data = await sendApiRequest(path, { key, ...args })
if (!data)
throw new Error(`No data returned from ${path}`)
// clean the response
return await cleanResponse(path, data, options ?? {})
}
const cleanResponseFunctions = {
'player': (data, options) => cleanPlayerResponse(data.player),
'skyblock/profile': (data: typedHypixelApi.SkyBlockProfileResponse, options) => cleanSkyblockProfileResponse(data.profile, options),
'skyblock/profiles': (data, options) => cleanSkyblockProfilesResponse(data.profiles),
'skyblock/auctions_ended': (data, options) => cleanEndedAuctions(data),
'resources/skyblock/election': (data, options) => cleanElectionResponse(data),
'resources/skyblock/items': (data, options) => cleanItemListResponse(data),
} as const
async function cleanResponse(
path: P,
data: typedHypixelApi.Requests[P]['response']['data'],
options: ApiOptions
): Promise>> {
// Cleans up an api response
const cleaningFunction: typeof cleanResponseFunctions[P] = cleanResponseFunctions[path]
// we do `as any` because typescript unfortunately doesn't know which path it is
const cleanedData = await cleaningFunction(data as any, options)
return cleanedData as Awaited>
}
/* ----------------------------- */
export interface UserAny {
user?: string
uuid?: string
username?: string
}
export interface CleanUser {
player: CleanPlayer | null
profiles?: CleanProfile[] | CleanBasicProfile[]
activeProfile?: string
online?: boolean
customization?: AccountCustomization
}
/**
* Higher level function that requests the api for a user, and returns the
* cleaned response. This is used by the /player/ route.
* This is safe to fetch many times because the results are cached!
* @param included lets you choose what is returned, so there's less processing required on the backend.
* used inclusions: player, profiles
*/
export async function fetchUser({ user, uuid, username }: UserAny, included: Included[] = ['player'], customization?: boolean): Promise {
if (!uuid) {
// If the uuid isn't provided, get it
if (!username && !user) return null
uuid = await cached.uuidFromUser((user ?? username)!)
}
if (!uuid) {
// the user doesn't exist.
if (debug) console.debug('error:', user, 'doesnt exist')
return null
}
const websiteAccountPromise = customization ? fetchAccount(uuid) : null
const includePlayers = included.includes('player')
const includeProfiles = included.includes('profiles')
let profilesData: CleanProfile[] | undefined
let basicProfilesData: CleanBasicProfile[] | undefined
let playerData: CleanPlayer | null = null
if (includePlayers) {
playerData = await cached.fetchPlayer(uuid, true)
// if not including profiles, include lightweight profiles just in case
if (!includeProfiles)
basicProfilesData = playerData?.profiles
// we don't want the `profiles` field in `player`
if (playerData)
delete playerData.profiles
}
if (includeProfiles)
profilesData = await cached.fetchSkyblockProfiles(uuid) ?? []
let activeProfile: CleanProfile
let lastOnline: number = 0
if (includeProfiles && profilesData !== undefined) {
for (const profile of profilesData) {
const member = profile.members?.find(member => member.uuid === uuid)
if (member && member.lastSave && member.lastSave > lastOnline) {
lastOnline = member.lastSave
activeProfile = profile
}
}
// we don't await so it happens in the background
removeDeletedProfilesFromLeaderboards(uuid, profilesData.map(p => p.uuid))
}
let websiteAccount: WithId | null = null
if (websiteAccountPromise)
websiteAccount = await websiteAccountPromise
return {
player: playerData,
profiles: profilesData ?? basicProfilesData,
activeProfile: includeProfiles ? activeProfile!?.uuid : undefined,
online: includeProfiles ? lastOnline > (Date.now() - saveInterval) : undefined,
customization: websiteAccount?.customization
}
}
/**
* Fetch a CleanMemberProfile from a user and string
* This is safe to use many times as the results are cached!
* @param user A username or uuid
* @param profile A profile name or profile uuid
* @param customization Whether stuff like the user's custom background will be returned
*/
export async function fetchMemberProfile(user: string, profile: string, customization: boolean): Promise {
const playerUuid = await cached.uuidFromUser(user)
if (!playerUuid) return null
// we don't await the promise immediately so it can load while we do other stuff
const websiteAccountPromise = customization ? fetchAccount(playerUuid) : null
const profileUuid = await cached.fetchProfileUuid(user, profile)
// if the profile or player doesn't have an id, just return
if (!profileUuid) return null
if (!playerUuid) return null
const player: CleanPlayer | null = await cached.fetchPlayer(playerUuid, true)
if (!player) return null // this should never happen, but if it does just return null
const cleanProfile = await cached.fetchProfile(playerUuid, profileUuid)
if (!cleanProfile) return null
const member = cleanProfile.members.find(m => m.uuid === playerUuid)
if (!member) return null // this should never happen, but if it does just return null
// remove unnecessary member data
const simpleMembers: CleanBasicMember[] = cleanProfile.members.map(m => {
return {
uuid: m.uuid,
username: m.username,
firstJoin: m.firstJoin,
lastSave: m.lastSave,
rank: m.rank,
left: m.left
}
})
const cleanProfileBasicMembers: CleanFullProfileBasicMembers = {
...cleanProfile,
members: simpleMembers
}
let websiteAccount: WithId | null = null
if (websiteAccountPromise)
websiteAccount = await websiteAccountPromise
return {
member: {
// the profile name is in member rather than profile since they sometimes differ for each member
profileName: cleanProfile.name!,
// add all the member data
...member,
// add all other data relating to the hypixel player, such as username, rank, etc
...player
},
profile: cleanProfileBasicMembers,
customization: websiteAccount?.customization
}
}
/**
* Fetches the Hypixel API to get a CleanFullProfile. This doesn't do any caching and you should use hypixelCached.fetchProfile instead
* @param playerUuid The UUID of the Minecraft player
* @param profileUuid The UUID of the Hypixel SkyBlock profile
*/
export async function fetchMemberProfileUncached(playerUuid: string, profileUuid: string): Promise {
const profile = await sendCleanApiRequest(
'skyblock/profile',
{ profile: profileUuid },
{ mainMemberUuid: playerUuid }
)
// we check for minions in profile to filter out the CleanProfile type (as opposed to CleanFullProfile)
if (!profile || !('minions' in profile)) return null
// queue updating the leaderboard positions for the member, eventually
if (profile.members)
for (const member of profile.members)
queueUpdateDatabaseMember(member, profile)
queueUpdateDatabaseProfile(profile)
return profile
}
/**
* Fetches the Hypixel API to get a CleanProfile from its id. This doesn't do any caching and you should use hypixelCached.fetchBasicProfileFromUuid instead
* @param playerUuid The UUID of the Minecraft player
* @param profileUuid The UUID of the Hypixel SkyBlock profile
*/
export async function fetchBasicProfileFromUuidUncached(profileUuid: string): Promise {
const profile = await sendCleanApiRequest(
'skyblock/profile',
{ profile: profileUuid },
{ basic: true }
)
return profile
}
export async function fetchMemberProfilesUncached(playerUuid: string): Promise {
const profiles = await sendCleanApiRequest(
'skyblock/profiles',
{ uuid: playerUuid },
{
// only the inventories for the main player are generated, this is for optimization purposes
mainMemberUuid: playerUuid
}
)
if (profiles === null)
return null
for (const profile of profiles) {
for (const member of profile.members) {
queueUpdateDatabaseMember(member, profile)
}
queueUpdateDatabaseProfile(profile)
}
return profiles
}
let isFetchingElection = false
let cachedElectionData: ElectionData | null = null
let nextElectionUpdate: Date = new Date(0)
export async function fetchElection(): Promise {
if (cachedElectionData && nextElectionUpdate > new Date())
return cachedElectionData
// if it's currently fetching the election data and it doesn't have it,
// wait until we do have the election data
if (isFetchingElection && !cachedElectionData) {
await new Promise(resolve => {
const interval = setInterval(() => {
if (cachedElectionData) {
clearInterval(interval)
resolve(cachedElectionData)
}
}, 100)
})
}
isFetchingElection = true
const election: ElectionData = await sendCleanApiRequest(
'resources/skyblock/election',
{}
)
isFetchingElection = false
cachedElectionData = election
// updates every 10 minutes
nextElectionUpdate = new Date((election.lastUpdated + 10 * 60) * 1000)
return election
}
let isFetchingItemList = false
let cachedItemListData: ItemListData | null = null
let nextItemListUpdate: Date = new Date(0)
export async function fetchItemList() {
if (cachedItemListData && nextItemListUpdate > new Date())
return cachedItemListData
// if it's currently fetching the election data and it doesn't have it,
// wait until we do have the election data
if (isFetchingItemList && !cachedItemListData) {
await new Promise(resolve => {
const interval = setInterval(() => {
if (cachedItemListData) {
clearInterval(interval)
resolve(cachedItemListData)
}
}, 100)
})
}
isFetchingItemList = true
const itemList: ItemListData = await sendCleanApiRequest(
'resources/skyblock/items',
{}
)
isFetchingItemList = false
cachedItemListData = itemList
// updates every 60 minutes
nextItemListUpdate = new Date((itemList.lastUpdated + 60 * 60) * 1000)
return itemList
}
// this function is called from database.ts so it starts when we connect to the database
// it should only ever be called once!
export async function periodicallyFetchRecentlyEndedAuctions() {
let previousAuctionIds = new Set()
while (true) {
const endedAuctions = await sendCleanApiRequest(
'skyblock/auctions_ended',
{}
)
let newAuctionUuids: Set = new Set()
let newAuctionItemIds: Set = new Set()
for (const auction of endedAuctions.auctions) {
if (previousAuctionIds.has(auction.id)) continue
newAuctionUuids.add(auction.id)
newAuctionItemIds.add(auction.item.id)
}
let updatedDatabaseAuctionItems: Map = new Map()
const itemsAuctions = await fetchItemsAuctions(Array.from(newAuctionItemIds))
for (const itemAuctions of itemsAuctions) {
updatedDatabaseAuctionItems[itemAuctions._id] = itemAuctions
}
for (const auction of endedAuctions.auctions) {
if (previousAuctionIds.has(auction.id)) continue
let auctions: SimpleAuctionSchema[]
if (!updatedDatabaseAuctionItems.has(auction.item.id)) {
updatedDatabaseAuctionItems.set(auction.item.id, {
_id: auction.item.id,
auctions: [],
})
auctions = []
} else {
auctions = updatedDatabaseAuctionItems.get(auction.item.id)!.auctions
}
const simpleAuction: SimpleAuctionSchema = {
success: true,
coins: auction.coins,
id: auction.id,
ts: Math.floor(auction.timestamp / 1000),
bin: auction.bin,
}
// make sure the auction isn't already in there
if (auctions.findIndex((a) => a.id === simpleAuction.id) === null) {
auctions.push(simpleAuction)
// keep only the last 100 items
if (auctions.length > 100)
auctions = auctions.slice(auctions.length - 100)
}
updatedDatabaseAuctionItems.set(auction.item.id, {
_id: auction.item.id,
auctions,
})
}
// we use a promise pool to set all the things fast but not overload the database
let tasks = Array.from(updatedDatabaseAuctionItems.values()).map(t => updateItemAuction(t))
async function doTasks() {
let hasTask = true
while (hasTask) {
const task = tasks.pop()
if (task) {
await task
} else
hasTask = false
}
}
// Promise.all 5 cycles
await Promise.all(Array(5).fill(0).map(_ => doTasks()))
previousAuctionIds = newAuctionUuids
let endedAgo = Date.now() - endedAuctions.lastUpdated
// +10 seconds just so we're sure we'll get the update
let refetchIn = 60 * 1000 - endedAgo + 10000
await new Promise(resolve => setTimeout(resolve, refetchIn))
}
}