/**
* Fetch the clean Hypixel API
*/
import {
cleanSkyblockProfileResponse,
CleanProfile,
CleanBasicProfile,
CleanFullProfile,
CleanFullProfileBasicMembers
} from './cleaners/skyblock/profile.js'
import {
AccountCustomization,
AccountSchema,
fetchAccount,
fetchItemsAuctions,
fetchItemsAuctionsIds,
ItemAuctionsSchema,
queueUpdateDatabaseMember,
queueUpdateDatabaseProfile,
removeDeletedProfilesFromLeaderboards,
SimpleAuctionSchema,
updateItemAuction
} from './database.js'
import { cleanElectionResponse, ElectionData } from './cleaners/skyblock/election.js'
import { cleanItemListResponse } from './cleaners/skyblock/itemList.js'
import { CleanBasicMember, CleanMemberProfile } from './cleaners/skyblock/member.js'
import { cleanSkyblockProfilesResponse } from './cleaners/skyblock/profiles.js'
import { CleanPlayer, cleanPlayerResponse } from './cleaners/player.js'
import { chooseApiKey, sendApiRequest } from './hypixelApi.js'
import typedHypixelApi from 'typed-hypixel-api'
import * as cached from './hypixelCached.js'
import { debug } from './index.js'
import { WithId } from 'mongodb'
import { cleanEndedAuctions } from './cleaners/skyblock/endedAuctions.js'
import { Auctions, cleanAuctions } from './cleaners/skyblock/auctions.js'
import { withCache } from './util.js'
import { Item } from './cleaners/skyblock/inventory.js'
export type Included = 'profiles' | 'player' | 'stats' | 'inventories' | undefined
// the interval at which the "last_save" parameter updates in the hypixel api, this is 3 minutes
export const saveInterval = 60 * 3 * 1000
/**
* Send a request to api.hypixel.net using a random key, clean it up to be more useable, and return it
*/
export interface ApiOptions {
mainMemberUuid?: string
/** Only get the most basic information, like uuids and names */
basic?: boolean
/** Some endpoints have pagination */
page?: number
}
/** Sends an API request to Hypixel and returns the response. */
export async function sendUncleanApiRequest
(path: P, args: Omit): Promise {
const key = await chooseApiKey()
const data = await sendApiRequest(path, { key, ...args })
if (!data)
throw new Error(`No data returned from ${path}`)
return data
}
/** Sends an API request to Hypixel and cleans it up. */
export async function sendCleanApiRequest(path: P, args: Omit, options?: ApiOptions): Promise>> {
const data = await sendUncleanApiRequest(path, args)
return await cleanResponse(path, data, options ?? {})
}
const cleanResponseFunctions = {
'player': (data, options) => cleanPlayerResponse(data.player),
'skyblock/profile': (data: typedHypixelApi.SkyBlockProfileResponse, options) => cleanSkyblockProfileResponse(data.profile, options),
'skyblock/profiles': (data, options) => cleanSkyblockProfilesResponse(data.profiles, options),
'skyblock/auctions_ended': (data, options) => cleanEndedAuctions(data),
'skyblock/auction': (data, options) => cleanAuctions(data, options.page ?? 0),
'resources/skyblock/election': (data, options) => cleanElectionResponse(data),
'resources/skyblock/items': (data, options) => cleanItemListResponse(data),
} as const
async function cleanResponse(
path: P,
data: typedHypixelApi.Requests[P]['response']['data'],
options: ApiOptions
): Promise>> {
// Cleans up an api response
const cleaningFunction: typeof cleanResponseFunctions[P] = cleanResponseFunctions[path]
// we do `as any` because typescript unfortunately doesn't know which path it is
const cleanedData = await cleaningFunction(data as any, options)
return cleanedData as Awaited>
}
/* ----------------------------- */
export interface UserAny {
user?: string
uuid?: string
username?: string
}
export interface CleanUser {
player: CleanPlayer | null
profiles?: CleanProfile[] | CleanBasicProfile[]
activeProfile?: string
online?: boolean
customization?: AccountCustomization
}
/**
* Higher level function that requests the api for a user, and returns the
* cleaned response. This is used by the /player/ route.
* This is safe to fetch many times because the results are cached!
* @param included lets you choose what is returned, so there's less processing required on the backend.
* used inclusions: player, profiles
*/
export async function fetchUser({ user, uuid, username }: UserAny, included: Included[] = ['player'], customization?: boolean): Promise {
if (!uuid) {
// If the uuid isn't provided, get it
if (!username && !user) return null
uuid = await cached.uuidFromUser((user ?? username)!)
}
if (!uuid) {
// the user doesn't exist.
if (debug) console.debug('error:', user, 'doesnt exist')
return null
}
const websiteAccountPromise = customization ? fetchAccount(uuid) : null
const includePlayers = included.includes('player')
const includeProfiles = included.includes('profiles')
let profilesData: CleanProfile[] | undefined
let basicProfilesData: CleanBasicProfile[] | undefined
let playerData: CleanPlayer | null = null
if (includePlayers) {
playerData = await cached.fetchPlayer(uuid, true)
// if not including profiles, include lightweight profiles just in case
if (!includeProfiles)
basicProfilesData = playerData?.profiles
// we don't want the `profiles` field in `player`
if (playerData)
delete playerData.profiles
}
if (includeProfiles)
profilesData = await cached.fetchSkyblockProfiles(uuid) ?? []
let activeProfile: CleanProfile
let lastOnline: number = 0
if (includeProfiles && profilesData !== undefined) {
for (const profile of profilesData) {
const member = profile.members?.find(member => member.uuid === uuid)
if (member && member.lastSave && member.lastSave > lastOnline) {
lastOnline = member.lastSave
activeProfile = profile
}
}
// we don't await so it happens in the background
removeDeletedProfilesFromLeaderboards(uuid, profilesData.map(p => p.uuid))
}
let websiteAccount: WithId | null = null
if (websiteAccountPromise)
websiteAccount = await websiteAccountPromise
return {
player: playerData,
profiles: profilesData ?? basicProfilesData,
activeProfile: includeProfiles ? activeProfile!?.uuid : undefined,
online: includeProfiles ? lastOnline > (Date.now() - saveInterval) : undefined,
customization: websiteAccount?.customization
}
}
/**
* Fetch a CleanMemberProfile from a user and string
* This is safe to use many times as the results are cached!
* @param user A username or uuid
* @param profile A profile name or profile uuid
* @param customization Whether stuff like the user's custom background will be returned
*/
export async function fetchMemberProfile(user: string, profile: string, customization: boolean): Promise {
const playerUuid = await cached.uuidFromUser(user)
if (!playerUuid) return null
// we don't await the promise immediately so it can load while we do other stuff
const websiteAccountPromise = customization ? fetchAccount(playerUuid) : null
const profileUuid = await cached.fetchProfileUuid(user, profile)
// if the profile or player doesn't have an id, just return
if (!profileUuid) return null
if (!playerUuid) return null
const player: CleanPlayer | null = await cached.fetchPlayer(playerUuid, true)
if (!player) return null // this should never happen, but if it does just return null
const cleanProfile = await cached.fetchProfile(playerUuid, profileUuid)
if (!cleanProfile) return null
const member = cleanProfile.members.find(m => m.uuid === playerUuid)
if (!member) return null // this should never happen, but if it does just return null
// remove unnecessary member data
const simpleMembers: CleanBasicMember[] = cleanProfile.members.map(m => {
return {
uuid: m.uuid,
username: m.username,
firstJoin: m.firstJoin,
lastSave: m.lastSave,
rank: m.rank,
left: m.left
}
})
const cleanProfileBasicMembers: CleanFullProfileBasicMembers = {
...cleanProfile,
members: simpleMembers
}
let websiteAccount: WithId | null = null
if (websiteAccountPromise)
websiteAccount = await websiteAccountPromise
return {
member: {
// the profile name is in member rather than profile since they sometimes differ for each member
profileName: cleanProfile.name!,
// add all the member data
...member,
// add all other data relating to the hypixel player, such as username, rank, etc
...player
},
profile: cleanProfileBasicMembers,
customization: websiteAccount?.customization
}
}
/**
* Fetches the Hypixel API to get a CleanFullProfile. This doesn't do any caching and you should use hypixelCached.fetchProfile instead
* @param playerUuid The UUID of the Minecraft player
* @param profileUuid The UUID of the Hypixel SkyBlock profile
*/
export async function fetchMemberProfileUncached(playerUuid: string, profileUuid: string): Promise {
const profile = await sendCleanApiRequest(
'skyblock/profile',
{ profile: profileUuid },
{ mainMemberUuid: playerUuid }
)
// we check for minions in profile to filter out the CleanProfile type (as opposed to CleanFullProfile)
if (!profile || !('minions' in profile)) return null
// queue updating the leaderboard positions for the member, eventually
if (profile.members)
for (const member of profile.members)
queueUpdateDatabaseMember(member, profile)
queueUpdateDatabaseProfile(profile)
return profile
}
/**
* Fetches the Hypixel API to get a CleanProfile from its id. This doesn't do any caching and you should use hypixelCached.fetchBasicProfileFromUuid instead
* @param playerUuid The UUID of the Minecraft player
* @param profileUuid The UUID of the Hypixel SkyBlock profile
*/
export async function fetchBasicProfileFromUuidUncached(profileUuid: string): Promise {
const profile = await sendCleanApiRequest(
'skyblock/profile',
{ profile: profileUuid },
{ basic: true }
)
return profile
}
export async function fetchMemberProfilesUncached(playerUuid: string): Promise {
const profiles = await sendCleanApiRequest(
'skyblock/profiles',
{ uuid: playerUuid },
{
// only the inventories for the main player are generated, this is for optimization purposes
mainMemberUuid: playerUuid
}
)
if (profiles === null)
return null
for (const profile of profiles) {
for (const member of profile.members) {
queueUpdateDatabaseMember(member, profile)
}
queueUpdateDatabaseProfile(profile)
}
return profiles
}
export async function fetchElection(): Promise {
return await withCache(
'election',
(r) => new Date(r.lastUpdated + 60 * 60 * 1000),
async () => {
return await sendCleanApiRequest(
'resources/skyblock/election',
{}
)
}
)
}
export async function fetchItemList() {
return await withCache(
'itemList',
(r) => new Date(r.lastUpdated + 60 * 60 * 1000),
async () => {
return await sendCleanApiRequest(
'resources/skyblock/items',
{}
)
}
)
}
export async function fetchAuctionUncached(uuid: string) {
const auctions = await sendCleanApiRequest(
'skyblock/auction',
{ uuid }
)
if (!auctions || auctions.auctions.length === 0) return null
return auctions.auctions[0]
}
/**
* Create an id that we use to differenciate between different items that are sold in auctions. This can also be used to filter out specific items by returning undefined.
*/
function createAuctionItemId(item: Item): string | undefined {
if (item.id === 'PET' && item.petInfo?.id)
return `${item.petInfo.id}_${item.id}`
if (item.id === 'ENCHANTED_BOOK') {
if (Object.keys(item.enchantments ?? {}).length !== 1)
// we only care about enchanted books that have a single enchantment
return
const [[enchantName, enchantValue]] = Object.entries(item.enchantments ?? {})
return `${item.id}_${enchantName.toUpperCase()}_${enchantValue}`
}
return item.id
}
// this function is called from database.ts so it starts when we connect to the database
// it should only ever be called once!
export async function periodicallyFetchRecentlyEndedAuctions() {
let previousAuctionIds = new Set()
while (true) {
const endedAuctions = await sendCleanApiRequest(
'skyblock/auctions_ended',
{}
)
let newAuctionUuids: Set = new Set()
let newAuctionItemIds: Set = new Set()
for (const auction of endedAuctions.auctions) {
if (previousAuctionIds.has(auction.id)) continue
const auctionItemId = createAuctionItemId(auction.item)
if (!auctionItemId) continue
newAuctionUuids.add(auction.id)
newAuctionItemIds.add(auctionItemId)
}
let updatedDatabaseAuctionItems: Map = new Map()
const itemsAuctions = await fetchItemsAuctions(Array.from(newAuctionItemIds), true)
for (const itemAuctions of itemsAuctions) {
updatedDatabaseAuctionItems.set(itemAuctions.id, itemAuctions)
}
for (const auction of endedAuctions.auctions) {
if (previousAuctionIds.has(auction.id)) continue
const auctionItemId = createAuctionItemId(auction.item)
if (!auctionItemId) continue
let auctions: SimpleAuctionSchema[]
if (!updatedDatabaseAuctionItems.has(auctionItemId)) {
auctions = []
} else {
auctions = updatedDatabaseAuctionItems.get(auctionItemId)!.auctions
}
const simpleAuction: SimpleAuctionSchema = {
s: true,
coins: Math.round(auction.coins / auction.item.count),
ts: Math.floor(auction.timestamp / 1000),
bin: auction.bin,
lore: auction.item.display.lore.join('\n')
}
// make sure the auction isn't already in there
// we use the timestamp because it's sorta unique enough
// (we don't store auction ids in the database because they're too big)
if (!auctions.find((a) => a.ts === simpleAuction.ts)) {
auctions.push(simpleAuction)
// keep only the last 100 items
if (auctions.length > 100)
auctions = auctions.slice(-100)
}
updatedDatabaseAuctionItems.set(auctionItemId, {
id: auctionItemId,
sbId: auction.item.id,
auctions,
})
}
// we use a promise pool to set all the things fast but not overload the database
let tasks = Array.from(updatedDatabaseAuctionItems.values()).map(t => updateItemAuction(t))
async function doTasks() {
let hasTask = true
while (hasTask) {
const task = tasks.pop()
if (task) {
await task
} else
hasTask = false
}
}
// Promise.all 5 cycles
await Promise.all(Array(5).fill(0).map(_ => doTasks()))
previousAuctionIds = newAuctionUuids
let endedAgo = Date.now() - endedAuctions.lastUpdated
// +10 seconds just so we're sure we'll get the update
let refetchIn = 60 * 1000 - endedAgo + 10000
await new Promise(resolve => setTimeout(resolve, refetchIn))
}
}
export async function fetchAuctionItems() {
return await withCache(
'auctionItems',
10 * 60 * 1000,
fetchAuctionItemsUncached
)
}
async function fetchAuctionItemsUncached() {
const auctionItemIds = await fetchItemsAuctionsIds(true)
if (!auctionItemIds) return undefined
const itemList = await fetchItemList()
const idsToData: Record = {}
for (const item of itemList.list)
// we only return items in auctionItemIds so the response isn't too big,
// since usually it would contain stuff that we don't care about like
// minions
if (auctionItemIds.includes(item.id))
idsToData[item.id] = {
display: {
name: item.display.name
},
vanillaId: item.vanillaId,
headTexture: item.headTexture
}
// if the item in the database isn't in the items api, just set the name to the id
for (const item of auctionItemIds)
if (!(item in idsToData))
idsToData[item] = {
display: {
name: (item.toLowerCase().replace(/^./, item[0].toUpperCase()).replace(/_/g, ' ')).replace(
/\w\S*/g,
w => w.charAt(0).toUpperCase() + w.slice(1).toLowerCase()
)
}
}
return idsToData
}
export async function fetchPlayerAuctions(user: string, page: number): Promise {
const playerUuid = await cached.uuidFromUser(user)
if (!playerUuid) return null
const playerAuctions = await sendCleanApiRequest(
'skyblock/auction',
{ player: playerUuid },
{ page }
)
return playerAuctions
}