diff options
| -rw-r--r-- | build/constants.js | 137 | ||||
| -rw-r--r-- | build/database.js | 4 | ||||
| -rw-r--r-- | build/hypixelCached.js | 12 |
3 files changed, 81 insertions, 72 deletions
diff --git a/build/constants.js b/build/constants.js index 961e998..6344a12 100644 --- a/build/constants.js +++ b/build/constants.js @@ -6,16 +6,22 @@ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.addCollections = exports.addStats = exports.fetchCollections = exports.fetchStats = void 0; +exports.addZones = exports.fetchZones = exports.addSkills = exports.fetchSkills = exports.addCollections = exports.fetchCollections = exports.addStats = exports.fetchStats = exports.addJSONConstants = void 0; const node_fetch_1 = __importDefault(require("node-fetch")); const https_1 = require("https"); const node_cache_1 = __importDefault(require("node-cache")); +const queue_promise_1 = __importDefault(require("queue-promise")); const httpsAgent = new https_1.Agent({ keepAlive: true }); const githubApiBase = 'https://api.github.com'; const owner = 'skyblockstats'; const repo = 'skyblock-constants'; +// we use a queue for editing so it doesnt hit the github ratelimit as much +const queue = new queue_promise_1.default({ + concurrent: 1, + interval: 500 +}); /** * Send a request to the GitHub API * @param method The HTTP method, for example GET, PUT, POST, etc @@ -48,7 +54,6 @@ async function fetchFile(path) { return fileCache.get(path); const r = await fetchGithubApi('GET', `/repos/${owner}/${repo}/contents/${path}`, { 'Accept': 'application/vnd.github.v3+json', - 'Authorization': undefined }); const data = await r.json(); const file = { @@ -79,9 +84,8 @@ async function editFile(file, message, newContent) { sha: data.content.sha }); } -/** Fetch all the known SkyBlock stats as an array of strings */ -async function fetchStats() { - const file = await fetchFile('stats.json'); +async function fetchJSONConstant(filename) { + const file = await fetchFile(filename); try { return JSON.parse(file.content); } @@ -90,72 +94,73 @@ async function fetchStats() { return []; } } -exports.fetchStats = fetchStats; -/** Fetch all the known SkyBlock collections as an array of strings */ -async function fetchCollections() { - const file = await fetchFile('collections.json'); - try { - return JSON.parse(file.content); - } - catch { - // probably invalid json, return an empty array - return []; - } +/** Add stats to skyblock-constants. This has caching so it's fine to call many times */ +async function addJSONConstants(filename, addingValues, units = 'stats') { + if (addingValues.length === 0) + return; // no stats provided, just return + queue.enqueue(async () => { + const file = await fetchFile(filename); + if (!file.path) + return; + let oldStats; + try { + oldStats = JSON.parse(file.content); + } + catch { + // invalid json, set it as an empty array + oldStats = []; + } + const updatedStats = oldStats + .concat(addingValues) + // remove duplicates + .filter((value, index, array) => array.indexOf(value) === index) + .sort((a, b) => a.localeCompare(b)); + const newStats = updatedStats.filter(value => !oldStats.includes(value)); + // there's not actually any new stats, just return + if (newStats.length === 0) + return; + const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new ${units}` : `Add '${newStats[0]}'`; + await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2)); + }); } -exports.fetchCollections = fetchCollections; +exports.addJSONConstants = addJSONConstants; +/** Fetch all the known SkyBlock stats as an array of strings */ +async function fetchStats() { + return await fetchJSONConstant('stats.json'); +} +exports.fetchStats = fetchStats; /** Add stats to skyblock-constants. This has caching so it's fine to call many times */ async function addStats(addingStats) { - if (addingStats.length === 0) - return; // no stats provided, just return - const file = await fetchFile('stats.json'); - if (!file.path) - return; - let oldStats; - try { - oldStats = JSON.parse(file.content); - } - catch { - // invalid json, set it as an empty array - oldStats = []; - } - const updatedStats = oldStats - .concat(addingStats) - // remove duplicates - .filter((value, index, array) => array.indexOf(value) === index) - .sort((a, b) => a.localeCompare(b)); - const newStats = updatedStats.filter(value => !oldStats.includes(value)); - // there's not actually any new stats, just return - if (newStats.length === 0) - return; - const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new stats` : `Add '${newStats[0]}'`; - await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2)); + await addJSONConstants('stats.json', addingStats, 'stats'); } exports.addStats = addStats; -/** Add stats to skyblock-constants. This has caching so it's fine to call many times */ +/** Fetch all the known SkyBlock collections as an array of strings */ +async function fetchCollections() { + return await fetchJSONConstant('collections.json'); +} +exports.fetchCollections = fetchCollections; +/** Add collections to skyblock-constants. This has caching so it's fine to call many times */ async function addCollections(addingCollections) { - if (addingCollections.length === 0) - return; // no stats provided, just return - const file = await fetchFile('collections.json'); - if (!file.path) - return; - let oldCollections; - try { - oldCollections = JSON.parse(file.content); - } - catch { - // invalid json, set it as an empty array - oldCollections = []; - } - const updatedCollections = oldCollections - .concat(addingCollections) - // remove duplicates - .filter((value, index, array) => array.indexOf(value) === index) - .sort((a, b) => a.localeCompare(b)); - const newCollections = updatedCollections.filter(value => !oldCollections.includes(value)); - // there's not actually any new stats, just return - if (newCollections.length === 0) - return; - const commitMessage = newCollections.length >= 2 ? `Add ${newCollections.length} new collections` : `Add '${newCollections[0]}'`; - await editFile(file, commitMessage, JSON.stringify(updatedCollections, null, 2)); + await addJSONConstants('collections.json', addingCollections, 'collections'); } exports.addCollections = addCollections; +/** Fetch all the known SkyBlock collections as an array of strings */ +async function fetchSkills() { + return await fetchJSONConstant('skills.json'); +} +exports.fetchSkills = fetchSkills; +/** Add skills to skyblock-constants. This has caching so it's fine to call many times */ +async function addSkills(addingSkills) { + await addJSONConstants('skills.json', addingSkills, 'skills'); +} +exports.addSkills = addSkills; +/** Fetch all the known SkyBlock collections as an array of strings */ +async function fetchZones() { + return await fetchJSONConstant('zones.json'); +} +exports.fetchZones = fetchZones; +/** Add skills to skyblock-constants. This has caching so it's fine to call many times */ +async function addZones(addingZones) { + await addJSONConstants('zones.json', addingZones, 'zones'); +} +exports.addZones = addZones; diff --git a/build/database.js b/build/database.js index f9f3509..ec1cbb2 100644 --- a/build/database.js +++ b/build/database.js @@ -175,7 +175,9 @@ async function updateDatabaseMember(member, profile) { // store the member in recentlyUpdated so it cant update for 3 more minutes recentlyUpdated.set(profile.uuid + member.uuid, true); await constants.addStats(Object.keys(member.rawHypixelStats)); - await constants.addCollections(member.collections.map(value => value.name)); + await constants.addCollections(member.collections.map(coll => coll.name)); + await constants.addSkills(member.skills.map(skill => skill.name)); + await constants.addZones(member.visited_zones.map(zone => zone.name)); const leaderboardAttributes = await getApplicableAttributes(member); await memberLeaderboardsCollection.updateOne({ uuid: member.uuid, diff --git a/build/hypixelCached.js b/build/hypixelCached.js index c4a3743..48b8ff0 100644 --- a/build/hypixelCached.js +++ b/build/hypixelCached.js @@ -40,12 +40,12 @@ const usernameCache = new node_cache_1.default({ const basicProfilesCache = new node_cache_1.default({ stdTTL: 60 * 10, checkperiod: 60, - useClones: false, + useClones: true, }); const playerCache = new node_cache_1.default({ stdTTL: 60, checkperiod: 10, - useClones: false, + useClones: true, }); const profileCache = new node_cache_1.default({ stdTTL: 30, @@ -78,6 +78,9 @@ function waitForSet(cache, key, value) { * @param user A user can be either a uuid or a username */ async function uuidFromUser(user) { + // if the user is 32 characters long, it has to be a uuid + if (util_1.undashUuid(user).length === 32) + return util_1.undashUuid(user); if (usernameCache.has(util_1.undashUuid(user))) { // check if the uuid is a key const username = usernameCache.get(util_1.undashUuid(user)); @@ -140,9 +143,8 @@ async function fetchPlayer(user) { if (!cleanPlayer) return; // clone in case it gets modified somehow later - const cleanPlayerClone = Object.assign({}, cleanPlayer); - playerCache.set(playerUuid, cleanPlayerClone); - usernameCache.set(playerUuid, cleanPlayerClone.username); + playerCache.set(playerUuid, cleanPlayer); + usernameCache.set(playerUuid, cleanPlayer.username); return cleanPlayer; } exports.fetchPlayer = fetchPlayer; |