diff --git a/scripts/populate-native-tokens.js b/scripts/populate-native-tokens.js deleted file mode 100644 index 156decd..0000000 --- a/scripts/populate-native-tokens.js +++ /dev/null @@ -1,191 +0,0 @@ -const fs = require('fs'); -const path = require('path'); -const axios = require('axios'); -require('dotenv').config(); - -const REGISTRY_PATH = path.join(__dirname, '../../l1-registry/data'); - -const GLACIER_API_BASE = process.env.GLACIER_API_BASE || 'https://glacier-api.avax.network/v1'; -const GLACIER_API_KEY = process.env.GLACIER_API_KEY; - -if (!GLACIER_API_KEY) { - console.error('Error: GLACIER_API_KEY environment variable is required'); - process.exit(1); -} - -const stats = { - total: 0, - glacierSuccess: 0, - noData: 0, - descriptionsAdded: 0, - rpcUrlsAdded: 0, - errors: [] -}; - -async function fetchGlacierData() { - console.log('Fetching chain data from Glacier API...'); - - try { - const response = await axios.get(`${GLACIER_API_BASE}/chains`, { - timeout: 60000, - headers: { - 'Accept': 'application/json', - 'User-Agent': 'l1beat-backend', - 'x-glacier-api-key': GLACIER_API_KEY - } - }); - - console.log(`Fetched ${response.data.chains.length} chains from Glacier API`); - return response.data.chains; - } catch (error) { - console.error('Error fetching from Glacier API:', error.message); - throw error; - } -} - -function processChain(folderName, chainData, glacierMap) { - const subnetId = chainData.subnetId; - const glacierChain = glacierMap.get(subnetId); - - let updated = false; - - if (!chainData.chains || chainData.chains.length === 0) { - stats.errors.push({ folder: folderName, error: 'No chains array found' }); - return false; - } - - chainData.chains.forEach(chain => { - delete chain.assets; - - if (glacierChain?.networkToken) { - chain.nativeToken = { - symbol: glacierChain.networkToken.symbol, - name: glacierChain.networkToken.name, - decimals: glacierChain.networkToken.decimals - }; - updated = true; - } else { - chain.nativeToken = {}; - } - - if (!chain.description && glacierChain?.description) { - chain.description = glacierChain.description; - stats.descriptionsAdded++; - } - - if (chain.rpcUrls.length === 0 && glacierChain?.rpcUrl) { - chain.rpcUrls = [glacierChain.rpcUrl]; - stats.rpcUrlsAdded++; - } - }); - - return updated; -} - -async function main() { - console.log('Starting population script...\n'); - - const glacierChains = await fetchGlacierData(); - const glacierMap = new Map(); - glacierChains.forEach(chain => { - glacierMap.set(chain.subnetId, chain); - }); - - console.log(`Created lookup map with ${glacierMap.size} Glacier chains\n`); - - if (!fs.existsSync(REGISTRY_PATH)) { - console.error(`Registry path not found: ${REGISTRY_PATH}`); - process.exit(1); - } - - const folders = fs.readdirSync(REGISTRY_PATH, { withFileTypes: true }) - .filter(dirent => dirent.isDirectory()) - .filter(dirent => !dirent.name.startsWith('.') && !dirent.name.startsWith('_')) - .map(dirent => dirent.name); - - console.log(`Found ${folders.length} chain folders in registry\n`); - stats.total = folders.length; - - for (let i = 0; i < folders.length; i++) { - const folderName = folders[i]; - const chainJsonPath = path.join(REGISTRY_PATH, folderName, 'chain.json'); - - try { - if (!fs.existsSync(chainJsonPath)) { - console.log(`[${i + 1}/${folders.length}] ⚠️ ${folderName}: chain.json not found`); - stats.errors.push({ folder: folderName, error: 'chain.json not found' }); - continue; - } - - const rawData = fs.readFileSync(chainJsonPath, 'utf8'); - const chainData = JSON.parse(rawData); - - const hasGlacierData = processChain(folderName, chainData, glacierMap); - - fs.writeFileSync(chainJsonPath, JSON.stringify(chainData, null, 2) + '\n'); - - const chainName = chainData.name || folderName; - const nativeToken = chainData.chains[0]?.nativeToken; - const hasToken = nativeToken?.symbol ? `${nativeToken.symbol}` : 'NONE'; - - if (hasGlacierData) { - console.log(`[${i + 1}/${folders.length}] ✓ ${chainName}: ${hasToken}`); - stats.glacierSuccess++; - } else { - console.log(`[${i + 1}/${folders.length}] ⚠️ ${chainName}: No Glacier data`); - stats.noData++; - } - - } catch (error) { - console.error(`[${i + 1}/${folders.length}] ✗ ${folderName}: ${error.message}`); - stats.errors.push({ folder: folderName, error: error.message }); - } - } - - console.log('\n' + '='.repeat(60)); - console.log('SUMMARY'); - console.log('='.repeat(60)); - console.log(`Total chains processed: ${stats.total}`); - console.log(`Glacier data found: ${stats.glacierSuccess}`); - console.log(`No Glacier data: ${stats.noData}`); - console.log(`Descriptions added: ${stats.descriptionsAdded}`); - console.log(`RPC URLs added: ${stats.rpcUrlsAdded}`); - console.log(`Errors: ${stats.errors.length}`); - - if (stats.errors.length > 0) { - console.log('\nErrors encountered:'); - stats.errors.forEach(({ folder, error }) => { - console.log(` - ${folder}: ${error}`); - }); - } - - if (stats.noData > 0) { - console.log('\nChains requiring manual review (no Glacier data):'); - const folders = fs.readdirSync(REGISTRY_PATH, { withFileTypes: true }) - .filter(dirent => dirent.isDirectory()) - .filter(dirent => !dirent.name.startsWith('.') && !dirent.name.startsWith('_')) - .map(dirent => dirent.name); - - for (const folder of folders) { - const chainJsonPath = path.join(REGISTRY_PATH, folder, 'chain.json'); - try { - const data = JSON.parse(fs.readFileSync(chainJsonPath, 'utf8')); - const nativeToken = data.chains[0]?.nativeToken; - if (!nativeToken?.symbol) { - console.log(` - ${folder} (${data.name})`); - } - } catch (e) { - // Skip - } - } - } - - console.log('\n' + '='.repeat(60)); - console.log('Migration complete!'); - console.log('='.repeat(60)); -} - -main().catch(error => { - console.error('Fatal error:', error); - process.exit(1); -}); diff --git a/src/routes/cumulativeTxCountRoutes.js b/src/routes/cumulativeTxCountRoutes.js index 74a09d0..c6d0f43 100644 --- a/src/routes/cumulativeTxCountRoutes.js +++ b/src/routes/cumulativeTxCountRoutes.js @@ -50,6 +50,60 @@ router.get('/chains/:chainId/cumulativeTxCount/latest', validate(validators.getC } }); +// Get latest cumulative tx count for all chains in a single call +router.get('/cumulativeTxCount/all/latest', async (req, res) => { + try { + // Get all chains with valid numeric IDs + const chains = await Chain.find({ + evmChainId: { $exists: true, $ne: null } + }).select('evmChainId chainName subnetId').lean(); + + const chainIds = chains.map(c => String(c.evmChainId)); + + // Fetch latest cumulative tx count for all chains in one aggregation + const txCountRecords = await CumulativeTxCount.aggregate([ + { $match: { chainId: { $in: chainIds } } }, + { $sort: { chainId: 1, timestamp: -1 } }, + { + $group: { + _id: '$chainId', + value: { $first: '$value' }, + timestamp: { $first: '$timestamp' } + } + } + ]); + + // Create a map for O(1) lookup + const txCountMap = new Map(txCountRecords.map(r => [r._id, r])); + + // Build response with chain info + const data = chains.map(chain => { + const txCount = txCountMap.get(String(chain.evmChainId)); + return { + evmChainId: chain.evmChainId, + subnetId: chain.subnetId, + chainName: chain.chainName, + cumulativeTxCount: txCount ? { + value: txCount.value, + timestamp: txCount.timestamp + } : null + }; + }); + + res.json({ + success: true, + count: data.length, + data + }); + } catch (error) { + logger.error('All CumulativeTxCount Error:', { error: error.message }); + res.status(500).json({ + success: false, + error: error.message + }); + } +}); + // Health check endpoint router.get('/cumulativeTxCount/health', async (req, res) => { try { diff --git a/src/routes/tpsRoutes.js b/src/routes/tpsRoutes.js index cac2ea5..6d9e9d1 100644 --- a/src/routes/tpsRoutes.js +++ b/src/routes/tpsRoutes.js @@ -48,6 +48,60 @@ router.get('/chains/:chainId/tps/latest', validate(validators.getLatestTps), asy } }); +// Get latest TPS for all chains in a single call +router.get('/tps/all/latest', async (req, res) => { + try { + // Get all chains with valid numeric IDs + const chains = await Chain.find({ + evmChainId: { $exists: true, $ne: null } + }).select('evmChainId chainName subnetId').lean(); + + const chainIds = chains.map(c => String(c.evmChainId)); + + // Fetch latest TPS for all chains in one aggregation + const tpsRecords = await TPS.aggregate([ + { $match: { chainId: { $in: chainIds } } }, + { $sort: { chainId: 1, timestamp: -1 } }, + { + $group: { + _id: '$chainId', + value: { $first: '$value' }, + timestamp: { $first: '$timestamp' } + } + } + ]); + + // Create a map for O(1) lookup + const tpsMap = new Map(tpsRecords.map(r => [r._id, r])); + + // Build response with chain info + const data = chains.map(chain => { + const tps = tpsMap.get(String(chain.evmChainId)); + return { + evmChainId: chain.evmChainId, + subnetId: chain.subnetId, + chainName: chain.chainName, + tps: tps ? { + value: parseFloat(tps.value).toFixed(2), + timestamp: tps.timestamp + } : null + }; + }); + + res.json({ + success: true, + count: data.length, + data + }); + } catch (error) { + logger.error('All TPS Error:', { error: error.message }); + res.status(500).json({ + success: false, + error: error.message + }); + } +}); + // Add new route for total network TPS router.get('/tps/network/latest', async (req, res) => { try { diff --git a/src/services/chainService.js b/src/services/chainService.js index 9ecc0ff..8e8415e 100644 --- a/src/services/chainService.js +++ b/src/services/chainService.js @@ -1,7 +1,6 @@ const Chain = require('../models/chain'); const axios = require('axios'); const config = require('../config/config'); -const tpsService = require('../services/tpsService'); const cacheManager = require('../utils/cacheManager'); const logger = require('../utils/logger'); @@ -49,103 +48,21 @@ class ChainService { }, { $project: { + _id: 0, + __v: 0, validators: 0, - registryMetadata: 0 + registryMetadata: 0, + assets: 0 } } ]); logger.info(`Fetched ${chains.length} chains in ${Date.now() - startTime}ms`); - // Collect all valid chain IDs for batch fetching - const chainIdMap = new Map(); // Maps chainIdForTps -> chain object - chains.forEach(chain => { - const chainIdForTps = chain.evmChainId || chain.chainId; - if (chainIdForTps && /^\d+$/.test(String(chainIdForTps))) { - chainIdMap.set(String(chainIdForTps), chain); - } - }); - - const chainIds = Array.from(chainIdMap.keys()); - logger.info(`Found ${chainIds.length} chains with valid numeric IDs for metrics fetch`); - - if (chainIds.length === 0) { - // No chains with valid IDs, return as-is - return chains; - } - - // Batch fetch latest TPS for all chains in a single aggregation query - const batchStartTime = Date.now(); - const TPS = require('../models/tps'); - const CumulativeTxCount = require('../models/cumulativeTxCount'); - - // Fetch all latest TPS records in one aggregation - const tpsRecords = await TPS.aggregate([ - { $match: { chainId: { $in: chainIds } } }, - { $sort: { chainId: 1, timestamp: -1 } }, - { - $group: { - _id: '$chainId', - value: { $first: '$value' }, - timestamp: { $first: '$timestamp' } - } - } - ]); - - // Fetch all latest TxCount records in one aggregation - const txCountRecords = await CumulativeTxCount.aggregate([ - { $match: { chainId: { $in: chainIds } } }, - { $sort: { chainId: 1, timestamp: -1 } }, - { - $group: { - _id: '$chainId', - value: { $first: '$value' }, - timestamp: { $first: '$timestamp' } - } - } - ]); - - logger.info(`Batch fetched metrics for ${chainIds.length} chains in ${Date.now() - batchStartTime}ms (TPS: ${tpsRecords.length}, TxCount: ${txCountRecords.length})`); - - // Create Maps for O(1) lookup - const tpsMap = new Map(tpsRecords.map(r => [r._id, { value: r.value, timestamp: r.timestamp }])); - const txCountMap = new Map(txCountRecords.map(r => [r._id, { value: r.value, timestamp: r.timestamp }])); - - // Merge metrics data with chains in-memory - const chainsWithMetrics = chains.map(chain => { - const chainIdForTps = String(chain.evmChainId || chain.chainId); - - if (!chainIdMap.has(chainIdForTps)) { - // Chain doesn't have valid numeric ID - return { - ...chain, - tps: null, - cumulativeTxCount: null - }; - } - - const tpsData = tpsMap.get(chainIdForTps); - const txCountData = txCountMap.get(chainIdForTps); - - return { - ...chain, - tps: tpsData ? { - value: parseFloat(tpsData.value).toFixed(2), - timestamp: tpsData.timestamp - } : null, - cumulativeTxCount: txCountData ? { - value: txCountData.value, - timestamp: txCountData.timestamp - } : null - }; - }); - - logger.info(`Total getAllChains execution time: ${Date.now() - startTime}ms`); - // Cache the result for 5 minutes - cacheManager.set(cacheKey, chainsWithMetrics, config.cache.chains); + cacheManager.set(cacheKey, chains, config.cache.chains); - return chainsWithMetrics; + return chains; } catch (error) { logger.error('Error fetching chains:', { error: error.message }); throw new Error(`Error fetching chains: ${error.message}`); diff --git a/src/services/teleporterService.js b/src/services/teleporterService.js index d38765e..0a464c6 100644 --- a/src/services/teleporterService.js +++ b/src/services/teleporterService.js @@ -177,23 +177,23 @@ class TeleporterService { // Check if the message is within our time range if (timestampInSeconds >= startTime) { validMessages.push(message); - } else { - // Found a message older than our time window, stop pagination - reachedTimeLimit = true; - logger.info(`[TELEPORTER ${updateLabel}] Found message older than ${hoursAgo} hours, stopping pagination`, { - messageTimestamp: new Date(timestampInSeconds * 1000).toISOString(), - startTime: new Date(startTime * 1000).toISOString(), - page: pageCount, - messageId: message.messageId || 'unknown' - }); - break; } + // Don't stop on individual old messages - they may not be chronological } // Add valid messages from this page to our collection allMessages = allMessages.concat(validMessages); nextPageToken = response.data?.nextPageToken; + // Only stop if we got a full page with NO valid messages + // This means we've definitely gone past our time range + if (validMessages.length === 0 && messages.length > 0) { + reachedTimeLimit = true; + logger.info(`[TELEPORTER ${updateLabel}] Full page of old messages (0/${messages.length} valid), stopping pagination`, { + page: pageCount + }); + } + // Calculate estimated time remaining for weekly updates const elapsedTime = (Date.now() - fetchStartTime) / 1000; const avgTimePerPage = elapsedTime / pageCount;