broccolini-bot/scripts/bulk-lookup-users-v2.js

#!/usr/bin/env node
/**
 * Bulk lookup Discord user information - IMPROVED VERSION
 * 
 * Features:
 * - Saves progress incrementally (every 100 users)
 * - Can resume from where it left off
 * - Better error handling
 * - Uses guild member cache when possible
 * 
 * Usage:
 *   node scripts/bulk-lookup-users-v2.js <input_file> <output_file>
 */

const fs = require('fs');
const path = require('path');
const { Client, GatewayIntentBits } = require('discord.js');

// Load environment variables
const envPath = path.join(__dirname, '../../.env');
const result = require('dotenv').config({ path: envPath });

const TOKEN = process.env.DISCORD_BOT_TOKEN || process.env.DISCORD_TOKEN;

if (!TOKEN) {
    console.error('Error: DISCORD_BOT_TOKEN must be set in .env');
    process.exit(1);
}

// Parse command line args
const args = process.argv.slice(2);
if (args.length < 2) {
    console.error('Usage: node scripts/bulk-lookup-users-v2.js <input_file> <output_file>');
    process.exit(1);
}

const inputFile = args[0];
const outputFile = args[1];

// Read user IDs from input file
const userIds = fs.readFileSync(inputFile, 'utf-8')
    .split('\n')
    .map(line => line.trim())
    .filter(line => line.length > 0);

console.log(`✅ Loaded ${userIds.length} user IDs from ${inputFile}`);

// Load existing results if any (for resume capability)
let results = {};
let processed = 0;
let errors = 0;

if (fs.existsSync(outputFile)) {
    try {
        const existing = JSON.parse(fs.readFileSync(outputFile, 'utf-8'));
        results = existing.users || {};
        processed = Object.keys(results).length;
        errors = existing.errors || 0;
        console.log(`📂 Found existing results: ${processed} users already processed`);
    } catch (e) {
        console.log(`⚠️  Could not load existing results, starting fresh`);
    }
}

const client = new Client({
    intents: [
        GatewayIntentBits.Guilds,
        GatewayIntentBits.GuildMembers
    ]
});

async function lookupUser(userId) {
    // Skip if already processed
    if (results[userId]) {
        return results[userId];
    }
    
    try {
        const user = await client.users.fetch(userId);
        return {
            success: true,
            id: user.id,
            username: user.username,
            globalName: user.globalName || user.username,
            tag: user.tag,
            bot: user.bot,
            avatar: user.displayAvatarURL()
        };
    } catch (error) {
        return {
            success: false,
            id: userId,
            error: error.message,
            username: null,
            globalName: null,
            tag: null,
            bot: false
        };
    }
}

function saveResults() {
    const output = {
        timestamp: new Date().toISOString(),
        total_users: userIds.length,
        processed: processed,
        successful: processed - errors,
        errors: errors,
        users: results
    };
    
    fs.writeFileSync(outputFile, JSON.stringify(output, null, 2));
}

async function processUsers() {
    console.log('\n🚀 Starting bulk lookup...');
    console.log(`   Progress will be saved every 100 users\n`);
    
    const startTime = Date.now();
    const startProcessed = processed;
    
    // Filter out already processed users
    const toProcess = userIds.filter(id => !results[id]);
    console.log(`   ${toProcess.length} users remaining to process\n`);
    
    // Process one at a time (safer and can still be reasonably fast)
    for (let i = 0; i < toProcess.length; i++) {
        const userId = toProcess[i];
        
        const result = await lookupUser(userId);
        results[result.id] = result;
        
        if (!result.success) {
            errors++;
        }
        processed++;
        
        // Save every 100 users
        if (processed % 100 === 0) {
            saveResults();
            const elapsed = ((Date.now() - startTime) / 1000);
            const rate = (processed - startProcessed) / elapsed;
            const remaining = (toProcess.length - i - 1) / rate;
            console.log(`💾 Progress: ${processed}/${userIds.length} (${errors} errors) - saved checkpoint - ~${remaining.toFixed(0)}s remaining`);
        }
        
        // Slower delay to avoid rate limits (500ms = 2 requests/second - more reliable)
        await new Promise(resolve => setTimeout(resolve, 500));
    }
    
    // Final save
    saveResults();
    
    const totalTime = ((Date.now() - startTime) / 1000);
    
    console.log(`\n${'='.repeat(70)}`);
    console.log(`✅ Lookup Complete!`);
    console.log(`${'='.repeat(70)}`);
    console.log(`   Total time: ${totalTime.toFixed(1)}s`);
    console.log(`   Total processed: ${processed}/${userIds.length}`);
    console.log(`   Successful: ${processed - errors} (${((processed - errors)/userIds.length*100).toFixed(1)}%)`);
    console.log(`   Errors: ${errors}`);
    console.log(`   Rate: ${((processed - startProcessed)/totalTime).toFixed(1)} users/second`);
    console.log(`\n💾 Saved to: ${outputFile}\n`);
    
    // Sample successful results
    const sample = Object.values(results).filter(r => r.success).slice(0, 5);
    if (sample.length > 0) {
        console.log('📋 Sample results:');
        sample.forEach(u => console.log(`   ${u.username} (${u.id})`));
    }
    
    process.exit(0);
}

client.once('ready', () => {
    console.log(`✅ Bot logged in as ${client.user.tag}\n`);
    processUsers();
});

client.on('error', (error) => {
    console.error('❌ Discord client error:', error);
});

// Handle graceful shutdown
process.on('SIGINT', () => {
    console.log('\n\n⚠️  Interrupted! Saving progress...');
    saveResults();
    console.log('✅ Progress saved. You can resume by running the same command again.\n');
    process.exit(0);
});

client.login(TOKEN);