test-enhanced.js

/**
 * Test Script for Enhanced Website Crawler API
 * This script tests the advanced features and JSON response functionality
 */

const API_BASE_URL = 'http://localhost:3000/api';

// Test configuration with advanced options
const testConfigs = [
    {
        name: 'Wikipedia Advanced Test',
        url: 'https://en.wikipedia.org/wiki/Artificial_intelligence',
        options: {
            extractHeadings: true,
            extractLinks: true,
            extractImages: true,
            extractText: false,
            extractMetadata: true,
            respectRobots: true,
            followRedirects: true,
            timeout: 30000,
            userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
            maxRedirects: 5
        },
        selectors: {
            title: '#firstHeading',
            summary: '#mw-content-text > div.mw-parser-output > p:first-of-type',
            tableOfContents: '#toc .toctext',
            categories: '#catlinks .mw-normal-catlinks ul li a',
            infobox: '.infobox',
            languages: '.interlanguage-link-target',
            lastModified: '#footer-info-lastmod'
        }
    },
    {
        name: 'Hacker News Test',
        url: 'https://news.ycombinator.com',
        options: {
            extractHeadings: true,
            extractLinks: true,
            extractImages: false,
            extractText: false,
            extractMetadata: true,
            timeout: 20000
        },
        selectors: {
            stories: '.titlelink',
            points: '.score',
            comments: '.subtext a[href*="item"]',
            users: '.hnuser',
            ranks: '.rank'
        }
    },
    {
        name: 'E-commerce Example',
        url: 'https://example.com',
        options: {
            extractHeadings: true,
            extractLinks: true,
            extractImages: true,
            extractText: true,
            extractMetadata: true,
            includeRawContent: false,
            timeout: 15000
        },
        selectors: {
            title: 'h1',
            description: 'meta[name="description"]',
            keywords: 'meta[name="keywords"]'
        }
    }
];

async function testCrawler(config) {
    console.log(`\n🕷️ Testing: ${config.name}`);
    console.log(`📍 URL: ${config.url}`);
    
    try {
        const startTime = Date.now();
        
        const response = await fetch(`${API_BASE_URL}/crawler/crawl`, {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
                'Accept': 'application/json'
            },
            body: JSON.stringify({
                url: config.url,
                ...config.options,
                selectors: config.selectors
            })
        });
        
        const endTime = Date.now();
        const requestDuration = endTime - startTime;
        
        if (!response.ok) {
            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
        }
        
        const data = await response.json();
        
        // Display results summary
        console.log('✅ Request successful');
        console.log(`⏱️ Request Duration: ${requestDuration}ms`);
        console.log(`🌐 Crawl Duration: ${data.duration}ms`);
        console.log(`📊 Status: ${data.status}`);
        console.log(`📦 Content Size: ${formatBytes(data.metadata?.contentLength || 0)}`);
        
        // Analyze extracted data
        const customData = data.html?.custom || {};
        const headings = data.html?.headings || {};
        const links = data.html?.links || [];
        const images = data.html?.images || [];
        
        console.log('\n📄 Extracted Data Summary:');
        console.log(`  🎯 Custom Fields: ${Object.keys(customData).length}`);
        console.log(`  📚 Headings: ${Object.values(headings).flat().length}`);
        console.log(`  🔗 Links: ${links.length}`);
        console.log(`  🖼️ Images: ${images.length}`);
        
        // Show custom extracted fields
        if (Object.keys(customData).length > 0) {
            console.log('\n🔍 Custom Extracted Fields:');
            Object.entries(customData).forEach(([key, value]) => {
                const preview = Array.isArray(value) ? 
                    `[${value.length} items]` : 
                    String(value).substring(0, 100) + (String(value).length > 100 ? '...' : '');
                console.log(`  ${key}: ${preview}`);
            });
        }
        
        // Calculate link analysis
        const internalLinks = links.filter(link => 
            link.href && (link.href.startsWith('/') || link.href.includes(new URL(config.url).hostname))
        ).length;
        const externalLinks = links.length - internalLinks;
        
        console.log('\n🔗 Link Analysis:');
        console.log(`  🏠 Internal Links: ${internalLinks}`);
        console.log(`  🌐 External Links: ${externalLinks}`);
        
        // JSON Response analysis
        const jsonString = JSON.stringify(data, null, 2);
        const jsonSize = new Blob([jsonString]).size;
        
        console.log('\n📋 JSON Response:');
        console.log(`  📦 Size: ${formatBytes(jsonSize)}`);
        console.log(`  🔧 Formatted: Yes (2-space indentation)`);
        console.log(`  ✅ Valid JSON: ${isValidJSON(jsonString)}`);
        
        // Performance evaluation
        const performance = evaluatePerformance(data.duration, jsonSize);
        console.log(`\n⚡ Performance: ${performance.grade} (${performance.description})`);
        
        return {
            success: true,
            data,
            requestDuration,
            jsonSize,
            performance
        };
        
    } catch (error) {
        console.error('❌ Test failed:', error.message);
        return {
            success: false,
            error: error.message
        };
    }
}

function formatBytes(bytes) {
    if (bytes === 0) return '0 B';
    const k = 1024;
    const sizes = ['B', 'KB', 'MB', 'GB'];
    const i = Math.floor(Math.log(bytes) / Math.log(k));
    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}

function isValidJSON(str) {
    try {
        JSON.parse(str);
        return true;
    } catch {
        return false;
    }
}

function evaluatePerformance(duration, size) {
    if (duration < 1000 && size < 100000) {
        return { grade: 'A+', description: 'Excellent - Fast response with compact data' };
    } else if (duration < 3000 && size < 500000) {
        return { grade: 'A', description: 'Very Good - Good balance of speed and data' };
    } else if (duration < 5000 && size < 1000000) {
        return { grade: 'B', description: 'Good - Acceptable performance' };
    } else if (duration < 10000) {
        return { grade: 'C', description: 'Fair - Could be optimized' };
    } else {
        return { grade: 'D', description: 'Poor - Needs optimization' };
    }
}

async function runAllTests() {
    console.log('🚀 Starting Enhanced Website Crawler API Tests');
    console.log('================================================');
    
    const results = [];
    
    for (const config of testConfigs) {
        const result = await testCrawler(config);
        results.push({ config: config.name, ...result });
        
        // Wait between tests to avoid rate limiting
        await new Promise(resolve => setTimeout(resolve, 2000));
    }
    
    // Summary report
    console.log('\n📊 Test Summary');
    console.log('================');
    
    const successful = results.filter(r => r.success).length;
    const failed = results.filter(r => !r.success).length;
    
    console.log(`✅ Successful: ${successful}/${results.length}`);
    console.log(`❌ Failed: ${failed}/${results.length}`);
    
    if (successful > 0) {
        const avgDuration = results
            .filter(r => r.success)
            .reduce((sum, r) => sum + r.data.duration, 0) / successful;
        
        const totalSize = results
            .filter(r => r.success)
            .reduce((sum, r) => sum + r.jsonSize, 0);
        
        console.log(`⚡ Average Response Time: ${Math.round(avgDuration)}ms`);
        console.log(`📦 Total JSON Size: ${formatBytes(totalSize)}`);
    }
    
    console.log('\n🏁 All tests completed!');
}

// Export the functions for use in other scripts
if (typeof module !== 'undefined' && module.exports) {
    module.exports = {
        testCrawler,
        runAllTests,
        testConfigs,
        formatBytes,
        evaluatePerformance
    };
} else {
    // Browser environment - run tests
    runAllTests().catch(console.error);
}