newsletter_spammer/src/processor.js

const { generateRandomData, parseLocator } = require('./config');
const { detectFormFields } = require('./fields');
const { detectSubmitButton } = require('./submit');
const { detectCaptcha, solveSimpleCaptcha } = require('./captcha');
const { classifyNewsletterForm } = require('./classifier');

async function processSite(browser, site, globalCreds, browserConfig, keepOpen = false) {
    let siteFailed = false;
    let failureReason = '';
    let page = null;
    
    try {
        let creds;
        if (site.credentials && site.credentials.username) {
            creds = { ...globalCreds, ...site.credentials };
        } else {
            creds = { ...globalCreds };
        }
        
        if (!creds.username || !creds.email || !creds.password) {
            const randomData = generateRandomData();
            creds = { ...randomData, ...creds };
        }

        console.log(`\n${site.url}`);
        console.log(`${'-'.repeat(70)}`);
        
        page = await browser.newPage();

        if (browserConfig.userAgent) {
            await page.setUserAgent(browserConfig.userAgent);
        }

        await page.goto(site.url, { waitUntil: 'networkidle2', timeout: 30000 });
        
        const captchaInfo = await detectCaptcha(page);
        if (captchaInfo.length > 0) {
            console.log(`\n   CAPTCHA DETECTED:`);
            for (const captcha of captchaInfo) {
                const visibility = captcha.isVisible ? '(visible)' : '(hidden)';
                console.log(`      - ${captcha.name} ${visibility}`);
            }
            
            if (captchaInfo.some(c => c.type === 'mathCaptcha')) {
                const solution = await solveSimpleCaptcha(page);
                if (solution && solution.answer) {
                    console.log(`   Auto-solving math: ${solution.text} = ${solution.answer}`);
                    try {
                        await page.evaluate((answer) => {
                            const input = document.querySelector('input[name*="captcha"], input[class*="captcha"], #captcha');
                            if (input) {
                                input.value = answer;
                                input.dispatchEvent(new Event('input', { bubbles: true }));
                            }
                        }, solution.answer);
                    } catch (e) {
                        console.warn(`   Failed to fill math answer: ${e.message}`);
                    }
                }
            }
            
            if (captchaInfo.some(c => c.type === 'imageCaptcha')) {
                const solution = await solveSimpleCaptcha(page);
                if (solution && solution.type === 'image' && solution.answer) {
                    console.log(`   OCR solved image CAPTCHA: "${solution.answer}"`);
                    try {
                        await page.evaluate((answer) => {
                            const input = document.querySelector('input[name*="captcha"], input[class*="captcha"], input[id*="captcha"], #captcha, .captcha-input');
                            if (input) {
                                input.value = answer;
                                input.dispatchEvent(new Event('input', { bubbles: true }));
                                input.dispatchEvent(new Event('change', { bubbles: true }));
                            }
                        }, solution.answer);
                    } catch (e) {
                        console.warn(`   Failed to fill OCR answer: ${e.message}`);
                    }
                }
            }
            
            const visibleRecaptcha = captchaInfo.filter(c => (c.type === 'recaptcha' || c.type === 'hcaptcha') && c.isVisible);
            if (visibleRecaptcha.length > 0) {
                siteFailed = true;
                failureReason = `CAPTCHA present: ${visibleRecaptcha.map(c => c.name).join(', ')}`;
                console.error(`   Cannot auto-solve ${visibleRecaptcha.map(c => c.name).join(', ')}`);
                return { siteFailed, failureReason, page };
            } else if (captchaInfo.some(c => c.type === 'recaptcha' || c.type === 'hcaptcha')) {
                console.log(`   CAPTCHA detected but not visible - attempting to proceed anyway`);
            }
        }
        
        const formClassification = await classifyNewsletterForm(page);
        console.log(`\n   Form Classification: ${formClassification.isNewsletter ? 'Newsletter' : 'Not newsletter'} (${formClassification.confidence}/100 confidence)`);
        
        if (formClassification.signals.positive.length > 0) {
            console.log('   Positive signals:');
            formClassification.signals.positive.forEach(signal => console.log(`      + ${signal}`));
        }
        
        if (formClassification.signals.negative.length > 0) {
            console.log('   Negative signals:');
            formClassification.signals.negative.forEach(signal => console.log(`      - ${signal}`));
        }
        
        if (!formClassification.isNewsletter && formClassification.confidence < 40) {
            siteFailed = true;
            failureReason = 'Not a newsletter signup form';
            console.error(`   ${failureReason}`);
        }
        
        console.log('\nDETECTED FIELDS:');
        
        let fieldsToFill = {};
        let detectedFields = {};
        const definedLocators = site.locators || {};
        
        if (site.autoDetect !== false) {
            detectedFields = await detectFormFields(page);
            for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
                fieldsToFill[fieldName] = fieldInfo.selector;
            }
        }
        
        const hasEmailField = fieldsToFill.email || fieldsToFill.confirmEmail;
        if (!hasEmailField) {
            siteFailed = true;
            failureReason = 'No email field detected';
            console.warn(`\n   ${failureReason}`);
        }
        
        const hasSearchInput = await page.evaluate(() => {
            const inputs = document.querySelectorAll('input[type="search"], input[name*="search"], input[id*="search"]');
            return inputs.length > 0;
        }).catch(() => false);
        
        if (hasSearchInput && !hasEmailField) {
            console.warn(`   Search form detected instead of newsletter`);
            siteFailed = true;
            failureReason = failureReason || 'Search form instead of newsletter';
        }
        
        for (const [fieldName, locator] of Object.entries(definedLocators)) {
            if (fieldName !== 'submit') {
                fieldsToFill[fieldName] = parseLocator(locator);
            }
        }
        
        if (Object.keys(fieldsToFill).length > 0) {
            console.log(`\n   ${'FIELD'.padEnd(12)} | ${'SELECTOR'.padEnd(35)} | ${'CONF'.padEnd(4)} | VALUE`);
            console.log('   ' + '-'.repeat(75));
            
            for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
                let valueToFill = creds[fieldName];
                
                if (fieldName === 'confirmEmail') {
                    valueToFill = creds['email'];
                }
                
                const selector = fieldInfo.selector.length > 34 ? fieldInfo.selector.substring(0, 31) + '...' : fieldInfo.selector;
                const value = valueToFill ? valueToFill.substring(0, 20) : 'N/A';
                
                console.log(`   ${fieldName.padEnd(12)} | ${selector.padEnd(35)} | ${fieldInfo.confidence.toString().padEnd(4)} | ${value}`);
                
                if (valueToFill) {
                    try {
                        await page.evaluate((sel, val) => {
                            const el = document.querySelector(sel);
                            if (el) {
                                el.value = val;
                                el.dispatchEvent(new Event('input', { bubbles: true }));
                                el.dispatchEvent(new Event('change', { bubbles: true }));
                            }
                        }, fieldInfo.selector, valueToFill);
                    } catch (e) {
                        console.warn(`   Failed to fill ${fieldName}: ${e.message}`);
                        siteFailed = true;
                        failureReason = `Failed to fill ${fieldName}: ${e.message}`;
                    }
                }
            }
            console.log('   ' + '-'.repeat(75));
        }

        let submitSelector = null;
        if (definedLocators.submit) {
            submitSelector = parseLocator(definedLocators.submit);
        } else if (site.autoDetect !== false) {
            submitSelector = await detectSubmitButton(page);
        }
        
        if (submitSelector) {
            try {
                await page.click(submitSelector);
                console.log(`   Submitted (${submitSelector})`);
            } catch (submitError) {
                siteFailed = true;
                failureReason = failureReason || `Submit failed: ${submitError.message}`;
                console.error(`   Submit failed: ${submitError.message}`);
            }
        } else {
            siteFailed = true;
            failureReason = failureReason || 'No submit button found';
            console.warn(`   ${failureReason}`);
        }
        
    } catch (error) {
        siteFailed = true;
        failureReason = error.message;
        console.error(`   Error: ${error.message}`);
    } finally {
        if (page && !keepOpen) {
            await page.close().catch(() => {});
        }
    }
    
    return { siteFailed, failureReason, page };
}

module.exports = {
    processSite
};