const { generateRandomData, parseLocator } = require('./config'); const { detectFormFields } = require('./fields'); const { detectSubmitButton } = require('./submit'); const { detectCaptcha, solveSimpleCaptcha } = require('./captcha'); const { classifyNewsletterForm } = require('./classifier'); async function processSite(browser, site, globalCreds, browserConfig, keepOpen = false) { let siteFailed = false; let failureReason = ''; let page = null; try { let creds; if (site.credentials && site.credentials.username) { creds = { ...globalCreds, ...site.credentials }; } else { creds = { ...globalCreds }; } if (!creds.username || !creds.email || !creds.password) { const randomData = generateRandomData(); creds = { ...randomData, ...creds }; } console.log(`\n${site.url}`); console.log(`${'-'.repeat(70)}`); page = await browser.newPage(); if (browserConfig.userAgent) { await page.setUserAgent(browserConfig.userAgent); } await page.goto(site.url, { waitUntil: 'networkidle2', timeout: 30000 }); const captchaInfo = await detectCaptcha(page); if (captchaInfo.length > 0) { console.log(`\n CAPTCHA DETECTED:`); for (const captcha of captchaInfo) { const visibility = captcha.isVisible ? '(visible)' : '(hidden)'; console.log(` - ${captcha.name} ${visibility}`); } if (captchaInfo.some(c => c.type === 'mathCaptcha')) { const solution = await solveSimpleCaptcha(page); if (solution && solution.answer) { console.log(` Auto-solving math: ${solution.text} = ${solution.answer}`); try { await page.evaluate((answer) => { const input = document.querySelector('input[name*="captcha"], input[class*="captcha"], #captcha'); if (input) { input.value = answer; input.dispatchEvent(new Event('input', { bubbles: true })); } }, solution.answer); } catch (e) { console.warn(` Failed to fill math answer: ${e.message}`); } } } if (captchaInfo.some(c => c.type === 'imageCaptcha')) { const solution = await solveSimpleCaptcha(page); if (solution && solution.type === 'image' && solution.answer) { console.log(` OCR solved image CAPTCHA: "${solution.answer}"`); try { await page.evaluate((answer) => { const input = document.querySelector('input[name*="captcha"], input[class*="captcha"], input[id*="captcha"], #captcha, .captcha-input'); if (input) { input.value = answer; input.dispatchEvent(new Event('input', { bubbles: true })); input.dispatchEvent(new Event('change', { bubbles: true })); } }, solution.answer); } catch (e) { console.warn(` Failed to fill OCR answer: ${e.message}`); } } } const visibleRecaptcha = captchaInfo.filter(c => (c.type === 'recaptcha' || c.type === 'hcaptcha') && c.isVisible); if (visibleRecaptcha.length > 0) { siteFailed = true; failureReason = `CAPTCHA present: ${visibleRecaptcha.map(c => c.name).join(', ')}`; console.error(` Cannot auto-solve ${visibleRecaptcha.map(c => c.name).join(', ')}`); return { siteFailed, failureReason, page }; } else if (captchaInfo.some(c => c.type === 'recaptcha' || c.type === 'hcaptcha')) { console.log(` CAPTCHA detected but not visible - attempting to proceed anyway`); } } const formClassification = await classifyNewsletterForm(page); console.log(`\n Form Classification: ${formClassification.isNewsletter ? 'Newsletter' : 'Not newsletter'} (${formClassification.confidence}/100 confidence)`); if (formClassification.signals.positive.length > 0) { console.log(' Positive signals:'); formClassification.signals.positive.forEach(signal => console.log(` + ${signal}`)); } if (formClassification.signals.negative.length > 0) { console.log(' Negative signals:'); formClassification.signals.negative.forEach(signal => console.log(` - ${signal}`)); } if (!formClassification.isNewsletter && formClassification.confidence < 40) { siteFailed = true; failureReason = 'Not a newsletter signup form'; console.error(` ${failureReason}`); } console.log('\nDETECTED FIELDS:'); let fieldsToFill = {}; let detectedFields = {}; const definedLocators = site.locators || {}; if (site.autoDetect !== false) { detectedFields = await detectFormFields(page); for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) { fieldsToFill[fieldName] = fieldInfo.selector; } } const hasEmailField = fieldsToFill.email || fieldsToFill.confirmEmail; if (!hasEmailField) { siteFailed = true; failureReason = 'No email field detected'; console.warn(`\n ${failureReason}`); } const hasSearchInput = await page.evaluate(() => { const inputs = document.querySelectorAll('input[type="search"], input[name*="search"], input[id*="search"]'); return inputs.length > 0; }).catch(() => false); if (hasSearchInput && !hasEmailField) { console.warn(` Search form detected instead of newsletter`); siteFailed = true; failureReason = failureReason || 'Search form instead of newsletter'; } for (const [fieldName, locator] of Object.entries(definedLocators)) { if (fieldName !== 'submit') { fieldsToFill[fieldName] = parseLocator(locator); } } if (Object.keys(fieldsToFill).length > 0) { console.log(`\n ${'FIELD'.padEnd(12)} | ${'SELECTOR'.padEnd(35)} | ${'CONF'.padEnd(4)} | VALUE`); console.log(' ' + '-'.repeat(75)); for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) { let valueToFill = creds[fieldName]; if (fieldName === 'confirmEmail') { valueToFill = creds['email']; } const selector = fieldInfo.selector.length > 34 ? fieldInfo.selector.substring(0, 31) + '...' : fieldInfo.selector; const value = valueToFill ? valueToFill.substring(0, 20) : 'N/A'; console.log(` ${fieldName.padEnd(12)} | ${selector.padEnd(35)} | ${fieldInfo.confidence.toString().padEnd(4)} | ${value}`); if (valueToFill) { try { await page.evaluate((sel, val) => { const el = document.querySelector(sel); if (el) { el.value = val; el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true })); } }, fieldInfo.selector, valueToFill); } catch (e) { console.warn(` Failed to fill ${fieldName}: ${e.message}`); siteFailed = true; failureReason = `Failed to fill ${fieldName}: ${e.message}`; } } } console.log(' ' + '-'.repeat(75)); } let submitSelector = null; if (definedLocators.submit) { submitSelector = parseLocator(definedLocators.submit); } else if (site.autoDetect !== false) { submitSelector = await detectSubmitButton(page); } if (submitSelector) { try { await page.click(submitSelector); console.log(` Submitted (${submitSelector})`); } catch (submitError) { siteFailed = true; failureReason = failureReason || `Submit failed: ${submitError.message}`; console.error(` Submit failed: ${submitError.message}`); } } else { siteFailed = true; failureReason = failureReason || 'No submit button found'; console.warn(` ${failureReason}`); } } catch (error) { siteFailed = true; failureReason = error.message; console.error(` Error: ${error.message}`); } finally { if (page && !keepOpen) { await page.close().catch(() => {}); } } return { siteFailed, failureReason, page }; } module.exports = { processSite };