diff --git a/failed_sites.txt b/failed_sites.txt new file mode 100644 index 0000000..e8e4f2a --- /dev/null +++ b/failed_sites.txt @@ -0,0 +1,19 @@ +DATE | TIME | WEBSITE | REASON +---------------------------------------------------------------------------------------------------- +14/02/2026 | 20:31:58 | Site_3 (https://kulalaland.com/) | No email field detected +14/02/2026 | 20:42:16 | Dutch News Test (https://www.dut | CAPTCHA present: reCAPTCHA +14/02/2026 | 20:43:17 | Site_1 (https://www.dutchnews.nl | CAPTCHA present: reCAPTCHA +14/02/2026 | 20:43:17 | Site_3 (https://kulalaland.com/) | No email field detected +14/02/2026 | 20:45:19 | Site_1 (https://www.dutchnews.nl | CAPTCHA present: reCAPTCHA +14/02/2026 | 20:45:19 | Site_2 (https://immaculatevegan. | Not a newsletter signup form +14/02/2026 | 20:45:19 | Site_3 (https://kulalaland.com/) | No email field detected +14/02/2026 | 20:46:06 | Site_1 (https://www.dutchnews.nl | CAPTCHA present: reCAPTCHA +14/02/2026 | 20:46:06 | Site_3 (https://kulalaland.com/) | No email field detected +14/02/2026 | 21:26:47 | Dutch News Test (https://www.dut | Navigating frame was detached +14/02/2026 | 21:26:47 | Immaculate Vegan (https://immacu | Connection closed. +14/02/2026 | 21:26:47 | Gate6 (https://gate6.vn/) | Connection closed. +14/02/2026 | 21:26:47 | Kulala Land (https://kulalaland. | Connection closed. +14/02/2026 | 21:26:47 | sneakywholefoods (https://www.sn | Connection closed. +14/02/2026 | 21:27:40 | Dutch News Test (https://www.dut | CAPTCHA present: reCAPTCHA +14/02/2026 | 21:27:40 | Kulala Land (https://kulalaland. | No email field detected +14/02/2026 | 21:27:40 | sneakywholefoods (https://www.sn | Not a newsletter signup form diff --git a/index.js b/index.js index a980805..d0eda46 100644 --- a/index.js +++ b/index.js @@ -1,437 +1,46 @@ const fs = require('fs'); const path = require('path'); -const toml = require('@iarna/toml'); const puppeteer = require('puppeteer-extra'); const StealthPlugin = require('puppeteer-extra-plugin-stealth'); puppeteer.use(StealthPlugin()); -const FIELD_PATTERNS = { - email: { - priority: 1, - types: ['email'], - names: ['email', 'e-mail', 'mail', 'user_email', 'login_email', 'email_address', 'emailaddress'], - ids: ['email', 'e-mail', 'mail', 'user-email', 'login-email'], - placeholders: ['email', 'e-mail', 'email address', 'your email', 'enter email'], - autocomplete: ['email'] - }, - confirmEmail: { - priority: 2, - types: ['email'], - names: ['confirm_email', 'confirmemail', 'email_confirm', 'verify_email', 'email_verify'], - ids: ['confirm-email', 'confirmemail', 'verify-email', 'email-confirm'], - placeholders: ['confirm email', 'verify email', 're-enter email', 'repeat email'], - mustHaveConfirmIndicator: true - }, - firstName: { - priority: 3, - types: ['text'], - names: ['first_name', 'firstname', 'fname', 'first-name', 'vorname', 'prenom', 'fname'], - ids: ['first-name', 'firstname', 'fname', 'firstName', 'mce-fname'], - placeholders: ['first name', 'firstname', 'first'], - autocomplete: ['given-name', 'first-name'] - }, - lastName: { - priority: 4, - types: ['text'], - names: ['last_name', 'lastname', 'lname', 'surname', 'family_name', 'last-name', 'lname'], - ids: ['last-name', 'lastname', 'lname', 'surname', 'lastName', 'mce-lname'], - placeholders: ['last name', 'lastname', 'surname', 'last'], - autocomplete: ['family-name', 'last-name'] - }, - username: { - priority: 5, - types: ['text'], - names: ['username', 'user_name', 'login', 'usr', 'uname', 'user-id', 'userid', 'account_name', 'login_id'], - ids: ['username', 'user-name', 'login', 'usr', 'uname'], - placeholders: ['username', 'user name', 'login', 'username or email'], - autocomplete: ['username'] - }, - password: { - priority: 6, - types: ['password'], - names: ['password', 'pass', 'pwd', 'user_password', 'login_password'], - ids: ['password', 'pass', 'pwd'], - placeholders: ['password', 'pass', 'enter password'], - autocomplete: ['current-password', 'new-password'] - }, - fullName: { - priority: 7, - types: ['text'], - names: ['full_name', 'fullname', 'name'], - ids: ['full-name', 'fullname', 'name'], - placeholders: ['full name', 'your name', 'complete name'], - autocomplete: ['name'] - }, - phone: { - priority: 8, - types: ['tel', 'text'], - names: ['phone', 'telephone', 'mobile', 'cell', 'cellphone', 'phone_number'], - ids: ['phone', 'telephone', 'mobile', 'cell', 'tel'], - placeholders: ['phone', 'telephone', 'mobile number'], - autocomplete: ['tel'] - }, - age: { - priority: 9, - types: ['number', 'text'], - names: ['age', 'user_age'], - ids: ['age'], - placeholders: ['age', 'your age'] - }, - company: { - priority: 10, - types: ['text'], - names: ['company', 'organization', 'company_name', 'employer'], - ids: ['company', 'organization', 'employer'], - placeholders: ['company', 'organization'], - autocomplete: ['organization'] - } -}; - -function loadConfig() { - try { - const configPath = path.join(__dirname, 'config.toml'); - const tomlString = fs.readFileSync(configPath, 'utf-8'); - const config = toml.parse(tomlString); - console.log('āœ… Configuration loaded successfully.'); - return config; - } catch (error) { - console.error('āŒ Failed to load or parse config.toml:', error.message); - process.exit(1); - } -} - -function generateRandomData() { - const randomString = Math.random().toString(36).substring(2, 10); - const username = `user${randomString}`; - const email = `${username}@gmail.com`; - const password = `pass${randomString}`; - const age = Math.floor(Math.random() * 30 + 20).toString(); - const firstName = 'John'; - const lastName = 'Doe'; - const phone = '+31610488717'; - const company = 'Example Corp'; - return { username, email, password, age, firstName, lastName, phone, company }; -} - -function parseLocator(locatorStr) { - locatorStr = locatorStr.trim(); - if (!locatorStr.startsWith('<')) { - return locatorStr; - } - - const getAttr = (attr) => { - const match = locatorStr.match(new RegExp(`${attr}=["']([^"']+)["']`)); - return match ? match[1] : null; - }; - - const tagName = (locatorStr.match(/^<([a-zA-Z0-9]+)/) || [])[1] || ''; - const id = getAttr('id'); - if (id) { - if (/^\d/.test(id)) { - return `${tagName}[id='${id}']`; - } - return `${tagName}#${id}`; - } - - const name = getAttr('name'); - if (name) return `${tagName}[name='${name}']`; - - const placeholder = getAttr('placeholder'); - if (placeholder) return `${tagName}[placeholder='${placeholder}']`; - - const type = getAttr('type'); - if (type) return `${tagName}[type='${type}']`; - - return tagName; -} - -async function detectFormFields(page) { - console.log('šŸ” Auto-detecting form fields...'); - - const detectedFields = await page.evaluate((patterns) => { - const results = {}; - const usedElements = new Set(); - - // Get all visible input fields, excluding hidden fields - const inputs = Array.from(document.querySelectorAll('input, select, textarea')).filter(input => { - const type = input.type?.toLowerCase() || 'text'; - const style = window.getComputedStyle(input); - const rect = input.getBoundingClientRect(); - - // Skip hidden fields and honeypots - if (type === 'hidden' || type === 'submit' || type === 'button') return false; - if (input.tabIndex === -1 && input.style.position === 'absolute') return false; // Likely honeypot - if (rect.width === 0 || rect.height === 0) return false; - if (style.display === 'none' || style.visibility === 'hidden') return false; - - return true; - }); - - const calculateScore = (input, pattern, fieldName) => { - let score = 0; - const inputType = (input.type || 'text').toLowerCase(); - const inputName = (input.name || '').toLowerCase(); - const inputId = (input.id || '').toLowerCase(); - const placeholder = (input.placeholder || '').toLowerCase(); - const autocomplete = (input.autocomplete || '').toLowerCase(); - - // Type matching - if (pattern.types && pattern.types.includes(inputType)) { - score += 5; - } - - // Name matching - exact matches get high scores - if (pattern.names) { - for (const name of pattern.names) { - const nameLower = name.toLowerCase(); - if (inputName === nameLower) { - score += 20; // Exact match - break; - } else if (inputName.includes(nameLower)) { - // Partial match - be careful not to match fname with username - if (fieldName === 'firstName' && (inputName.includes('user') || inputName.includes('login'))) { - score -= 5; // Penalize if it looks like username - } else if (fieldName === 'lastName' && (inputName.includes('user') || inputName.includes('login'))) { - score -= 5; - } else { - score += 10; - } - break; - } - } - } - - // ID matching - very specific - if (pattern.ids) { - for (const id of pattern.ids) { - const idLower = id.toLowerCase(); - if (inputId === idLower) { - score += 20; // Exact match - break; - } else if (inputId.includes(idLower)) { - score += 10; - break; - } - } - } - - // Placeholder matching - if (pattern.placeholders) { - for (const ph of pattern.placeholders) { - const phLower = ph.toLowerCase(); - if (placeholder === phLower) { - score += 15; - break; - } else if (placeholder.includes(phLower)) { - score += 8; - break; - } - } - } - - // Autocomplete matching - if (pattern.autocomplete) { - for (const ac of pattern.autocomplete) { - if (autocomplete === ac.toLowerCase()) { - score += 12; - break; - } - } - } - - // Check label text - const labels = input.labels; - if (labels && labels.length > 0) { - const labelText = labels[0].textContent.toLowerCase(); - const fieldNameLower = fieldName.toLowerCase().replace(/([A-Z])/g, ' $1').trim(); - if (labelText.includes(fieldNameLower) || labelText.includes(pattern.names?.[0] || '')) { - score += 15; - } - } - - // Penalize if field appears to be a honeypot - if (inputName.includes('b_') || inputId.includes('b_')) { - score -= 30; // Likely Mailchimp honeypot - } - - // Penalize disabled/readonly - if (input.disabled || input.readOnly) { - score -= 20; - } - - // Special handling for confirmEmail - if (pattern.mustHaveConfirmIndicator) { - const hasConfirm = inputId.includes('confirm') || inputId.includes('verify') || - inputName.includes('confirm') || inputName.includes('verify') || - placeholder.includes('confirm') || placeholder.includes('verify') || - placeholder.includes('re-') || placeholder.includes('again'); - if (!hasConfirm) { - score -= 15; - } - } - - return score; - }; - - // Sort patterns by priority - const sortedPatterns = Object.entries(patterns).sort((a, b) => a[1].priority - b[1].priority); - - for (const [fieldName, pattern] of sortedPatterns) { - let bestMatch = null; - let bestScore = 0; - let bestIndex = -1; - - for (let i = 0; i < inputs.length; i++) { - if (usedElements.has(i)) continue; - - const input = inputs[i]; - const score = calculateScore(input, pattern, fieldName); - - if (score > bestScore && score >= 10) { - bestScore = score; - bestMatch = input; - bestIndex = i; - } - } - - if (bestMatch && bestIndex >= 0) { - usedElements.add(bestIndex); - - // Generate selector - let selector = ''; - if (bestMatch.id) { - selector = `#${bestMatch.id}`; - } else if (bestMatch.name) { - selector = `[name="${bestMatch.name}"]`; - } else if (bestMatch.placeholder) { - selector = `[placeholder="${bestMatch.placeholder}"]`; - } else { - const tagName = bestMatch.tagName.toLowerCase(); - const type = bestMatch.type ? `[type="${bestMatch.type}"]` : ''; - const allSame = Array.from(document.querySelectorAll(`${tagName}${type}`)); - const index = allSame.indexOf(bestMatch) + 1; - selector = `${tagName}${type}:nth-of-type(${index})`; - } - - results[fieldName] = { - selector, - confidence: bestScore, - name: bestMatch.name, - id: bestMatch.id - }; - } - } - - return results; - }, FIELD_PATTERNS); - - const detectedCount = Object.keys(detectedFields).length; - if (detectedCount === 0) { - console.log(' No fields detected'); - } - - return detectedFields; -} - -async function detectSubmitButton(page) { - console.log('šŸ” Auto-detecting submit button...'); - - const submitSelector = await page.evaluate(() => { - const buttons = Array.from(document.querySelectorAll('button[type="submit"], input[type="submit"], button, input[type="button"]')); - let bestMatch = null; - let bestScore = 0; - - buttons.forEach(btn => { - const text = (btn.textContent || btn.value || '').toLowerCase().trim(); - const type = btn.type?.toLowerCase() || ''; - const className = (btn.className || '').toLowerCase(); - let score = 0; - - if (type === 'submit') score += 20; - - if (text.includes('subscribe')) score += 15; - else if (text.includes('sign up')) score += 12; - else if (text.includes('submit')) score += 10; - else if (text.includes('join')) score += 8; - else if (text.includes('register')) score += 8; - else if (text.includes('send')) score += 5; - - if (className.includes('submit')) score += 8; - if (className.includes('subscribe')) score += 8; - - if (className.includes('close') || text.includes('close')) score -= 20; - if (className.includes('cancel') || text.includes('cancel')) score -= 20; - if (className.includes('back') || text.includes('back')) score -= 15; - if (className.includes('drawer')) score -= 15; - - const rect = btn.getBoundingClientRect(); - if (rect.width > 0 && rect.height > 0 && rect.top > 0) { - score += 3; - } - - if (score > bestScore) { - bestScore = score; - bestMatch = btn; - } - }); - - if (bestMatch && bestScore >= 10) { - if (bestMatch.id) return `#${bestMatch.id}`; - if (bestMatch.name) return `[name="${bestMatch.name}"]`; - - const className = bestMatch.className; - if (className) { - const classes = className.split(' ').filter(c => c.length > 0); - for (const cls of classes) { - const clsLower = cls.toLowerCase(); - if (clsLower.includes('submit') || clsLower.includes('subscribe')) { - return `.${cls}`; - } - } - for (const cls of classes) { - if (!cls.toLowerCase().includes('close') && !cls.toLowerCase().includes('cancel')) { - return `.${cls}`; - } - } - } - - const allBtns = Array.from(document.querySelectorAll('button')); - const index = allBtns.indexOf(bestMatch) + 1; - return `button:nth-of-type(${index})`; - } - - return null; - }); - - if (submitSelector) { - console.log(` āœ… Submit button found: ${submitSelector}`); - } else { - console.log(' āŒ No submit button found'); - } - - return submitSelector; -} +const { loadConfig, loadSitesFromFile } = require('./src/config'); +const { processSite } = require('./src/processor'); +const { createCLI, parseCLI } = require('./src/cli'); async function main() { + const cliOptions = parseCLI(); + + if (cliOptions.help) { + createCLI().help(); + return; + } + const config = loadConfig(); let sitesToProcess = config.sites || []; - const targetSite = process.argv[2]; - const logToFile = process.argv.includes('--log'); - const logFile = logToFile ? path.join(__dirname, `run_${new Date().toISOString().replace(/[:.]/g, '-')}.log`) : null; + if (cliOptions.file) { + console.log(`Loading sites from: ${cliOptions.file}`); + sitesToProcess = loadSitesFromFile(cliOptions.file); + console.log(`Loaded ${sitesToProcess.length} sites from file`); + } - if (targetSite && !targetSite.startsWith('--')) { + const logFile = cliOptions.log ? path.join(__dirname, `run_${new Date().toISOString().replace(/[:.]/g, '-')}.log`) : null; + + if (cliOptions.target) { sitesToProcess = sitesToProcess.filter(site => - site.name.toLowerCase().includes(targetSite.toLowerCase()) || - site.url.toLowerCase().includes(targetSite.toLowerCase()) + site.name.toLowerCase().includes(cliOptions.target.toLowerCase()) || + site.url.toLowerCase().includes(cliOptions.target.toLowerCase()) ); if (sitesToProcess.length === 0) { - console.error(`āŒ No site found matching "${targetSite}"`); - console.log('Available sites:'); - config.sites.forEach(site => console.log(` - ${site.name}: ${site.url}`)); + console.error(`No site found matching "${cliOptions.target}"`); return; } - console.log(`šŸŽÆ Filtering to ${sitesToProcess.length} site(s) matching "${targetSite}"`); + console.log(`Targeting ${sitesToProcess.length} site(s)`); + } + + if (cliOptions.keepOpen) { + console.log('Keep-open mode: Tabs will stay open for verification'); } const browserConfig = config.browser || {}; @@ -439,11 +48,11 @@ async function main() { const userDataDir = browserConfig.userDataDir; if (!userDataDir) { - console.error('āŒ userDataDir is not set in config.toml. This is required.'); + console.error('userDataDir not set in config.toml'); return; } - if (logToFile) { + if (cliOptions.log && logFile) { const originalLog = console.log; const originalError = console.error; const originalWarn = console.warn; @@ -453,173 +62,58 @@ async function main() { fs.appendFileSync(logFile, `[${new Date().toISOString()}] [${type}] ${message}\n`); }; - console.log = (...args) => { - writeToLog('LOG', args); - originalLog.apply(console, args); - }; - - console.error = (...args) => { - writeToLog('ERROR', args); - originalError.apply(console, args); - }; - - console.warn = (...args) => { - writeToLog('WARN', args); - originalWarn.apply(console, args); - }; - - console.log(`šŸ“ Logging to file: ${logFile}`); + console.log = (...args) => { writeToLog('LOG', args); originalLog.apply(console, args); }; + console.error = (...args) => { writeToLog('ERROR', args); originalError.apply(console, args); }; + console.warn = (...args) => { writeToLog('WARN', args); originalWarn.apply(console, args); }; + console.log(`Logging to: ${logFile}`); } const failedSites = []; + const processedCount = { success: 0, failed: 0 }; const browser = await puppeteer.launch({ headless: false, userDataDir: userDataDir, - args: ["--start-maximized"] + args: ['--start-maximized', '--no-sandbox', '--disable-setuid-sandbox'] }); try { - let isFirstSite = true; - for (const site of sitesToProcess) { - let siteFailed = false; - let failureReason = ''; - let creds; - if (site.credentials && site.credentials.username) { - creds = { ...globalCreds, ...site.credentials }; - console.log(`Using credentials specified for site: ${site.name}`); - } else { - creds = { ...globalCreds }; - console.log(`Using global credentials for site: ${site.name}`); - } + console.log(`\nProcessing ${sitesToProcess.length} sites...\n`); + + for (let i = 0; i < sitesToProcess.length; i++) { + const site = sitesToProcess[i]; + console.log(`[${i + 1}/${sitesToProcess.length}]`); - if (!creds.username || !creds.email || !creds.password) { - const randomData = generateRandomData(); - creds = { ...randomData, ...creds }; - console.log('Filled in missing credentials with random data.'); - } - - console.log(`\n${site.url}`); - console.log(`${'-'.repeat(70)}`); + const result = await processSite(browser, site, globalCreds, browserConfig, cliOptions.keepOpen); - const page = isFirstSite ? (await browser.pages())[0] : await browser.newPage(); - isFirstSite = false; - - if (browserConfig.userAgent) { - await page.setUserAgent(browserConfig.userAgent); - } - - await page.goto(site.url, { waitUntil: 'networkidle2' }); - - console.log('\nšŸ” DETECTED FIELDS:'); - - let fieldsToFill = {}; - let detectedFields = {}; - const definedLocators = site.locators || {}; - - if (site.autoDetect !== false) { - detectedFields = await detectFormFields(page); - for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) { - fieldsToFill[fieldName] = fieldInfo.selector; - } - } - - const hasEmailField = fieldsToFill.email || fieldsToFill.confirmEmail; - if (!hasEmailField) { - siteFailed = true; - failureReason = 'No email field detected'; - console.warn(`\n āŒ ${failureReason}`); - } - - const hasSearchInput = await page.evaluate(() => { - const inputs = document.querySelectorAll('input[type="search"], input[name*="search"], input[id*="search"]'); - return inputs.length > 0; - }); - - if (hasSearchInput && !hasEmailField) { - console.warn(` āš ļø WARNING: This appears to be a SEARCH form, not a newsletter signup!`); - siteFailed = true; - failureReason = failureReason || 'Detected search form instead of newsletter'; - } - - for (const [fieldName, locator] of Object.entries(definedLocators)) { - if (fieldName !== 'submit') { - fieldsToFill[fieldName] = parseLocator(locator); - } - } - - if (Object.keys(fieldsToFill).length > 0) { - console.log(`\n ${'FIELD'.padEnd(12)} | ${'SELECTOR'.padEnd(35)} | ${'CONF'.padEnd(4)} | VALUE`); - console.log(' ' + '-'.repeat(75)); - - for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) { - let valueToFill = creds[fieldName]; - - if (fieldName === 'confirmEmail') { - valueToFill = creds['email']; - } - - const selector = fieldInfo.selector.length > 34 ? fieldInfo.selector.substring(0, 31) + '...' : fieldInfo.selector; - const value = valueToFill ? valueToFill.substring(0, 20) : 'N/A'; - - console.log(` ${fieldName.padEnd(12)} | ${selector.padEnd(35)} | ${fieldInfo.confidence.toString().padEnd(4)} | ${value}`); - - if (valueToFill) { - try { - await page.evaluate((sel, val) => { - const el = document.querySelector(sel); - if (el) { - el.value = val; - el.dispatchEvent(new Event('input', { bubbles: true })); - el.dispatchEvent(new Event('change', { bubbles: true })); - } - }, fieldInfo.selector, valueToFill); - } catch (e) { - console.warn(` āœ— Failed to fill ${fieldName}: ${e.message}`); - siteFailed = true; - failureReason = `Failed to fill ${fieldName}: ${e.message}`; - } - } - } - console.log(' ' + '-'.repeat(75)); - } - - let submitSelector = null; - if (definedLocators.submit) { - submitSelector = parseLocator(definedLocators.submit); - } else if (site.autoDetect !== false) { - submitSelector = await detectSubmitButton(page); - } - - if (submitSelector) { - try { - await page.click(submitSelector); - console.log(` āœ“ Submitted (${submitSelector})`); - } catch (submitError) { - siteFailed = true; - failureReason = failureReason || `Submit failed: ${submitError.message}`; - console.error(` āœ— Submit failed: ${submitError.message}`); - } - } else { - siteFailed = true; - failureReason = failureReason || 'No submit button found'; - console.warn(` āœ— ${failureReason}`); - } - - if (siteFailed) { + if (result.siteFailed) { failedSites.push({ name: site.name, url: site.url, - reason: failureReason, + reason: result.failureReason, timestamp: new Date().toISOString() }); + processedCount.failed++; + } else { + processedCount.success++; } - await new Promise(r => setTimeout(r, 2000)); + await new Promise(r => setTimeout(r, 1000)); } + + console.log(`\n${'='.repeat(70)}`); + console.log(`Done: ${processedCount.success} success, ${processedCount.failed} failed`); + } catch(error) { - console.error('Error:', error.message); + console.error('Fatal error:', error.message); } finally { + if (cliOptions.keepOpen) { + console.log(`\nBrowser kept open for verification`); + console.log(` Close browser manually when done`); + } else { + await browser.close(); + } + if (failedSites.length > 0) { const reportPath = path.join(__dirname, 'failed_sites.txt'); const now = new Date(); @@ -638,7 +132,7 @@ async function main() { } fs.appendFileSync(reportPath, reportContent); - console.log(`\n${failedSites.length} site(s) failed. See: failed_sites.txt`); + console.log(`${failedSites.length} failed. See: failed_sites.txt`); } } } diff --git a/package-lock.json b/package-lock.json index b3f013f..8f4b886 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,11 +10,13 @@ "license": "ISC", "dependencies": { "@iarna/toml": "^2.2.5", + "commander": "^14.0.3", "playwright": "^1.56.1", "playwright-extra": "^4.3.6", "puppeteer": "^24.30.0", "puppeteer-extra": "^3.3.6", - "puppeteer-extra-plugin-stealth": "^2.11.2" + "puppeteer-extra-plugin-stealth": "^2.11.2", + "tesseract.js": "^7.0.0" } }, "node_modules/@babel/code-frame": { @@ -288,6 +290,12 @@ "node": ">=10.0.0" } }, + "node_modules/bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -377,6 +385,15 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "license": "MIT" }, + "node_modules/commander": { + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -748,6 +765,12 @@ "node": ">= 14" } }, + "node_modules/idb-keyval": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.2.tgz", + "integrity": "sha512-yjD9nARJ/jb1g+CvD0tlhUHOrJ9Sy0P8T9MF3YaLlHnSRpwPfpTX0XIvpmw3gAJUmEu3FiICLBDPXVwyEvrleg==", + "license": "Apache-2.0" + }, "node_modules/import-fresh": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", @@ -832,6 +855,12 @@ "node": ">=0.10.0" } }, + "node_modules/is-url": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", + "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==", + "license": "MIT" + }, "node_modules/isobject": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", @@ -982,6 +1011,26 @@ "node": ">= 0.4.0" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -991,6 +1040,15 @@ "wrappy": "1" } }, + "node_modules/opencollective-postinstall": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", + "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", + "license": "MIT", + "bin": { + "opencollective-postinstall": "index.js" + } + }, "node_modules/pac-proxy-agent": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", @@ -1347,6 +1405,12 @@ } } }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT" + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -1539,6 +1603,30 @@ "streamx": "^2.15.0" } }, + "node_modules/tesseract.js": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-7.0.0.tgz", + "integrity": "sha512-exPBkd+z+wM1BuMkx/Bjv43OeLBxhL5kKWsz/9JY+DXcXdiBjiAch0V49QR3oAJqCaL5qURE0vx9Eo+G5YE7mA==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "bmp-js": "^0.1.0", + "idb-keyval": "^6.2.0", + "is-url": "^1.2.4", + "node-fetch": "^2.6.9", + "opencollective-postinstall": "^2.0.3", + "regenerator-runtime": "^0.13.3", + "tesseract.js-core": "^7.0.0", + "wasm-feature-detect": "^1.8.0", + "zlibjs": "^0.3.1" + } + }, + "node_modules/tesseract.js-core": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-7.0.0.tgz", + "integrity": "sha512-WnNH518NzmbSq9zgTPeoF8c+xmilS8rFIl1YKbk/ptuuc7p6cLNELNuPAzcmsYw450ca6bLa8j3t0VAtq435Vw==", + "license": "Apache-2.0" + }, "node_modules/text-decoder": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.6.tgz", @@ -1548,6 +1636,12 @@ "b4a": "^1.6.4" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -1576,12 +1670,34 @@ "node": ">= 10.0.0" } }, + "node_modules/wasm-feature-detect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz", + "integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==", + "license": "Apache-2.0" + }, "node_modules/webdriver-bidi-protocol": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz", "integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==", "license": "Apache-2.0" }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1672,6 +1788,15 @@ "fd-slicer": "~1.1.0" } }, + "node_modules/zlibjs": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz", + "integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/package.json b/package.json index 4adee5e..6d298f6 100644 --- a/package.json +++ b/package.json @@ -12,10 +12,12 @@ "type": "commonjs", "dependencies": { "@iarna/toml": "^2.2.5", + "commander": "^14.0.3", "playwright": "^1.56.1", "playwright-extra": "^4.3.6", "puppeteer": "^24.30.0", "puppeteer-extra": "^3.3.6", - "puppeteer-extra-plugin-stealth": "^2.11.2" + "puppeteer-extra-plugin-stealth": "^2.11.2", + "tesseract.js": "^7.0.0" } } diff --git a/sites.txt b/sites.txt new file mode 100644 index 0000000..a1de280 --- /dev/null +++ b/sites.txt @@ -0,0 +1,6 @@ +# Newsletter signup sites +# Lines starting with # are ignored + +https://www.dutchnews.nl/dutchnews-newsletter-signup/ +https://immaculatevegan.com/blogs/magazine +https://kulalaland.com/ \ No newline at end of file diff --git a/src/captcha.js b/src/captcha.js new file mode 100644 index 0000000..92dee14 --- /dev/null +++ b/src/captcha.js @@ -0,0 +1,128 @@ +const Tesseract = require('tesseract.js'); + +const CAPTCHA_PATTERNS = { + recaptcha: { + selectors: ['.g-recaptcha', '[data-sitekey]', 'iframe[src*="recaptcha"]', '#recaptcha'], + name: 'reCAPTCHA' + }, + hcaptcha: { + selectors: ['.h-captcha', '[data-hcaptcha-sitekey]', 'iframe[src*="hcaptcha"]', '#hcaptcha'], + name: 'hCaptcha' + }, + textCaptcha: { + selectors: ['input[name*="captcha"]', 'input[id*="captcha"]', '.captcha-input', '#captcha'], + name: 'Text CAPTCHA' + }, + imageCaptcha: { + selectors: ['img[src*="captcha"]', '.captcha-image', '#captcha-image', 'img[alt*="captcha" i]'], + name: 'Image CAPTCHA' + }, + mathCaptcha: { + selectors: ['.math-captcha', '[name*="math"]', 'input[placeholder*="math" i]'], + name: 'Math CAPTCHA' + } +}; + +async function detectCaptcha(page) { + const captchaInfo = await page.evaluate((patterns) => { + const results = []; + + for (const [type, config] of Object.entries(patterns)) { + for (const selector of config.selectors) { + const elements = document.querySelectorAll(selector); + if (elements.length > 0) { + results.push({ + type: type, + name: config.name, + selector: selector, + count: elements.length + }); + break; + } + } + } + + return results; + }, CAPTCHA_PATTERNS); + + return captchaInfo; +} + +async function solveSimpleCaptcha(page) { + const mathSolution = await page.evaluate(() => { + const mathText = document.querySelector('.math-captcha, .captcha-question, [class*="math"]'); + if (mathText) { + const text = mathText.textContent || ''; + const match = text.match(/(\d+)\s*([+\-*/])\s*(\d+)/); + if (match) { + const [_, a, op, b] = match; + let result; + switch(op) { + case '+': result = parseInt(a) + parseInt(b); break; + case '-': result = parseInt(a) - parseInt(b); break; + case '*': result = parseInt(a) * parseInt(b); break; + case '/': result = Math.floor(parseInt(a) / parseInt(b)); break; + } + return { type: 'math', answer: result, text: text }; + } + } + return null; + }); + + if (mathSolution) { + return mathSolution; + } + + const imageCaptchaInfo = await page.evaluate(() => { + const selectors = [ + 'img[src*="captcha"]', + '.captcha-image img', + '#captcha-image', + 'img[alt*="captcha" i]', + 'img[class*="captcha" i]' + ]; + + for (const selector of selectors) { + const img = document.querySelector(selector); + if (img && img.src) { + return { src: img.src, selector: selector }; + } + } + return null; + }); + + if (imageCaptchaInfo) { + console.log(' Attempting OCR-based CAPTCHA solving...'); + try { + const { data: { text } } = await Tesseract.recognize( + imageCaptchaInfo.src, + 'eng', + { logger: m => {} } + ); + + const cleanedText = text.replace(/[^a-zA-Z0-9]/g, '').trim(); + + if (cleanedText && cleanedText.length >= 3) { + console.log(` OCR read: "${cleanedText}"`); + return { + type: 'image', + answer: cleanedText, + originalText: text, + imageSrc: imageCaptchaInfo.src + }; + } + } catch (ocrError) { + console.log(` OCR failed: ${ocrError.message}`); + } + + return { type: 'image', message: 'Image CAPTCHA detected - OCR failed, manual solving required' }; + } + + return null; +} + +module.exports = { + CAPTCHA_PATTERNS, + detectCaptcha, + solveSimpleCaptcha +}; diff --git a/src/classifier.js b/src/classifier.js new file mode 100644 index 0000000..652317b --- /dev/null +++ b/src/classifier.js @@ -0,0 +1,109 @@ +async function classifyNewsletterForm(page) { + const formInfo = await page.evaluate(() => { + const results = { + hasEmailInput: false, + hasNewsletterKeywords: false, + hasSearchInput: false, + hasLoginInput: false, + hasContactForm: false, + hasOnlyEmail: false, + formCount: 0, + emailInputCount: 0, + totalInputCount: 0, + confidence: 0, + reasons: [], + signals: { + positive: [], + negative: [] + } + }; + + const allInputs = document.querySelectorAll('input:not([type="hidden"]):not([type="submit"]):not([type="button"])'); + results.totalInputCount = allInputs.length; + + const emailInputs = document.querySelectorAll('input[type="email"], input[name*="email" i], input[id*="email" i], input[placeholder*="email" i]'); + results.emailInputCount = emailInputs.length; + results.hasEmailInput = emailInputs.length > 0; + if (results.hasEmailInput) { + results.signals.positive.push(`Found ${emailInputs.length} email input(s)`); + } + + const pageText = document.body.innerText.toLowerCase(); + const titleText = document.title.toLowerCase(); + const combinedText = pageText + ' ' + titleText; + + const newsletterKeywords = [ + 'newsletter', 'subscribe', 'signup', 'sign up', 'join our', 'stay updated', + 'get updates', 'email updates', 'weekly digest', 'monthly newsletter', + 'sign up for updates', 'join the list', 'email list', 'mailing list' + ]; + + for (const keyword of newsletterKeywords) { + if (combinedText.includes(keyword)) { + results.hasNewsletterKeywords = true; + results.signals.positive.push(`Keyword: "${keyword}"`); + } + } + + const searchInputs = document.querySelectorAll('input[type="search"], input[name*="search" i], input[id*="search" i], input[placeholder*="search" i]'); + results.hasSearchInput = searchInputs.length > 0; + if (results.hasSearchInput) { + results.signals.negative.push('Search input detected'); + results.reasons.push('Search input detected'); + } + + const passwordInputs = document.querySelectorAll('input[type="password"]'); + const usernameInputs = document.querySelectorAll('input[name*="username" i], input[name*="login" i], input[id*="username" i]'); + results.hasLoginInput = passwordInputs.length > 0 || usernameInputs.length > 0; + if (results.hasLoginInput) { + results.signals.negative.push('Login/password fields detected'); + results.reasons.push('Login/password fields detected'); + } + + const contactKeywords = ['contact us', 'get in touch', 'send message', 'inquiry']; + for (const keyword of contactKeywords) { + if (combinedText.includes(keyword)) { + results.hasContactForm = true; + results.signals.negative.push(`Contact form keyword: "${keyword}"`); + break; + } + } + + const nonEmailInputs = Array.from(allInputs).filter(input => { + const type = input.type?.toLowerCase(); + const name = input.name?.toLowerCase() || ''; + return type !== 'email' && !name.includes('email'); + }); + + results.hasOnlyEmail = results.hasEmailInput && nonEmailInputs.length === 0; + if (results.hasOnlyEmail) { + results.signals.positive.push('Only email field (no other inputs)'); + } + + let score = 0; + + if (results.hasEmailInput) score += 40; + if (results.hasNewsletterKeywords) score += 35; + if (results.hasOnlyEmail) score += 15; + if (results.totalInputCount <= 3) score += 10; + + if (!results.hasSearchInput) score += 10; + if (!results.hasLoginInput) score += 15; + if (!results.hasContactForm) score += 5; + + if (results.hasSearchInput) score -= 30; + if (results.hasLoginInput) score -= 50; + if (results.hasContactForm && !results.hasNewsletterKeywords) score -= 20; + + results.confidence = Math.max(0, Math.min(100, score)); + results.isNewsletter = results.confidence >= 55; + + return results; + }); + + return formInfo; +} + +module.exports = { + classifyNewsletterForm +}; diff --git a/src/cli.js b/src/cli.js new file mode 100644 index 0000000..78b4b13 --- /dev/null +++ b/src/cli.js @@ -0,0 +1,69 @@ +const { Command } = require('commander'); + +function createCLI() { + const program = new Command(); + + program + .name('newsletter-signup') + .description('Newsletter Signup Automation Tool') + .version('1.0.0') + .option('-f, --file ', 'Load sites from text file (one URL per line)') + .option('-k, --keep-open', 'Keep browser tabs open after processing for verification') + .option('-l, --log', 'Save output to timestamped log file') + .option('-h, --help', 'Show help message') + .argument('[target]', 'Site name or URL to process (filters from config.toml sites)'); + + program.on('option:help', () => { + console.log(` +Newsletter Signup Automation Tool + +Usage: node index.js [options] [site-name-or-url] + +Options: + -f, --file= Load sites from text file (one URL per line) + -k, --keep-open Keep browser tabs open after processing for verification + -l, --log Save output to timestamped log file + -h, --help Show this help message + +Examples: + node index.js Process all sites from config.toml + node index.js --file=sites.txt Process sites from file + node index.js "Dutch News" Process specific site + node index.js --file=sites.txt --keep-open Keep tabs open for verification + node index.js --log Save output to log file + +Text file format (sites.txt): + # Lines starting with # are comments + https://example.com/newsletter + https://another-site.com/signup + +Configuration: + Edit config.toml to set credentials and default sites +`); + process.exit(0); + }); + + return program; +} + +function parseCLI(args = process.argv.slice(2)) { + const program = createCLI(); + + const fullArgs = ['node', 'index.js', ...args]; + program.parse(fullArgs); + + const opts = program.opts(); + + return { + target: program.args[0] || null, + file: opts.file || null, + keepOpen: opts.keepOpen || false, + log: opts.log || false, + help: opts.help || false + }; +} + +module.exports = { + createCLI, + parseCLI +}; diff --git a/src/config.js b/src/config.js new file mode 100644 index 0000000..ccc01d5 --- /dev/null +++ b/src/config.js @@ -0,0 +1,80 @@ +const fs = require('fs'); +const path = require('path'); +const toml = require('@iarna/toml'); + +function loadConfig() { + try { + const configPath = path.join(__dirname, '..', 'config.toml'); + const tomlString = fs.readFileSync(configPath, 'utf-8'); + const config = toml.parse(tomlString); + console.log('Configuration loaded successfully.'); + return config; + } catch (error) { + console.error('Failed to load or parse config.toml:', error.message); + process.exit(1); + } +} + +function generateRandomData() { + const randomString = Math.random().toString(36).substring(2, 10); + const username = `user${randomString}`; + const email = `${username}@gmail.com`; + const password = `pass${randomString}`; + const age = Math.floor(Math.random() * 30 + 20).toString(); + const firstName = 'John'; + const lastName = 'Doe'; + const phone = '+31610488717'; + const company = 'Example Corp'; + return { username, email, password, age, firstName, lastName, phone, company }; +} + +function parseLocator(locatorStr) { + locatorStr = locatorStr.trim(); + if (!locatorStr.startsWith('<')) { + return locatorStr; + } + + const getAttr = (attr) => { + const match = locatorStr.match(new RegExp(`${attr}=["']([^"']+)["']`)); + return match ? match[1] : null; + }; + + const tagName = (locatorStr.match(/^<([a-zA-Z0-9]+)/) || [])[1] || ''; + const id = getAttr('id'); + if (id) { + if (/^\d/.test(id)) { + return `${tagName}[id='${id}']`; + } + return `${tagName}#${id}`; + } + + const name = getAttr('name'); + if (name) return `${tagName}[name='${name}']`; + + const placeholder = getAttr('placeholder'); + if (placeholder) return `${tagName}[placeholder='${placeholder}']`; + + const type = getAttr('type'); + if (type) return `${tagName}[type='${type}']`; + + return tagName; +} + +function loadSitesFromFile(filePath) { + const fullPath = path.isAbsolute(filePath) ? filePath : path.join(__dirname, '..', filePath); + const content = fs.readFileSync(fullPath, 'utf-8'); + const urls = content.split('\n').filter(line => line.trim() && !line.startsWith('#')); + return urls.map((url, index) => ({ + name: `Site_${index + 1}`, + url: url.trim(), + category: 'Newsletter', + needsPreSubmitAction: false + })); +} + +module.exports = { + loadConfig, + generateRandomData, + parseLocator, + loadSitesFromFile +}; diff --git a/src/fields.js b/src/fields.js new file mode 100644 index 0000000..fd393b4 --- /dev/null +++ b/src/fields.js @@ -0,0 +1,260 @@ +const FIELD_PATTERNS = { + email: { + priority: 1, + types: ['email'], + names: ['email', 'e-mail', 'mail', 'user_email', 'login_email', 'email_address', 'emailaddress'], + ids: ['email', 'e-mail', 'mail', 'user-email', 'login-email'], + placeholders: ['email', 'e-mail', 'email address', 'your email', 'enter email'], + autocomplete: ['email'] + }, + confirmEmail: { + priority: 2, + types: ['email'], + names: ['confirm_email', 'confirmemail', 'email_confirm', 'verify_email', 'email_verify'], + ids: ['confirm-email', 'confirmemail', 'verify-email', 'email-confirm'], + placeholders: ['confirm email', 'verify email', 're-enter email', 'repeat email'], + mustHaveConfirmIndicator: true + }, + firstName: { + priority: 3, + types: ['text'], + names: ['first_name', 'firstname', 'fname', 'first-name', 'vorname', 'prenom', 'fname'], + ids: ['first-name', 'firstname', 'fname', 'firstName', 'mce-fname'], + placeholders: ['first name', 'firstname', 'first'], + autocomplete: ['given-name', 'first-name'] + }, + lastName: { + priority: 4, + types: ['text'], + names: ['last_name', 'lastname', 'lname', 'surname', 'family_name', 'last-name', 'lname'], + ids: ['last-name', 'lastname', 'lname', 'surname', 'lastName', 'mce-lname'], + placeholders: ['last name', 'lastname', 'surname', 'last'], + autocomplete: ['family-name', 'last-name'] + }, + username: { + priority: 5, + types: ['text'], + names: ['username', 'user_name', 'login', 'usr', 'uname', 'user-id', 'userid', 'account_name', 'login_id'], + ids: ['username', 'user-name', 'login', 'usr', 'uname'], + placeholders: ['username', 'user name', 'login', 'username or email'], + autocomplete: ['username'] + }, + password: { + priority: 6, + types: ['password'], + names: ['password', 'pass', 'pwd', 'user_password', 'login_password'], + ids: ['password', 'pass', 'pwd'], + placeholders: ['password', 'pass', 'enter password'], + autocomplete: ['current-password', 'new-password'] + }, + fullName: { + priority: 7, + types: ['text'], + names: ['full_name', 'fullname', 'name'], + ids: ['full-name', 'fullname', 'name'], + placeholders: ['full name', 'your name', 'complete name'], + autocomplete: ['name'] + }, + phone: { + priority: 8, + types: ['tel', 'text'], + names: ['phone', 'telephone', 'mobile', 'cell', 'cellphone', 'phone_number'], + ids: ['phone', 'telephone', 'mobile', 'cell', 'tel'], + placeholders: ['phone', 'telephone', 'mobile number'], + autocomplete: ['tel'] + }, + age: { + priority: 9, + types: ['number', 'text'], + names: ['age', 'user_age'], + ids: ['age'], + placeholders: ['age', 'your age'] + }, + company: { + priority: 10, + types: ['text'], + names: ['company', 'organization', 'company_name', 'employer'], + ids: ['company', 'organization', 'employer'], + placeholders: ['company', 'organization'], + autocomplete: ['organization'] + } +}; + +async function detectFormFields(page) { + console.log('Auto-detecting form fields...'); + + const detectedFields = await page.evaluate((patterns) => { + const results = {}; + const usedElements = new Set(); + + const inputs = Array.from(document.querySelectorAll('input, select, textarea')).filter(input => { + const type = input.type?.toLowerCase() || 'text'; + const style = window.getComputedStyle(input); + const rect = input.getBoundingClientRect(); + + if (type === 'hidden' || type === 'submit' || type === 'button') return false; + if (input.tabIndex === -1 && input.style.position === 'absolute') return false; + if (rect.width === 0 || rect.height === 0) return false; + if (style.display === 'none' || style.visibility === 'hidden') return false; + + return true; + }); + + const calculateScore = (input, pattern, fieldName) => { + let score = 0; + const inputType = (input.type || 'text').toLowerCase(); + const inputName = (input.name || '').toLowerCase(); + const inputId = (input.id || '').toLowerCase(); + const placeholder = (input.placeholder || '').toLowerCase(); + const autocomplete = (input.autocomplete || '').toLowerCase(); + + if (pattern.types && pattern.types.includes(inputType)) { + score += 5; + } + + if (pattern.names) { + for (const name of pattern.names) { + const nameLower = name.toLowerCase(); + if (inputName === nameLower) { + score += 20; + break; + } else if (inputName.includes(nameLower)) { + if (fieldName === 'firstName' && (inputName.includes('user') || inputName.includes('login'))) { + score -= 5; + } else if (fieldName === 'lastName' && (inputName.includes('user') || inputName.includes('login'))) { + score -= 5; + } else { + score += 10; + } + break; + } + } + } + + if (pattern.ids) { + for (const id of pattern.ids) { + const idLower = id.toLowerCase(); + if (inputId === idLower) { + score += 20; + break; + } else if (inputId.includes(idLower)) { + score += 10; + break; + } + } + } + + if (pattern.placeholders) { + for (const ph of pattern.placeholders) { + const phLower = ph.toLowerCase(); + if (placeholder === phLower) { + score += 15; + break; + } else if (placeholder.includes(phLower)) { + score += 8; + break; + } + } + } + + if (pattern.autocomplete) { + for (const ac of pattern.autocomplete) { + if (autocomplete === ac.toLowerCase()) { + score += 12; + break; + } + } + } + + const labels = input.labels; + if (labels && labels.length > 0) { + const labelText = labels[0].textContent.toLowerCase(); + const fieldNameLower = fieldName.toLowerCase().replace(/([A-Z])/g, ' $1').trim(); + if (labelText.includes(fieldNameLower) || labelText.includes(pattern.names?.[0] || '')) { + score += 15; + } + } + + if (inputName.includes('b_') || inputId.includes('b_')) { + score -= 30; + } + + if (input.disabled || input.readOnly) { + score -= 20; + } + + if (pattern.mustHaveConfirmIndicator) { + const hasConfirm = inputId.includes('confirm') || inputId.includes('verify') || + inputName.includes('confirm') || inputName.includes('verify') || + placeholder.includes('confirm') || placeholder.includes('verify') || + placeholder.includes('re-') || placeholder.includes('again'); + if (!hasConfirm) { + score -= 15; + } + } + + return score; + }; + + const sortedPatterns = Object.entries(patterns).sort((a, b) => a[1].priority - b[1].priority); + + for (const [fieldName, pattern] of sortedPatterns) { + let bestMatch = null; + let bestScore = 0; + let bestIndex = -1; + + for (let i = 0; i < inputs.length; i++) { + if (usedElements.has(i)) continue; + + const input = inputs[i]; + const score = calculateScore(input, pattern, fieldName); + + if (score > bestScore && score >= 10) { + bestScore = score; + bestMatch = input; + bestIndex = i; + } + } + + if (bestMatch && bestIndex >= 0) { + usedElements.add(bestIndex); + + let selector = ''; + if (bestMatch.id) { + selector = `#${bestMatch.id}`; + } else if (bestMatch.name) { + selector = `[name="${bestMatch.name}"]`; + } else if (bestMatch.placeholder) { + selector = `[placeholder="${bestMatch.placeholder}"]`; + } else { + const tagName = bestMatch.tagName.toLowerCase(); + const type = bestMatch.type ? `[type="${bestMatch.type}"]` : ''; + const allSame = Array.from(document.querySelectorAll(`${tagName}${type}`)); + const index = allSame.indexOf(bestMatch) + 1; + selector = `${tagName}${type}:nth-of-type(${index})`; + } + + results[fieldName] = { + selector, + confidence: bestScore, + name: bestMatch.name, + id: bestMatch.id + }; + } + } + + return results; + }, FIELD_PATTERNS); + + const detectedCount = Object.keys(detectedFields).length; + if (detectedCount === 0) { + console.log(' No fields detected'); + } + + return detectedFields; +} + +module.exports = { + FIELD_PATTERNS, + detectFormFields +}; diff --git a/src/processor.js b/src/processor.js new file mode 100644 index 0000000..56df9d4 --- /dev/null +++ b/src/processor.js @@ -0,0 +1,217 @@ +const { generateRandomData, parseLocator } = require('./config'); +const { detectFormFields } = require('./fields'); +const { detectSubmitButton } = require('./submit'); +const { detectCaptcha, solveSimpleCaptcha } = require('./captcha'); +const { classifyNewsletterForm } = require('./classifier'); + +async function processSite(browser, site, globalCreds, browserConfig, keepOpen = false) { + let siteFailed = false; + let failureReason = ''; + let page = null; + + try { + let creds; + if (site.credentials && site.credentials.username) { + creds = { ...globalCreds, ...site.credentials }; + } else { + creds = { ...globalCreds }; + } + + if (!creds.username || !creds.email || !creds.password) { + const randomData = generateRandomData(); + creds = { ...randomData, ...creds }; + } + + console.log(`\n${site.url}`); + console.log(`${'-'.repeat(70)}`); + + page = await browser.newPage(); + + if (browserConfig.userAgent) { + await page.setUserAgent(browserConfig.userAgent); + } + + await page.goto(site.url, { waitUntil: 'networkidle2', timeout: 30000 }); + + const captchaInfo = await detectCaptcha(page); + if (captchaInfo.length > 0) { + console.log(`\n CAPTCHA DETECTED:`); + for (const captcha of captchaInfo) { + console.log(` - ${captcha.name}`); + } + + if (captchaInfo.some(c => c.type === 'mathCaptcha')) { + const solution = await solveSimpleCaptcha(page); + if (solution && solution.answer) { + console.log(` Auto-solving math: ${solution.text} = ${solution.answer}`); + try { + await page.evaluate((answer) => { + const input = document.querySelector('input[name*="captcha"], input[class*="captcha"], #captcha'); + if (input) { + input.value = answer; + input.dispatchEvent(new Event('input', { bubbles: true })); + } + }, solution.answer); + } catch (e) { + console.warn(` Failed to fill math answer: ${e.message}`); + } + } + } + + if (captchaInfo.some(c => c.type === 'imageCaptcha')) { + const solution = await solveSimpleCaptcha(page); + if (solution && solution.type === 'image' && solution.answer) { + console.log(` OCR solved image CAPTCHA: "${solution.answer}"`); + try { + await page.evaluate((answer) => { + const input = document.querySelector('input[name*="captcha"], input[class*="captcha"], input[id*="captcha"], #captcha, .captcha-input'); + if (input) { + input.value = answer; + input.dispatchEvent(new Event('input', { bubbles: true })); + input.dispatchEvent(new Event('change', { bubbles: true })); + } + }, solution.answer); + } catch (e) { + console.warn(` Failed to fill OCR answer: ${e.message}`); + } + } + } + + if (captchaInfo.some(c => c.type === 'recaptcha' || c.type === 'hcaptcha')) { + siteFailed = true; + failureReason = `CAPTCHA present: ${captchaInfo.map(c => c.name).join(', ')}`; + console.error(` Cannot auto-solve ${captchaInfo.map(c => c.name).join(', ')}`); + return { siteFailed, failureReason, page }; + } + } + + const formClassification = await classifyNewsletterForm(page); + console.log(`\n Form Classification: ${formClassification.isNewsletter ? 'Newsletter' : 'Not newsletter'} (${formClassification.confidence}/100 confidence)`); + + if (formClassification.signals.positive.length > 0) { + console.log(' Positive signals:'); + formClassification.signals.positive.forEach(signal => console.log(` + ${signal}`)); + } + + if (formClassification.signals.negative.length > 0) { + console.log(' Negative signals:'); + formClassification.signals.negative.forEach(signal => console.log(` - ${signal}`)); + } + + if (!formClassification.isNewsletter && formClassification.confidence < 40) { + siteFailed = true; + failureReason = 'Not a newsletter signup form'; + console.error(` ${failureReason}`); + } + + console.log('\nDETECTED FIELDS:'); + + let fieldsToFill = {}; + let detectedFields = {}; + const definedLocators = site.locators || {}; + + if (site.autoDetect !== false) { + detectedFields = await detectFormFields(page); + for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) { + fieldsToFill[fieldName] = fieldInfo.selector; + } + } + + const hasEmailField = fieldsToFill.email || fieldsToFill.confirmEmail; + if (!hasEmailField) { + siteFailed = true; + failureReason = 'No email field detected'; + console.warn(`\n ${failureReason}`); + } + + const hasSearchInput = await page.evaluate(() => { + const inputs = document.querySelectorAll('input[type="search"], input[name*="search"], input[id*="search"]'); + return inputs.length > 0; + }).catch(() => false); + + if (hasSearchInput && !hasEmailField) { + console.warn(` Search form detected instead of newsletter`); + siteFailed = true; + failureReason = failureReason || 'Search form instead of newsletter'; + } + + for (const [fieldName, locator] of Object.entries(definedLocators)) { + if (fieldName !== 'submit') { + fieldsToFill[fieldName] = parseLocator(locator); + } + } + + if (Object.keys(fieldsToFill).length > 0) { + console.log(`\n ${'FIELD'.padEnd(12)} | ${'SELECTOR'.padEnd(35)} | ${'CONF'.padEnd(4)} | VALUE`); + console.log(' ' + '-'.repeat(75)); + + for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) { + let valueToFill = creds[fieldName]; + + if (fieldName === 'confirmEmail') { + valueToFill = creds['email']; + } + + const selector = fieldInfo.selector.length > 34 ? fieldInfo.selector.substring(0, 31) + '...' : fieldInfo.selector; + const value = valueToFill ? valueToFill.substring(0, 20) : 'N/A'; + + console.log(` ${fieldName.padEnd(12)} | ${selector.padEnd(35)} | ${fieldInfo.confidence.toString().padEnd(4)} | ${value}`); + + if (valueToFill) { + try { + await page.evaluate((sel, val) => { + const el = document.querySelector(sel); + if (el) { + el.value = val; + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + } + }, fieldInfo.selector, valueToFill); + } catch (e) { + console.warn(` Failed to fill ${fieldName}: ${e.message}`); + siteFailed = true; + failureReason = `Failed to fill ${fieldName}: ${e.message}`; + } + } + } + console.log(' ' + '-'.repeat(75)); + } + + let submitSelector = null; + if (definedLocators.submit) { + submitSelector = parseLocator(definedLocators.submit); + } else if (site.autoDetect !== false) { + submitSelector = await detectSubmitButton(page); + } + + if (submitSelector) { + try { + await page.click(submitSelector); + console.log(` Submitted (${submitSelector})`); + } catch (submitError) { + siteFailed = true; + failureReason = failureReason || `Submit failed: ${submitError.message}`; + console.error(` Submit failed: ${submitError.message}`); + } + } else { + siteFailed = true; + failureReason = failureReason || 'No submit button found'; + console.warn(` ${failureReason}`); + } + + } catch (error) { + siteFailed = true; + failureReason = error.message; + console.error(` Error: ${error.message}`); + } finally { + if (page && !keepOpen) { + await page.close().catch(() => {}); + } + } + + return { siteFailed, failureReason, page }; +} + +module.exports = { + processSite +}; diff --git a/src/submit.js b/src/submit.js new file mode 100644 index 0000000..3f5def6 --- /dev/null +++ b/src/submit.js @@ -0,0 +1,82 @@ +async function detectSubmitButton(page) { + console.log('Auto-detecting submit button...'); + + const submitSelector = await page.evaluate(() => { + const buttons = Array.from(document.querySelectorAll('button[type="submit"], input[type="submit"], button, input[type="button"]')); + let bestMatch = null; + let bestScore = 0; + + buttons.forEach(btn => { + const text = (btn.textContent || btn.value || '').toLowerCase().trim(); + const type = btn.type?.toLowerCase() || ''; + const className = (btn.className || '').toLowerCase(); + let score = 0; + + if (type === 'submit') score += 20; + + if (text.includes('subscribe')) score += 15; + else if (text.includes('sign up')) score += 12; + else if (text.includes('submit')) score += 10; + else if (text.includes('join')) score += 8; + else if (text.includes('register')) score += 8; + else if (text.includes('send')) score += 5; + + if (className.includes('submit')) score += 8; + if (className.includes('subscribe')) score += 8; + + if (className.includes('close') || text.includes('close')) score -= 20; + if (className.includes('cancel') || text.includes('cancel')) score -= 20; + if (className.includes('back') || text.includes('back')) score -= 15; + if (className.includes('drawer')) score -= 15; + + const rect = btn.getBoundingClientRect(); + if (rect.width > 0 && rect.height > 0 && rect.top > 0) { + score += 3; + } + + if (score > bestScore) { + bestScore = score; + bestMatch = btn; + } + }); + + if (bestMatch && bestScore >= 10) { + if (bestMatch.id) return `#${bestMatch.id}`; + if (bestMatch.name) return `[name="${bestMatch.name}"]`; + + const className = bestMatch.className; + if (className) { + const classes = className.split(' ').filter(c => c.length > 0); + for (const cls of classes) { + const clsLower = cls.toLowerCase(); + if (clsLower.includes('submit') || clsLower.includes('subscribe')) { + return `.${cls}`; + } + } + for (const cls of classes) { + if (!cls.toLowerCase().includes('close') && !cls.toLowerCase().includes('cancel')) { + return `.${cls}`; + } + } + } + + const allBtns = Array.from(document.querySelectorAll('button')); + const index = allBtns.indexOf(bestMatch) + 1; + return `button:nth-of-type(${index})`; + } + + return null; + }); + + if (submitSelector) { + console.log(` Submit button found: ${submitSelector}`); + } else { + console.log(' No submit button found'); + } + + return submitSelector; +} + +module.exports = { + detectSubmitButton +};