changed to individual files instead one big ass script and added auto captcha solver functionality
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
DATE | TIME | WEBSITE | REASON
|
||||
----------------------------------------------------------------------------------------------------
|
||||
14/02/2026 | 20:31:58 | Site_3 (https://kulalaland.com/) | No email field detected
|
||||
14/02/2026 | 20:42:16 | Dutch News Test (https://www.dut | CAPTCHA present: reCAPTCHA
|
||||
14/02/2026 | 20:43:17 | Site_1 (https://www.dutchnews.nl | CAPTCHA present: reCAPTCHA
|
||||
14/02/2026 | 20:43:17 | Site_3 (https://kulalaland.com/) | No email field detected
|
||||
14/02/2026 | 20:45:19 | Site_1 (https://www.dutchnews.nl | CAPTCHA present: reCAPTCHA
|
||||
14/02/2026 | 20:45:19 | Site_2 (https://immaculatevegan. | Not a newsletter signup form
|
||||
14/02/2026 | 20:45:19 | Site_3 (https://kulalaland.com/) | No email field detected
|
||||
14/02/2026 | 20:46:06 | Site_1 (https://www.dutchnews.nl | CAPTCHA present: reCAPTCHA
|
||||
14/02/2026 | 20:46:06 | Site_3 (https://kulalaland.com/) | No email field detected
|
||||
14/02/2026 | 21:26:47 | Dutch News Test (https://www.dut | Navigating frame was detached
|
||||
14/02/2026 | 21:26:47 | Immaculate Vegan (https://immacu | Connection closed.
|
||||
14/02/2026 | 21:26:47 | Gate6 (https://gate6.vn/) | Connection closed.
|
||||
14/02/2026 | 21:26:47 | Kulala Land (https://kulalaland. | Connection closed.
|
||||
14/02/2026 | 21:26:47 | sneakywholefoods (https://www.sn | Connection closed.
|
||||
14/02/2026 | 21:27:40 | Dutch News Test (https://www.dut | CAPTCHA present: reCAPTCHA
|
||||
14/02/2026 | 21:27:40 | Kulala Land (https://kulalaland. | No email field detected
|
||||
14/02/2026 | 21:27:40 | sneakywholefoods (https://www.sn | Not a newsletter signup form
|
||||
@@ -1,437 +1,46 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const toml = require('@iarna/toml');
|
||||
const puppeteer = require('puppeteer-extra');
|
||||
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
||||
puppeteer.use(StealthPlugin());
|
||||
|
||||
const FIELD_PATTERNS = {
|
||||
email: {
|
||||
priority: 1,
|
||||
types: ['email'],
|
||||
names: ['email', 'e-mail', 'mail', 'user_email', 'login_email', 'email_address', 'emailaddress'],
|
||||
ids: ['email', 'e-mail', 'mail', 'user-email', 'login-email'],
|
||||
placeholders: ['email', 'e-mail', 'email address', 'your email', 'enter email'],
|
||||
autocomplete: ['email']
|
||||
},
|
||||
confirmEmail: {
|
||||
priority: 2,
|
||||
types: ['email'],
|
||||
names: ['confirm_email', 'confirmemail', 'email_confirm', 'verify_email', 'email_verify'],
|
||||
ids: ['confirm-email', 'confirmemail', 'verify-email', 'email-confirm'],
|
||||
placeholders: ['confirm email', 'verify email', 're-enter email', 'repeat email'],
|
||||
mustHaveConfirmIndicator: true
|
||||
},
|
||||
firstName: {
|
||||
priority: 3,
|
||||
types: ['text'],
|
||||
names: ['first_name', 'firstname', 'fname', 'first-name', 'vorname', 'prenom', 'fname'],
|
||||
ids: ['first-name', 'firstname', 'fname', 'firstName', 'mce-fname'],
|
||||
placeholders: ['first name', 'firstname', 'first'],
|
||||
autocomplete: ['given-name', 'first-name']
|
||||
},
|
||||
lastName: {
|
||||
priority: 4,
|
||||
types: ['text'],
|
||||
names: ['last_name', 'lastname', 'lname', 'surname', 'family_name', 'last-name', 'lname'],
|
||||
ids: ['last-name', 'lastname', 'lname', 'surname', 'lastName', 'mce-lname'],
|
||||
placeholders: ['last name', 'lastname', 'surname', 'last'],
|
||||
autocomplete: ['family-name', 'last-name']
|
||||
},
|
||||
username: {
|
||||
priority: 5,
|
||||
types: ['text'],
|
||||
names: ['username', 'user_name', 'login', 'usr', 'uname', 'user-id', 'userid', 'account_name', 'login_id'],
|
||||
ids: ['username', 'user-name', 'login', 'usr', 'uname'],
|
||||
placeholders: ['username', 'user name', 'login', 'username or email'],
|
||||
autocomplete: ['username']
|
||||
},
|
||||
password: {
|
||||
priority: 6,
|
||||
types: ['password'],
|
||||
names: ['password', 'pass', 'pwd', 'user_password', 'login_password'],
|
||||
ids: ['password', 'pass', 'pwd'],
|
||||
placeholders: ['password', 'pass', 'enter password'],
|
||||
autocomplete: ['current-password', 'new-password']
|
||||
},
|
||||
fullName: {
|
||||
priority: 7,
|
||||
types: ['text'],
|
||||
names: ['full_name', 'fullname', 'name'],
|
||||
ids: ['full-name', 'fullname', 'name'],
|
||||
placeholders: ['full name', 'your name', 'complete name'],
|
||||
autocomplete: ['name']
|
||||
},
|
||||
phone: {
|
||||
priority: 8,
|
||||
types: ['tel', 'text'],
|
||||
names: ['phone', 'telephone', 'mobile', 'cell', 'cellphone', 'phone_number'],
|
||||
ids: ['phone', 'telephone', 'mobile', 'cell', 'tel'],
|
||||
placeholders: ['phone', 'telephone', 'mobile number'],
|
||||
autocomplete: ['tel']
|
||||
},
|
||||
age: {
|
||||
priority: 9,
|
||||
types: ['number', 'text'],
|
||||
names: ['age', 'user_age'],
|
||||
ids: ['age'],
|
||||
placeholders: ['age', 'your age']
|
||||
},
|
||||
company: {
|
||||
priority: 10,
|
||||
types: ['text'],
|
||||
names: ['company', 'organization', 'company_name', 'employer'],
|
||||
ids: ['company', 'organization', 'employer'],
|
||||
placeholders: ['company', 'organization'],
|
||||
autocomplete: ['organization']
|
||||
}
|
||||
};
|
||||
|
||||
function loadConfig() {
|
||||
try {
|
||||
const configPath = path.join(__dirname, 'config.toml');
|
||||
const tomlString = fs.readFileSync(configPath, 'utf-8');
|
||||
const config = toml.parse(tomlString);
|
||||
console.log('✅ Configuration loaded successfully.');
|
||||
return config;
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to load or parse config.toml:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
function generateRandomData() {
|
||||
const randomString = Math.random().toString(36).substring(2, 10);
|
||||
const username = `user${randomString}`;
|
||||
const email = `${username}@gmail.com`;
|
||||
const password = `pass${randomString}`;
|
||||
const age = Math.floor(Math.random() * 30 + 20).toString();
|
||||
const firstName = 'John';
|
||||
const lastName = 'Doe';
|
||||
const phone = '+31610488717';
|
||||
const company = 'Example Corp';
|
||||
return { username, email, password, age, firstName, lastName, phone, company };
|
||||
}
|
||||
|
||||
function parseLocator(locatorStr) {
|
||||
locatorStr = locatorStr.trim();
|
||||
if (!locatorStr.startsWith('<')) {
|
||||
return locatorStr;
|
||||
}
|
||||
|
||||
const getAttr = (attr) => {
|
||||
const match = locatorStr.match(new RegExp(`${attr}=["']([^"']+)["']`));
|
||||
return match ? match[1] : null;
|
||||
};
|
||||
|
||||
const tagName = (locatorStr.match(/^<([a-zA-Z0-9]+)/) || [])[1] || '';
|
||||
const id = getAttr('id');
|
||||
if (id) {
|
||||
if (/^\d/.test(id)) {
|
||||
return `${tagName}[id='${id}']`;
|
||||
}
|
||||
return `${tagName}#${id}`;
|
||||
}
|
||||
|
||||
const name = getAttr('name');
|
||||
if (name) return `${tagName}[name='${name}']`;
|
||||
|
||||
const placeholder = getAttr('placeholder');
|
||||
if (placeholder) return `${tagName}[placeholder='${placeholder}']`;
|
||||
|
||||
const type = getAttr('type');
|
||||
if (type) return `${tagName}[type='${type}']`;
|
||||
|
||||
return tagName;
|
||||
}
|
||||
|
||||
async function detectFormFields(page) {
|
||||
console.log('🔍 Auto-detecting form fields...');
|
||||
|
||||
const detectedFields = await page.evaluate((patterns) => {
|
||||
const results = {};
|
||||
const usedElements = new Set();
|
||||
|
||||
// Get all visible input fields, excluding hidden fields
|
||||
const inputs = Array.from(document.querySelectorAll('input, select, textarea')).filter(input => {
|
||||
const type = input.type?.toLowerCase() || 'text';
|
||||
const style = window.getComputedStyle(input);
|
||||
const rect = input.getBoundingClientRect();
|
||||
|
||||
// Skip hidden fields and honeypots
|
||||
if (type === 'hidden' || type === 'submit' || type === 'button') return false;
|
||||
if (input.tabIndex === -1 && input.style.position === 'absolute') return false; // Likely honeypot
|
||||
if (rect.width === 0 || rect.height === 0) return false;
|
||||
if (style.display === 'none' || style.visibility === 'hidden') return false;
|
||||
|
||||
return true;
|
||||
});
|
||||
|
||||
const calculateScore = (input, pattern, fieldName) => {
|
||||
let score = 0;
|
||||
const inputType = (input.type || 'text').toLowerCase();
|
||||
const inputName = (input.name || '').toLowerCase();
|
||||
const inputId = (input.id || '').toLowerCase();
|
||||
const placeholder = (input.placeholder || '').toLowerCase();
|
||||
const autocomplete = (input.autocomplete || '').toLowerCase();
|
||||
|
||||
// Type matching
|
||||
if (pattern.types && pattern.types.includes(inputType)) {
|
||||
score += 5;
|
||||
}
|
||||
|
||||
// Name matching - exact matches get high scores
|
||||
if (pattern.names) {
|
||||
for (const name of pattern.names) {
|
||||
const nameLower = name.toLowerCase();
|
||||
if (inputName === nameLower) {
|
||||
score += 20; // Exact match
|
||||
break;
|
||||
} else if (inputName.includes(nameLower)) {
|
||||
// Partial match - be careful not to match fname with username
|
||||
if (fieldName === 'firstName' && (inputName.includes('user') || inputName.includes('login'))) {
|
||||
score -= 5; // Penalize if it looks like username
|
||||
} else if (fieldName === 'lastName' && (inputName.includes('user') || inputName.includes('login'))) {
|
||||
score -= 5;
|
||||
} else {
|
||||
score += 10;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ID matching - very specific
|
||||
if (pattern.ids) {
|
||||
for (const id of pattern.ids) {
|
||||
const idLower = id.toLowerCase();
|
||||
if (inputId === idLower) {
|
||||
score += 20; // Exact match
|
||||
break;
|
||||
} else if (inputId.includes(idLower)) {
|
||||
score += 10;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Placeholder matching
|
||||
if (pattern.placeholders) {
|
||||
for (const ph of pattern.placeholders) {
|
||||
const phLower = ph.toLowerCase();
|
||||
if (placeholder === phLower) {
|
||||
score += 15;
|
||||
break;
|
||||
} else if (placeholder.includes(phLower)) {
|
||||
score += 8;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Autocomplete matching
|
||||
if (pattern.autocomplete) {
|
||||
for (const ac of pattern.autocomplete) {
|
||||
if (autocomplete === ac.toLowerCase()) {
|
||||
score += 12;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check label text
|
||||
const labels = input.labels;
|
||||
if (labels && labels.length > 0) {
|
||||
const labelText = labels[0].textContent.toLowerCase();
|
||||
const fieldNameLower = fieldName.toLowerCase().replace(/([A-Z])/g, ' $1').trim();
|
||||
if (labelText.includes(fieldNameLower) || labelText.includes(pattern.names?.[0] || '')) {
|
||||
score += 15;
|
||||
}
|
||||
}
|
||||
|
||||
// Penalize if field appears to be a honeypot
|
||||
if (inputName.includes('b_') || inputId.includes('b_')) {
|
||||
score -= 30; // Likely Mailchimp honeypot
|
||||
}
|
||||
|
||||
// Penalize disabled/readonly
|
||||
if (input.disabled || input.readOnly) {
|
||||
score -= 20;
|
||||
}
|
||||
|
||||
// Special handling for confirmEmail
|
||||
if (pattern.mustHaveConfirmIndicator) {
|
||||
const hasConfirm = inputId.includes('confirm') || inputId.includes('verify') ||
|
||||
inputName.includes('confirm') || inputName.includes('verify') ||
|
||||
placeholder.includes('confirm') || placeholder.includes('verify') ||
|
||||
placeholder.includes('re-') || placeholder.includes('again');
|
||||
if (!hasConfirm) {
|
||||
score -= 15;
|
||||
}
|
||||
}
|
||||
|
||||
return score;
|
||||
};
|
||||
|
||||
// Sort patterns by priority
|
||||
const sortedPatterns = Object.entries(patterns).sort((a, b) => a[1].priority - b[1].priority);
|
||||
|
||||
for (const [fieldName, pattern] of sortedPatterns) {
|
||||
let bestMatch = null;
|
||||
let bestScore = 0;
|
||||
let bestIndex = -1;
|
||||
|
||||
for (let i = 0; i < inputs.length; i++) {
|
||||
if (usedElements.has(i)) continue;
|
||||
|
||||
const input = inputs[i];
|
||||
const score = calculateScore(input, pattern, fieldName);
|
||||
|
||||
if (score > bestScore && score >= 10) {
|
||||
bestScore = score;
|
||||
bestMatch = input;
|
||||
bestIndex = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (bestMatch && bestIndex >= 0) {
|
||||
usedElements.add(bestIndex);
|
||||
|
||||
// Generate selector
|
||||
let selector = '';
|
||||
if (bestMatch.id) {
|
||||
selector = `#${bestMatch.id}`;
|
||||
} else if (bestMatch.name) {
|
||||
selector = `[name="${bestMatch.name}"]`;
|
||||
} else if (bestMatch.placeholder) {
|
||||
selector = `[placeholder="${bestMatch.placeholder}"]`;
|
||||
} else {
|
||||
const tagName = bestMatch.tagName.toLowerCase();
|
||||
const type = bestMatch.type ? `[type="${bestMatch.type}"]` : '';
|
||||
const allSame = Array.from(document.querySelectorAll(`${tagName}${type}`));
|
||||
const index = allSame.indexOf(bestMatch) + 1;
|
||||
selector = `${tagName}${type}:nth-of-type(${index})`;
|
||||
}
|
||||
|
||||
results[fieldName] = {
|
||||
selector,
|
||||
confidence: bestScore,
|
||||
name: bestMatch.name,
|
||||
id: bestMatch.id
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}, FIELD_PATTERNS);
|
||||
|
||||
const detectedCount = Object.keys(detectedFields).length;
|
||||
if (detectedCount === 0) {
|
||||
console.log(' No fields detected');
|
||||
}
|
||||
|
||||
return detectedFields;
|
||||
}
|
||||
|
||||
async function detectSubmitButton(page) {
|
||||
console.log('🔍 Auto-detecting submit button...');
|
||||
|
||||
const submitSelector = await page.evaluate(() => {
|
||||
const buttons = Array.from(document.querySelectorAll('button[type="submit"], input[type="submit"], button, input[type="button"]'));
|
||||
let bestMatch = null;
|
||||
let bestScore = 0;
|
||||
|
||||
buttons.forEach(btn => {
|
||||
const text = (btn.textContent || btn.value || '').toLowerCase().trim();
|
||||
const type = btn.type?.toLowerCase() || '';
|
||||
const className = (btn.className || '').toLowerCase();
|
||||
let score = 0;
|
||||
|
||||
if (type === 'submit') score += 20;
|
||||
|
||||
if (text.includes('subscribe')) score += 15;
|
||||
else if (text.includes('sign up')) score += 12;
|
||||
else if (text.includes('submit')) score += 10;
|
||||
else if (text.includes('join')) score += 8;
|
||||
else if (text.includes('register')) score += 8;
|
||||
else if (text.includes('send')) score += 5;
|
||||
|
||||
if (className.includes('submit')) score += 8;
|
||||
if (className.includes('subscribe')) score += 8;
|
||||
|
||||
if (className.includes('close') || text.includes('close')) score -= 20;
|
||||
if (className.includes('cancel') || text.includes('cancel')) score -= 20;
|
||||
if (className.includes('back') || text.includes('back')) score -= 15;
|
||||
if (className.includes('drawer')) score -= 15;
|
||||
|
||||
const rect = btn.getBoundingClientRect();
|
||||
if (rect.width > 0 && rect.height > 0 && rect.top > 0) {
|
||||
score += 3;
|
||||
}
|
||||
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
bestMatch = btn;
|
||||
}
|
||||
});
|
||||
|
||||
if (bestMatch && bestScore >= 10) {
|
||||
if (bestMatch.id) return `#${bestMatch.id}`;
|
||||
if (bestMatch.name) return `[name="${bestMatch.name}"]`;
|
||||
|
||||
const className = bestMatch.className;
|
||||
if (className) {
|
||||
const classes = className.split(' ').filter(c => c.length > 0);
|
||||
for (const cls of classes) {
|
||||
const clsLower = cls.toLowerCase();
|
||||
if (clsLower.includes('submit') || clsLower.includes('subscribe')) {
|
||||
return `.${cls}`;
|
||||
}
|
||||
}
|
||||
for (const cls of classes) {
|
||||
if (!cls.toLowerCase().includes('close') && !cls.toLowerCase().includes('cancel')) {
|
||||
return `.${cls}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const allBtns = Array.from(document.querySelectorAll('button'));
|
||||
const index = allBtns.indexOf(bestMatch) + 1;
|
||||
return `button:nth-of-type(${index})`;
|
||||
}
|
||||
|
||||
return null;
|
||||
});
|
||||
|
||||
if (submitSelector) {
|
||||
console.log(` ✅ Submit button found: ${submitSelector}`);
|
||||
} else {
|
||||
console.log(' ❌ No submit button found');
|
||||
}
|
||||
|
||||
return submitSelector;
|
||||
}
|
||||
const { loadConfig, loadSitesFromFile } = require('./src/config');
|
||||
const { processSite } = require('./src/processor');
|
||||
const { createCLI, parseCLI } = require('./src/cli');
|
||||
|
||||
async function main() {
|
||||
const cliOptions = parseCLI();
|
||||
|
||||
if (cliOptions.help) {
|
||||
createCLI().help();
|
||||
return;
|
||||
}
|
||||
|
||||
const config = loadConfig();
|
||||
let sitesToProcess = config.sites || [];
|
||||
|
||||
const targetSite = process.argv[2];
|
||||
const logToFile = process.argv.includes('--log');
|
||||
const logFile = logToFile ? path.join(__dirname, `run_${new Date().toISOString().replace(/[:.]/g, '-')}.log`) : null;
|
||||
if (cliOptions.file) {
|
||||
console.log(`Loading sites from: ${cliOptions.file}`);
|
||||
sitesToProcess = loadSitesFromFile(cliOptions.file);
|
||||
console.log(`Loaded ${sitesToProcess.length} sites from file`);
|
||||
}
|
||||
|
||||
if (targetSite && !targetSite.startsWith('--')) {
|
||||
const logFile = cliOptions.log ? path.join(__dirname, `run_${new Date().toISOString().replace(/[:.]/g, '-')}.log`) : null;
|
||||
|
||||
if (cliOptions.target) {
|
||||
sitesToProcess = sitesToProcess.filter(site =>
|
||||
site.name.toLowerCase().includes(targetSite.toLowerCase()) ||
|
||||
site.url.toLowerCase().includes(targetSite.toLowerCase())
|
||||
site.name.toLowerCase().includes(cliOptions.target.toLowerCase()) ||
|
||||
site.url.toLowerCase().includes(cliOptions.target.toLowerCase())
|
||||
);
|
||||
if (sitesToProcess.length === 0) {
|
||||
console.error(`❌ No site found matching "${targetSite}"`);
|
||||
console.log('Available sites:');
|
||||
config.sites.forEach(site => console.log(` - ${site.name}: ${site.url}`));
|
||||
console.error(`No site found matching "${cliOptions.target}"`);
|
||||
return;
|
||||
}
|
||||
console.log(`🎯 Filtering to ${sitesToProcess.length} site(s) matching "${targetSite}"`);
|
||||
console.log(`Targeting ${sitesToProcess.length} site(s)`);
|
||||
}
|
||||
|
||||
if (cliOptions.keepOpen) {
|
||||
console.log('Keep-open mode: Tabs will stay open for verification');
|
||||
}
|
||||
|
||||
const browserConfig = config.browser || {};
|
||||
@@ -439,11 +48,11 @@ async function main() {
|
||||
const userDataDir = browserConfig.userDataDir;
|
||||
|
||||
if (!userDataDir) {
|
||||
console.error('❌ userDataDir is not set in config.toml. This is required.');
|
||||
console.error('userDataDir not set in config.toml');
|
||||
return;
|
||||
}
|
||||
|
||||
if (logToFile) {
|
||||
if (cliOptions.log && logFile) {
|
||||
const originalLog = console.log;
|
||||
const originalError = console.error;
|
||||
const originalWarn = console.warn;
|
||||
@@ -453,173 +62,58 @@ async function main() {
|
||||
fs.appendFileSync(logFile, `[${new Date().toISOString()}] [${type}] ${message}\n`);
|
||||
};
|
||||
|
||||
console.log = (...args) => {
|
||||
writeToLog('LOG', args);
|
||||
originalLog.apply(console, args);
|
||||
};
|
||||
|
||||
console.error = (...args) => {
|
||||
writeToLog('ERROR', args);
|
||||
originalError.apply(console, args);
|
||||
};
|
||||
|
||||
console.warn = (...args) => {
|
||||
writeToLog('WARN', args);
|
||||
originalWarn.apply(console, args);
|
||||
};
|
||||
|
||||
console.log(`📝 Logging to file: ${logFile}`);
|
||||
console.log = (...args) => { writeToLog('LOG', args); originalLog.apply(console, args); };
|
||||
console.error = (...args) => { writeToLog('ERROR', args); originalError.apply(console, args); };
|
||||
console.warn = (...args) => { writeToLog('WARN', args); originalWarn.apply(console, args); };
|
||||
console.log(`Logging to: ${logFile}`);
|
||||
}
|
||||
|
||||
const failedSites = [];
|
||||
const processedCount = { success: 0, failed: 0 };
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: false,
|
||||
userDataDir: userDataDir,
|
||||
args: ["--start-maximized"]
|
||||
args: ['--start-maximized', '--no-sandbox', '--disable-setuid-sandbox']
|
||||
});
|
||||
|
||||
try {
|
||||
let isFirstSite = true;
|
||||
for (const site of sitesToProcess) {
|
||||
let siteFailed = false;
|
||||
let failureReason = '';
|
||||
let creds;
|
||||
if (site.credentials && site.credentials.username) {
|
||||
creds = { ...globalCreds, ...site.credentials };
|
||||
console.log(`Using credentials specified for site: ${site.name}`);
|
||||
} else {
|
||||
creds = { ...globalCreds };
|
||||
console.log(`Using global credentials for site: ${site.name}`);
|
||||
}
|
||||
console.log(`\nProcessing ${sitesToProcess.length} sites...\n`);
|
||||
|
||||
for (let i = 0; i < sitesToProcess.length; i++) {
|
||||
const site = sitesToProcess[i];
|
||||
console.log(`[${i + 1}/${sitesToProcess.length}]`);
|
||||
|
||||
if (!creds.username || !creds.email || !creds.password) {
|
||||
const randomData = generateRandomData();
|
||||
creds = { ...randomData, ...creds };
|
||||
console.log('Filled in missing credentials with random data.');
|
||||
}
|
||||
|
||||
console.log(`\n${site.url}`);
|
||||
console.log(`${'-'.repeat(70)}`);
|
||||
const result = await processSite(browser, site, globalCreds, browserConfig, cliOptions.keepOpen);
|
||||
|
||||
const page = isFirstSite ? (await browser.pages())[0] : await browser.newPage();
|
||||
isFirstSite = false;
|
||||
|
||||
if (browserConfig.userAgent) {
|
||||
await page.setUserAgent(browserConfig.userAgent);
|
||||
}
|
||||
|
||||
await page.goto(site.url, { waitUntil: 'networkidle2' });
|
||||
|
||||
console.log('\n🔍 DETECTED FIELDS:');
|
||||
|
||||
let fieldsToFill = {};
|
||||
let detectedFields = {};
|
||||
const definedLocators = site.locators || {};
|
||||
|
||||
if (site.autoDetect !== false) {
|
||||
detectedFields = await detectFormFields(page);
|
||||
for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
|
||||
fieldsToFill[fieldName] = fieldInfo.selector;
|
||||
}
|
||||
}
|
||||
|
||||
const hasEmailField = fieldsToFill.email || fieldsToFill.confirmEmail;
|
||||
if (!hasEmailField) {
|
||||
siteFailed = true;
|
||||
failureReason = 'No email field detected';
|
||||
console.warn(`\n ❌ ${failureReason}`);
|
||||
}
|
||||
|
||||
const hasSearchInput = await page.evaluate(() => {
|
||||
const inputs = document.querySelectorAll('input[type="search"], input[name*="search"], input[id*="search"]');
|
||||
return inputs.length > 0;
|
||||
});
|
||||
|
||||
if (hasSearchInput && !hasEmailField) {
|
||||
console.warn(` ⚠️ WARNING: This appears to be a SEARCH form, not a newsletter signup!`);
|
||||
siteFailed = true;
|
||||
failureReason = failureReason || 'Detected search form instead of newsletter';
|
||||
}
|
||||
|
||||
for (const [fieldName, locator] of Object.entries(definedLocators)) {
|
||||
if (fieldName !== 'submit') {
|
||||
fieldsToFill[fieldName] = parseLocator(locator);
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(fieldsToFill).length > 0) {
|
||||
console.log(`\n ${'FIELD'.padEnd(12)} | ${'SELECTOR'.padEnd(35)} | ${'CONF'.padEnd(4)} | VALUE`);
|
||||
console.log(' ' + '-'.repeat(75));
|
||||
|
||||
for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
|
||||
let valueToFill = creds[fieldName];
|
||||
|
||||
if (fieldName === 'confirmEmail') {
|
||||
valueToFill = creds['email'];
|
||||
}
|
||||
|
||||
const selector = fieldInfo.selector.length > 34 ? fieldInfo.selector.substring(0, 31) + '...' : fieldInfo.selector;
|
||||
const value = valueToFill ? valueToFill.substring(0, 20) : 'N/A';
|
||||
|
||||
console.log(` ${fieldName.padEnd(12)} | ${selector.padEnd(35)} | ${fieldInfo.confidence.toString().padEnd(4)} | ${value}`);
|
||||
|
||||
if (valueToFill) {
|
||||
try {
|
||||
await page.evaluate((sel, val) => {
|
||||
const el = document.querySelector(sel);
|
||||
if (el) {
|
||||
el.value = val;
|
||||
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
el.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}
|
||||
}, fieldInfo.selector, valueToFill);
|
||||
} catch (e) {
|
||||
console.warn(` ✗ Failed to fill ${fieldName}: ${e.message}`);
|
||||
siteFailed = true;
|
||||
failureReason = `Failed to fill ${fieldName}: ${e.message}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(' ' + '-'.repeat(75));
|
||||
}
|
||||
|
||||
let submitSelector = null;
|
||||
if (definedLocators.submit) {
|
||||
submitSelector = parseLocator(definedLocators.submit);
|
||||
} else if (site.autoDetect !== false) {
|
||||
submitSelector = await detectSubmitButton(page);
|
||||
}
|
||||
|
||||
if (submitSelector) {
|
||||
try {
|
||||
await page.click(submitSelector);
|
||||
console.log(` ✓ Submitted (${submitSelector})`);
|
||||
} catch (submitError) {
|
||||
siteFailed = true;
|
||||
failureReason = failureReason || `Submit failed: ${submitError.message}`;
|
||||
console.error(` ✗ Submit failed: ${submitError.message}`);
|
||||
}
|
||||
} else {
|
||||
siteFailed = true;
|
||||
failureReason = failureReason || 'No submit button found';
|
||||
console.warn(` ✗ ${failureReason}`);
|
||||
}
|
||||
|
||||
if (siteFailed) {
|
||||
if (result.siteFailed) {
|
||||
failedSites.push({
|
||||
name: site.name,
|
||||
url: site.url,
|
||||
reason: failureReason,
|
||||
reason: result.failureReason,
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
processedCount.failed++;
|
||||
} else {
|
||||
processedCount.success++;
|
||||
}
|
||||
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
}
|
||||
|
||||
console.log(`\n${'='.repeat(70)}`);
|
||||
console.log(`Done: ${processedCount.success} success, ${processedCount.failed} failed`);
|
||||
|
||||
} catch(error) {
|
||||
console.error('Error:', error.message);
|
||||
console.error('Fatal error:', error.message);
|
||||
} finally {
|
||||
if (cliOptions.keepOpen) {
|
||||
console.log(`\nBrowser kept open for verification`);
|
||||
console.log(` Close browser manually when done`);
|
||||
} else {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
if (failedSites.length > 0) {
|
||||
const reportPath = path.join(__dirname, 'failed_sites.txt');
|
||||
const now = new Date();
|
||||
@@ -638,7 +132,7 @@ async function main() {
|
||||
}
|
||||
|
||||
fs.appendFileSync(reportPath, reportContent);
|
||||
console.log(`\n${failedSites.length} site(s) failed. See: failed_sites.txt`);
|
||||
console.log(`${failedSites.length} failed. See: failed_sites.txt`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+126
-1
@@ -10,11 +10,13 @@
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@iarna/toml": "^2.2.5",
|
||||
"commander": "^14.0.3",
|
||||
"playwright": "^1.56.1",
|
||||
"playwright-extra": "^4.3.6",
|
||||
"puppeteer": "^24.30.0",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2"
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||
"tesseract.js": "^7.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/code-frame": {
|
||||
@@ -288,6 +290,12 @@
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bmp-js": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
|
||||
"integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "1.1.12",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
|
||||
@@ -377,6 +385,15 @@
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/commander": {
|
||||
"version": "14.0.3",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz",
|
||||
"integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/concat-map": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
|
||||
@@ -748,6 +765,12 @@
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/idb-keyval": {
|
||||
"version": "6.2.2",
|
||||
"resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.2.tgz",
|
||||
"integrity": "sha512-yjD9nARJ/jb1g+CvD0tlhUHOrJ9Sy0P8T9MF3YaLlHnSRpwPfpTX0XIvpmw3gAJUmEu3FiICLBDPXVwyEvrleg==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/import-fresh": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz",
|
||||
@@ -832,6 +855,12 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/is-url": {
|
||||
"version": "1.2.4",
|
||||
"resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz",
|
||||
"integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/isobject": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
|
||||
@@ -982,6 +1011,26 @@
|
||||
"node": ">= 0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"encoding": "^0.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"encoding": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
@@ -991,6 +1040,15 @@
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"node_modules/opencollective-postinstall": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz",
|
||||
"integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==",
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"opencollective-postinstall": "index.js"
|
||||
}
|
||||
},
|
||||
"node_modules/pac-proxy-agent": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz",
|
||||
@@ -1347,6 +1405,12 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/regenerator-runtime": {
|
||||
"version": "0.13.11",
|
||||
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz",
|
||||
"integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/require-directory": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||
@@ -1539,6 +1603,30 @@
|
||||
"streamx": "^2.15.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tesseract.js": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-7.0.0.tgz",
|
||||
"integrity": "sha512-exPBkd+z+wM1BuMkx/Bjv43OeLBxhL5kKWsz/9JY+DXcXdiBjiAch0V49QR3oAJqCaL5qURE0vx9Eo+G5YE7mA==",
|
||||
"hasInstallScript": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"bmp-js": "^0.1.0",
|
||||
"idb-keyval": "^6.2.0",
|
||||
"is-url": "^1.2.4",
|
||||
"node-fetch": "^2.6.9",
|
||||
"opencollective-postinstall": "^2.0.3",
|
||||
"regenerator-runtime": "^0.13.3",
|
||||
"tesseract.js-core": "^7.0.0",
|
||||
"wasm-feature-detect": "^1.8.0",
|
||||
"zlibjs": "^0.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/tesseract.js-core": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-7.0.0.tgz",
|
||||
"integrity": "sha512-WnNH518NzmbSq9zgTPeoF8c+xmilS8rFIl1YKbk/ptuuc7p6cLNELNuPAzcmsYw450ca6bLa8j3t0VAtq435Vw==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/text-decoder": {
|
||||
"version": "1.2.6",
|
||||
"resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.6.tgz",
|
||||
@@ -1548,6 +1636,12 @@
|
||||
"b4a": "^1.6.4"
|
||||
}
|
||||
},
|
||||
"node_modules/tr46": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.8.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||
@@ -1576,12 +1670,34 @@
|
||||
"node": ">= 10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/wasm-feature-detect": {
|
||||
"version": "1.8.0",
|
||||
"resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz",
|
||||
"integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/webdriver-bidi-protocol": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz",
|
||||
"integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/whatwg-url": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tr46": "~0.0.3",
|
||||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/wrap-ansi": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
|
||||
@@ -1672,6 +1788,15 @@
|
||||
"fd-slicer": "~1.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/zlibjs": {
|
||||
"version": "0.3.1",
|
||||
"resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz",
|
||||
"integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/zod": {
|
||||
"version": "3.25.76",
|
||||
"resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
|
||||
|
||||
+3
-1
@@ -12,10 +12,12 @@
|
||||
"type": "commonjs",
|
||||
"dependencies": {
|
||||
"@iarna/toml": "^2.2.5",
|
||||
"commander": "^14.0.3",
|
||||
"playwright": "^1.56.1",
|
||||
"playwright-extra": "^4.3.6",
|
||||
"puppeteer": "^24.30.0",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2"
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||
"tesseract.js": "^7.0.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# Newsletter signup sites
|
||||
# Lines starting with # are ignored
|
||||
|
||||
https://www.dutchnews.nl/dutchnews-newsletter-signup/
|
||||
https://immaculatevegan.com/blogs/magazine
|
||||
https://kulalaland.com/
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
const Tesseract = require('tesseract.js');
|
||||
|
||||
const CAPTCHA_PATTERNS = {
|
||||
recaptcha: {
|
||||
selectors: ['.g-recaptcha', '[data-sitekey]', 'iframe[src*="recaptcha"]', '#recaptcha'],
|
||||
name: 'reCAPTCHA'
|
||||
},
|
||||
hcaptcha: {
|
||||
selectors: ['.h-captcha', '[data-hcaptcha-sitekey]', 'iframe[src*="hcaptcha"]', '#hcaptcha'],
|
||||
name: 'hCaptcha'
|
||||
},
|
||||
textCaptcha: {
|
||||
selectors: ['input[name*="captcha"]', 'input[id*="captcha"]', '.captcha-input', '#captcha'],
|
||||
name: 'Text CAPTCHA'
|
||||
},
|
||||
imageCaptcha: {
|
||||
selectors: ['img[src*="captcha"]', '.captcha-image', '#captcha-image', 'img[alt*="captcha" i]'],
|
||||
name: 'Image CAPTCHA'
|
||||
},
|
||||
mathCaptcha: {
|
||||
selectors: ['.math-captcha', '[name*="math"]', 'input[placeholder*="math" i]'],
|
||||
name: 'Math CAPTCHA'
|
||||
}
|
||||
};
|
||||
|
||||
async function detectCaptcha(page) {
|
||||
const captchaInfo = await page.evaluate((patterns) => {
|
||||
const results = [];
|
||||
|
||||
for (const [type, config] of Object.entries(patterns)) {
|
||||
for (const selector of config.selectors) {
|
||||
const elements = document.querySelectorAll(selector);
|
||||
if (elements.length > 0) {
|
||||
results.push({
|
||||
type: type,
|
||||
name: config.name,
|
||||
selector: selector,
|
||||
count: elements.length
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}, CAPTCHA_PATTERNS);
|
||||
|
||||
return captchaInfo;
|
||||
}
|
||||
|
||||
async function solveSimpleCaptcha(page) {
|
||||
const mathSolution = await page.evaluate(() => {
|
||||
const mathText = document.querySelector('.math-captcha, .captcha-question, [class*="math"]');
|
||||
if (mathText) {
|
||||
const text = mathText.textContent || '';
|
||||
const match = text.match(/(\d+)\s*([+\-*/])\s*(\d+)/);
|
||||
if (match) {
|
||||
const [_, a, op, b] = match;
|
||||
let result;
|
||||
switch(op) {
|
||||
case '+': result = parseInt(a) + parseInt(b); break;
|
||||
case '-': result = parseInt(a) - parseInt(b); break;
|
||||
case '*': result = parseInt(a) * parseInt(b); break;
|
||||
case '/': result = Math.floor(parseInt(a) / parseInt(b)); break;
|
||||
}
|
||||
return { type: 'math', answer: result, text: text };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
});
|
||||
|
||||
if (mathSolution) {
|
||||
return mathSolution;
|
||||
}
|
||||
|
||||
const imageCaptchaInfo = await page.evaluate(() => {
|
||||
const selectors = [
|
||||
'img[src*="captcha"]',
|
||||
'.captcha-image img',
|
||||
'#captcha-image',
|
||||
'img[alt*="captcha" i]',
|
||||
'img[class*="captcha" i]'
|
||||
];
|
||||
|
||||
for (const selector of selectors) {
|
||||
const img = document.querySelector(selector);
|
||||
if (img && img.src) {
|
||||
return { src: img.src, selector: selector };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
});
|
||||
|
||||
if (imageCaptchaInfo) {
|
||||
console.log(' Attempting OCR-based CAPTCHA solving...');
|
||||
try {
|
||||
const { data: { text } } = await Tesseract.recognize(
|
||||
imageCaptchaInfo.src,
|
||||
'eng',
|
||||
{ logger: m => {} }
|
||||
);
|
||||
|
||||
const cleanedText = text.replace(/[^a-zA-Z0-9]/g, '').trim();
|
||||
|
||||
if (cleanedText && cleanedText.length >= 3) {
|
||||
console.log(` OCR read: "${cleanedText}"`);
|
||||
return {
|
||||
type: 'image',
|
||||
answer: cleanedText,
|
||||
originalText: text,
|
||||
imageSrc: imageCaptchaInfo.src
|
||||
};
|
||||
}
|
||||
} catch (ocrError) {
|
||||
console.log(` OCR failed: ${ocrError.message}`);
|
||||
}
|
||||
|
||||
return { type: 'image', message: 'Image CAPTCHA detected - OCR failed, manual solving required' };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
CAPTCHA_PATTERNS,
|
||||
detectCaptcha,
|
||||
solveSimpleCaptcha
|
||||
};
|
||||
@@ -0,0 +1,109 @@
|
||||
async function classifyNewsletterForm(page) {
|
||||
const formInfo = await page.evaluate(() => {
|
||||
const results = {
|
||||
hasEmailInput: false,
|
||||
hasNewsletterKeywords: false,
|
||||
hasSearchInput: false,
|
||||
hasLoginInput: false,
|
||||
hasContactForm: false,
|
||||
hasOnlyEmail: false,
|
||||
formCount: 0,
|
||||
emailInputCount: 0,
|
||||
totalInputCount: 0,
|
||||
confidence: 0,
|
||||
reasons: [],
|
||||
signals: {
|
||||
positive: [],
|
||||
negative: []
|
||||
}
|
||||
};
|
||||
|
||||
const allInputs = document.querySelectorAll('input:not([type="hidden"]):not([type="submit"]):not([type="button"])');
|
||||
results.totalInputCount = allInputs.length;
|
||||
|
||||
const emailInputs = document.querySelectorAll('input[type="email"], input[name*="email" i], input[id*="email" i], input[placeholder*="email" i]');
|
||||
results.emailInputCount = emailInputs.length;
|
||||
results.hasEmailInput = emailInputs.length > 0;
|
||||
if (results.hasEmailInput) {
|
||||
results.signals.positive.push(`Found ${emailInputs.length} email input(s)`);
|
||||
}
|
||||
|
||||
const pageText = document.body.innerText.toLowerCase();
|
||||
const titleText = document.title.toLowerCase();
|
||||
const combinedText = pageText + ' ' + titleText;
|
||||
|
||||
const newsletterKeywords = [
|
||||
'newsletter', 'subscribe', 'signup', 'sign up', 'join our', 'stay updated',
|
||||
'get updates', 'email updates', 'weekly digest', 'monthly newsletter',
|
||||
'sign up for updates', 'join the list', 'email list', 'mailing list'
|
||||
];
|
||||
|
||||
for (const keyword of newsletterKeywords) {
|
||||
if (combinedText.includes(keyword)) {
|
||||
results.hasNewsletterKeywords = true;
|
||||
results.signals.positive.push(`Keyword: "${keyword}"`);
|
||||
}
|
||||
}
|
||||
|
||||
const searchInputs = document.querySelectorAll('input[type="search"], input[name*="search" i], input[id*="search" i], input[placeholder*="search" i]');
|
||||
results.hasSearchInput = searchInputs.length > 0;
|
||||
if (results.hasSearchInput) {
|
||||
results.signals.negative.push('Search input detected');
|
||||
results.reasons.push('Search input detected');
|
||||
}
|
||||
|
||||
const passwordInputs = document.querySelectorAll('input[type="password"]');
|
||||
const usernameInputs = document.querySelectorAll('input[name*="username" i], input[name*="login" i], input[id*="username" i]');
|
||||
results.hasLoginInput = passwordInputs.length > 0 || usernameInputs.length > 0;
|
||||
if (results.hasLoginInput) {
|
||||
results.signals.negative.push('Login/password fields detected');
|
||||
results.reasons.push('Login/password fields detected');
|
||||
}
|
||||
|
||||
const contactKeywords = ['contact us', 'get in touch', 'send message', 'inquiry'];
|
||||
for (const keyword of contactKeywords) {
|
||||
if (combinedText.includes(keyword)) {
|
||||
results.hasContactForm = true;
|
||||
results.signals.negative.push(`Contact form keyword: "${keyword}"`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const nonEmailInputs = Array.from(allInputs).filter(input => {
|
||||
const type = input.type?.toLowerCase();
|
||||
const name = input.name?.toLowerCase() || '';
|
||||
return type !== 'email' && !name.includes('email');
|
||||
});
|
||||
|
||||
results.hasOnlyEmail = results.hasEmailInput && nonEmailInputs.length === 0;
|
||||
if (results.hasOnlyEmail) {
|
||||
results.signals.positive.push('Only email field (no other inputs)');
|
||||
}
|
||||
|
||||
let score = 0;
|
||||
|
||||
if (results.hasEmailInput) score += 40;
|
||||
if (results.hasNewsletterKeywords) score += 35;
|
||||
if (results.hasOnlyEmail) score += 15;
|
||||
if (results.totalInputCount <= 3) score += 10;
|
||||
|
||||
if (!results.hasSearchInput) score += 10;
|
||||
if (!results.hasLoginInput) score += 15;
|
||||
if (!results.hasContactForm) score += 5;
|
||||
|
||||
if (results.hasSearchInput) score -= 30;
|
||||
if (results.hasLoginInput) score -= 50;
|
||||
if (results.hasContactForm && !results.hasNewsletterKeywords) score -= 20;
|
||||
|
||||
results.confidence = Math.max(0, Math.min(100, score));
|
||||
results.isNewsletter = results.confidence >= 55;
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
return formInfo;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
classifyNewsletterForm
|
||||
};
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
const { Command } = require('commander');
|
||||
|
||||
function createCLI() {
|
||||
const program = new Command();
|
||||
|
||||
program
|
||||
.name('newsletter-signup')
|
||||
.description('Newsletter Signup Automation Tool')
|
||||
.version('1.0.0')
|
||||
.option('-f, --file <path>', 'Load sites from text file (one URL per line)')
|
||||
.option('-k, --keep-open', 'Keep browser tabs open after processing for verification')
|
||||
.option('-l, --log', 'Save output to timestamped log file')
|
||||
.option('-h, --help', 'Show help message')
|
||||
.argument('[target]', 'Site name or URL to process (filters from config.toml sites)');
|
||||
|
||||
program.on('option:help', () => {
|
||||
console.log(`
|
||||
Newsletter Signup Automation Tool
|
||||
|
||||
Usage: node index.js [options] [site-name-or-url]
|
||||
|
||||
Options:
|
||||
-f, --file=<path> Load sites from text file (one URL per line)
|
||||
-k, --keep-open Keep browser tabs open after processing for verification
|
||||
-l, --log Save output to timestamped log file
|
||||
-h, --help Show this help message
|
||||
|
||||
Examples:
|
||||
node index.js Process all sites from config.toml
|
||||
node index.js --file=sites.txt Process sites from file
|
||||
node index.js "Dutch News" Process specific site
|
||||
node index.js --file=sites.txt --keep-open Keep tabs open for verification
|
||||
node index.js --log Save output to log file
|
||||
|
||||
Text file format (sites.txt):
|
||||
# Lines starting with # are comments
|
||||
https://example.com/newsletter
|
||||
https://another-site.com/signup
|
||||
|
||||
Configuration:
|
||||
Edit config.toml to set credentials and default sites
|
||||
`);
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
return program;
|
||||
}
|
||||
|
||||
function parseCLI(args = process.argv.slice(2)) {
|
||||
const program = createCLI();
|
||||
|
||||
const fullArgs = ['node', 'index.js', ...args];
|
||||
program.parse(fullArgs);
|
||||
|
||||
const opts = program.opts();
|
||||
|
||||
return {
|
||||
target: program.args[0] || null,
|
||||
file: opts.file || null,
|
||||
keepOpen: opts.keepOpen || false,
|
||||
log: opts.log || false,
|
||||
help: opts.help || false
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createCLI,
|
||||
parseCLI
|
||||
};
|
||||
@@ -0,0 +1,80 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const toml = require('@iarna/toml');
|
||||
|
||||
function loadConfig() {
|
||||
try {
|
||||
const configPath = path.join(__dirname, '..', 'config.toml');
|
||||
const tomlString = fs.readFileSync(configPath, 'utf-8');
|
||||
const config = toml.parse(tomlString);
|
||||
console.log('Configuration loaded successfully.');
|
||||
return config;
|
||||
} catch (error) {
|
||||
console.error('Failed to load or parse config.toml:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
function generateRandomData() {
|
||||
const randomString = Math.random().toString(36).substring(2, 10);
|
||||
const username = `user${randomString}`;
|
||||
const email = `${username}@gmail.com`;
|
||||
const password = `pass${randomString}`;
|
||||
const age = Math.floor(Math.random() * 30 + 20).toString();
|
||||
const firstName = 'John';
|
||||
const lastName = 'Doe';
|
||||
const phone = '+31610488717';
|
||||
const company = 'Example Corp';
|
||||
return { username, email, password, age, firstName, lastName, phone, company };
|
||||
}
|
||||
|
||||
function parseLocator(locatorStr) {
|
||||
locatorStr = locatorStr.trim();
|
||||
if (!locatorStr.startsWith('<')) {
|
||||
return locatorStr;
|
||||
}
|
||||
|
||||
const getAttr = (attr) => {
|
||||
const match = locatorStr.match(new RegExp(`${attr}=["']([^"']+)["']`));
|
||||
return match ? match[1] : null;
|
||||
};
|
||||
|
||||
const tagName = (locatorStr.match(/^<([a-zA-Z0-9]+)/) || [])[1] || '';
|
||||
const id = getAttr('id');
|
||||
if (id) {
|
||||
if (/^\d/.test(id)) {
|
||||
return `${tagName}[id='${id}']`;
|
||||
}
|
||||
return `${tagName}#${id}`;
|
||||
}
|
||||
|
||||
const name = getAttr('name');
|
||||
if (name) return `${tagName}[name='${name}']`;
|
||||
|
||||
const placeholder = getAttr('placeholder');
|
||||
if (placeholder) return `${tagName}[placeholder='${placeholder}']`;
|
||||
|
||||
const type = getAttr('type');
|
||||
if (type) return `${tagName}[type='${type}']`;
|
||||
|
||||
return tagName;
|
||||
}
|
||||
|
||||
function loadSitesFromFile(filePath) {
|
||||
const fullPath = path.isAbsolute(filePath) ? filePath : path.join(__dirname, '..', filePath);
|
||||
const content = fs.readFileSync(fullPath, 'utf-8');
|
||||
const urls = content.split('\n').filter(line => line.trim() && !line.startsWith('#'));
|
||||
return urls.map((url, index) => ({
|
||||
name: `Site_${index + 1}`,
|
||||
url: url.trim(),
|
||||
category: 'Newsletter',
|
||||
needsPreSubmitAction: false
|
||||
}));
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
loadConfig,
|
||||
generateRandomData,
|
||||
parseLocator,
|
||||
loadSitesFromFile
|
||||
};
|
||||
+260
@@ -0,0 +1,260 @@
|
||||
const FIELD_PATTERNS = {
|
||||
email: {
|
||||
priority: 1,
|
||||
types: ['email'],
|
||||
names: ['email', 'e-mail', 'mail', 'user_email', 'login_email', 'email_address', 'emailaddress'],
|
||||
ids: ['email', 'e-mail', 'mail', 'user-email', 'login-email'],
|
||||
placeholders: ['email', 'e-mail', 'email address', 'your email', 'enter email'],
|
||||
autocomplete: ['email']
|
||||
},
|
||||
confirmEmail: {
|
||||
priority: 2,
|
||||
types: ['email'],
|
||||
names: ['confirm_email', 'confirmemail', 'email_confirm', 'verify_email', 'email_verify'],
|
||||
ids: ['confirm-email', 'confirmemail', 'verify-email', 'email-confirm'],
|
||||
placeholders: ['confirm email', 'verify email', 're-enter email', 'repeat email'],
|
||||
mustHaveConfirmIndicator: true
|
||||
},
|
||||
firstName: {
|
||||
priority: 3,
|
||||
types: ['text'],
|
||||
names: ['first_name', 'firstname', 'fname', 'first-name', 'vorname', 'prenom', 'fname'],
|
||||
ids: ['first-name', 'firstname', 'fname', 'firstName', 'mce-fname'],
|
||||
placeholders: ['first name', 'firstname', 'first'],
|
||||
autocomplete: ['given-name', 'first-name']
|
||||
},
|
||||
lastName: {
|
||||
priority: 4,
|
||||
types: ['text'],
|
||||
names: ['last_name', 'lastname', 'lname', 'surname', 'family_name', 'last-name', 'lname'],
|
||||
ids: ['last-name', 'lastname', 'lname', 'surname', 'lastName', 'mce-lname'],
|
||||
placeholders: ['last name', 'lastname', 'surname', 'last'],
|
||||
autocomplete: ['family-name', 'last-name']
|
||||
},
|
||||
username: {
|
||||
priority: 5,
|
||||
types: ['text'],
|
||||
names: ['username', 'user_name', 'login', 'usr', 'uname', 'user-id', 'userid', 'account_name', 'login_id'],
|
||||
ids: ['username', 'user-name', 'login', 'usr', 'uname'],
|
||||
placeholders: ['username', 'user name', 'login', 'username or email'],
|
||||
autocomplete: ['username']
|
||||
},
|
||||
password: {
|
||||
priority: 6,
|
||||
types: ['password'],
|
||||
names: ['password', 'pass', 'pwd', 'user_password', 'login_password'],
|
||||
ids: ['password', 'pass', 'pwd'],
|
||||
placeholders: ['password', 'pass', 'enter password'],
|
||||
autocomplete: ['current-password', 'new-password']
|
||||
},
|
||||
fullName: {
|
||||
priority: 7,
|
||||
types: ['text'],
|
||||
names: ['full_name', 'fullname', 'name'],
|
||||
ids: ['full-name', 'fullname', 'name'],
|
||||
placeholders: ['full name', 'your name', 'complete name'],
|
||||
autocomplete: ['name']
|
||||
},
|
||||
phone: {
|
||||
priority: 8,
|
||||
types: ['tel', 'text'],
|
||||
names: ['phone', 'telephone', 'mobile', 'cell', 'cellphone', 'phone_number'],
|
||||
ids: ['phone', 'telephone', 'mobile', 'cell', 'tel'],
|
||||
placeholders: ['phone', 'telephone', 'mobile number'],
|
||||
autocomplete: ['tel']
|
||||
},
|
||||
age: {
|
||||
priority: 9,
|
||||
types: ['number', 'text'],
|
||||
names: ['age', 'user_age'],
|
||||
ids: ['age'],
|
||||
placeholders: ['age', 'your age']
|
||||
},
|
||||
company: {
|
||||
priority: 10,
|
||||
types: ['text'],
|
||||
names: ['company', 'organization', 'company_name', 'employer'],
|
||||
ids: ['company', 'organization', 'employer'],
|
||||
placeholders: ['company', 'organization'],
|
||||
autocomplete: ['organization']
|
||||
}
|
||||
};
|
||||
|
||||
async function detectFormFields(page) {
|
||||
console.log('Auto-detecting form fields...');
|
||||
|
||||
const detectedFields = await page.evaluate((patterns) => {
|
||||
const results = {};
|
||||
const usedElements = new Set();
|
||||
|
||||
const inputs = Array.from(document.querySelectorAll('input, select, textarea')).filter(input => {
|
||||
const type = input.type?.toLowerCase() || 'text';
|
||||
const style = window.getComputedStyle(input);
|
||||
const rect = input.getBoundingClientRect();
|
||||
|
||||
if (type === 'hidden' || type === 'submit' || type === 'button') return false;
|
||||
if (input.tabIndex === -1 && input.style.position === 'absolute') return false;
|
||||
if (rect.width === 0 || rect.height === 0) return false;
|
||||
if (style.display === 'none' || style.visibility === 'hidden') return false;
|
||||
|
||||
return true;
|
||||
});
|
||||
|
||||
const calculateScore = (input, pattern, fieldName) => {
|
||||
let score = 0;
|
||||
const inputType = (input.type || 'text').toLowerCase();
|
||||
const inputName = (input.name || '').toLowerCase();
|
||||
const inputId = (input.id || '').toLowerCase();
|
||||
const placeholder = (input.placeholder || '').toLowerCase();
|
||||
const autocomplete = (input.autocomplete || '').toLowerCase();
|
||||
|
||||
if (pattern.types && pattern.types.includes(inputType)) {
|
||||
score += 5;
|
||||
}
|
||||
|
||||
if (pattern.names) {
|
||||
for (const name of pattern.names) {
|
||||
const nameLower = name.toLowerCase();
|
||||
if (inputName === nameLower) {
|
||||
score += 20;
|
||||
break;
|
||||
} else if (inputName.includes(nameLower)) {
|
||||
if (fieldName === 'firstName' && (inputName.includes('user') || inputName.includes('login'))) {
|
||||
score -= 5;
|
||||
} else if (fieldName === 'lastName' && (inputName.includes('user') || inputName.includes('login'))) {
|
||||
score -= 5;
|
||||
} else {
|
||||
score += 10;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (pattern.ids) {
|
||||
for (const id of pattern.ids) {
|
||||
const idLower = id.toLowerCase();
|
||||
if (inputId === idLower) {
|
||||
score += 20;
|
||||
break;
|
||||
} else if (inputId.includes(idLower)) {
|
||||
score += 10;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (pattern.placeholders) {
|
||||
for (const ph of pattern.placeholders) {
|
||||
const phLower = ph.toLowerCase();
|
||||
if (placeholder === phLower) {
|
||||
score += 15;
|
||||
break;
|
||||
} else if (placeholder.includes(phLower)) {
|
||||
score += 8;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (pattern.autocomplete) {
|
||||
for (const ac of pattern.autocomplete) {
|
||||
if (autocomplete === ac.toLowerCase()) {
|
||||
score += 12;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const labels = input.labels;
|
||||
if (labels && labels.length > 0) {
|
||||
const labelText = labels[0].textContent.toLowerCase();
|
||||
const fieldNameLower = fieldName.toLowerCase().replace(/([A-Z])/g, ' $1').trim();
|
||||
if (labelText.includes(fieldNameLower) || labelText.includes(pattern.names?.[0] || '')) {
|
||||
score += 15;
|
||||
}
|
||||
}
|
||||
|
||||
if (inputName.includes('b_') || inputId.includes('b_')) {
|
||||
score -= 30;
|
||||
}
|
||||
|
||||
if (input.disabled || input.readOnly) {
|
||||
score -= 20;
|
||||
}
|
||||
|
||||
if (pattern.mustHaveConfirmIndicator) {
|
||||
const hasConfirm = inputId.includes('confirm') || inputId.includes('verify') ||
|
||||
inputName.includes('confirm') || inputName.includes('verify') ||
|
||||
placeholder.includes('confirm') || placeholder.includes('verify') ||
|
||||
placeholder.includes('re-') || placeholder.includes('again');
|
||||
if (!hasConfirm) {
|
||||
score -= 15;
|
||||
}
|
||||
}
|
||||
|
||||
return score;
|
||||
};
|
||||
|
||||
const sortedPatterns = Object.entries(patterns).sort((a, b) => a[1].priority - b[1].priority);
|
||||
|
||||
for (const [fieldName, pattern] of sortedPatterns) {
|
||||
let bestMatch = null;
|
||||
let bestScore = 0;
|
||||
let bestIndex = -1;
|
||||
|
||||
for (let i = 0; i < inputs.length; i++) {
|
||||
if (usedElements.has(i)) continue;
|
||||
|
||||
const input = inputs[i];
|
||||
const score = calculateScore(input, pattern, fieldName);
|
||||
|
||||
if (score > bestScore && score >= 10) {
|
||||
bestScore = score;
|
||||
bestMatch = input;
|
||||
bestIndex = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (bestMatch && bestIndex >= 0) {
|
||||
usedElements.add(bestIndex);
|
||||
|
||||
let selector = '';
|
||||
if (bestMatch.id) {
|
||||
selector = `#${bestMatch.id}`;
|
||||
} else if (bestMatch.name) {
|
||||
selector = `[name="${bestMatch.name}"]`;
|
||||
} else if (bestMatch.placeholder) {
|
||||
selector = `[placeholder="${bestMatch.placeholder}"]`;
|
||||
} else {
|
||||
const tagName = bestMatch.tagName.toLowerCase();
|
||||
const type = bestMatch.type ? `[type="${bestMatch.type}"]` : '';
|
||||
const allSame = Array.from(document.querySelectorAll(`${tagName}${type}`));
|
||||
const index = allSame.indexOf(bestMatch) + 1;
|
||||
selector = `${tagName}${type}:nth-of-type(${index})`;
|
||||
}
|
||||
|
||||
results[fieldName] = {
|
||||
selector,
|
||||
confidence: bestScore,
|
||||
name: bestMatch.name,
|
||||
id: bestMatch.id
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}, FIELD_PATTERNS);
|
||||
|
||||
const detectedCount = Object.keys(detectedFields).length;
|
||||
if (detectedCount === 0) {
|
||||
console.log(' No fields detected');
|
||||
}
|
||||
|
||||
return detectedFields;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
FIELD_PATTERNS,
|
||||
detectFormFields
|
||||
};
|
||||
@@ -0,0 +1,217 @@
|
||||
const { generateRandomData, parseLocator } = require('./config');
|
||||
const { detectFormFields } = require('./fields');
|
||||
const { detectSubmitButton } = require('./submit');
|
||||
const { detectCaptcha, solveSimpleCaptcha } = require('./captcha');
|
||||
const { classifyNewsletterForm } = require('./classifier');
|
||||
|
||||
async function processSite(browser, site, globalCreds, browserConfig, keepOpen = false) {
|
||||
let siteFailed = false;
|
||||
let failureReason = '';
|
||||
let page = null;
|
||||
|
||||
try {
|
||||
let creds;
|
||||
if (site.credentials && site.credentials.username) {
|
||||
creds = { ...globalCreds, ...site.credentials };
|
||||
} else {
|
||||
creds = { ...globalCreds };
|
||||
}
|
||||
|
||||
if (!creds.username || !creds.email || !creds.password) {
|
||||
const randomData = generateRandomData();
|
||||
creds = { ...randomData, ...creds };
|
||||
}
|
||||
|
||||
console.log(`\n${site.url}`);
|
||||
console.log(`${'-'.repeat(70)}`);
|
||||
|
||||
page = await browser.newPage();
|
||||
|
||||
if (browserConfig.userAgent) {
|
||||
await page.setUserAgent(browserConfig.userAgent);
|
||||
}
|
||||
|
||||
await page.goto(site.url, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||
|
||||
const captchaInfo = await detectCaptcha(page);
|
||||
if (captchaInfo.length > 0) {
|
||||
console.log(`\n CAPTCHA DETECTED:`);
|
||||
for (const captcha of captchaInfo) {
|
||||
console.log(` - ${captcha.name}`);
|
||||
}
|
||||
|
||||
if (captchaInfo.some(c => c.type === 'mathCaptcha')) {
|
||||
const solution = await solveSimpleCaptcha(page);
|
||||
if (solution && solution.answer) {
|
||||
console.log(` Auto-solving math: ${solution.text} = ${solution.answer}`);
|
||||
try {
|
||||
await page.evaluate((answer) => {
|
||||
const input = document.querySelector('input[name*="captcha"], input[class*="captcha"], #captcha');
|
||||
if (input) {
|
||||
input.value = answer;
|
||||
input.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
}, solution.answer);
|
||||
} catch (e) {
|
||||
console.warn(` Failed to fill math answer: ${e.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (captchaInfo.some(c => c.type === 'imageCaptcha')) {
|
||||
const solution = await solveSimpleCaptcha(page);
|
||||
if (solution && solution.type === 'image' && solution.answer) {
|
||||
console.log(` OCR solved image CAPTCHA: "${solution.answer}"`);
|
||||
try {
|
||||
await page.evaluate((answer) => {
|
||||
const input = document.querySelector('input[name*="captcha"], input[class*="captcha"], input[id*="captcha"], #captcha, .captcha-input');
|
||||
if (input) {
|
||||
input.value = answer;
|
||||
input.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
input.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}
|
||||
}, solution.answer);
|
||||
} catch (e) {
|
||||
console.warn(` Failed to fill OCR answer: ${e.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (captchaInfo.some(c => c.type === 'recaptcha' || c.type === 'hcaptcha')) {
|
||||
siteFailed = true;
|
||||
failureReason = `CAPTCHA present: ${captchaInfo.map(c => c.name).join(', ')}`;
|
||||
console.error(` Cannot auto-solve ${captchaInfo.map(c => c.name).join(', ')}`);
|
||||
return { siteFailed, failureReason, page };
|
||||
}
|
||||
}
|
||||
|
||||
const formClassification = await classifyNewsletterForm(page);
|
||||
console.log(`\n Form Classification: ${formClassification.isNewsletter ? 'Newsletter' : 'Not newsletter'} (${formClassification.confidence}/100 confidence)`);
|
||||
|
||||
if (formClassification.signals.positive.length > 0) {
|
||||
console.log(' Positive signals:');
|
||||
formClassification.signals.positive.forEach(signal => console.log(` + ${signal}`));
|
||||
}
|
||||
|
||||
if (formClassification.signals.negative.length > 0) {
|
||||
console.log(' Negative signals:');
|
||||
formClassification.signals.negative.forEach(signal => console.log(` - ${signal}`));
|
||||
}
|
||||
|
||||
if (!formClassification.isNewsletter && formClassification.confidence < 40) {
|
||||
siteFailed = true;
|
||||
failureReason = 'Not a newsletter signup form';
|
||||
console.error(` ${failureReason}`);
|
||||
}
|
||||
|
||||
console.log('\nDETECTED FIELDS:');
|
||||
|
||||
let fieldsToFill = {};
|
||||
let detectedFields = {};
|
||||
const definedLocators = site.locators || {};
|
||||
|
||||
if (site.autoDetect !== false) {
|
||||
detectedFields = await detectFormFields(page);
|
||||
for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
|
||||
fieldsToFill[fieldName] = fieldInfo.selector;
|
||||
}
|
||||
}
|
||||
|
||||
const hasEmailField = fieldsToFill.email || fieldsToFill.confirmEmail;
|
||||
if (!hasEmailField) {
|
||||
siteFailed = true;
|
||||
failureReason = 'No email field detected';
|
||||
console.warn(`\n ${failureReason}`);
|
||||
}
|
||||
|
||||
const hasSearchInput = await page.evaluate(() => {
|
||||
const inputs = document.querySelectorAll('input[type="search"], input[name*="search"], input[id*="search"]');
|
||||
return inputs.length > 0;
|
||||
}).catch(() => false);
|
||||
|
||||
if (hasSearchInput && !hasEmailField) {
|
||||
console.warn(` Search form detected instead of newsletter`);
|
||||
siteFailed = true;
|
||||
failureReason = failureReason || 'Search form instead of newsletter';
|
||||
}
|
||||
|
||||
for (const [fieldName, locator] of Object.entries(definedLocators)) {
|
||||
if (fieldName !== 'submit') {
|
||||
fieldsToFill[fieldName] = parseLocator(locator);
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(fieldsToFill).length > 0) {
|
||||
console.log(`\n ${'FIELD'.padEnd(12)} | ${'SELECTOR'.padEnd(35)} | ${'CONF'.padEnd(4)} | VALUE`);
|
||||
console.log(' ' + '-'.repeat(75));
|
||||
|
||||
for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
|
||||
let valueToFill = creds[fieldName];
|
||||
|
||||
if (fieldName === 'confirmEmail') {
|
||||
valueToFill = creds['email'];
|
||||
}
|
||||
|
||||
const selector = fieldInfo.selector.length > 34 ? fieldInfo.selector.substring(0, 31) + '...' : fieldInfo.selector;
|
||||
const value = valueToFill ? valueToFill.substring(0, 20) : 'N/A';
|
||||
|
||||
console.log(` ${fieldName.padEnd(12)} | ${selector.padEnd(35)} | ${fieldInfo.confidence.toString().padEnd(4)} | ${value}`);
|
||||
|
||||
if (valueToFill) {
|
||||
try {
|
||||
await page.evaluate((sel, val) => {
|
||||
const el = document.querySelector(sel);
|
||||
if (el) {
|
||||
el.value = val;
|
||||
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
el.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}
|
||||
}, fieldInfo.selector, valueToFill);
|
||||
} catch (e) {
|
||||
console.warn(` Failed to fill ${fieldName}: ${e.message}`);
|
||||
siteFailed = true;
|
||||
failureReason = `Failed to fill ${fieldName}: ${e.message}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(' ' + '-'.repeat(75));
|
||||
}
|
||||
|
||||
let submitSelector = null;
|
||||
if (definedLocators.submit) {
|
||||
submitSelector = parseLocator(definedLocators.submit);
|
||||
} else if (site.autoDetect !== false) {
|
||||
submitSelector = await detectSubmitButton(page);
|
||||
}
|
||||
|
||||
if (submitSelector) {
|
||||
try {
|
||||
await page.click(submitSelector);
|
||||
console.log(` Submitted (${submitSelector})`);
|
||||
} catch (submitError) {
|
||||
siteFailed = true;
|
||||
failureReason = failureReason || `Submit failed: ${submitError.message}`;
|
||||
console.error(` Submit failed: ${submitError.message}`);
|
||||
}
|
||||
} else {
|
||||
siteFailed = true;
|
||||
failureReason = failureReason || 'No submit button found';
|
||||
console.warn(` ${failureReason}`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
siteFailed = true;
|
||||
failureReason = error.message;
|
||||
console.error(` Error: ${error.message}`);
|
||||
} finally {
|
||||
if (page && !keepOpen) {
|
||||
await page.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
return { siteFailed, failureReason, page };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
processSite
|
||||
};
|
||||
@@ -0,0 +1,82 @@
|
||||
async function detectSubmitButton(page) {
|
||||
console.log('Auto-detecting submit button...');
|
||||
|
||||
const submitSelector = await page.evaluate(() => {
|
||||
const buttons = Array.from(document.querySelectorAll('button[type="submit"], input[type="submit"], button, input[type="button"]'));
|
||||
let bestMatch = null;
|
||||
let bestScore = 0;
|
||||
|
||||
buttons.forEach(btn => {
|
||||
const text = (btn.textContent || btn.value || '').toLowerCase().trim();
|
||||
const type = btn.type?.toLowerCase() || '';
|
||||
const className = (btn.className || '').toLowerCase();
|
||||
let score = 0;
|
||||
|
||||
if (type === 'submit') score += 20;
|
||||
|
||||
if (text.includes('subscribe')) score += 15;
|
||||
else if (text.includes('sign up')) score += 12;
|
||||
else if (text.includes('submit')) score += 10;
|
||||
else if (text.includes('join')) score += 8;
|
||||
else if (text.includes('register')) score += 8;
|
||||
else if (text.includes('send')) score += 5;
|
||||
|
||||
if (className.includes('submit')) score += 8;
|
||||
if (className.includes('subscribe')) score += 8;
|
||||
|
||||
if (className.includes('close') || text.includes('close')) score -= 20;
|
||||
if (className.includes('cancel') || text.includes('cancel')) score -= 20;
|
||||
if (className.includes('back') || text.includes('back')) score -= 15;
|
||||
if (className.includes('drawer')) score -= 15;
|
||||
|
||||
const rect = btn.getBoundingClientRect();
|
||||
if (rect.width > 0 && rect.height > 0 && rect.top > 0) {
|
||||
score += 3;
|
||||
}
|
||||
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
bestMatch = btn;
|
||||
}
|
||||
});
|
||||
|
||||
if (bestMatch && bestScore >= 10) {
|
||||
if (bestMatch.id) return `#${bestMatch.id}`;
|
||||
if (bestMatch.name) return `[name="${bestMatch.name}"]`;
|
||||
|
||||
const className = bestMatch.className;
|
||||
if (className) {
|
||||
const classes = className.split(' ').filter(c => c.length > 0);
|
||||
for (const cls of classes) {
|
||||
const clsLower = cls.toLowerCase();
|
||||
if (clsLower.includes('submit') || clsLower.includes('subscribe')) {
|
||||
return `.${cls}`;
|
||||
}
|
||||
}
|
||||
for (const cls of classes) {
|
||||
if (!cls.toLowerCase().includes('close') && !cls.toLowerCase().includes('cancel')) {
|
||||
return `.${cls}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const allBtns = Array.from(document.querySelectorAll('button'));
|
||||
const index = allBtns.indexOf(bestMatch) + 1;
|
||||
return `button:nth-of-type(${index})`;
|
||||
}
|
||||
|
||||
return null;
|
||||
});
|
||||
|
||||
if (submitSelector) {
|
||||
console.log(` Submit button found: ${submitSelector}`);
|
||||
} else {
|
||||
console.log(' No submit button found');
|
||||
}
|
||||
|
||||
return submitSelector;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
detectSubmitButton
|
||||
};
|
||||
Reference in New Issue
Block a user