changed to individual files instead one big ass script and added auto captcha solver functionality
This commit is contained in:
@@ -0,0 +1,19 @@
|
|||||||
|
DATE | TIME | WEBSITE | REASON
|
||||||
|
----------------------------------------------------------------------------------------------------
|
||||||
|
14/02/2026 | 20:31:58 | Site_3 (https://kulalaland.com/) | No email field detected
|
||||||
|
14/02/2026 | 20:42:16 | Dutch News Test (https://www.dut | CAPTCHA present: reCAPTCHA
|
||||||
|
14/02/2026 | 20:43:17 | Site_1 (https://www.dutchnews.nl | CAPTCHA present: reCAPTCHA
|
||||||
|
14/02/2026 | 20:43:17 | Site_3 (https://kulalaland.com/) | No email field detected
|
||||||
|
14/02/2026 | 20:45:19 | Site_1 (https://www.dutchnews.nl | CAPTCHA present: reCAPTCHA
|
||||||
|
14/02/2026 | 20:45:19 | Site_2 (https://immaculatevegan. | Not a newsletter signup form
|
||||||
|
14/02/2026 | 20:45:19 | Site_3 (https://kulalaland.com/) | No email field detected
|
||||||
|
14/02/2026 | 20:46:06 | Site_1 (https://www.dutchnews.nl | CAPTCHA present: reCAPTCHA
|
||||||
|
14/02/2026 | 20:46:06 | Site_3 (https://kulalaland.com/) | No email field detected
|
||||||
|
14/02/2026 | 21:26:47 | Dutch News Test (https://www.dut | Navigating frame was detached
|
||||||
|
14/02/2026 | 21:26:47 | Immaculate Vegan (https://immacu | Connection closed.
|
||||||
|
14/02/2026 | 21:26:47 | Gate6 (https://gate6.vn/) | Connection closed.
|
||||||
|
14/02/2026 | 21:26:47 | Kulala Land (https://kulalaland. | Connection closed.
|
||||||
|
14/02/2026 | 21:26:47 | sneakywholefoods (https://www.sn | Connection closed.
|
||||||
|
14/02/2026 | 21:27:40 | Dutch News Test (https://www.dut | CAPTCHA present: reCAPTCHA
|
||||||
|
14/02/2026 | 21:27:40 | Kulala Land (https://kulalaland. | No email field detected
|
||||||
|
14/02/2026 | 21:27:40 | sneakywholefoods (https://www.sn | Not a newsletter signup form
|
||||||
@@ -1,437 +1,46 @@
|
|||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
const toml = require('@iarna/toml');
|
|
||||||
const puppeteer = require('puppeteer-extra');
|
const puppeteer = require('puppeteer-extra');
|
||||||
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
||||||
puppeteer.use(StealthPlugin());
|
puppeteer.use(StealthPlugin());
|
||||||
|
|
||||||
const FIELD_PATTERNS = {
|
const { loadConfig, loadSitesFromFile } = require('./src/config');
|
||||||
email: {
|
const { processSite } = require('./src/processor');
|
||||||
priority: 1,
|
const { createCLI, parseCLI } = require('./src/cli');
|
||||||
types: ['email'],
|
|
||||||
names: ['email', 'e-mail', 'mail', 'user_email', 'login_email', 'email_address', 'emailaddress'],
|
|
||||||
ids: ['email', 'e-mail', 'mail', 'user-email', 'login-email'],
|
|
||||||
placeholders: ['email', 'e-mail', 'email address', 'your email', 'enter email'],
|
|
||||||
autocomplete: ['email']
|
|
||||||
},
|
|
||||||
confirmEmail: {
|
|
||||||
priority: 2,
|
|
||||||
types: ['email'],
|
|
||||||
names: ['confirm_email', 'confirmemail', 'email_confirm', 'verify_email', 'email_verify'],
|
|
||||||
ids: ['confirm-email', 'confirmemail', 'verify-email', 'email-confirm'],
|
|
||||||
placeholders: ['confirm email', 'verify email', 're-enter email', 'repeat email'],
|
|
||||||
mustHaveConfirmIndicator: true
|
|
||||||
},
|
|
||||||
firstName: {
|
|
||||||
priority: 3,
|
|
||||||
types: ['text'],
|
|
||||||
names: ['first_name', 'firstname', 'fname', 'first-name', 'vorname', 'prenom', 'fname'],
|
|
||||||
ids: ['first-name', 'firstname', 'fname', 'firstName', 'mce-fname'],
|
|
||||||
placeholders: ['first name', 'firstname', 'first'],
|
|
||||||
autocomplete: ['given-name', 'first-name']
|
|
||||||
},
|
|
||||||
lastName: {
|
|
||||||
priority: 4,
|
|
||||||
types: ['text'],
|
|
||||||
names: ['last_name', 'lastname', 'lname', 'surname', 'family_name', 'last-name', 'lname'],
|
|
||||||
ids: ['last-name', 'lastname', 'lname', 'surname', 'lastName', 'mce-lname'],
|
|
||||||
placeholders: ['last name', 'lastname', 'surname', 'last'],
|
|
||||||
autocomplete: ['family-name', 'last-name']
|
|
||||||
},
|
|
||||||
username: {
|
|
||||||
priority: 5,
|
|
||||||
types: ['text'],
|
|
||||||
names: ['username', 'user_name', 'login', 'usr', 'uname', 'user-id', 'userid', 'account_name', 'login_id'],
|
|
||||||
ids: ['username', 'user-name', 'login', 'usr', 'uname'],
|
|
||||||
placeholders: ['username', 'user name', 'login', 'username or email'],
|
|
||||||
autocomplete: ['username']
|
|
||||||
},
|
|
||||||
password: {
|
|
||||||
priority: 6,
|
|
||||||
types: ['password'],
|
|
||||||
names: ['password', 'pass', 'pwd', 'user_password', 'login_password'],
|
|
||||||
ids: ['password', 'pass', 'pwd'],
|
|
||||||
placeholders: ['password', 'pass', 'enter password'],
|
|
||||||
autocomplete: ['current-password', 'new-password']
|
|
||||||
},
|
|
||||||
fullName: {
|
|
||||||
priority: 7,
|
|
||||||
types: ['text'],
|
|
||||||
names: ['full_name', 'fullname', 'name'],
|
|
||||||
ids: ['full-name', 'fullname', 'name'],
|
|
||||||
placeholders: ['full name', 'your name', 'complete name'],
|
|
||||||
autocomplete: ['name']
|
|
||||||
},
|
|
||||||
phone: {
|
|
||||||
priority: 8,
|
|
||||||
types: ['tel', 'text'],
|
|
||||||
names: ['phone', 'telephone', 'mobile', 'cell', 'cellphone', 'phone_number'],
|
|
||||||
ids: ['phone', 'telephone', 'mobile', 'cell', 'tel'],
|
|
||||||
placeholders: ['phone', 'telephone', 'mobile number'],
|
|
||||||
autocomplete: ['tel']
|
|
||||||
},
|
|
||||||
age: {
|
|
||||||
priority: 9,
|
|
||||||
types: ['number', 'text'],
|
|
||||||
names: ['age', 'user_age'],
|
|
||||||
ids: ['age'],
|
|
||||||
placeholders: ['age', 'your age']
|
|
||||||
},
|
|
||||||
company: {
|
|
||||||
priority: 10,
|
|
||||||
types: ['text'],
|
|
||||||
names: ['company', 'organization', 'company_name', 'employer'],
|
|
||||||
ids: ['company', 'organization', 'employer'],
|
|
||||||
placeholders: ['company', 'organization'],
|
|
||||||
autocomplete: ['organization']
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
function loadConfig() {
|
|
||||||
try {
|
|
||||||
const configPath = path.join(__dirname, 'config.toml');
|
|
||||||
const tomlString = fs.readFileSync(configPath, 'utf-8');
|
|
||||||
const config = toml.parse(tomlString);
|
|
||||||
console.log('✅ Configuration loaded successfully.');
|
|
||||||
return config;
|
|
||||||
} catch (error) {
|
|
||||||
console.error('❌ Failed to load or parse config.toml:', error.message);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function generateRandomData() {
|
|
||||||
const randomString = Math.random().toString(36).substring(2, 10);
|
|
||||||
const username = `user${randomString}`;
|
|
||||||
const email = `${username}@gmail.com`;
|
|
||||||
const password = `pass${randomString}`;
|
|
||||||
const age = Math.floor(Math.random() * 30 + 20).toString();
|
|
||||||
const firstName = 'John';
|
|
||||||
const lastName = 'Doe';
|
|
||||||
const phone = '+31610488717';
|
|
||||||
const company = 'Example Corp';
|
|
||||||
return { username, email, password, age, firstName, lastName, phone, company };
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseLocator(locatorStr) {
|
|
||||||
locatorStr = locatorStr.trim();
|
|
||||||
if (!locatorStr.startsWith('<')) {
|
|
||||||
return locatorStr;
|
|
||||||
}
|
|
||||||
|
|
||||||
const getAttr = (attr) => {
|
|
||||||
const match = locatorStr.match(new RegExp(`${attr}=["']([^"']+)["']`));
|
|
||||||
return match ? match[1] : null;
|
|
||||||
};
|
|
||||||
|
|
||||||
const tagName = (locatorStr.match(/^<([a-zA-Z0-9]+)/) || [])[1] || '';
|
|
||||||
const id = getAttr('id');
|
|
||||||
if (id) {
|
|
||||||
if (/^\d/.test(id)) {
|
|
||||||
return `${tagName}[id='${id}']`;
|
|
||||||
}
|
|
||||||
return `${tagName}#${id}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const name = getAttr('name');
|
|
||||||
if (name) return `${tagName}[name='${name}']`;
|
|
||||||
|
|
||||||
const placeholder = getAttr('placeholder');
|
|
||||||
if (placeholder) return `${tagName}[placeholder='${placeholder}']`;
|
|
||||||
|
|
||||||
const type = getAttr('type');
|
|
||||||
if (type) return `${tagName}[type='${type}']`;
|
|
||||||
|
|
||||||
return tagName;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function detectFormFields(page) {
|
|
||||||
console.log('🔍 Auto-detecting form fields...');
|
|
||||||
|
|
||||||
const detectedFields = await page.evaluate((patterns) => {
|
|
||||||
const results = {};
|
|
||||||
const usedElements = new Set();
|
|
||||||
|
|
||||||
// Get all visible input fields, excluding hidden fields
|
|
||||||
const inputs = Array.from(document.querySelectorAll('input, select, textarea')).filter(input => {
|
|
||||||
const type = input.type?.toLowerCase() || 'text';
|
|
||||||
const style = window.getComputedStyle(input);
|
|
||||||
const rect = input.getBoundingClientRect();
|
|
||||||
|
|
||||||
// Skip hidden fields and honeypots
|
|
||||||
if (type === 'hidden' || type === 'submit' || type === 'button') return false;
|
|
||||||
if (input.tabIndex === -1 && input.style.position === 'absolute') return false; // Likely honeypot
|
|
||||||
if (rect.width === 0 || rect.height === 0) return false;
|
|
||||||
if (style.display === 'none' || style.visibility === 'hidden') return false;
|
|
||||||
|
|
||||||
return true;
|
|
||||||
});
|
|
||||||
|
|
||||||
const calculateScore = (input, pattern, fieldName) => {
|
|
||||||
let score = 0;
|
|
||||||
const inputType = (input.type || 'text').toLowerCase();
|
|
||||||
const inputName = (input.name || '').toLowerCase();
|
|
||||||
const inputId = (input.id || '').toLowerCase();
|
|
||||||
const placeholder = (input.placeholder || '').toLowerCase();
|
|
||||||
const autocomplete = (input.autocomplete || '').toLowerCase();
|
|
||||||
|
|
||||||
// Type matching
|
|
||||||
if (pattern.types && pattern.types.includes(inputType)) {
|
|
||||||
score += 5;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Name matching - exact matches get high scores
|
|
||||||
if (pattern.names) {
|
|
||||||
for (const name of pattern.names) {
|
|
||||||
const nameLower = name.toLowerCase();
|
|
||||||
if (inputName === nameLower) {
|
|
||||||
score += 20; // Exact match
|
|
||||||
break;
|
|
||||||
} else if (inputName.includes(nameLower)) {
|
|
||||||
// Partial match - be careful not to match fname with username
|
|
||||||
if (fieldName === 'firstName' && (inputName.includes('user') || inputName.includes('login'))) {
|
|
||||||
score -= 5; // Penalize if it looks like username
|
|
||||||
} else if (fieldName === 'lastName' && (inputName.includes('user') || inputName.includes('login'))) {
|
|
||||||
score -= 5;
|
|
||||||
} else {
|
|
||||||
score += 10;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ID matching - very specific
|
|
||||||
if (pattern.ids) {
|
|
||||||
for (const id of pattern.ids) {
|
|
||||||
const idLower = id.toLowerCase();
|
|
||||||
if (inputId === idLower) {
|
|
||||||
score += 20; // Exact match
|
|
||||||
break;
|
|
||||||
} else if (inputId.includes(idLower)) {
|
|
||||||
score += 10;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Placeholder matching
|
|
||||||
if (pattern.placeholders) {
|
|
||||||
for (const ph of pattern.placeholders) {
|
|
||||||
const phLower = ph.toLowerCase();
|
|
||||||
if (placeholder === phLower) {
|
|
||||||
score += 15;
|
|
||||||
break;
|
|
||||||
} else if (placeholder.includes(phLower)) {
|
|
||||||
score += 8;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Autocomplete matching
|
|
||||||
if (pattern.autocomplete) {
|
|
||||||
for (const ac of pattern.autocomplete) {
|
|
||||||
if (autocomplete === ac.toLowerCase()) {
|
|
||||||
score += 12;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check label text
|
|
||||||
const labels = input.labels;
|
|
||||||
if (labels && labels.length > 0) {
|
|
||||||
const labelText = labels[0].textContent.toLowerCase();
|
|
||||||
const fieldNameLower = fieldName.toLowerCase().replace(/([A-Z])/g, ' $1').trim();
|
|
||||||
if (labelText.includes(fieldNameLower) || labelText.includes(pattern.names?.[0] || '')) {
|
|
||||||
score += 15;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Penalize if field appears to be a honeypot
|
|
||||||
if (inputName.includes('b_') || inputId.includes('b_')) {
|
|
||||||
score -= 30; // Likely Mailchimp honeypot
|
|
||||||
}
|
|
||||||
|
|
||||||
// Penalize disabled/readonly
|
|
||||||
if (input.disabled || input.readOnly) {
|
|
||||||
score -= 20;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Special handling for confirmEmail
|
|
||||||
if (pattern.mustHaveConfirmIndicator) {
|
|
||||||
const hasConfirm = inputId.includes('confirm') || inputId.includes('verify') ||
|
|
||||||
inputName.includes('confirm') || inputName.includes('verify') ||
|
|
||||||
placeholder.includes('confirm') || placeholder.includes('verify') ||
|
|
||||||
placeholder.includes('re-') || placeholder.includes('again');
|
|
||||||
if (!hasConfirm) {
|
|
||||||
score -= 15;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return score;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Sort patterns by priority
|
|
||||||
const sortedPatterns = Object.entries(patterns).sort((a, b) => a[1].priority - b[1].priority);
|
|
||||||
|
|
||||||
for (const [fieldName, pattern] of sortedPatterns) {
|
|
||||||
let bestMatch = null;
|
|
||||||
let bestScore = 0;
|
|
||||||
let bestIndex = -1;
|
|
||||||
|
|
||||||
for (let i = 0; i < inputs.length; i++) {
|
|
||||||
if (usedElements.has(i)) continue;
|
|
||||||
|
|
||||||
const input = inputs[i];
|
|
||||||
const score = calculateScore(input, pattern, fieldName);
|
|
||||||
|
|
||||||
if (score > bestScore && score >= 10) {
|
|
||||||
bestScore = score;
|
|
||||||
bestMatch = input;
|
|
||||||
bestIndex = i;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (bestMatch && bestIndex >= 0) {
|
|
||||||
usedElements.add(bestIndex);
|
|
||||||
|
|
||||||
// Generate selector
|
|
||||||
let selector = '';
|
|
||||||
if (bestMatch.id) {
|
|
||||||
selector = `#${bestMatch.id}`;
|
|
||||||
} else if (bestMatch.name) {
|
|
||||||
selector = `[name="${bestMatch.name}"]`;
|
|
||||||
} else if (bestMatch.placeholder) {
|
|
||||||
selector = `[placeholder="${bestMatch.placeholder}"]`;
|
|
||||||
} else {
|
|
||||||
const tagName = bestMatch.tagName.toLowerCase();
|
|
||||||
const type = bestMatch.type ? `[type="${bestMatch.type}"]` : '';
|
|
||||||
const allSame = Array.from(document.querySelectorAll(`${tagName}${type}`));
|
|
||||||
const index = allSame.indexOf(bestMatch) + 1;
|
|
||||||
selector = `${tagName}${type}:nth-of-type(${index})`;
|
|
||||||
}
|
|
||||||
|
|
||||||
results[fieldName] = {
|
|
||||||
selector,
|
|
||||||
confidence: bestScore,
|
|
||||||
name: bestMatch.name,
|
|
||||||
id: bestMatch.id
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return results;
|
|
||||||
}, FIELD_PATTERNS);
|
|
||||||
|
|
||||||
const detectedCount = Object.keys(detectedFields).length;
|
|
||||||
if (detectedCount === 0) {
|
|
||||||
console.log(' No fields detected');
|
|
||||||
}
|
|
||||||
|
|
||||||
return detectedFields;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function detectSubmitButton(page) {
|
|
||||||
console.log('🔍 Auto-detecting submit button...');
|
|
||||||
|
|
||||||
const submitSelector = await page.evaluate(() => {
|
|
||||||
const buttons = Array.from(document.querySelectorAll('button[type="submit"], input[type="submit"], button, input[type="button"]'));
|
|
||||||
let bestMatch = null;
|
|
||||||
let bestScore = 0;
|
|
||||||
|
|
||||||
buttons.forEach(btn => {
|
|
||||||
const text = (btn.textContent || btn.value || '').toLowerCase().trim();
|
|
||||||
const type = btn.type?.toLowerCase() || '';
|
|
||||||
const className = (btn.className || '').toLowerCase();
|
|
||||||
let score = 0;
|
|
||||||
|
|
||||||
if (type === 'submit') score += 20;
|
|
||||||
|
|
||||||
if (text.includes('subscribe')) score += 15;
|
|
||||||
else if (text.includes('sign up')) score += 12;
|
|
||||||
else if (text.includes('submit')) score += 10;
|
|
||||||
else if (text.includes('join')) score += 8;
|
|
||||||
else if (text.includes('register')) score += 8;
|
|
||||||
else if (text.includes('send')) score += 5;
|
|
||||||
|
|
||||||
if (className.includes('submit')) score += 8;
|
|
||||||
if (className.includes('subscribe')) score += 8;
|
|
||||||
|
|
||||||
if (className.includes('close') || text.includes('close')) score -= 20;
|
|
||||||
if (className.includes('cancel') || text.includes('cancel')) score -= 20;
|
|
||||||
if (className.includes('back') || text.includes('back')) score -= 15;
|
|
||||||
if (className.includes('drawer')) score -= 15;
|
|
||||||
|
|
||||||
const rect = btn.getBoundingClientRect();
|
|
||||||
if (rect.width > 0 && rect.height > 0 && rect.top > 0) {
|
|
||||||
score += 3;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (score > bestScore) {
|
|
||||||
bestScore = score;
|
|
||||||
bestMatch = btn;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
if (bestMatch && bestScore >= 10) {
|
|
||||||
if (bestMatch.id) return `#${bestMatch.id}`;
|
|
||||||
if (bestMatch.name) return `[name="${bestMatch.name}"]`;
|
|
||||||
|
|
||||||
const className = bestMatch.className;
|
|
||||||
if (className) {
|
|
||||||
const classes = className.split(' ').filter(c => c.length > 0);
|
|
||||||
for (const cls of classes) {
|
|
||||||
const clsLower = cls.toLowerCase();
|
|
||||||
if (clsLower.includes('submit') || clsLower.includes('subscribe')) {
|
|
||||||
return `.${cls}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (const cls of classes) {
|
|
||||||
if (!cls.toLowerCase().includes('close') && !cls.toLowerCase().includes('cancel')) {
|
|
||||||
return `.${cls}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const allBtns = Array.from(document.querySelectorAll('button'));
|
|
||||||
const index = allBtns.indexOf(bestMatch) + 1;
|
|
||||||
return `button:nth-of-type(${index})`;
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
});
|
|
||||||
|
|
||||||
if (submitSelector) {
|
|
||||||
console.log(` ✅ Submit button found: ${submitSelector}`);
|
|
||||||
} else {
|
|
||||||
console.log(' ❌ No submit button found');
|
|
||||||
}
|
|
||||||
|
|
||||||
return submitSelector;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function main() {
|
async function main() {
|
||||||
|
const cliOptions = parseCLI();
|
||||||
|
|
||||||
|
if (cliOptions.help) {
|
||||||
|
createCLI().help();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const config = loadConfig();
|
const config = loadConfig();
|
||||||
let sitesToProcess = config.sites || [];
|
let sitesToProcess = config.sites || [];
|
||||||
|
|
||||||
const targetSite = process.argv[2];
|
if (cliOptions.file) {
|
||||||
const logToFile = process.argv.includes('--log');
|
console.log(`Loading sites from: ${cliOptions.file}`);
|
||||||
const logFile = logToFile ? path.join(__dirname, `run_${new Date().toISOString().replace(/[:.]/g, '-')}.log`) : null;
|
sitesToProcess = loadSitesFromFile(cliOptions.file);
|
||||||
|
console.log(`Loaded ${sitesToProcess.length} sites from file`);
|
||||||
|
}
|
||||||
|
|
||||||
if (targetSite && !targetSite.startsWith('--')) {
|
const logFile = cliOptions.log ? path.join(__dirname, `run_${new Date().toISOString().replace(/[:.]/g, '-')}.log`) : null;
|
||||||
|
|
||||||
|
if (cliOptions.target) {
|
||||||
sitesToProcess = sitesToProcess.filter(site =>
|
sitesToProcess = sitesToProcess.filter(site =>
|
||||||
site.name.toLowerCase().includes(targetSite.toLowerCase()) ||
|
site.name.toLowerCase().includes(cliOptions.target.toLowerCase()) ||
|
||||||
site.url.toLowerCase().includes(targetSite.toLowerCase())
|
site.url.toLowerCase().includes(cliOptions.target.toLowerCase())
|
||||||
);
|
);
|
||||||
if (sitesToProcess.length === 0) {
|
if (sitesToProcess.length === 0) {
|
||||||
console.error(`❌ No site found matching "${targetSite}"`);
|
console.error(`No site found matching "${cliOptions.target}"`);
|
||||||
console.log('Available sites:');
|
|
||||||
config.sites.forEach(site => console.log(` - ${site.name}: ${site.url}`));
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
console.log(`🎯 Filtering to ${sitesToProcess.length} site(s) matching "${targetSite}"`);
|
console.log(`Targeting ${sitesToProcess.length} site(s)`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cliOptions.keepOpen) {
|
||||||
|
console.log('Keep-open mode: Tabs will stay open for verification');
|
||||||
}
|
}
|
||||||
|
|
||||||
const browserConfig = config.browser || {};
|
const browserConfig = config.browser || {};
|
||||||
@@ -439,11 +48,11 @@ async function main() {
|
|||||||
const userDataDir = browserConfig.userDataDir;
|
const userDataDir = browserConfig.userDataDir;
|
||||||
|
|
||||||
if (!userDataDir) {
|
if (!userDataDir) {
|
||||||
console.error('❌ userDataDir is not set in config.toml. This is required.');
|
console.error('userDataDir not set in config.toml');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (logToFile) {
|
if (cliOptions.log && logFile) {
|
||||||
const originalLog = console.log;
|
const originalLog = console.log;
|
||||||
const originalError = console.error;
|
const originalError = console.error;
|
||||||
const originalWarn = console.warn;
|
const originalWarn = console.warn;
|
||||||
@@ -453,173 +62,58 @@ async function main() {
|
|||||||
fs.appendFileSync(logFile, `[${new Date().toISOString()}] [${type}] ${message}\n`);
|
fs.appendFileSync(logFile, `[${new Date().toISOString()}] [${type}] ${message}\n`);
|
||||||
};
|
};
|
||||||
|
|
||||||
console.log = (...args) => {
|
console.log = (...args) => { writeToLog('LOG', args); originalLog.apply(console, args); };
|
||||||
writeToLog('LOG', args);
|
console.error = (...args) => { writeToLog('ERROR', args); originalError.apply(console, args); };
|
||||||
originalLog.apply(console, args);
|
console.warn = (...args) => { writeToLog('WARN', args); originalWarn.apply(console, args); };
|
||||||
};
|
console.log(`Logging to: ${logFile}`);
|
||||||
|
|
||||||
console.error = (...args) => {
|
|
||||||
writeToLog('ERROR', args);
|
|
||||||
originalError.apply(console, args);
|
|
||||||
};
|
|
||||||
|
|
||||||
console.warn = (...args) => {
|
|
||||||
writeToLog('WARN', args);
|
|
||||||
originalWarn.apply(console, args);
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log(`📝 Logging to file: ${logFile}`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const failedSites = [];
|
const failedSites = [];
|
||||||
|
const processedCount = { success: 0, failed: 0 };
|
||||||
|
|
||||||
const browser = await puppeteer.launch({
|
const browser = await puppeteer.launch({
|
||||||
headless: false,
|
headless: false,
|
||||||
userDataDir: userDataDir,
|
userDataDir: userDataDir,
|
||||||
args: ["--start-maximized"]
|
args: ['--start-maximized', '--no-sandbox', '--disable-setuid-sandbox']
|
||||||
});
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
let isFirstSite = true;
|
console.log(`\nProcessing ${sitesToProcess.length} sites...\n`);
|
||||||
for (const site of sitesToProcess) {
|
|
||||||
let siteFailed = false;
|
|
||||||
let failureReason = '';
|
|
||||||
let creds;
|
|
||||||
if (site.credentials && site.credentials.username) {
|
|
||||||
creds = { ...globalCreds, ...site.credentials };
|
|
||||||
console.log(`Using credentials specified for site: ${site.name}`);
|
|
||||||
} else {
|
|
||||||
creds = { ...globalCreds };
|
|
||||||
console.log(`Using global credentials for site: ${site.name}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!creds.username || !creds.email || !creds.password) {
|
for (let i = 0; i < sitesToProcess.length; i++) {
|
||||||
const randomData = generateRandomData();
|
const site = sitesToProcess[i];
|
||||||
creds = { ...randomData, ...creds };
|
console.log(`[${i + 1}/${sitesToProcess.length}]`);
|
||||||
console.log('Filled in missing credentials with random data.');
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`\n${site.url}`);
|
const result = await processSite(browser, site, globalCreds, browserConfig, cliOptions.keepOpen);
|
||||||
console.log(`${'-'.repeat(70)}`);
|
|
||||||
|
|
||||||
const page = isFirstSite ? (await browser.pages())[0] : await browser.newPage();
|
if (result.siteFailed) {
|
||||||
isFirstSite = false;
|
|
||||||
|
|
||||||
if (browserConfig.userAgent) {
|
|
||||||
await page.setUserAgent(browserConfig.userAgent);
|
|
||||||
}
|
|
||||||
|
|
||||||
await page.goto(site.url, { waitUntil: 'networkidle2' });
|
|
||||||
|
|
||||||
console.log('\n🔍 DETECTED FIELDS:');
|
|
||||||
|
|
||||||
let fieldsToFill = {};
|
|
||||||
let detectedFields = {};
|
|
||||||
const definedLocators = site.locators || {};
|
|
||||||
|
|
||||||
if (site.autoDetect !== false) {
|
|
||||||
detectedFields = await detectFormFields(page);
|
|
||||||
for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
|
|
||||||
fieldsToFill[fieldName] = fieldInfo.selector;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const hasEmailField = fieldsToFill.email || fieldsToFill.confirmEmail;
|
|
||||||
if (!hasEmailField) {
|
|
||||||
siteFailed = true;
|
|
||||||
failureReason = 'No email field detected';
|
|
||||||
console.warn(`\n ❌ ${failureReason}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const hasSearchInput = await page.evaluate(() => {
|
|
||||||
const inputs = document.querySelectorAll('input[type="search"], input[name*="search"], input[id*="search"]');
|
|
||||||
return inputs.length > 0;
|
|
||||||
});
|
|
||||||
|
|
||||||
if (hasSearchInput && !hasEmailField) {
|
|
||||||
console.warn(` ⚠️ WARNING: This appears to be a SEARCH form, not a newsletter signup!`);
|
|
||||||
siteFailed = true;
|
|
||||||
failureReason = failureReason || 'Detected search form instead of newsletter';
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const [fieldName, locator] of Object.entries(definedLocators)) {
|
|
||||||
if (fieldName !== 'submit') {
|
|
||||||
fieldsToFill[fieldName] = parseLocator(locator);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Object.keys(fieldsToFill).length > 0) {
|
|
||||||
console.log(`\n ${'FIELD'.padEnd(12)} | ${'SELECTOR'.padEnd(35)} | ${'CONF'.padEnd(4)} | VALUE`);
|
|
||||||
console.log(' ' + '-'.repeat(75));
|
|
||||||
|
|
||||||
for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
|
|
||||||
let valueToFill = creds[fieldName];
|
|
||||||
|
|
||||||
if (fieldName === 'confirmEmail') {
|
|
||||||
valueToFill = creds['email'];
|
|
||||||
}
|
|
||||||
|
|
||||||
const selector = fieldInfo.selector.length > 34 ? fieldInfo.selector.substring(0, 31) + '...' : fieldInfo.selector;
|
|
||||||
const value = valueToFill ? valueToFill.substring(0, 20) : 'N/A';
|
|
||||||
|
|
||||||
console.log(` ${fieldName.padEnd(12)} | ${selector.padEnd(35)} | ${fieldInfo.confidence.toString().padEnd(4)} | ${value}`);
|
|
||||||
|
|
||||||
if (valueToFill) {
|
|
||||||
try {
|
|
||||||
await page.evaluate((sel, val) => {
|
|
||||||
const el = document.querySelector(sel);
|
|
||||||
if (el) {
|
|
||||||
el.value = val;
|
|
||||||
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
||||||
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
||||||
}
|
|
||||||
}, fieldInfo.selector, valueToFill);
|
|
||||||
} catch (e) {
|
|
||||||
console.warn(` ✗ Failed to fill ${fieldName}: ${e.message}`);
|
|
||||||
siteFailed = true;
|
|
||||||
failureReason = `Failed to fill ${fieldName}: ${e.message}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
console.log(' ' + '-'.repeat(75));
|
|
||||||
}
|
|
||||||
|
|
||||||
let submitSelector = null;
|
|
||||||
if (definedLocators.submit) {
|
|
||||||
submitSelector = parseLocator(definedLocators.submit);
|
|
||||||
} else if (site.autoDetect !== false) {
|
|
||||||
submitSelector = await detectSubmitButton(page);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (submitSelector) {
|
|
||||||
try {
|
|
||||||
await page.click(submitSelector);
|
|
||||||
console.log(` ✓ Submitted (${submitSelector})`);
|
|
||||||
} catch (submitError) {
|
|
||||||
siteFailed = true;
|
|
||||||
failureReason = failureReason || `Submit failed: ${submitError.message}`;
|
|
||||||
console.error(` ✗ Submit failed: ${submitError.message}`);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
siteFailed = true;
|
|
||||||
failureReason = failureReason || 'No submit button found';
|
|
||||||
console.warn(` ✗ ${failureReason}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (siteFailed) {
|
|
||||||
failedSites.push({
|
failedSites.push({
|
||||||
name: site.name,
|
name: site.name,
|
||||||
url: site.url,
|
url: site.url,
|
||||||
reason: failureReason,
|
reason: result.failureReason,
|
||||||
timestamp: new Date().toISOString()
|
timestamp: new Date().toISOString()
|
||||||
});
|
});
|
||||||
|
processedCount.failed++;
|
||||||
|
} else {
|
||||||
|
processedCount.success++;
|
||||||
}
|
}
|
||||||
|
|
||||||
await new Promise(r => setTimeout(r, 2000));
|
await new Promise(r => setTimeout(r, 1000));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
console.log(`\n${'='.repeat(70)}`);
|
||||||
|
console.log(`Done: ${processedCount.success} success, ${processedCount.failed} failed`);
|
||||||
|
|
||||||
} catch(error) {
|
} catch(error) {
|
||||||
console.error('Error:', error.message);
|
console.error('Fatal error:', error.message);
|
||||||
} finally {
|
} finally {
|
||||||
|
if (cliOptions.keepOpen) {
|
||||||
|
console.log(`\nBrowser kept open for verification`);
|
||||||
|
console.log(` Close browser manually when done`);
|
||||||
|
} else {
|
||||||
|
await browser.close();
|
||||||
|
}
|
||||||
|
|
||||||
if (failedSites.length > 0) {
|
if (failedSites.length > 0) {
|
||||||
const reportPath = path.join(__dirname, 'failed_sites.txt');
|
const reportPath = path.join(__dirname, 'failed_sites.txt');
|
||||||
const now = new Date();
|
const now = new Date();
|
||||||
@@ -638,7 +132,7 @@ async function main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fs.appendFileSync(reportPath, reportContent);
|
fs.appendFileSync(reportPath, reportContent);
|
||||||
console.log(`\n${failedSites.length} site(s) failed. See: failed_sites.txt`);
|
console.log(`${failedSites.length} failed. See: failed_sites.txt`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Generated
+126
-1
@@ -10,11 +10,13 @@
|
|||||||
"license": "ISC",
|
"license": "ISC",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@iarna/toml": "^2.2.5",
|
"@iarna/toml": "^2.2.5",
|
||||||
|
"commander": "^14.0.3",
|
||||||
"playwright": "^1.56.1",
|
"playwright": "^1.56.1",
|
||||||
"playwright-extra": "^4.3.6",
|
"playwright-extra": "^4.3.6",
|
||||||
"puppeteer": "^24.30.0",
|
"puppeteer": "^24.30.0",
|
||||||
"puppeteer-extra": "^3.3.6",
|
"puppeteer-extra": "^3.3.6",
|
||||||
"puppeteer-extra-plugin-stealth": "^2.11.2"
|
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||||
|
"tesseract.js": "^7.0.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@babel/code-frame": {
|
"node_modules/@babel/code-frame": {
|
||||||
@@ -288,6 +290,12 @@
|
|||||||
"node": ">=10.0.0"
|
"node": ">=10.0.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/bmp-js": {
|
||||||
|
"version": "0.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
|
||||||
|
"integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/brace-expansion": {
|
"node_modules/brace-expansion": {
|
||||||
"version": "1.1.12",
|
"version": "1.1.12",
|
||||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
|
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
|
||||||
@@ -377,6 +385,15 @@
|
|||||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/commander": {
|
||||||
|
"version": "14.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz",
|
||||||
|
"integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/concat-map": {
|
"node_modules/concat-map": {
|
||||||
"version": "0.0.1",
|
"version": "0.0.1",
|
||||||
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
|
||||||
@@ -748,6 +765,12 @@
|
|||||||
"node": ">= 14"
|
"node": ">= 14"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/idb-keyval": {
|
||||||
|
"version": "6.2.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.2.tgz",
|
||||||
|
"integrity": "sha512-yjD9nARJ/jb1g+CvD0tlhUHOrJ9Sy0P8T9MF3YaLlHnSRpwPfpTX0XIvpmw3gAJUmEu3FiICLBDPXVwyEvrleg==",
|
||||||
|
"license": "Apache-2.0"
|
||||||
|
},
|
||||||
"node_modules/import-fresh": {
|
"node_modules/import-fresh": {
|
||||||
"version": "3.3.1",
|
"version": "3.3.1",
|
||||||
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz",
|
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz",
|
||||||
@@ -832,6 +855,12 @@
|
|||||||
"node": ">=0.10.0"
|
"node": ">=0.10.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/is-url": {
|
||||||
|
"version": "1.2.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz",
|
||||||
|
"integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/isobject": {
|
"node_modules/isobject": {
|
||||||
"version": "3.0.1",
|
"version": "3.0.1",
|
||||||
"resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
|
||||||
@@ -982,6 +1011,26 @@
|
|||||||
"node": ">= 0.4.0"
|
"node": ">= 0.4.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/node-fetch": {
|
||||||
|
"version": "2.7.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||||
|
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"whatwg-url": "^5.0.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": "4.x || >=6.0.0"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"encoding": "^0.1.0"
|
||||||
|
},
|
||||||
|
"peerDependenciesMeta": {
|
||||||
|
"encoding": {
|
||||||
|
"optional": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/once": {
|
"node_modules/once": {
|
||||||
"version": "1.4.0",
|
"version": "1.4.0",
|
||||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||||
@@ -991,6 +1040,15 @@
|
|||||||
"wrappy": "1"
|
"wrappy": "1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/opencollective-postinstall": {
|
||||||
|
"version": "2.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz",
|
||||||
|
"integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==",
|
||||||
|
"license": "MIT",
|
||||||
|
"bin": {
|
||||||
|
"opencollective-postinstall": "index.js"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/pac-proxy-agent": {
|
"node_modules/pac-proxy-agent": {
|
||||||
"version": "7.2.0",
|
"version": "7.2.0",
|
||||||
"resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz",
|
"resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz",
|
||||||
@@ -1347,6 +1405,12 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/regenerator-runtime": {
|
||||||
|
"version": "0.13.11",
|
||||||
|
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz",
|
||||||
|
"integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/require-directory": {
|
"node_modules/require-directory": {
|
||||||
"version": "2.1.1",
|
"version": "2.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||||
@@ -1539,6 +1603,30 @@
|
|||||||
"streamx": "^2.15.0"
|
"streamx": "^2.15.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/tesseract.js": {
|
||||||
|
"version": "7.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-7.0.0.tgz",
|
||||||
|
"integrity": "sha512-exPBkd+z+wM1BuMkx/Bjv43OeLBxhL5kKWsz/9JY+DXcXdiBjiAch0V49QR3oAJqCaL5qURE0vx9Eo+G5YE7mA==",
|
||||||
|
"hasInstallScript": true,
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"dependencies": {
|
||||||
|
"bmp-js": "^0.1.0",
|
||||||
|
"idb-keyval": "^6.2.0",
|
||||||
|
"is-url": "^1.2.4",
|
||||||
|
"node-fetch": "^2.6.9",
|
||||||
|
"opencollective-postinstall": "^2.0.3",
|
||||||
|
"regenerator-runtime": "^0.13.3",
|
||||||
|
"tesseract.js-core": "^7.0.0",
|
||||||
|
"wasm-feature-detect": "^1.8.0",
|
||||||
|
"zlibjs": "^0.3.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/tesseract.js-core": {
|
||||||
|
"version": "7.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-7.0.0.tgz",
|
||||||
|
"integrity": "sha512-WnNH518NzmbSq9zgTPeoF8c+xmilS8rFIl1YKbk/ptuuc7p6cLNELNuPAzcmsYw450ca6bLa8j3t0VAtq435Vw==",
|
||||||
|
"license": "Apache-2.0"
|
||||||
|
},
|
||||||
"node_modules/text-decoder": {
|
"node_modules/text-decoder": {
|
||||||
"version": "1.2.6",
|
"version": "1.2.6",
|
||||||
"resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.6.tgz",
|
"resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.6.tgz",
|
||||||
@@ -1548,6 +1636,12 @@
|
|||||||
"b4a": "^1.6.4"
|
"b4a": "^1.6.4"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/tr46": {
|
||||||
|
"version": "0.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||||
|
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/tslib": {
|
"node_modules/tslib": {
|
||||||
"version": "2.8.1",
|
"version": "2.8.1",
|
||||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||||
@@ -1576,12 +1670,34 @@
|
|||||||
"node": ">= 10.0.0"
|
"node": ">= 10.0.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/wasm-feature-detect": {
|
||||||
|
"version": "1.8.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz",
|
||||||
|
"integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==",
|
||||||
|
"license": "Apache-2.0"
|
||||||
|
},
|
||||||
"node_modules/webdriver-bidi-protocol": {
|
"node_modules/webdriver-bidi-protocol": {
|
||||||
"version": "0.4.1",
|
"version": "0.4.1",
|
||||||
"resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz",
|
"resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz",
|
||||||
"integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==",
|
"integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==",
|
||||||
"license": "Apache-2.0"
|
"license": "Apache-2.0"
|
||||||
},
|
},
|
||||||
|
"node_modules/webidl-conversions": {
|
||||||
|
"version": "3.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||||
|
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
|
||||||
|
"license": "BSD-2-Clause"
|
||||||
|
},
|
||||||
|
"node_modules/whatwg-url": {
|
||||||
|
"version": "5.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||||
|
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"tr46": "~0.0.3",
|
||||||
|
"webidl-conversions": "^3.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/wrap-ansi": {
|
"node_modules/wrap-ansi": {
|
||||||
"version": "7.0.0",
|
"version": "7.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
|
||||||
@@ -1672,6 +1788,15 @@
|
|||||||
"fd-slicer": "~1.1.0"
|
"fd-slicer": "~1.1.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/zlibjs": {
|
||||||
|
"version": "0.3.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz",
|
||||||
|
"integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/zod": {
|
"node_modules/zod": {
|
||||||
"version": "3.25.76",
|
"version": "3.25.76",
|
||||||
"resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
|
"resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
|
||||||
|
|||||||
+3
-1
@@ -12,10 +12,12 @@
|
|||||||
"type": "commonjs",
|
"type": "commonjs",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@iarna/toml": "^2.2.5",
|
"@iarna/toml": "^2.2.5",
|
||||||
|
"commander": "^14.0.3",
|
||||||
"playwright": "^1.56.1",
|
"playwright": "^1.56.1",
|
||||||
"playwright-extra": "^4.3.6",
|
"playwright-extra": "^4.3.6",
|
||||||
"puppeteer": "^24.30.0",
|
"puppeteer": "^24.30.0",
|
||||||
"puppeteer-extra": "^3.3.6",
|
"puppeteer-extra": "^3.3.6",
|
||||||
"puppeteer-extra-plugin-stealth": "^2.11.2"
|
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||||
|
"tesseract.js": "^7.0.0"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Newsletter signup sites
|
||||||
|
# Lines starting with # are ignored
|
||||||
|
|
||||||
|
https://www.dutchnews.nl/dutchnews-newsletter-signup/
|
||||||
|
https://immaculatevegan.com/blogs/magazine
|
||||||
|
https://kulalaland.com/
|
||||||
+128
@@ -0,0 +1,128 @@
|
|||||||
|
const Tesseract = require('tesseract.js');
|
||||||
|
|
||||||
|
const CAPTCHA_PATTERNS = {
|
||||||
|
recaptcha: {
|
||||||
|
selectors: ['.g-recaptcha', '[data-sitekey]', 'iframe[src*="recaptcha"]', '#recaptcha'],
|
||||||
|
name: 'reCAPTCHA'
|
||||||
|
},
|
||||||
|
hcaptcha: {
|
||||||
|
selectors: ['.h-captcha', '[data-hcaptcha-sitekey]', 'iframe[src*="hcaptcha"]', '#hcaptcha'],
|
||||||
|
name: 'hCaptcha'
|
||||||
|
},
|
||||||
|
textCaptcha: {
|
||||||
|
selectors: ['input[name*="captcha"]', 'input[id*="captcha"]', '.captcha-input', '#captcha'],
|
||||||
|
name: 'Text CAPTCHA'
|
||||||
|
},
|
||||||
|
imageCaptcha: {
|
||||||
|
selectors: ['img[src*="captcha"]', '.captcha-image', '#captcha-image', 'img[alt*="captcha" i]'],
|
||||||
|
name: 'Image CAPTCHA'
|
||||||
|
},
|
||||||
|
mathCaptcha: {
|
||||||
|
selectors: ['.math-captcha', '[name*="math"]', 'input[placeholder*="math" i]'],
|
||||||
|
name: 'Math CAPTCHA'
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
async function detectCaptcha(page) {
|
||||||
|
const captchaInfo = await page.evaluate((patterns) => {
|
||||||
|
const results = [];
|
||||||
|
|
||||||
|
for (const [type, config] of Object.entries(patterns)) {
|
||||||
|
for (const selector of config.selectors) {
|
||||||
|
const elements = document.querySelectorAll(selector);
|
||||||
|
if (elements.length > 0) {
|
||||||
|
results.push({
|
||||||
|
type: type,
|
||||||
|
name: config.name,
|
||||||
|
selector: selector,
|
||||||
|
count: elements.length
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}, CAPTCHA_PATTERNS);
|
||||||
|
|
||||||
|
return captchaInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function solveSimpleCaptcha(page) {
|
||||||
|
const mathSolution = await page.evaluate(() => {
|
||||||
|
const mathText = document.querySelector('.math-captcha, .captcha-question, [class*="math"]');
|
||||||
|
if (mathText) {
|
||||||
|
const text = mathText.textContent || '';
|
||||||
|
const match = text.match(/(\d+)\s*([+\-*/])\s*(\d+)/);
|
||||||
|
if (match) {
|
||||||
|
const [_, a, op, b] = match;
|
||||||
|
let result;
|
||||||
|
switch(op) {
|
||||||
|
case '+': result = parseInt(a) + parseInt(b); break;
|
||||||
|
case '-': result = parseInt(a) - parseInt(b); break;
|
||||||
|
case '*': result = parseInt(a) * parseInt(b); break;
|
||||||
|
case '/': result = Math.floor(parseInt(a) / parseInt(b)); break;
|
||||||
|
}
|
||||||
|
return { type: 'math', answer: result, text: text };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (mathSolution) {
|
||||||
|
return mathSolution;
|
||||||
|
}
|
||||||
|
|
||||||
|
const imageCaptchaInfo = await page.evaluate(() => {
|
||||||
|
const selectors = [
|
||||||
|
'img[src*="captcha"]',
|
||||||
|
'.captcha-image img',
|
||||||
|
'#captcha-image',
|
||||||
|
'img[alt*="captcha" i]',
|
||||||
|
'img[class*="captcha" i]'
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const selector of selectors) {
|
||||||
|
const img = document.querySelector(selector);
|
||||||
|
if (img && img.src) {
|
||||||
|
return { src: img.src, selector: selector };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (imageCaptchaInfo) {
|
||||||
|
console.log(' Attempting OCR-based CAPTCHA solving...');
|
||||||
|
try {
|
||||||
|
const { data: { text } } = await Tesseract.recognize(
|
||||||
|
imageCaptchaInfo.src,
|
||||||
|
'eng',
|
||||||
|
{ logger: m => {} }
|
||||||
|
);
|
||||||
|
|
||||||
|
const cleanedText = text.replace(/[^a-zA-Z0-9]/g, '').trim();
|
||||||
|
|
||||||
|
if (cleanedText && cleanedText.length >= 3) {
|
||||||
|
console.log(` OCR read: "${cleanedText}"`);
|
||||||
|
return {
|
||||||
|
type: 'image',
|
||||||
|
answer: cleanedText,
|
||||||
|
originalText: text,
|
||||||
|
imageSrc: imageCaptchaInfo.src
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch (ocrError) {
|
||||||
|
console.log(` OCR failed: ${ocrError.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { type: 'image', message: 'Image CAPTCHA detected - OCR failed, manual solving required' };
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
CAPTCHA_PATTERNS,
|
||||||
|
detectCaptcha,
|
||||||
|
solveSimpleCaptcha
|
||||||
|
};
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
async function classifyNewsletterForm(page) {
|
||||||
|
const formInfo = await page.evaluate(() => {
|
||||||
|
const results = {
|
||||||
|
hasEmailInput: false,
|
||||||
|
hasNewsletterKeywords: false,
|
||||||
|
hasSearchInput: false,
|
||||||
|
hasLoginInput: false,
|
||||||
|
hasContactForm: false,
|
||||||
|
hasOnlyEmail: false,
|
||||||
|
formCount: 0,
|
||||||
|
emailInputCount: 0,
|
||||||
|
totalInputCount: 0,
|
||||||
|
confidence: 0,
|
||||||
|
reasons: [],
|
||||||
|
signals: {
|
||||||
|
positive: [],
|
||||||
|
negative: []
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const allInputs = document.querySelectorAll('input:not([type="hidden"]):not([type="submit"]):not([type="button"])');
|
||||||
|
results.totalInputCount = allInputs.length;
|
||||||
|
|
||||||
|
const emailInputs = document.querySelectorAll('input[type="email"], input[name*="email" i], input[id*="email" i], input[placeholder*="email" i]');
|
||||||
|
results.emailInputCount = emailInputs.length;
|
||||||
|
results.hasEmailInput = emailInputs.length > 0;
|
||||||
|
if (results.hasEmailInput) {
|
||||||
|
results.signals.positive.push(`Found ${emailInputs.length} email input(s)`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const pageText = document.body.innerText.toLowerCase();
|
||||||
|
const titleText = document.title.toLowerCase();
|
||||||
|
const combinedText = pageText + ' ' + titleText;
|
||||||
|
|
||||||
|
const newsletterKeywords = [
|
||||||
|
'newsletter', 'subscribe', 'signup', 'sign up', 'join our', 'stay updated',
|
||||||
|
'get updates', 'email updates', 'weekly digest', 'monthly newsletter',
|
||||||
|
'sign up for updates', 'join the list', 'email list', 'mailing list'
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const keyword of newsletterKeywords) {
|
||||||
|
if (combinedText.includes(keyword)) {
|
||||||
|
results.hasNewsletterKeywords = true;
|
||||||
|
results.signals.positive.push(`Keyword: "${keyword}"`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const searchInputs = document.querySelectorAll('input[type="search"], input[name*="search" i], input[id*="search" i], input[placeholder*="search" i]');
|
||||||
|
results.hasSearchInput = searchInputs.length > 0;
|
||||||
|
if (results.hasSearchInput) {
|
||||||
|
results.signals.negative.push('Search input detected');
|
||||||
|
results.reasons.push('Search input detected');
|
||||||
|
}
|
||||||
|
|
||||||
|
const passwordInputs = document.querySelectorAll('input[type="password"]');
|
||||||
|
const usernameInputs = document.querySelectorAll('input[name*="username" i], input[name*="login" i], input[id*="username" i]');
|
||||||
|
results.hasLoginInput = passwordInputs.length > 0 || usernameInputs.length > 0;
|
||||||
|
if (results.hasLoginInput) {
|
||||||
|
results.signals.negative.push('Login/password fields detected');
|
||||||
|
results.reasons.push('Login/password fields detected');
|
||||||
|
}
|
||||||
|
|
||||||
|
const contactKeywords = ['contact us', 'get in touch', 'send message', 'inquiry'];
|
||||||
|
for (const keyword of contactKeywords) {
|
||||||
|
if (combinedText.includes(keyword)) {
|
||||||
|
results.hasContactForm = true;
|
||||||
|
results.signals.negative.push(`Contact form keyword: "${keyword}"`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const nonEmailInputs = Array.from(allInputs).filter(input => {
|
||||||
|
const type = input.type?.toLowerCase();
|
||||||
|
const name = input.name?.toLowerCase() || '';
|
||||||
|
return type !== 'email' && !name.includes('email');
|
||||||
|
});
|
||||||
|
|
||||||
|
results.hasOnlyEmail = results.hasEmailInput && nonEmailInputs.length === 0;
|
||||||
|
if (results.hasOnlyEmail) {
|
||||||
|
results.signals.positive.push('Only email field (no other inputs)');
|
||||||
|
}
|
||||||
|
|
||||||
|
let score = 0;
|
||||||
|
|
||||||
|
if (results.hasEmailInput) score += 40;
|
||||||
|
if (results.hasNewsletterKeywords) score += 35;
|
||||||
|
if (results.hasOnlyEmail) score += 15;
|
||||||
|
if (results.totalInputCount <= 3) score += 10;
|
||||||
|
|
||||||
|
if (!results.hasSearchInput) score += 10;
|
||||||
|
if (!results.hasLoginInput) score += 15;
|
||||||
|
if (!results.hasContactForm) score += 5;
|
||||||
|
|
||||||
|
if (results.hasSearchInput) score -= 30;
|
||||||
|
if (results.hasLoginInput) score -= 50;
|
||||||
|
if (results.hasContactForm && !results.hasNewsletterKeywords) score -= 20;
|
||||||
|
|
||||||
|
results.confidence = Math.max(0, Math.min(100, score));
|
||||||
|
results.isNewsletter = results.confidence >= 55;
|
||||||
|
|
||||||
|
return results;
|
||||||
|
});
|
||||||
|
|
||||||
|
return formInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
classifyNewsletterForm
|
||||||
|
};
|
||||||
+69
@@ -0,0 +1,69 @@
|
|||||||
|
const { Command } = require('commander');
|
||||||
|
|
||||||
|
function createCLI() {
|
||||||
|
const program = new Command();
|
||||||
|
|
||||||
|
program
|
||||||
|
.name('newsletter-signup')
|
||||||
|
.description('Newsletter Signup Automation Tool')
|
||||||
|
.version('1.0.0')
|
||||||
|
.option('-f, --file <path>', 'Load sites from text file (one URL per line)')
|
||||||
|
.option('-k, --keep-open', 'Keep browser tabs open after processing for verification')
|
||||||
|
.option('-l, --log', 'Save output to timestamped log file')
|
||||||
|
.option('-h, --help', 'Show help message')
|
||||||
|
.argument('[target]', 'Site name or URL to process (filters from config.toml sites)');
|
||||||
|
|
||||||
|
program.on('option:help', () => {
|
||||||
|
console.log(`
|
||||||
|
Newsletter Signup Automation Tool
|
||||||
|
|
||||||
|
Usage: node index.js [options] [site-name-or-url]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-f, --file=<path> Load sites from text file (one URL per line)
|
||||||
|
-k, --keep-open Keep browser tabs open after processing for verification
|
||||||
|
-l, --log Save output to timestamped log file
|
||||||
|
-h, --help Show this help message
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
node index.js Process all sites from config.toml
|
||||||
|
node index.js --file=sites.txt Process sites from file
|
||||||
|
node index.js "Dutch News" Process specific site
|
||||||
|
node index.js --file=sites.txt --keep-open Keep tabs open for verification
|
||||||
|
node index.js --log Save output to log file
|
||||||
|
|
||||||
|
Text file format (sites.txt):
|
||||||
|
# Lines starting with # are comments
|
||||||
|
https://example.com/newsletter
|
||||||
|
https://another-site.com/signup
|
||||||
|
|
||||||
|
Configuration:
|
||||||
|
Edit config.toml to set credentials and default sites
|
||||||
|
`);
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
return program;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseCLI(args = process.argv.slice(2)) {
|
||||||
|
const program = createCLI();
|
||||||
|
|
||||||
|
const fullArgs = ['node', 'index.js', ...args];
|
||||||
|
program.parse(fullArgs);
|
||||||
|
|
||||||
|
const opts = program.opts();
|
||||||
|
|
||||||
|
return {
|
||||||
|
target: program.args[0] || null,
|
||||||
|
file: opts.file || null,
|
||||||
|
keepOpen: opts.keepOpen || false,
|
||||||
|
log: opts.log || false,
|
||||||
|
help: opts.help || false
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
createCLI,
|
||||||
|
parseCLI
|
||||||
|
};
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const toml = require('@iarna/toml');
|
||||||
|
|
||||||
|
function loadConfig() {
|
||||||
|
try {
|
||||||
|
const configPath = path.join(__dirname, '..', 'config.toml');
|
||||||
|
const tomlString = fs.readFileSync(configPath, 'utf-8');
|
||||||
|
const config = toml.parse(tomlString);
|
||||||
|
console.log('Configuration loaded successfully.');
|
||||||
|
return config;
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to load or parse config.toml:', error.message);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function generateRandomData() {
|
||||||
|
const randomString = Math.random().toString(36).substring(2, 10);
|
||||||
|
const username = `user${randomString}`;
|
||||||
|
const email = `${username}@gmail.com`;
|
||||||
|
const password = `pass${randomString}`;
|
||||||
|
const age = Math.floor(Math.random() * 30 + 20).toString();
|
||||||
|
const firstName = 'John';
|
||||||
|
const lastName = 'Doe';
|
||||||
|
const phone = '+31610488717';
|
||||||
|
const company = 'Example Corp';
|
||||||
|
return { username, email, password, age, firstName, lastName, phone, company };
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseLocator(locatorStr) {
|
||||||
|
locatorStr = locatorStr.trim();
|
||||||
|
if (!locatorStr.startsWith('<')) {
|
||||||
|
return locatorStr;
|
||||||
|
}
|
||||||
|
|
||||||
|
const getAttr = (attr) => {
|
||||||
|
const match = locatorStr.match(new RegExp(`${attr}=["']([^"']+)["']`));
|
||||||
|
return match ? match[1] : null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const tagName = (locatorStr.match(/^<([a-zA-Z0-9]+)/) || [])[1] || '';
|
||||||
|
const id = getAttr('id');
|
||||||
|
if (id) {
|
||||||
|
if (/^\d/.test(id)) {
|
||||||
|
return `${tagName}[id='${id}']`;
|
||||||
|
}
|
||||||
|
return `${tagName}#${id}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const name = getAttr('name');
|
||||||
|
if (name) return `${tagName}[name='${name}']`;
|
||||||
|
|
||||||
|
const placeholder = getAttr('placeholder');
|
||||||
|
if (placeholder) return `${tagName}[placeholder='${placeholder}']`;
|
||||||
|
|
||||||
|
const type = getAttr('type');
|
||||||
|
if (type) return `${tagName}[type='${type}']`;
|
||||||
|
|
||||||
|
return tagName;
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadSitesFromFile(filePath) {
|
||||||
|
const fullPath = path.isAbsolute(filePath) ? filePath : path.join(__dirname, '..', filePath);
|
||||||
|
const content = fs.readFileSync(fullPath, 'utf-8');
|
||||||
|
const urls = content.split('\n').filter(line => line.trim() && !line.startsWith('#'));
|
||||||
|
return urls.map((url, index) => ({
|
||||||
|
name: `Site_${index + 1}`,
|
||||||
|
url: url.trim(),
|
||||||
|
category: 'Newsletter',
|
||||||
|
needsPreSubmitAction: false
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
loadConfig,
|
||||||
|
generateRandomData,
|
||||||
|
parseLocator,
|
||||||
|
loadSitesFromFile
|
||||||
|
};
|
||||||
+260
@@ -0,0 +1,260 @@
|
|||||||
|
const FIELD_PATTERNS = {
|
||||||
|
email: {
|
||||||
|
priority: 1,
|
||||||
|
types: ['email'],
|
||||||
|
names: ['email', 'e-mail', 'mail', 'user_email', 'login_email', 'email_address', 'emailaddress'],
|
||||||
|
ids: ['email', 'e-mail', 'mail', 'user-email', 'login-email'],
|
||||||
|
placeholders: ['email', 'e-mail', 'email address', 'your email', 'enter email'],
|
||||||
|
autocomplete: ['email']
|
||||||
|
},
|
||||||
|
confirmEmail: {
|
||||||
|
priority: 2,
|
||||||
|
types: ['email'],
|
||||||
|
names: ['confirm_email', 'confirmemail', 'email_confirm', 'verify_email', 'email_verify'],
|
||||||
|
ids: ['confirm-email', 'confirmemail', 'verify-email', 'email-confirm'],
|
||||||
|
placeholders: ['confirm email', 'verify email', 're-enter email', 'repeat email'],
|
||||||
|
mustHaveConfirmIndicator: true
|
||||||
|
},
|
||||||
|
firstName: {
|
||||||
|
priority: 3,
|
||||||
|
types: ['text'],
|
||||||
|
names: ['first_name', 'firstname', 'fname', 'first-name', 'vorname', 'prenom', 'fname'],
|
||||||
|
ids: ['first-name', 'firstname', 'fname', 'firstName', 'mce-fname'],
|
||||||
|
placeholders: ['first name', 'firstname', 'first'],
|
||||||
|
autocomplete: ['given-name', 'first-name']
|
||||||
|
},
|
||||||
|
lastName: {
|
||||||
|
priority: 4,
|
||||||
|
types: ['text'],
|
||||||
|
names: ['last_name', 'lastname', 'lname', 'surname', 'family_name', 'last-name', 'lname'],
|
||||||
|
ids: ['last-name', 'lastname', 'lname', 'surname', 'lastName', 'mce-lname'],
|
||||||
|
placeholders: ['last name', 'lastname', 'surname', 'last'],
|
||||||
|
autocomplete: ['family-name', 'last-name']
|
||||||
|
},
|
||||||
|
username: {
|
||||||
|
priority: 5,
|
||||||
|
types: ['text'],
|
||||||
|
names: ['username', 'user_name', 'login', 'usr', 'uname', 'user-id', 'userid', 'account_name', 'login_id'],
|
||||||
|
ids: ['username', 'user-name', 'login', 'usr', 'uname'],
|
||||||
|
placeholders: ['username', 'user name', 'login', 'username or email'],
|
||||||
|
autocomplete: ['username']
|
||||||
|
},
|
||||||
|
password: {
|
||||||
|
priority: 6,
|
||||||
|
types: ['password'],
|
||||||
|
names: ['password', 'pass', 'pwd', 'user_password', 'login_password'],
|
||||||
|
ids: ['password', 'pass', 'pwd'],
|
||||||
|
placeholders: ['password', 'pass', 'enter password'],
|
||||||
|
autocomplete: ['current-password', 'new-password']
|
||||||
|
},
|
||||||
|
fullName: {
|
||||||
|
priority: 7,
|
||||||
|
types: ['text'],
|
||||||
|
names: ['full_name', 'fullname', 'name'],
|
||||||
|
ids: ['full-name', 'fullname', 'name'],
|
||||||
|
placeholders: ['full name', 'your name', 'complete name'],
|
||||||
|
autocomplete: ['name']
|
||||||
|
},
|
||||||
|
phone: {
|
||||||
|
priority: 8,
|
||||||
|
types: ['tel', 'text'],
|
||||||
|
names: ['phone', 'telephone', 'mobile', 'cell', 'cellphone', 'phone_number'],
|
||||||
|
ids: ['phone', 'telephone', 'mobile', 'cell', 'tel'],
|
||||||
|
placeholders: ['phone', 'telephone', 'mobile number'],
|
||||||
|
autocomplete: ['tel']
|
||||||
|
},
|
||||||
|
age: {
|
||||||
|
priority: 9,
|
||||||
|
types: ['number', 'text'],
|
||||||
|
names: ['age', 'user_age'],
|
||||||
|
ids: ['age'],
|
||||||
|
placeholders: ['age', 'your age']
|
||||||
|
},
|
||||||
|
company: {
|
||||||
|
priority: 10,
|
||||||
|
types: ['text'],
|
||||||
|
names: ['company', 'organization', 'company_name', 'employer'],
|
||||||
|
ids: ['company', 'organization', 'employer'],
|
||||||
|
placeholders: ['company', 'organization'],
|
||||||
|
autocomplete: ['organization']
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
async function detectFormFields(page) {
|
||||||
|
console.log('Auto-detecting form fields...');
|
||||||
|
|
||||||
|
const detectedFields = await page.evaluate((patterns) => {
|
||||||
|
const results = {};
|
||||||
|
const usedElements = new Set();
|
||||||
|
|
||||||
|
const inputs = Array.from(document.querySelectorAll('input, select, textarea')).filter(input => {
|
||||||
|
const type = input.type?.toLowerCase() || 'text';
|
||||||
|
const style = window.getComputedStyle(input);
|
||||||
|
const rect = input.getBoundingClientRect();
|
||||||
|
|
||||||
|
if (type === 'hidden' || type === 'submit' || type === 'button') return false;
|
||||||
|
if (input.tabIndex === -1 && input.style.position === 'absolute') return false;
|
||||||
|
if (rect.width === 0 || rect.height === 0) return false;
|
||||||
|
if (style.display === 'none' || style.visibility === 'hidden') return false;
|
||||||
|
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
|
||||||
|
const calculateScore = (input, pattern, fieldName) => {
|
||||||
|
let score = 0;
|
||||||
|
const inputType = (input.type || 'text').toLowerCase();
|
||||||
|
const inputName = (input.name || '').toLowerCase();
|
||||||
|
const inputId = (input.id || '').toLowerCase();
|
||||||
|
const placeholder = (input.placeholder || '').toLowerCase();
|
||||||
|
const autocomplete = (input.autocomplete || '').toLowerCase();
|
||||||
|
|
||||||
|
if (pattern.types && pattern.types.includes(inputType)) {
|
||||||
|
score += 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pattern.names) {
|
||||||
|
for (const name of pattern.names) {
|
||||||
|
const nameLower = name.toLowerCase();
|
||||||
|
if (inputName === nameLower) {
|
||||||
|
score += 20;
|
||||||
|
break;
|
||||||
|
} else if (inputName.includes(nameLower)) {
|
||||||
|
if (fieldName === 'firstName' && (inputName.includes('user') || inputName.includes('login'))) {
|
||||||
|
score -= 5;
|
||||||
|
} else if (fieldName === 'lastName' && (inputName.includes('user') || inputName.includes('login'))) {
|
||||||
|
score -= 5;
|
||||||
|
} else {
|
||||||
|
score += 10;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pattern.ids) {
|
||||||
|
for (const id of pattern.ids) {
|
||||||
|
const idLower = id.toLowerCase();
|
||||||
|
if (inputId === idLower) {
|
||||||
|
score += 20;
|
||||||
|
break;
|
||||||
|
} else if (inputId.includes(idLower)) {
|
||||||
|
score += 10;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pattern.placeholders) {
|
||||||
|
for (const ph of pattern.placeholders) {
|
||||||
|
const phLower = ph.toLowerCase();
|
||||||
|
if (placeholder === phLower) {
|
||||||
|
score += 15;
|
||||||
|
break;
|
||||||
|
} else if (placeholder.includes(phLower)) {
|
||||||
|
score += 8;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pattern.autocomplete) {
|
||||||
|
for (const ac of pattern.autocomplete) {
|
||||||
|
if (autocomplete === ac.toLowerCase()) {
|
||||||
|
score += 12;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const labels = input.labels;
|
||||||
|
if (labels && labels.length > 0) {
|
||||||
|
const labelText = labels[0].textContent.toLowerCase();
|
||||||
|
const fieldNameLower = fieldName.toLowerCase().replace(/([A-Z])/g, ' $1').trim();
|
||||||
|
if (labelText.includes(fieldNameLower) || labelText.includes(pattern.names?.[0] || '')) {
|
||||||
|
score += 15;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (inputName.includes('b_') || inputId.includes('b_')) {
|
||||||
|
score -= 30;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (input.disabled || input.readOnly) {
|
||||||
|
score -= 20;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pattern.mustHaveConfirmIndicator) {
|
||||||
|
const hasConfirm = inputId.includes('confirm') || inputId.includes('verify') ||
|
||||||
|
inputName.includes('confirm') || inputName.includes('verify') ||
|
||||||
|
placeholder.includes('confirm') || placeholder.includes('verify') ||
|
||||||
|
placeholder.includes('re-') || placeholder.includes('again');
|
||||||
|
if (!hasConfirm) {
|
||||||
|
score -= 15;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return score;
|
||||||
|
};
|
||||||
|
|
||||||
|
const sortedPatterns = Object.entries(patterns).sort((a, b) => a[1].priority - b[1].priority);
|
||||||
|
|
||||||
|
for (const [fieldName, pattern] of sortedPatterns) {
|
||||||
|
let bestMatch = null;
|
||||||
|
let bestScore = 0;
|
||||||
|
let bestIndex = -1;
|
||||||
|
|
||||||
|
for (let i = 0; i < inputs.length; i++) {
|
||||||
|
if (usedElements.has(i)) continue;
|
||||||
|
|
||||||
|
const input = inputs[i];
|
||||||
|
const score = calculateScore(input, pattern, fieldName);
|
||||||
|
|
||||||
|
if (score > bestScore && score >= 10) {
|
||||||
|
bestScore = score;
|
||||||
|
bestMatch = input;
|
||||||
|
bestIndex = i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bestMatch && bestIndex >= 0) {
|
||||||
|
usedElements.add(bestIndex);
|
||||||
|
|
||||||
|
let selector = '';
|
||||||
|
if (bestMatch.id) {
|
||||||
|
selector = `#${bestMatch.id}`;
|
||||||
|
} else if (bestMatch.name) {
|
||||||
|
selector = `[name="${bestMatch.name}"]`;
|
||||||
|
} else if (bestMatch.placeholder) {
|
||||||
|
selector = `[placeholder="${bestMatch.placeholder}"]`;
|
||||||
|
} else {
|
||||||
|
const tagName = bestMatch.tagName.toLowerCase();
|
||||||
|
const type = bestMatch.type ? `[type="${bestMatch.type}"]` : '';
|
||||||
|
const allSame = Array.from(document.querySelectorAll(`${tagName}${type}`));
|
||||||
|
const index = allSame.indexOf(bestMatch) + 1;
|
||||||
|
selector = `${tagName}${type}:nth-of-type(${index})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
results[fieldName] = {
|
||||||
|
selector,
|
||||||
|
confidence: bestScore,
|
||||||
|
name: bestMatch.name,
|
||||||
|
id: bestMatch.id
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}, FIELD_PATTERNS);
|
||||||
|
|
||||||
|
const detectedCount = Object.keys(detectedFields).length;
|
||||||
|
if (detectedCount === 0) {
|
||||||
|
console.log(' No fields detected');
|
||||||
|
}
|
||||||
|
|
||||||
|
return detectedFields;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
FIELD_PATTERNS,
|
||||||
|
detectFormFields
|
||||||
|
};
|
||||||
@@ -0,0 +1,217 @@
|
|||||||
|
const { generateRandomData, parseLocator } = require('./config');
|
||||||
|
const { detectFormFields } = require('./fields');
|
||||||
|
const { detectSubmitButton } = require('./submit');
|
||||||
|
const { detectCaptcha, solveSimpleCaptcha } = require('./captcha');
|
||||||
|
const { classifyNewsletterForm } = require('./classifier');
|
||||||
|
|
||||||
|
async function processSite(browser, site, globalCreds, browserConfig, keepOpen = false) {
|
||||||
|
let siteFailed = false;
|
||||||
|
let failureReason = '';
|
||||||
|
let page = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
let creds;
|
||||||
|
if (site.credentials && site.credentials.username) {
|
||||||
|
creds = { ...globalCreds, ...site.credentials };
|
||||||
|
} else {
|
||||||
|
creds = { ...globalCreds };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!creds.username || !creds.email || !creds.password) {
|
||||||
|
const randomData = generateRandomData();
|
||||||
|
creds = { ...randomData, ...creds };
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\n${site.url}`);
|
||||||
|
console.log(`${'-'.repeat(70)}`);
|
||||||
|
|
||||||
|
page = await browser.newPage();
|
||||||
|
|
||||||
|
if (browserConfig.userAgent) {
|
||||||
|
await page.setUserAgent(browserConfig.userAgent);
|
||||||
|
}
|
||||||
|
|
||||||
|
await page.goto(site.url, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||||
|
|
||||||
|
const captchaInfo = await detectCaptcha(page);
|
||||||
|
if (captchaInfo.length > 0) {
|
||||||
|
console.log(`\n CAPTCHA DETECTED:`);
|
||||||
|
for (const captcha of captchaInfo) {
|
||||||
|
console.log(` - ${captcha.name}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (captchaInfo.some(c => c.type === 'mathCaptcha')) {
|
||||||
|
const solution = await solveSimpleCaptcha(page);
|
||||||
|
if (solution && solution.answer) {
|
||||||
|
console.log(` Auto-solving math: ${solution.text} = ${solution.answer}`);
|
||||||
|
try {
|
||||||
|
await page.evaluate((answer) => {
|
||||||
|
const input = document.querySelector('input[name*="captcha"], input[class*="captcha"], #captcha');
|
||||||
|
if (input) {
|
||||||
|
input.value = answer;
|
||||||
|
input.dispatchEvent(new Event('input', { bubbles: true }));
|
||||||
|
}
|
||||||
|
}, solution.answer);
|
||||||
|
} catch (e) {
|
||||||
|
console.warn(` Failed to fill math answer: ${e.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (captchaInfo.some(c => c.type === 'imageCaptcha')) {
|
||||||
|
const solution = await solveSimpleCaptcha(page);
|
||||||
|
if (solution && solution.type === 'image' && solution.answer) {
|
||||||
|
console.log(` OCR solved image CAPTCHA: "${solution.answer}"`);
|
||||||
|
try {
|
||||||
|
await page.evaluate((answer) => {
|
||||||
|
const input = document.querySelector('input[name*="captcha"], input[class*="captcha"], input[id*="captcha"], #captcha, .captcha-input');
|
||||||
|
if (input) {
|
||||||
|
input.value = answer;
|
||||||
|
input.dispatchEvent(new Event('input', { bubbles: true }));
|
||||||
|
input.dispatchEvent(new Event('change', { bubbles: true }));
|
||||||
|
}
|
||||||
|
}, solution.answer);
|
||||||
|
} catch (e) {
|
||||||
|
console.warn(` Failed to fill OCR answer: ${e.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (captchaInfo.some(c => c.type === 'recaptcha' || c.type === 'hcaptcha')) {
|
||||||
|
siteFailed = true;
|
||||||
|
failureReason = `CAPTCHA present: ${captchaInfo.map(c => c.name).join(', ')}`;
|
||||||
|
console.error(` Cannot auto-solve ${captchaInfo.map(c => c.name).join(', ')}`);
|
||||||
|
return { siteFailed, failureReason, page };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const formClassification = await classifyNewsletterForm(page);
|
||||||
|
console.log(`\n Form Classification: ${formClassification.isNewsletter ? 'Newsletter' : 'Not newsletter'} (${formClassification.confidence}/100 confidence)`);
|
||||||
|
|
||||||
|
if (formClassification.signals.positive.length > 0) {
|
||||||
|
console.log(' Positive signals:');
|
||||||
|
formClassification.signals.positive.forEach(signal => console.log(` + ${signal}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (formClassification.signals.negative.length > 0) {
|
||||||
|
console.log(' Negative signals:');
|
||||||
|
formClassification.signals.negative.forEach(signal => console.log(` - ${signal}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!formClassification.isNewsletter && formClassification.confidence < 40) {
|
||||||
|
siteFailed = true;
|
||||||
|
failureReason = 'Not a newsletter signup form';
|
||||||
|
console.error(` ${failureReason}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('\nDETECTED FIELDS:');
|
||||||
|
|
||||||
|
let fieldsToFill = {};
|
||||||
|
let detectedFields = {};
|
||||||
|
const definedLocators = site.locators || {};
|
||||||
|
|
||||||
|
if (site.autoDetect !== false) {
|
||||||
|
detectedFields = await detectFormFields(page);
|
||||||
|
for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
|
||||||
|
fieldsToFill[fieldName] = fieldInfo.selector;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const hasEmailField = fieldsToFill.email || fieldsToFill.confirmEmail;
|
||||||
|
if (!hasEmailField) {
|
||||||
|
siteFailed = true;
|
||||||
|
failureReason = 'No email field detected';
|
||||||
|
console.warn(`\n ${failureReason}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const hasSearchInput = await page.evaluate(() => {
|
||||||
|
const inputs = document.querySelectorAll('input[type="search"], input[name*="search"], input[id*="search"]');
|
||||||
|
return inputs.length > 0;
|
||||||
|
}).catch(() => false);
|
||||||
|
|
||||||
|
if (hasSearchInput && !hasEmailField) {
|
||||||
|
console.warn(` Search form detected instead of newsletter`);
|
||||||
|
siteFailed = true;
|
||||||
|
failureReason = failureReason || 'Search form instead of newsletter';
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [fieldName, locator] of Object.entries(definedLocators)) {
|
||||||
|
if (fieldName !== 'submit') {
|
||||||
|
fieldsToFill[fieldName] = parseLocator(locator);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Object.keys(fieldsToFill).length > 0) {
|
||||||
|
console.log(`\n ${'FIELD'.padEnd(12)} | ${'SELECTOR'.padEnd(35)} | ${'CONF'.padEnd(4)} | VALUE`);
|
||||||
|
console.log(' ' + '-'.repeat(75));
|
||||||
|
|
||||||
|
for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
|
||||||
|
let valueToFill = creds[fieldName];
|
||||||
|
|
||||||
|
if (fieldName === 'confirmEmail') {
|
||||||
|
valueToFill = creds['email'];
|
||||||
|
}
|
||||||
|
|
||||||
|
const selector = fieldInfo.selector.length > 34 ? fieldInfo.selector.substring(0, 31) + '...' : fieldInfo.selector;
|
||||||
|
const value = valueToFill ? valueToFill.substring(0, 20) : 'N/A';
|
||||||
|
|
||||||
|
console.log(` ${fieldName.padEnd(12)} | ${selector.padEnd(35)} | ${fieldInfo.confidence.toString().padEnd(4)} | ${value}`);
|
||||||
|
|
||||||
|
if (valueToFill) {
|
||||||
|
try {
|
||||||
|
await page.evaluate((sel, val) => {
|
||||||
|
const el = document.querySelector(sel);
|
||||||
|
if (el) {
|
||||||
|
el.value = val;
|
||||||
|
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||||||
|
el.dispatchEvent(new Event('change', { bubbles: true }));
|
||||||
|
}
|
||||||
|
}, fieldInfo.selector, valueToFill);
|
||||||
|
} catch (e) {
|
||||||
|
console.warn(` Failed to fill ${fieldName}: ${e.message}`);
|
||||||
|
siteFailed = true;
|
||||||
|
failureReason = `Failed to fill ${fieldName}: ${e.message}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log(' ' + '-'.repeat(75));
|
||||||
|
}
|
||||||
|
|
||||||
|
let submitSelector = null;
|
||||||
|
if (definedLocators.submit) {
|
||||||
|
submitSelector = parseLocator(definedLocators.submit);
|
||||||
|
} else if (site.autoDetect !== false) {
|
||||||
|
submitSelector = await detectSubmitButton(page);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (submitSelector) {
|
||||||
|
try {
|
||||||
|
await page.click(submitSelector);
|
||||||
|
console.log(` Submitted (${submitSelector})`);
|
||||||
|
} catch (submitError) {
|
||||||
|
siteFailed = true;
|
||||||
|
failureReason = failureReason || `Submit failed: ${submitError.message}`;
|
||||||
|
console.error(` Submit failed: ${submitError.message}`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
siteFailed = true;
|
||||||
|
failureReason = failureReason || 'No submit button found';
|
||||||
|
console.warn(` ${failureReason}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
siteFailed = true;
|
||||||
|
failureReason = error.message;
|
||||||
|
console.error(` Error: ${error.message}`);
|
||||||
|
} finally {
|
||||||
|
if (page && !keepOpen) {
|
||||||
|
await page.close().catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { siteFailed, failureReason, page };
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
processSite
|
||||||
|
};
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
async function detectSubmitButton(page) {
|
||||||
|
console.log('Auto-detecting submit button...');
|
||||||
|
|
||||||
|
const submitSelector = await page.evaluate(() => {
|
||||||
|
const buttons = Array.from(document.querySelectorAll('button[type="submit"], input[type="submit"], button, input[type="button"]'));
|
||||||
|
let bestMatch = null;
|
||||||
|
let bestScore = 0;
|
||||||
|
|
||||||
|
buttons.forEach(btn => {
|
||||||
|
const text = (btn.textContent || btn.value || '').toLowerCase().trim();
|
||||||
|
const type = btn.type?.toLowerCase() || '';
|
||||||
|
const className = (btn.className || '').toLowerCase();
|
||||||
|
let score = 0;
|
||||||
|
|
||||||
|
if (type === 'submit') score += 20;
|
||||||
|
|
||||||
|
if (text.includes('subscribe')) score += 15;
|
||||||
|
else if (text.includes('sign up')) score += 12;
|
||||||
|
else if (text.includes('submit')) score += 10;
|
||||||
|
else if (text.includes('join')) score += 8;
|
||||||
|
else if (text.includes('register')) score += 8;
|
||||||
|
else if (text.includes('send')) score += 5;
|
||||||
|
|
||||||
|
if (className.includes('submit')) score += 8;
|
||||||
|
if (className.includes('subscribe')) score += 8;
|
||||||
|
|
||||||
|
if (className.includes('close') || text.includes('close')) score -= 20;
|
||||||
|
if (className.includes('cancel') || text.includes('cancel')) score -= 20;
|
||||||
|
if (className.includes('back') || text.includes('back')) score -= 15;
|
||||||
|
if (className.includes('drawer')) score -= 15;
|
||||||
|
|
||||||
|
const rect = btn.getBoundingClientRect();
|
||||||
|
if (rect.width > 0 && rect.height > 0 && rect.top > 0) {
|
||||||
|
score += 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (score > bestScore) {
|
||||||
|
bestScore = score;
|
||||||
|
bestMatch = btn;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (bestMatch && bestScore >= 10) {
|
||||||
|
if (bestMatch.id) return `#${bestMatch.id}`;
|
||||||
|
if (bestMatch.name) return `[name="${bestMatch.name}"]`;
|
||||||
|
|
||||||
|
const className = bestMatch.className;
|
||||||
|
if (className) {
|
||||||
|
const classes = className.split(' ').filter(c => c.length > 0);
|
||||||
|
for (const cls of classes) {
|
||||||
|
const clsLower = cls.toLowerCase();
|
||||||
|
if (clsLower.includes('submit') || clsLower.includes('subscribe')) {
|
||||||
|
return `.${cls}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const cls of classes) {
|
||||||
|
if (!cls.toLowerCase().includes('close') && !cls.toLowerCase().includes('cancel')) {
|
||||||
|
return `.${cls}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const allBtns = Array.from(document.querySelectorAll('button'));
|
||||||
|
const index = allBtns.indexOf(bestMatch) + 1;
|
||||||
|
return `button:nth-of-type(${index})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (submitSelector) {
|
||||||
|
console.log(` Submit button found: ${submitSelector}`);
|
||||||
|
} else {
|
||||||
|
console.log(' No submit button found');
|
||||||
|
}
|
||||||
|
|
||||||
|
return submitSelector;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
detectSubmitButton
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user