better logging and detecting

2026-02-14 20:30:12 +01:00
parent 6e5cfe02fa
commit ae87077519
2 changed files with 172 additions and 38 deletions
@@ -25,3 +25,22 @@ name = "Immaculate Vegan"
 category = "Newsletter"
 url = "https://immaculatevegan.com/blogs/magazine"
 needsPreSubmitAction = false
+
+[[sites]]
+name = "Gate6"
+category = "Newsletter"
+url = "https://gate6.vn/"
+needsPreSubmitAction = false
+
+[[sites]]
+name = "Kulala Land"
+category = "Newsletter"
+url = "https://kulalaland.com/"
+needsPreSubmitAction = false
+
+[[sites]]
+name = "sneakywholefoods"
+category = "Newsletter"
+url = "https://www.sneakywholefoods.com/"
+needsPreSubmitAction = false
+
@@ -326,13 +326,8 @@ async function detectFormFields(page) {
    }, FIELD_PATTERNS);
    
    const detectedCount = Object.keys(detectedFields).length;
-    if (detectedCount > 0) {
-        console.log(`   ✅ Auto-detected ${detectedCount} fields:`);
-        for (const [field, info] of Object.entries(detectedFields)) {
-            console.log(`      - ${field}: ${info.selector} (score: ${info.confidence})`);
-        }
-    } else {
-        console.log('   ⚠️  No fields auto-detected');
+    if (detectedCount === 0) {
+        console.log('   No fields detected');
    }
    
    return detectedFields;
@@ -409,7 +404,9 @@ async function detectSubmitButton(page) {
    });
    
    if (submitSelector) {
-        console.log(`   ✅ Auto-detected submit button: ${submitSelector}`);
+        console.log(`   ✅ Submit button found: ${submitSelector}`);
+    } else {
+        console.log('   ❌ No submit button found');
    }
    
    return submitSelector;
@@ -417,7 +414,26 @@ async function detectSubmitButton(page) {

 async function main() {
    const config = loadConfig();
-    const sitesToProcess = config.sites || [];
+    let sitesToProcess = config.sites || [];
+    
+    const targetSite = process.argv[2];
+    const logToFile = process.argv.includes('--log');
+    const logFile = logToFile ? path.join(__dirname, `run_${new Date().toISOString().replace(/[:.]/g, '-')}.log`) : null;
+    
+    if (targetSite && !targetSite.startsWith('--')) {
+        sitesToProcess = sitesToProcess.filter(site => 
+            site.name.toLowerCase().includes(targetSite.toLowerCase()) ||
+            site.url.toLowerCase().includes(targetSite.toLowerCase())
+        );
+        if (sitesToProcess.length === 0) {
+            console.error(`❌ No site found matching "${targetSite}"`);
+            console.log('Available sites:');
+            config.sites.forEach(site => console.log(`  - ${site.name}: ${site.url}`));
+            return;
+        }
+        console.log(`🎯 Filtering to ${sitesToProcess.length} site(s) matching "${targetSite}"`);
+    }
+    
    const browserConfig = config.browser || {};
    const globalCreds = config.global_credentials || {};
    const userDataDir = browserConfig.userDataDir;
@@ -427,6 +443,36 @@ async function main() {
        return;
    }
    
+    if (logToFile) {
+        const originalLog = console.log;
+        const originalError = console.error;
+        const originalWarn = console.warn;
+        
+        const writeToLog = (type, args) => {
+            const message = args.map(arg => typeof arg === 'string' ? arg : JSON.stringify(arg)).join(' ');
+            fs.appendFileSync(logFile, `[${new Date().toISOString()}] [${type}] ${message}\n`);
+        };
+        
+        console.log = (...args) => {
+            writeToLog('LOG', args);
+            originalLog.apply(console, args);
+        };
+        
+        console.error = (...args) => {
+            writeToLog('ERROR', args);
+            originalError.apply(console, args);
+        };
+        
+        console.warn = (...args) => {
+            writeToLog('WARN', args);
+            originalWarn.apply(console, args);
+        };
+        
+        console.log(`📝 Logging to file: ${logFile}`);
+    }
+    
+    const failedSites = [];
+    
    const browser = await puppeteer.launch({
        headless: false,
        userDataDir: userDataDir,
@@ -436,6 +482,8 @@ async function main() {
    try {
        let isFirstSite = true;
        for (const site of sitesToProcess) {
+            let siteFailed = false;
+            let failureReason = '';
            let creds;
            if (site.credentials && site.credentials.username) {
                creds = { ...globalCreds, ...site.credentials };
@@ -451,7 +499,8 @@ async function main() {
                console.log('Filled in missing credentials with random data.');
            }

-            console.log(`\n--- Processing Site: ${site.name} (Category: ${site.category || 'N/A'}) ---`);
+            console.log(`\n${site.url}`);
+            console.log(`${'-'.repeat(70)}`);
            
            const page = isFirstSite ? (await browser.pages())[0] : await browser.newPage();
            isFirstSite = false;
@@ -462,46 +511,77 @@ async function main() {

            await page.goto(site.url, { waitUntil: 'networkidle2' });
            
-            console.log('Dynamically filling out the form...');
+            console.log('\n🔍 DETECTED FIELDS:');
            
            let fieldsToFill = {};
+            let detectedFields = {};
            const definedLocators = site.locators || {};
            
            if (site.autoDetect !== false) {
-                const detectedFields = await detectFormFields(page);
+                detectedFields = await detectFormFields(page);
                for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
                    fieldsToFill[fieldName] = fieldInfo.selector;
                }
            }
            
+            const hasEmailField = fieldsToFill.email || fieldsToFill.confirmEmail;
+            if (!hasEmailField) {
+                siteFailed = true;
+                failureReason = 'No email field detected';
+                console.warn(`\n   ❌ ${failureReason}`);
+            }
+            
+            const hasSearchInput = await page.evaluate(() => {
+                const inputs = document.querySelectorAll('input[type="search"], input[name*="search"], input[id*="search"]');
+                return inputs.length > 0;
+            });
+            
+            if (hasSearchInput && !hasEmailField) {
+                console.warn(`   ⚠️  WARNING: This appears to be a SEARCH form, not a newsletter signup!`);
+                siteFailed = true;
+                failureReason = failureReason || 'Detected search form instead of newsletter';
+            }
+            
            for (const [fieldName, locator] of Object.entries(definedLocators)) {
                if (fieldName !== 'submit') {
                    fieldsToFill[fieldName] = parseLocator(locator);
                }
            }
            
-            for (const [fieldName, selector] of Object.entries(fieldsToFill)) {
-                let valueToFill = creds[fieldName];
+            if (Object.keys(fieldsToFill).length > 0) {
+                console.log(`\n   ${'FIELD'.padEnd(12)} | ${'SELECTOR'.padEnd(35)} | ${'CONF'.padEnd(4)} | VALUE`);
+                console.log('   ' + '-'.repeat(75));
                
-                if (fieldName === 'confirmEmail') {
-                    valueToFill = creds['email'];
-                }
+                for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
+                    let valueToFill = creds[fieldName];
                    
-                if (selector && valueToFill) {
-                    try {
-                        console.log(`  - Filling field: ${fieldName} with "${valueToFill}"`);
-                        await page.evaluate((sel, val) => {
-                            const el = document.querySelector(sel);
-                            if (el) {
-                                el.value = val;
-                                el.dispatchEvent(new Event('input', { bubbles: true }));
-                                el.dispatchEvent(new Event('change', { bubbles: true }));
-                            }
-                        }, selector, valueToFill);
-                    } catch (e) {
-                        console.warn(`  - Could not fill field '${fieldName}': ${e.message}`);
+                    if (fieldName === 'confirmEmail') {
+                        valueToFill = creds['email'];
+                    }
+                    
+                    const selector = fieldInfo.selector.length > 34 ? fieldInfo.selector.substring(0, 31) + '...' : fieldInfo.selector;
+                    const value = valueToFill ? valueToFill.substring(0, 20) : 'N/A';
+                    
+                    console.log(`   ${fieldName.padEnd(12)} | ${selector.padEnd(35)} | ${fieldInfo.confidence.toString().padEnd(4)} | ${value}`);
+                    
+                    if (valueToFill) {
+                        try {
+                            await page.evaluate((sel, val) => {
+                                const el = document.querySelector(sel);
+                                if (el) {
+                                    el.value = val;
+                                    el.dispatchEvent(new Event('input', { bubbles: true }));
+                                    el.dispatchEvent(new Event('change', { bubbles: true }));
+                                }
+                            }, fieldInfo.selector, valueToFill);
+                        } catch (e) {
+                            console.warn(`   ✗ Failed to fill ${fieldName}: ${e.message}`);
+                            siteFailed = true;
+                            failureReason = `Failed to fill ${fieldName}: ${e.message}`;
+                        }
                    }
                }
+                console.log('   ' + '-'.repeat(75));
            }

            let submitSelector = null;
@@ -512,19 +592,54 @@ async function main() {
            }
            
            if (submitSelector) {
-                console.log('Form filled. Attempting to submit...');
-                await page.click(submitSelector);
-                console.log(`✅ Submission attempt for ${site.name} is complete.`);
+                try {
+                    await page.click(submitSelector);
+                    console.log(`   ✓ Submitted (${submitSelector})`);
+                } catch (submitError) {
+                    siteFailed = true;
+                    failureReason = failureReason || `Submit failed: ${submitError.message}`;
+                    console.error(`   ✗ Submit failed: ${submitError.message}`);
+                }
            } else {
-                console.warn('⚠️  No submit button found.');
+                siteFailed = true;
+                failureReason = failureReason || 'No submit button found';
+                console.warn(`   ✗ ${failureReason}`);
+            }
+            
+            if (siteFailed) {
+                failedSites.push({
+                    name: site.name,
+                    url: site.url,
+                    reason: failureReason,
+                    timestamp: new Date().toISOString()
+                });
            }
            
            await new Promise(r => setTimeout(r, 2000));
        }
    } catch(error) {
-        console.error('❌ A critical error occurred:', error.message);
+        console.error('Error:', error.message);
    } finally {
-        console.log('\n🎉 All tasks complete.');
+        if (failedSites.length > 0) {
+            const reportPath = path.join(__dirname, 'failed_sites.txt');
+            const now = new Date();
+            const dateStr = now.toLocaleDateString('en-GB');
+            const timeStr = now.toLocaleTimeString('en-GB');
+            
+            let reportContent = '';
+            if (!fs.existsSync(reportPath)) {
+                reportContent = 'DATE       | TIME     | WEBSITE                          | REASON\n';
+                reportContent += '-'.repeat(100) + '\n';
+            }
+            
+            for (const site of failedSites) {
+                const website = `${site.name} (${site.url})`.substring(0, 32).padEnd(32);
+                reportContent += `${dateStr} | ${timeStr} | ${website} | ${site.reason}\n`;
+            }
+            
+            fs.appendFileSync(reportPath, reportContent);
+            console.log(`\n${failedSites.length} site(s) failed. See: failed_sites.txt`);
+        }
    }
 }