better logging and detecting

config.toml

@@ -25,3 +25,22 @@ name = "Immaculate Vegan"
 category = "Newsletter"
 url = "https://immaculatevegan.com/blogs/magazine"
 needsPreSubmitAction = false
+
+[[sites]]
+name = "Gate6"
+category = "Newsletter"
+url = "https://gate6.vn/"
+needsPreSubmitAction = false
+
+[[sites]]
+name = "Kulala Land"
+category = "Newsletter"
+url = "https://kulalaland.com/"
+needsPreSubmitAction = false
+
+[[sites]]
+name = "sneakywholefoods"
+category = "Newsletter"
+url = "https://www.sneakywholefoods.com/"
+needsPreSubmitAction = false
+

index.js

@@ -326,13 +326,8 @@ async function detectFormFields(page) {
     }, FIELD_PATTERNS);
     
     const detectedCount = Object.keys(detectedFields).length;
-    if (detectedCount > 0) {
+    if (detectedCount === 0) {
-        console.log(`   ✅ Auto-detected ${detectedCount} fields:`);
+        console.log('   No fields detected');
-        for (const [field, info] of Object.entries(detectedFields)) {
-            console.log(`      - ${field}: ${info.selector} (score: ${info.confidence})`);
-        }
-    } else {
-        console.log('   ⚠️  No fields auto-detected');
     }
     
     return detectedFields;
@@ -409,7 +404,9 @@ async function detectSubmitButton(page) {
     });
     
     if (submitSelector) {
-        console.log(`   ✅ Auto-detected submit button: ${submitSelector}`);
+        console.log(`   ✅ Submit button found: ${submitSelector}`);
+    } else {
+        console.log('   ❌ No submit button found');
     }
     
     return submitSelector;
@@ -417,7 +414,26 @@ async function detectSubmitButton(page) {
 
 async function main() {
     const config = loadConfig();
-    const sitesToProcess = config.sites || [];
+    let sitesToProcess = config.sites || [];
+    
+    const targetSite = process.argv[2];
+    const logToFile = process.argv.includes('--log');
+    const logFile = logToFile ? path.join(__dirname, `run_${new Date().toISOString().replace(/[:.]/g, '-')}.log`) : null;
+    
+    if (targetSite && !targetSite.startsWith('--')) {
+        sitesToProcess = sitesToProcess.filter(site => 
+            site.name.toLowerCase().includes(targetSite.toLowerCase()) ||
+            site.url.toLowerCase().includes(targetSite.toLowerCase())
+        );
+        if (sitesToProcess.length === 0) {
+            console.error(`❌ No site found matching "${targetSite}"`);
+            console.log('Available sites:');
+            config.sites.forEach(site => console.log(`  - ${site.name}: ${site.url}`));
+            return;
+        }
+        console.log(`🎯 Filtering to ${sitesToProcess.length} site(s) matching "${targetSite}"`);
+    }
+    
     const browserConfig = config.browser || {};
     const globalCreds = config.global_credentials || {};
     const userDataDir = browserConfig.userDataDir;
@@ -427,6 +443,36 @@ async function main() {
         return;
     }
     
+    if (logToFile) {
+        const originalLog = console.log;
+        const originalError = console.error;
+        const originalWarn = console.warn;
+        
+        const writeToLog = (type, args) => {
+            const message = args.map(arg => typeof arg === 'string' ? arg : JSON.stringify(arg)).join(' ');
+            fs.appendFileSync(logFile, `[${new Date().toISOString()}] [${type}] ${message}\n`);
+        };
+        
+        console.log = (...args) => {
+            writeToLog('LOG', args);
+            originalLog.apply(console, args);
+        };
+        
+        console.error = (...args) => {
+            writeToLog('ERROR', args);
+            originalError.apply(console, args);
+        };
+        
+        console.warn = (...args) => {
+            writeToLog('WARN', args);
+            originalWarn.apply(console, args);
+        };
+        
+        console.log(`📝 Logging to file: ${logFile}`);
+    }
+    
+    const failedSites = [];
+    
     const browser = await puppeteer.launch({
         headless: false,
         userDataDir: userDataDir,
@@ -436,6 +482,8 @@ async function main() {
     try {
         let isFirstSite = true;
         for (const site of sitesToProcess) {
+            let siteFailed = false;
+            let failureReason = '';
             let creds;
             if (site.credentials && site.credentials.username) {
                 creds = { ...globalCreds, ...site.credentials };
@@ -451,7 +499,8 @@ async function main() {
                 console.log('Filled in missing credentials with random data.');
             }
 
-            console.log(`\n--- Processing Site: ${site.name} (Category: ${site.category || 'N/A'}) ---`);
+            console.log(`\n${site.url}`);
+            console.log(`${'-'.repeat(70)}`);
             
             const page = isFirstSite ? (await browser.pages())[0] : await browser.newPage();
             isFirstSite = false;
@@ -461,47 +510,78 @@ async function main() {
             }
 
             await page.goto(site.url, { waitUntil: 'networkidle2' });
-
+            
-            console.log('Dynamically filling out the form...');
+            console.log('\n🔍 DETECTED FIELDS:');
             
             let fieldsToFill = {};
+            let detectedFields = {};
             const definedLocators = site.locators || {};
             
             if (site.autoDetect !== false) {
-                const detectedFields = await detectFormFields(page);
+                detectedFields = await detectFormFields(page);
                 for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
                     fieldsToFill[fieldName] = fieldInfo.selector;
                 }
             }
             
+            const hasEmailField = fieldsToFill.email || fieldsToFill.confirmEmail;
+            if (!hasEmailField) {
+                siteFailed = true;
+                failureReason = 'No email field detected';
+                console.warn(`\n   ❌ ${failureReason}`);
+            }
+            
+            const hasSearchInput = await page.evaluate(() => {
+                const inputs = document.querySelectorAll('input[type="search"], input[name*="search"], input[id*="search"]');
+                return inputs.length > 0;
+            });
+            
+            if (hasSearchInput && !hasEmailField) {
+                console.warn(`   ⚠️  WARNING: This appears to be a SEARCH form, not a newsletter signup!`);
+                siteFailed = true;
+                failureReason = failureReason || 'Detected search form instead of newsletter';
+            }
+            
             for (const [fieldName, locator] of Object.entries(definedLocators)) {
                 if (fieldName !== 'submit') {
                     fieldsToFill[fieldName] = parseLocator(locator);
                 }
             }
             
-            for (const [fieldName, selector] of Object.entries(fieldsToFill)) {
+            if (Object.keys(fieldsToFill).length > 0) {
-                let valueToFill = creds[fieldName];
+                console.log(`\n   ${'FIELD'.padEnd(12)} | ${'SELECTOR'.padEnd(35)} | ${'CONF'.padEnd(4)} | VALUE`);
+                console.log('   ' + '-'.repeat(75));
                 
-                if (fieldName === 'confirmEmail') {
+                for (const [fieldName, fieldInfo] of Object.entries(detectedFields)) {
-                    valueToFill = creds['email'];
+                    let valueToFill = creds[fieldName];
-                }
+                    
-                
+                    if (fieldName === 'confirmEmail') {
-                if (selector && valueToFill) {
+                        valueToFill = creds['email'];
-                    try {
+                    }
-                        console.log(`  - Filling field: ${fieldName} with "${valueToFill}"`);
+                    
-                        await page.evaluate((sel, val) => {
+                    const selector = fieldInfo.selector.length > 34 ? fieldInfo.selector.substring(0, 31) + '...' : fieldInfo.selector;
-                            const el = document.querySelector(sel);
+                    const value = valueToFill ? valueToFill.substring(0, 20) : 'N/A';
-                            if (el) {
+                    
-                                el.value = val;
+                    console.log(`   ${fieldName.padEnd(12)} | ${selector.padEnd(35)} | ${fieldInfo.confidence.toString().padEnd(4)} | ${value}`);
-                                el.dispatchEvent(new Event('input', { bubbles: true }));
+                    
-                                el.dispatchEvent(new Event('change', { bubbles: true }));
+                    if (valueToFill) {
-                            }
+                        try {
-                        }, selector, valueToFill);
+                            await page.evaluate((sel, val) => {
-                    } catch (e) {
+                                const el = document.querySelector(sel);
-                        console.warn(`  - Could not fill field '${fieldName}': ${e.message}`);
+                                if (el) {
+                                    el.value = val;
+                                    el.dispatchEvent(new Event('input', { bubbles: true }));
+                                    el.dispatchEvent(new Event('change', { bubbles: true }));
+                                }
+                            }, fieldInfo.selector, valueToFill);
+                        } catch (e) {
+                            console.warn(`   ✗ Failed to fill ${fieldName}: ${e.message}`);
+                            siteFailed = true;
+                            failureReason = `Failed to fill ${fieldName}: ${e.message}`;
+                        }
                     }
                 }
+                console.log('   ' + '-'.repeat(75));
             }
 
             let submitSelector = null;
@@ -512,19 +592,54 @@ async function main() {
             }
             
             if (submitSelector) {
-                console.log('Form filled. Attempting to submit...');
+                try {
-                await page.click(submitSelector);
+                    await page.click(submitSelector);
-                console.log(`✅ Submission attempt for ${site.name} is complete.`);
+                    console.log(`   ✓ Submitted (${submitSelector})`);
+                } catch (submitError) {
+                    siteFailed = true;
+                    failureReason = failureReason || `Submit failed: ${submitError.message}`;
+                    console.error(`   ✗ Submit failed: ${submitError.message}`);
+                }
             } else {
-                console.warn('⚠️  No submit button found.');
+                siteFailed = true;
+                failureReason = failureReason || 'No submit button found';
+                console.warn(`   ✗ ${failureReason}`);
+            }
+            
+            if (siteFailed) {
+                failedSites.push({
+                    name: site.name,
+                    url: site.url,
+                    reason: failureReason,
+                    timestamp: new Date().toISOString()
+                });
             }
             
             await new Promise(r => setTimeout(r, 2000));
         }
     } catch(error) {
-        console.error('❌ A critical error occurred:', error.message);
+        console.error('Error:', error.message);
     } finally {
-        console.log('\n🎉 All tasks complete.');
+        if (failedSites.length > 0) {
+            const reportPath = path.join(__dirname, 'failed_sites.txt');
+            const now = new Date();
+            const dateStr = now.toLocaleDateString('en-GB');
+            const timeStr = now.toLocaleTimeString('en-GB');
+            
+            let reportContent = '';
+            if (!fs.existsSync(reportPath)) {
+                reportContent = 'DATE       | TIME     | WEBSITE                          | REASON\n';
+                reportContent += '-'.repeat(100) + '\n';
+            }
+            
+            for (const site of failedSites) {
+                const website = `${site.name} (${site.url})`.substring(0, 32).padEnd(32);
+                reportContent += `${dateStr} | ${timeStr} | ${website} | ${site.reason}\n`;
+            }
+            
+            fs.appendFileSync(reportPath, reportContent);
+            console.log(`\n${failedSites.length} site(s) failed. See: failed_sites.txt`);
+        }
     }
 }