from typing import Dict, Any, Optional
from ...utils.logging import save_debug_artifact


def should_replace_dividend_value(existing_value: Optional[str], new_value: Optional[str]) -> bool:
    """
    Decide whether to replace an existing dividend field value with a new one.

    Rules:
    - Never replace with empty/None values
    - Replace if there is no existing value
    - Replace if the existing value is "Show More" or contains "Show More"
    - Otherwise, keep the existing (good) data
    """
    if not new_value or not str(new_value).strip():
        return False
    if not existing_value:
        return True
    existing_text = str(existing_value)
    if existing_text == 'Show More' or 'Show More' in existing_text:
        return True
    return False


async def extract_dividend_data(page, debug: bool = False) -> Dict[str, Any]:
    """
    Extract dividend information from Schwab stock page.
    Returns dictionary with dividend data fields.
    """
    dividend_data: Dict[str, Any] = {}

    try:
        if debug:
            print("DEBUG: Starting dividend data extraction...")
            # Take initial screenshot to see page state
            png = await page.screenshot(full_page=True)
            path = save_debug_artifact("debug_dividend_start.png", png)
            print(f"DEBUG: Initial screenshot saved as {path}")

        # Wait for the dividends section to load dynamically
        if debug:
            print("DEBUG: Waiting for dividends section to load...")

        try:
            # First wait for the dividends panel to appear
            await page.wait_for_selector('#dividends', timeout=15000)
            if debug:
                print("DEBUG: #dividends panel found")

            # Wait for dividend content to load dynamically
            dividend_loaded = False
            max_attempts = 5  # Reduced from 10 for faster tests
            attempt = 0

            while not dividend_loaded and attempt < max_attempts:
                attempt += 1
                if debug:
                    print(f"DEBUG: Attempt {attempt}/{max_attempts} - Waiting for dynamic dividend content...")

                # Check if the dividends section has been populated with actual content
                dividend_status = await page.evaluate('''
                    () => {
                        const result = { loaded: false, debug: {} };

                        // Look for the dividends panel content that should be populated
                        const dividendsPanel = document.querySelector('#dividends');
                        if (dividendsPanel) {
                            const panelBody = dividendsPanel.querySelector('.sdps-panel__body');
                            if (panelBody) {
                                const textContent = panelBody.textContent || '';
                                result.debug.panelBodyLength = textContent.length;
                                result.debug.panelBodySample = textContent.substring(0, 200);

                                // Check if the panel has been populated with actual dividend text
                                // (not just empty comments)
                                const hasRealContent = textContent.length > 50 && (
                                    textContent.includes('Previous Dividend') ||
                                    textContent.includes('Pay Date') ||
                                    textContent.includes('Ex-Date') ||
                                    textContent.includes('Frequency') ||
                                    textContent.includes('Annual Dividend') ||
                                    textContent.includes('$') ||
                                    textContent.includes('%')
                                );

                                if (hasRealContent) {
                                    result.loaded = true;
                                    return result;
                                }
                            }
                        }

                        // Alternative: check for stock-dividends component
                        const stockDividends = document.querySelector('stock-dividends');
                        if (stockDividends) {
                            const text = stockDividends.textContent || '';
                            result.debug.stockDividendsLength = text.length;
                            result.debug.stockDividendsSample = text.substring(0, 100);

                            if (text.length > 20 && text.includes('$')) {
                                result.loaded = true;
                                return result;
                            }
                        }

                        // Alternative: check for any elements with dividend-related content
                        const allElements = document.querySelectorAll('#dividends *');
                        result.debug.totalElements = allElements.length;

                        for (let elem of allElements) {
                            const text = elem.textContent || '';
                            if (text.includes('Previous Dividend Payment') || 
                                (text.includes('$') && text.includes('.'))) {
                                result.loaded = true;
                                result.debug.foundInElement = elem.tagName + '.' + elem.className;
                                return result;
                            }
                        }

                        return result;
                    }
                ''')

                if debug:
                    print(f"DEBUG: Dividend status: {dividend_status}")

                dividend_loaded = dividend_status.get('loaded', False)

                if dividend_loaded:
                    if debug:
                        print("DEBUG: Dynamic dividend content loaded!")
                        png = await page.screenshot(full_page=True)
                        path = save_debug_artifact("debug_dividend_content_loaded.png", png)
                        print(f"DEBUG: Screenshot after content loaded: {path}")
                    break

                # Wait between attempts to allow for async loading
                await page.wait_for_timeout(1000)  # Reduced from 2000ms for faster tests

            if not dividend_loaded:
                if debug:
                    print("DEBUG: Basic dividend content did not auto-load - this suggests the page is not behaving as expected")
                    print("DEBUG: Expected behavior: Basic dividend info should be visible without clicking 'Show More'")

                    # Try to force a page refresh or trigger loading
                    print("DEBUG: Attempting to trigger dividend content loading...")
                    try:
                        # Try scrolling to the dividend section to trigger lazy loading
                        await page.evaluate('''
                            () => {
                                const dividendsPanel = document.querySelector('#dividends');
                                if (dividendsPanel) {
                                    dividendsPanel.scrollIntoView({ behavior: 'smooth', block: 'center' });
                                }
                            }
                        ''')
                        await page.wait_for_timeout(3000)

                        # Try clicking on the dividends panel header to ensure it's active
                        try:
                            dividends_header = await page.query_selector('#dividends h2, #dividends .sdps-panel__title')
                            if dividends_header:
                                await dividends_header.click()
                                await page.wait_for_timeout(2000)
                                print("DEBUG: Clicked on dividends panel header")
                        except:
                            pass

                        # Check one more time if content loaded
                        final_status = await page.evaluate('''
                            () => {
                                const dividendsPanel = document.querySelector('#dividends');
                                if (dividendsPanel) {
                                    const panelBody = dividendsPanel.querySelector('.sdps-panel__body');
                                    if (panelBody) {
                                        const textContent = panelBody.textContent || '';
                                        return {
                                            length: textContent.length,
                                            sample: textContent.substring(0, 500),
                                            hasBasicData: textContent.includes('$') && (
                                                textContent.includes('Previous') || 
                                                textContent.includes('Pay Date') ||
                                                textContent.includes('Ex-Date')
                                            )
                                        };
                                    }
                                }
                                return { length: 0, sample: '', hasBasicData: false };
                            }
                        ''')

                        if debug:
                            print(f"DEBUG: Final dividend panel status: {final_status}")

                        if final_status.get('hasBasicData'):
                            print("DEBUG: Basic dividend data now detected after manual triggering!")
                            dividend_loaded = True

                            # Extract the data immediately while it's loaded
                            immediate_extraction = await page.evaluate(r'''
                                () => {
                                    const results = {};
                                    const dividendsPanel = document.querySelector('#dividends');

                                    if (dividendsPanel) {
                                        const panelBody = dividendsPanel.querySelector('.sdps-panel__body');
                                        if (panelBody) {
                                            const fullText = panelBody.textContent || '';

                                            // Extract data using pattern matching from the full text
                                            const patterns = {
                                                'Previous Dividend Payment': /Previous Dividend Payment\s*\$([0-9]+\.[0-9]+)/,
                                                'Previous Pay Date': /Previous Pay Date\s*([A-Za-z]+ [0-9]{1,2}, [0-9]{4})/,
                                                'Previous Ex-Date': /Previous Ex-Date\s*([A-Za-z]+ [0-9]{1,2}, [0-9]{4})/,
                                                'Frequency': /Frequency\s*([A-Za-z]+)/,
                                                'Annual Dividend Rate': /(?:Annual Dividend Rate|IAD).*?\$([0-9]+\.[0-9]+)/,
                                                'Annual Dividend Yield': /([0-9]+\.[0-9]+%)(?=\s|Annual|$)/
                                            };

                                            for (const [field, pattern] of Object.entries(patterns)) {
                                                const match = fullText.match(pattern);
                                                if (match) {
                                                    if (field === 'Previous Dividend Payment' || field === 'Annual Dividend Rate') {
                                                        results[field] = '$' + match[1];
                                                    } else {
                                                        results[field] = match[1];
                                                    }
                                                }
                                            }
                                        }
                                    }

                                    return results;
                                }
                            ''')

                            if debug:
                                print(f"DEBUG: Immediate extraction results: {immediate_extraction}")

                            if immediate_extraction:
                                dividend_data.update(immediate_extraction)
                                # Clean up the Frequency field if it has extra text
                                if 'Frequency' in dividend_data and 'Quarterly' in dividend_data['Frequency']:
                                    dividend_data['Frequency'] = 'Quarterly'

                    except Exception as e:
                        if debug:
                            print(f"DEBUG: Error during manual triggering: {e}")

                    png = await page.screenshot(full_page=True)
                    path = save_debug_artifact("debug_dividend_timeout.png", png)
                    print(f"DEBUG: Screenshot after timeout: {path}")

        except Exception as e:
            if debug:
                print(f"DEBUG: Error waiting for dividend content: {e}")

        # Check for dividend grid directly without clicking
        if debug:
            print("DEBUG: Checking for #dividend-grid...")

        dividend_grid_found = False
        try:
            await page.wait_for_selector('#dividend-grid', timeout=10000)
            dividend_grid_found = True
            if debug:
                print("DEBUG: #dividend-grid found!")
                png = await page.screenshot(full_page=True)
                path = save_debug_artifact("debug_dividend_grid_found.png", png)
                print(f"DEBUG: Screenshot with dividend grid: {path}")
        except:
            if debug:
                print("DEBUG: #dividend-grid not found initially")
                png = await page.screenshot(full_page=True)
                path = save_debug_artifact("debug_dividend_no_grid.png", png)
                print(f"DEBUG: Screenshot without grid: {path}")

        # Try to scroll to the dividend section to ensure it's in view
        if debug:
            print("DEBUG: Scrolling to stock-dividends component...")

        try:
            await page.evaluate('''
                () => {
                    const stockDividends = document.querySelector('stock-dividends');
                    if (stockDividends) {
                        stockDividends.scrollIntoView({ behavior: 'smooth', block: 'center' });
                    }
                }
            ''')
            await page.wait_for_timeout(3000)

            if debug:
                png = await page.screenshot(full_page=True)
                path = save_debug_artifact("debug_dividend_after_scroll.png", png)
                print(f"DEBUG: Screenshot after scroll: {path}")

            # Check again for dividend grid after scrolling
            try:
                await page.wait_for_selector('#dividend-grid', timeout=5000)
                dividend_grid_found = True
                if debug:
                    print("DEBUG: #dividend-grid found after scroll!")
                    png = await page.screenshot(full_page=True)
                    path = save_debug_artifact("debug_dividend_grid_after_scroll.png", png)
                    print(f"DEBUG: Screenshot with grid after scroll: {path}")
            except:
                if debug:
                    print("DEBUG: #dividend-grid still not found after scroll")

        except Exception as e:
            if debug:
                print(f"DEBUG: Error during scroll attempt: {e}")

        # Common dividend section selectors used by financial websites
        dividend_selectors = [
            '#dividend-grid',  # Primary target based on user feedback
            'stock-dividends',  # Secondary target - the web component
            '#dividend-section',
            '#dividends-section', 
            '.dividend-summary',
            '.dividends-summary',
            'div[data-testid*="dividend"]',
            'div[aria-label*="dividend"]',
            '[class*="dividend"]',
            'section:has-text("Dividend")',
            'div:has-text("Previous Dividend Payment")'
        ]

        # Try to find dividend section
        dividend_section = None
        for selector in dividend_selectors:
            try:
                if await page.is_visible(selector):
                    dividend_section = selector
                    if debug:
                        print(f"DEBUG: Found dividend section with selector: {selector}")
                    break
            except:
                continue

        if not dividend_section:
            if debug:
                print("DEBUG: No dividend section found, trying broader search...")

                # In debug mode, capture the page content to help identify selectors
                page_content = await page.content()
                path_html = save_debug_artifact("debug_dividend_page.html", page_content)
                print(f"DEBUG: Page HTML saved to {path_html} for analysis")

                # Also save a screenshot to see the visual layout
                png = await page.screenshot(full_page=True)
                path_png = save_debug_artifact("debug_dividend_page.png", png)
                print(f"DEBUG: Page screenshot saved to {path_png}")

            # Fallback: look for dividend-related text anywhere on page
            dividend_text_exists = await page.evaluate('''
                () => {
                    const text = document.body.innerText.toLowerCase();
                    return text.includes('dividend') || text.includes('ex-date') || text.includes('pay date') || text.includes('previous dividend') || text.includes('iad');
                }
            ''')

            if debug:
                print(f"DEBUG: Dividend-related text found on page: {dividend_text_exists}")

                # Try scrolling down to reveal more content
                await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
                await page.wait_for_timeout(2000)

                # Extract all text content that might contain dividend info
                dividend_related_text = await page.evaluate('''
                    () => {
                        const text = document.body.innerText;
                        const lines = text.split('\n');
                        const dividendLines = lines.filter(line => {
                            const lower = line.toLowerCase();
                            return lower.includes('dividend') || lower.includes('ex-date') || 
                                   lower.includes('pay date') || lower.includes('previous') ||
                                   lower.includes('iad') || lower.includes('frequency') ||
                                   lower.includes('quarterly') || lower.includes('$0.26') ||
                                   lower.includes('0.4865%') || lower.includes('$1.04') ||
                                   lower.includes('annual dividend') || lower.includes('yield');
                        });
                        return dividendLines;
                    }
                ''')
                print(f"DEBUG: Found dividend-related text lines: {dividend_related_text}")

                # Try a more comprehensive search for dividend data
                all_dividend_info = await page.evaluate('''
                    () => {
                        // Look for elements containing common dividend field names
                        const fieldNames = [
                            'Previous Dividend Payment', 'Next Dividend Payment',
                            'Previous Pay Date', 'Next Pay Date',
                            'Previous Ex-Date', 'Next Ex-Date', 'Ex-Date',
                            'Frequency', 'Annual Dividend Rate', 'IAD',
                            'Annual Dividend Yield', 'Dividend Yield'
                        ];

                        const results = {};

                        fieldNames.forEach(fieldName => {
                            // Search for elements containing this field name
                            const elements = Array.from(document.querySelectorAll('*')).filter(el => 
                                el.textContent && el.textContent.includes(fieldName) && 
                                el.children.length === 0  // Text nodes only
                            );

                            elements.forEach(el => {
                                // Look for value in nearby elements
                                const parent = el.parentElement;
                                if (parent) {
                                    const siblings = Array.from(parent.children);
                                    const currentIndex = siblings.indexOf(el);

                                    // Check next siblings for values
                                    for (let i = currentIndex + 1; i < siblings.length; i++) {
                                        const sibling = siblings[i];
                                        const text = sibling.textContent.trim();
                                        if (text && text !== fieldName && text.length > 0 && text.length < 50) {
                                            results[fieldName] = text;
                                            break;
                                        }
                                    }

                                    // Check same element for values after the field name
                                    const fullText = el.textContent;
                                    const fieldIndex = fullText.indexOf(fieldName);
                                    if (fieldIndex >= 0) {
                                        const afterField = fullText.substring(fieldIndex + fieldName.length).trim();
                                        if (afterField && afterField.length > 0 && afterField.length < 50) {
                                            results[fieldName] = afterField;
                                        }
                                    }
                                }
                            });
                        });

                        return results;
                    }
                ''')
                print(f"DEBUG: Comprehensive dividend search results: {all_dividend_info}")

                # If we found data in the comprehensive search, use it only if we don't already have good data
                if all_dividend_info:
                    for field, value in all_dividend_info.items():
                        if value and value.strip():
                            existing_value = dividend_data.get(field, '')
                            if should_replace_dividend_value(existing_value, value):
                                dividend_data[field] = value.strip()
                                if debug:
                                    print(f"DEBUG: Added dividend field from comprehensive search: {field} = {value}")
                            elif debug:
                                print(f"DEBUG: Keeping existing good data for {field}: {existing_value} (ignoring comprehensive search value: {value})")

            if not dividend_text_exists:
                if debug:
                    print("DEBUG: No dividend-related content found on page")
                return dividend_data

            # Use body as fallback section for broad search
            dividend_section = 'body'
            if debug:
                print("DEBUG: Using body as dividend section for broad search")

        # If we found the dividend grid, use specific selectors based on user feedback
        if dividend_section == '#dividend-grid':
            if debug:
                print("DEBUG: Using specific dividend grid selectors...")

            try:
                # First check if dividend grid is actually present and populated
                grid_status = await page.evaluate('''
                    () => {
                        const dividendGrid = document.querySelector('#dividend-grid');
                        if (!dividendGrid) return { found: false, message: 'No #dividend-grid element found' };

                        const textContent = dividendGrid.textContent || '';
                        const hasContent = textContent.trim().length > 50;
                        const childCount = dividendGrid.children.length;

                        return {
                            found: true,
                            hasContent,
                            textLength: textContent.length,
                            childCount,
                            preview: textContent.substring(0, 200),
                            message: `Grid found with ${childCount} children, ${textContent.length} chars`
                        };
                    }
                ''')

                if debug:
                    print(f"DEBUG: Dividend grid status: {grid_status}")

                # Extract dividend data using improved selectors
                specific_dividend_data = await page.evaluate(r'''
                    () => {
                        const results = {};

                        // Check if dividend grid exists and has content
                        const dividendGrid = document.querySelector('#dividend-grid');
                        if (dividendGrid) {
                            const allGridText = dividendGrid.textContent || '';
                            const lines = allGridText.split('\n').map(line => line.trim()).filter(line => line.length > 0);

                            // Try structured approach first - look for rows/cells
                            const dividendRows = dividendGrid.querySelectorAll('div[class*="row"], tr, .dividend-row, div:has(div)');
                            dividendRows.forEach((row, rowIndex) => {
                                const rowText = row.textContent || '';

                                // Look for dividend payment info
                                if (rowText.includes('Dividend Payment') || (rowText.includes('Previous') && rowText.includes('$'))) {
                                    const amountMatch = rowText.match(/\$[0-9]+\.[0-9]+/);
                                    if (amountMatch && !results['Previous Dividend Payment']) {
                                        results['Previous Dividend Payment'] = amountMatch[0];
                                    }

                                    // Look for dates in the same row
                                    const dateMatches = rowText.match(/([A-Za-z]+ [0-9]{1,2}, [0-9]{4})/g);
                                    if (dateMatches) {
                                        if (dateMatches.length >= 1 && !results['Previous Pay Date']) results['Previous Pay Date'] = dateMatches[0];
                                        if (dateMatches.length >= 2 && !results['Previous Ex-Date']) results['Previous Ex-Date'] = dateMatches[1];
                                    }
                                }
                            });

                            // Fallback: Parse all lines systematically
                            for (let i = 0; i < lines.length; i++) {
                                const line = lines[i];
                                const nextLine = i + 1 < lines.length ? lines[i + 1] : '';

                                // Match dividend payment
                                if ((line.includes('Previous Dividend Payment') || line.includes('Dividend Payment')) && !results['Previous Dividend Payment']) {
                                    const amountPattern = /\$[0-9]+\.[0-9]+/;
                                    let amount = line.match(amountPattern) || nextLine.match(amountPattern);
                                    if (amount) results['Previous Dividend Payment'] = amount[0];
                                }

                                // Match pay date
                                if (line.includes('Pay Date') && !results['Previous Pay Date']) {
                                    const datePattern = /[A-Za-z]{3,9} [0-9]{1,2}, [0-9]{4}/;
                                    let date = line.match(datePattern) || nextLine.match(datePattern);
                                    if (date) results['Previous Pay Date'] = date[0];
                                }

                                // Match ex-date
                                if (line.includes('Ex-Date') && !results['Previous Ex-Date']) {
                                    const datePattern = /[A-Za-z]{3,9} [0-9]{1,2}, [0-9]{4}/;
                                    let date = line.match(datePattern) || nextLine.match(datePattern);
                                    if (date) results['Previous Ex-Date'] = date[0];
                                }

                                // Match frequency
                                if (line.includes('Frequency') && !results['Frequency']) {
                                    const freqLine = line + ' ' + nextLine;
                                    if (freqLine.toLowerCase().includes('quarterly')) results['Frequency'] = 'Quarterly';
                                    else if (freqLine.toLowerCase().includes('monthly')) results['Frequency'] = 'Monthly';
                                    else if (freqLine.toLowerCase().includes('annual')) results['Frequency'] = 'Annual';
                                    else if (freqLine.toLowerCase().includes('semi')) results['Frequency'] = 'Semi-Annual';
                                }

                                // Match annual dividend rate
                                if ((line.includes('Annual Dividend Rate') || line.includes('IAD')) && !results['Annual Dividend Rate']) {
                                    const amountPattern = /\$[0-9]+\.[0-9]+/;
                                    let amount = line.match(amountPattern) || nextLine.match(amountPattern);
                                    if (amount) results['Annual Dividend Rate'] = amount[0];
                                }

                                // Match annual dividend yield
                                if (line.includes('Annual Dividend Yield') && !results['Annual Dividend Yield']) {
                                    const percentPattern = /[0-9]+\.[0-9]+%/;
                                    let percent = line.match(percentPattern) || nextLine.match(percentPattern);
                                    if (percent) results['Annual Dividend Yield'] = percent[0];
                                }
                            }
                        }

                        return results;
                    }
                ''')

                if debug:
                    print(f"DEBUG: Specific dividend grid extraction results: {specific_dividend_data}")

                # Add the extracted data to dividend_data only if we don't already have good data
                if specific_dividend_data:
                    for field, value in specific_dividend_data.items():
                        existing_value = dividend_data.get(field, '')
                        if should_replace_dividend_value(existing_value, value):
                            dividend_data[field] = value
                            if debug:
                                print(f"DEBUG: Updated {field} from specific extraction: {value}")
                        elif debug:
                            print(f"DEBUG: Keeping existing good data for {field}: {existing_value} (ignoring specific extraction value: {value})")

            except Exception as e:
                if debug:
                    print(f"DEBUG: Error in specific dividend grid extraction: {e}")

        # Extract dividend data using the correct structure from gemini analysis
        if debug:
            print("DEBUG: Extracting dividend data from dividend-grid structure...")

        # First try to extract data from the dynamically loaded dividend content
        try:
            dividend_dynamic_data = await page.evaluate(r'''
                () => {
                    const results = {};

                    // Strategy 1: Look for any dividend grid structure that was loaded
                    const dividendGrid = document.querySelector('#dividend-grid');
                    if (dividendGrid) {
                        const rows = dividendGrid.querySelectorAll('div.sdps-row, .row');

                        for (let row of rows) {
                            const cells = row.querySelectorAll('div[class*="col-"]');
                            if (cells.length >= 2) {
                                const label = cells[0].textContent.trim();
                                const value = cells[1].textContent.trim();

                                // Map the labels to our expected field names
                                if (label.includes('Previous Dividend Payment') || label.includes('Dividend Payment')) {
                                    results['Previous Dividend Payment'] = value;
                                } else if (label.includes('Previous Pay Date') || label.includes('Pay Date')) {
                                    results['Previous Pay Date'] = value;
                                } else if (label.includes('Previous Ex-Date') || label.includes('Ex-Date')) {
                                    results['Previous Ex-Date'] = value;
                                } else if (label.includes('Frequency')) {
                                    results['Frequency'] = value;
                                } else if (label.includes('Annual Dividend Rate') || label.includes('IAD')) {
                                    results['Annual Dividend Rate'] = value;
                                } else if (label.includes('Annual Dividend Yield')) {
                                    results['Annual Dividend Yield'] = value;
                                }
                            }
                        }

                        if (Object.keys(results).length > 0) {
                            return results;
                        }
                    }

                    // Strategy 2: Look for stock-dividends component content
                    const stockDividends = document.querySelector('stock-dividends');
                    if (stockDividends) {
                        const allText = stockDividends.textContent || '';
                        const lines = allText.split('\n').map(line => line.trim()).filter(line => line);

                        for (let i = 0; i < lines.length; i++) {
                            const line = lines[i];
                            const nextLine = i + 1 < lines.length ? lines[i + 1] : '';

                            if (line.includes('Previous Dividend Payment') || line.includes('Dividend Payment')) {
                                const amountMatch = (line + ' ' + nextLine).match(/\$[0-9]+\.[0-9]+/);
                                if (amountMatch) results['Previous Dividend Payment'] = amountMatch[0];
                            } else if (line.includes('Pay Date')) {
                                const dateMatch = (line + ' ' + nextLine).match(/[A-Za-z]+ [0-9]{1,2}, [0-9]{4}/);
                                if (dateMatch) results['Previous Pay Date'] = dateMatch[0];
                            } else if (line.includes('Ex-Date')) {
                                const dateMatch = (line + ' ' + nextLine).match(/[A-Za-z]+ [0-9]{1,2}, [0-9]{4}/);
                                if (dateMatch) results['Previous Ex-Date'] = dateMatch[0];
                            } else if (line.includes('Frequency')) {
                                if (line.toLowerCase().includes('quarterly') || nextLine.toLowerCase().includes('quarterly')) {
                                    results['Frequency'] = 'Quarterly';
                                } else if (line.toLowerCase().includes('monthly') || nextLine.toLowerCase().includes('monthly')) {
                                    results['Frequency'] = 'Monthly';
                                } else if (line.toLowerCase().includes('annual') || nextLine.toLowerCase().includes('annual')) {
                                    results['Frequency'] = 'Annual';
                                }
                            } else if (line.includes('Annual Dividend Rate') || line.includes('IAD')) {
                                const amountMatch = (line + ' ' + nextLine).match(/\$[0-9]+\.[0-9]+/);
                                if (amountMatch) results['Annual Dividend Rate'] = amountMatch[0];
                            } else if (line.includes('Annual Dividend Yield')) {
                                const percentMatch = (line + ' ' + nextLine).match(/[0-9]+\.[0-9]+%/);
                                if (percentMatch) results['Annual Dividend Yield'] = percentMatch[0];
                            }
                        }

                        if (Object.keys(results).length > 0) {
                            return results;
                        }
                    }

                    // Strategy 3: Look within entire dividends panel for any structured content
                    const dividendsPanel = document.querySelector('#dividends');
                    if (dividendsPanel) {
                        const allElements = dividendsPanel.querySelectorAll('*');

                        for (let elem of allElements) {
                            const text = elem.textContent || '';

                            // Look for dollar amounts near dividend-related text
                            if (text.includes('Previous Dividend Payment') || text.includes('Dividend Payment')) {
                                const parent = elem.parentElement;
                                if (parent) {
                                    const siblings = Array.from(parent.children);
                                    const currentIndex = siblings.indexOf(elem);

                                    // Check next siblings for values
                                    for (let j = currentIndex + 1; j < siblings.length; j++) {
                                        const sibling = siblings[j];
                                        const siblingText = sibling.textContent.trim();
                                        const amountMatch = siblingText.match(/\$[0-9]+\.[0-9]+/);
                                        if (amountMatch) {
                                            results['Previous Dividend Payment'] = amountMatch[0];
                                            break;
                                        }
                                    }
                                }
                            }

                            // Similar logic for other fields...
                            // (truncated for brevity but would include Pay Date, Ex-Date, etc.)
                        }
                    }

                    return results;
                }
            ''')

            if debug:
                print(f"DEBUG: Dynamic dividend extraction results: {dividend_dynamic_data}")

            if dividend_dynamic_data:
                for field, value in dividend_dynamic_data.items():
                    existing_value = dividend_data.get(field, '')
                    if should_replace_dividend_value(existing_value, value):
                        dividend_data[field] = value
                        if debug:
                            print(f"DEBUG: Updated {field} from dynamic extraction: {value}")
                    elif debug:
                        print(f"DEBUG: Keeping existing good data for {field}: {existing_value} (ignoring dynamic extraction value: {value})")

        except Exception as e:
            if debug:
                print(f"DEBUG: Error in dynamic dividend extraction: {e}")

        # Define dividend fields and their possible selectors as fallback
        dividend_fields = {
            'Previous Dividend Payment': [
                '#dividend-grid div:has-text("Previous Dividend Payment") ~ div',
                '#dividend-grid div:has-text("Dividend Payment") ~ div',
                '#dividends span:has-text("Previous Dividend Payment") + span',
                '#dividends div:has-text("Previous Dividend Payment") + div',
                '#dividends *:has-text("Previous Dividend Payment") ~ *',
                'stock-dividends span:has-text("Previous Dividend Payment") + span',
                'stock-dividends div:has-text("Previous Dividend Payment") + div',
                'span:has-text("Previous Dividend Payment") + span',
                'div:has-text("Previous Dividend Payment") + div',
                '*:has-text("Previous Dividend Payment") ~ *',
                'span:has-text("Next Dividend Payment") + span',
                'div:has-text("Next Dividend Payment") + div',
                '*:has-text("Next Dividend Payment") ~ *',
                '[data-field="dividend-payment"]',
                '.dividend-payment'
            ],
            'Previous Pay Date': [
                '#dividend-grid div:has-text("Previous Pay Date") ~ div',
                '#dividend-grid div:has-text("Pay Date") ~ div',
                '#dividends span:has-text("Previous Pay Date") + span',
                '#dividends div:has-text("Previous Pay Date") + div',
                '#dividends *:has-text("Previous Pay Date") ~ *',
                'stock-dividends span:has-text("Previous Pay Date") + span',
                'stock-dividends div:has-text("Previous Pay Date") + div',
                'span:has-text("Previous Pay Date") + span',
                'div:has-text("Previous Pay Date") + div', 
                '*:has-text("Previous Pay Date") ~ *',
                'span:has-text("Next Pay Date") + span',
                'div:has-text("Next Pay Date") + div', 
                '*:has-text("Next Pay Date") ~ *',
                '*:has-text("Pay Date") ~ *',
                '[data-field="pay-date"]',
                '.pay-date'
            ],
            'Previous Ex-Date': [
                '#dividend-grid div:has-text("Previous Ex-Date") ~ div',
                '#dividend-grid div:has-text("Ex-Date") ~ div',
                '#dividends span:has-text("Previous Ex-Date") + span',
                '#dividends div:has-text("Previous Ex-Date") + div',
                '#dividends *:has-text("Previous Ex-Date") ~ *',
                'stock-dividends span:has-text("Previous Ex-Date") + span',
                'stock-dividends div:has-text("Previous Ex-Date") + div',
                'span:has-text("Previous Ex-Date") + span',
                'div:has-text("Previous Ex-Date") + div',
                '*:has-text("Previous Ex-Date") ~ *',
                'span:has-text("Next Ex-Date") + span',
                'div:has-text("Next Ex-Date") + div',
                '*:has-text("Next Ex-Date") ~ *',
                '*:has-text("Ex-Date") ~ *',
                '[data-field="ex-date"]',
                '.ex-date'
            ],
            'Frequency': [
                '#dividend-grid div:has-text("Frequency") ~ div',
                '#dividends span:has-text("Frequency") + span',
                '#dividends div:has-text("Frequency") + div',
                '#dividends *:has-text("Frequency") ~ *',
                'stock-dividends span:has-text("Frequency") + span',
                'stock-dividends div:has-text("Frequency") + div',
                'span:has-text("Frequency") + span',
                'div:has-text("Frequency") + div',
                '*:has-text("Frequency") ~ *',
                '[data-field="frequency"]',
                '.dividend-frequency',
                '.frequency'
            ],
            'Annual Dividend Rate': [
                '#dividend-grid div:has-text("Annual Dividend Rate") ~ div',
                '#dividend-grid div:has-text("IAD") ~ div',
                '#dividends span:has-text("Annual Dividend Rate") + span',
                '#dividends div:has-text("Annual Dividend Rate") + div',
                '#dividends *:has-text("Annual Dividend Rate") ~ *',
                '#dividends span:has-text("IAD") + span',
                '#dividends *:has-text("IAD") ~ *',
                'stock-dividends span:has-text("Annual Dividend Rate") + span',
                'stock-dividends div:has-text("Annual Dividend Rate") + div',
                'stock-dividends span:has-text("IAD") + span',
                'span:has-text("Annual Dividend Rate") + span',
                'div:has-text("Annual Dividend Rate") + div',
                '*:has-text("Annual Dividend Rate") ~ *',
                'span:has-text("IAD") + span',
                '*:has-text("IAD") ~ *',
                '[data-field="annual-rate"]',
                '.annual-dividend-rate'
            ],
            'Annual Dividend Yield': [
                '#dividend-grid div:has-text("Annual Dividend Yield") ~ div',
                '#dividends span:has-text("Annual Dividend Yield") + span',
                '#dividends div:has-text("Annual Dividend Yield") + div',
                '#dividends *:has-text("Annual Dividend Yield") ~ *',
                'stock-dividends span:has-text("Annual Dividend Yield") + span',
                'stock-dividends div:has-text("Annual Dividend Yield") + div',
                'span:has-text("Annual Dividend Yield") + span',
                'div:has-text("Annual Dividend Yield") + div',
                '*:has-text("Annual Dividend Yield") ~ *',
                '[data-field="dividend-yield"]',
                '.dividend-yield'
            ]
        }

        # Extract each dividend field using multiple selector strategies
        for field_name, selectors in dividend_fields.items():
            field_found = False

            # Try each selector for this field
            for selector in selectors:
                if field_found:
                    break

                try:
                    # Scope search within dividend section if found, otherwise search whole page
                    full_selector = f'{dividend_section} {selector}' if dividend_section != 'body' else selector

                    if await page.is_visible(full_selector, timeout=1000):
                        value = await page.inner_text(full_selector)
                        clean_value = value.strip()

                        if clean_value and clean_value != field_name:  # Ensure we got actual value, not the label
                            existing_value = dividend_data.get(field_name, '')
                            if should_replace_dividend_value(existing_value, clean_value):
                                dividend_data[field_name] = clean_value
                                field_found = True
                                if debug:
                                    print(f"DEBUG: Found {field_name}: {clean_value} (selector: {full_selector})")
                            elif debug:
                                print(f"DEBUG: Keeping existing good data for {field_name}: {existing_value} (ignoring selector-based value: {clean_value})")
                            break
                except:
                    continue

            # If standard selectors failed, try JavaScript-based text search as fallback
            if not field_found:
                try:
                    # Try multiple variations of the field name
                    search_terms = [field_name]
                    if "Previous" in field_name:
                        search_terms.append(field_name.replace("Previous", "Next"))
                    if "Annual Dividend Rate" in field_name:
                        search_terms.append("IAD")
                    if "Annual Dividend Yield" in field_name:
                        search_terms.append("Dividend Yield")

                    for search_term in search_terms:
                        if field_found:
                            break

                        value = await page.evaluate(rf'''
                            () => {{
                                const searchText = "{search_term}";

                                // First check within the dividends section specifically
                                const dividendsPanel = document.querySelector('#dividends');
                                const stockDividends = document.querySelector('stock-dividends');
                                const searchContainers = [dividendsPanel, stockDividends, document];

                                for (let container of searchContainers) {{
                                    if (!container) continue;

                                    const elements = Array.from(container.querySelectorAll('*'));

                                    for (let elem of elements) {{
                                        if (elem.textContent && elem.textContent.includes(searchText)) {{
                                            // Look for next sibling or nearby element with value
                                            let candidate = elem.nextElementSibling;
                                            if (candidate && candidate.textContent && 
                                                !candidate.textContent.includes(searchText) &&
                                                candidate.textContent.trim().length > 0) {{
                                                return candidate.textContent.trim();
                                            }}

                                            // Try parent's next sibling
                                            candidate = elem.parentElement?.nextElementSibling;
                                            if (candidate && candidate.textContent && 
                                                !candidate.textContent.includes(searchText) &&
                                                candidate.textContent.trim().length > 0) {{
                                                return candidate.textContent.trim();
                                            }}

                                            // Try looking in the same element's parent for nearby text
                                            const parent = elem.parentElement;
                                            if (parent) {{
                                                const parentText = parent.textContent;
                                                const lines = parentText.split('\n');
                                                for (let i = 0; i < lines.length; i++) {{
                                                    if (lines[i].includes(searchText) && i + 1 < lines.length) {{
                                                        const nextLine = lines[i + 1].trim();
                                                        if (nextLine && !nextLine.includes(searchText)) {{
                                                            return nextLine;
                                                        }}
                                                    }}
                                                }}
                                            }}
                                        }}
                                    }}

                                    // If found in this container, stop searching
                                    if (container !== document) {{
                                        break;
                                    }}
                                }}
                                return null;
                            }}
                        ''')

                        if value and value.strip():
                            existing_value = dividend_data.get(field_name, '')
                            if should_replace_dividend_value(existing_value, value):
                                dividend_data[field_name] = value.strip()
                                field_found = True
                                if debug:
                                    print(f"DEBUG: Found {field_name} via JS search with term '{search_term}': {value}")
                            elif debug:
                                print(f"DEBUG: Keeping existing good data for {field_name}: {existing_value} (ignoring JS search value: {value})")
                            break

                except Exception as e:
                    if debug:
                        print(f"DEBUG: Could not find {field_name}: {e}")
                    continue

        if debug:
            print(f"DEBUG: Extracted dividend data: {dividend_data}")

        return dividend_data

    except Exception as e:
        if debug:
            print(f"DEBUG: Error extracting dividend data: {e}")
        return dividend_data


async def extract(page, debug: bool = False) -> Dict[str, Any]:
    """Compatibility wrapper to call `extract_dividend_data`"""
    return await extract_dividend_data(page, debug=debug)