import pandas as pd
import numpy as np
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import json
from datetime import datetime
from django.db import connection
from django.apps import apps
import ast

class AdvancedDjangoDataAnalyzer:
    def __init__(self, model_class, estate_type_field='estate_type'):
        """
        Initialize analyzer with Django model
        
        Args:
            model_class: Django model class (e.g., AdsManual)
            estate_type_field: Field name for grouping (default: 'estate_type')
        """
        self.model = model_class
        self.estate_type_field = estate_type_field
        self.table_name = model_class._meta.db_table
        self.model_name = model_class.__name__
        
        # Get all field information
        self.all_fields = self._get_all_fields()
        self.description_vars = set()
        # Per-record mapping of fields filled by the description scraper
        self.desc_vars_by_pk = {}
        
        print(f"🔍 Analyzing Django Model: {self.model_name}")
        print(f"📊 Table: {self.table_name}")
        print(f"📝 Total Fields: {len(self.all_fields)}")
        
    def _get_all_fields(self):
        """Extract all field names from Django model"""
        fields = []
        for field in self.model._meta.get_fields():
            # Skip reverse relations
            if field.concrete:
                fields.append(field.name)
        return fields
    
    def _extract_description_variables(self, queryset):
        """Extract variables from description_scraped_variables column"""
        print("🔎 Extracting description-based scraped variables...")
        
        description_vars = set()
        
        # Prefer new column 'description_based_scraper' (comma-separated), but support legacy
        fields_to_check = []
        if 'description_based_scraper' in self.all_fields:
            fields_to_check.append('description_based_scraper')
        if 'description_scraped_variables' in self.all_fields:
            fields_to_check.append('description_scraped_variables')

        if fields_to_check:
            for obj in queryset:
                combined_keys = set()
                for col in fields_to_check:
                    var_data = getattr(obj, col, None)
                    if not var_data:
                        continue
                    try:
                        # JSON or dict
                        if isinstance(var_data, str):
                            var_dict = json.loads(var_data)
                        elif isinstance(var_data, dict):
                            var_dict = var_data
                        else:
                            var_dict = None

                        if isinstance(var_dict, dict):
                            combined_keys.update(set(var_dict.keys()))
                            continue
                    except (json.JSONDecodeError, AttributeError):
                        # not JSON; try literal_eval or fallback comma-split
                        pass

                    try:
                        var_dict = ast.literal_eval(str(var_data))
                        if isinstance(var_dict, dict):
                            combined_keys.update(set(var_dict.keys()))
                        elif isinstance(var_dict, (list, set, tuple)):
                            combined_keys.update({str(x).strip() for x in var_dict if str(x).strip()})
                        else:
                            # string fallback below
                            raise ValueError
                    except Exception:
                        if isinstance(var_data, str):
                            parts = [p.strip() for p in var_data.split(',') if p.strip()]
                            combined_keys.update(parts)

                if combined_keys:
                    description_vars.update(combined_keys)
                    self.desc_vars_by_pk[obj.pk] = combined_keys
        
        self.description_vars = description_vars
        print(f"✅ Found {len(description_vars)} description-based variables: {description_vars}")
        return description_vars
    
    def analyze_data_quality(self):
        """Main analysis method - replaces SQL null analysis"""
        print("\n" + "="*80)
        print("🚀 Starting Comprehensive Data Quality Analysis")
        print("="*80 + "\n")
        
        # Get all data
        queryset = self.model.objects.all()
        total_records = queryset.count()
        
        print(f"📊 Total Records: {total_records:,}")
        
        if total_records == 0:
            print("❌ No records found in database!")
            return None
        
        # Extract description variables first
        self._extract_description_variables(queryset)
        
        # Group by estate type
        estate_types = queryset.values_list(self.estate_type_field, flat=True).distinct()
        
        print(f"🏠 Estate Types Found: {len(list(estate_types))}")
        
        # Analyze each estate type
        analysis_results = []
        
        for estate_type in estate_types:
            if estate_type is None:
                estate_type = 'unknown'
            
            print(f"\n📋 Analyzing Estate Type: {estate_type}")
            
            # Get records for this estate type
            estate_qs = queryset.filter(**{self.estate_type_field: estate_type})
            estate_count = estate_qs.count()
            
            print(f"   Records: {estate_count:,}")
            
            # Preload PKs for description contribution counting
            estate_pks = list(estate_qs.values_list('pk', flat=True))

            # Analyze each field
            field_analysis = {}
            
            for field_name in self.all_fields:
                # Skip the grouping field itself
                if field_name == self.estate_type_field:
                    continue
                
                # Count nulls/empty values
                null_count = estate_qs.filter(**{f"{field_name}__isnull": True}).count()

                # Only check empty string for string-like fields to avoid type errors on numeric/relations
                try:
                    field_obj = self.model._meta.get_field(field_name)
                    field_type = getattr(field_obj, 'get_internal_type', lambda: str(type(field_obj)))()
                    string_like_types = {"CharField", "TextField", "EmailField", "URLField", "SlugField"}
                    if field_type in string_like_types:
                        empty_count = estate_qs.filter(**{f"{field_name}": ''}).count()
                    else:
                        empty_count = 0
                except Exception:
                    # If field lookup fails for any reason, be conservative
                    empty_count = 0
                
                # Total missing
                total_missing = null_count + empty_count
                missing_pct = (total_missing / estate_count * 100) if estate_count > 0 else 0
                present_count = estate_count - total_missing
                
                # Check if it's a description variable
                is_description_var = field_name in self.description_vars
                # Count how many values for this field were filled by the description scraper
                desc_filled = sum(1 for pk in estate_pks if field_name in self.desc_vars_by_pk.get(pk, set()))
                manual_present = max(present_count - desc_filled, 0)
                
                field_analysis[field_name] = {
                    'missing': total_missing,
                    'missing_pct': missing_pct,
                    'null_count': null_count,
                    'empty_count': empty_count,
                    'present': present_count,
                    'desc_filled': desc_filled,
                    'manual_present': manual_present,
                    'total': estate_count,
                    'is_description_var': is_description_var
                }
            
            analysis_results.append({
                'estate_type': estate_type,
                'total_listings': estate_count,
                'fields': field_analysis
            })
        
        self.analysis_results = analysis_results
        return analysis_results
    
    def create_advanced_visualizations(self):
        """Create comprehensive visualizations with description variable highlighting"""
        
        if not hasattr(self, 'analysis_results') or not self.analysis_results:
            print("❌ No analysis results available. Run analyze_data_quality() first.")
            return None
        
        # Aggregate statistics
        all_fields = set()
        for estate in self.analysis_results:
            all_fields.update(estate['fields'].keys())
        
        # Categorize fields
        regular_fields = [f for f in all_fields if f not in self.description_vars]
        desc_fields = [f for f in all_fields if f in self.description_vars]
        
        print(f"\n📊 Field Categories:")
        print(f"   Regular Fields: {len(regular_fields)}")
        print(f"   Description-based Fields: {len(desc_fields)}")
        
        # Calculate field statistics
        field_stats = self._calculate_field_statistics(all_fields)
        estate_performance = self._calculate_estate_performance()
        
        # Create visualizations
        fig = make_subplots(
            rows=4, cols=2,
            subplot_titles=[
                "📊 Overall Data Quality by Field Type",
                "🏠 Estate Type Performance Comparison",
                "🔥 Top 10 Critical Fields (Missing Data)",
                "⭐ Top 10 Complete Fields (Best Quality)",
                "📈 Fill Rate Distribution (All Fields)",
                "🎯 Priority Actions by Severity",
                "🔍 Description Variables vs Regular Fields",
                "📉 Missing Data by Estate Type"
            ],
            specs=[
                [{"type": "pie"}, {"type": "bar"}],
                [{"type": "bar"}, {"type": "bar"}],
                [{"type": "histogram"}, {"type": "bar"}],
                [{"type": "bar"}, {"type": "bar"}]
            ],
            vertical_spacing=0.10,
            horizontal_spacing=0.12
        )
        
        # 1. Data Quality by Field Type (with description var highlighting)
        quality_by_type = self._get_quality_by_field_type(field_stats)
        
        fig.add_trace(
            go.Pie(
                labels=list(quality_by_type.keys()),
                values=list(quality_by_type.values()),
                marker_colors=['#e74c3c', '#e67e22', '#f39c12', '#27ae60', '#2ecc71'],
                textinfo='label+percent',
                hovertemplate="<b>%{label}</b><br>Fields: %{value}<br>Percentage: %{percent}<extra></extra>"
            ),
            row=1, col=1
        )
        
        # 2. Estate Performance
        estate_names = list(estate_performance.keys())
        estate_scores = [estate_performance[name]['avg_fill_rate'] for name in estate_names]
        
        fig.add_trace(
            go.Bar(
                x=estate_names,
                y=estate_scores,
                marker_color=['#27ae60' if s >= 70 else '#f39c12' if s >= 50 else '#e74c3c' for s in estate_scores],
                text=[f"{score:.1f}%<br>{estate_performance[name]['total_listings']:,} records" 
                      for name, score in zip(estate_names, estate_scores)],
                textposition='outside',
                hovertemplate="<b>%{x}</b><br>Fill Rate: %{y:.1f}%<extra></extra>"
            ),
            row=1, col=2
        )
        
        # 3. Most Critical Fields
        critical_fields = sorted(
            [(f, s['avg_missing_pct'], s['is_description_var']) for f, s in field_stats.items()],
            key=lambda x: x[1],
            reverse=True
        )[:10]
        
        if critical_fields:
            crit_names, crit_pcts, is_desc = zip(*critical_fields)
            colors = ['#ff6b6b' if desc else '#e74c3c' for desc in is_desc]
            
            fig.add_trace(
                go.Bar(
                    x=list(crit_names),
                    y=list(crit_pcts),
                    marker_color=colors,
                    text=[f"{pct:.1f}%<br>{'📝 Desc' if desc else '📊 Regular'}" 
                          for pct, desc in zip(crit_pcts, is_desc)],
                    textposition='outside',
                    hovertemplate="<b>%{x}</b><br>Missing: %{y:.1f}%<extra></extra>"
                ),
                row=2, col=1
            )
        
        # 4. Best Performing Fields
        best_fields = sorted(
            [(f, s['avg_fill_rate'], s['is_description_var']) for f, s in field_stats.items()],
            key=lambda x: x[1],
            reverse=True
        )[:10]
        
        if best_fields:
            best_names, best_rates, is_desc = zip(*best_fields)
            colors = ['#66ff66' if desc else '#27ae60' for desc in is_desc]
            
            fig.add_trace(
                go.Bar(
                    x=list(best_names),
                    y=list(best_rates),
                    marker_color=colors,
                    text=[f"{rate:.1f}%<br>{'📝 Desc' if desc else '📊 Regular'}" 
                          for rate, desc in zip(best_rates, is_desc)],
                    textposition='outside',
                    hovertemplate="<b>%{x}</b><br>Fill Rate: %{y:.1f}%<extra></extra>"
                ),
                row=2, col=2
            )
        
        # 5. Fill Rate Distribution
        all_fill_rates = [s['avg_fill_rate'] for s in field_stats.values()]
        
        fig.add_trace(
            go.Histogram(
                x=all_fill_rates,
                nbinsx=20,
                marker_color='#3498db',
                opacity=0.75,
                hovertemplate="Fill Rate: %{x:.1f}%<br>Fields: %{y}<extra></extra>"
            ),
            row=3, col=1
        )
        
        # 6. Priority Actions
        priority_counts = self._get_priority_counts(field_stats)
        
        fig.add_trace(
            go.Bar(
                x=list(priority_counts.keys()),
                y=list(priority_counts.values()),
                marker_color=['#e74c3c', '#e67e22', '#f39c12', '#f1c40f', '#27ae60'],
                text=list(priority_counts.values()),
                textposition='outside'
            ),
            row=3, col=2
        )
        
    # 7. Description vs Regular Fields Comparison
        desc_vs_regular = self._compare_field_types(field_stats)
        
        fig.add_trace(
            go.Bar(
                x=['Regular Fields', 'Description Variables'],
                y=[desc_vs_regular['regular_avg'], desc_vs_regular['desc_avg']],
                marker_color=['#3498db', '#9b59b6'],
                text=[f"{desc_vs_regular['regular_avg']:.1f}%<br>{desc_vs_regular['regular_count']} fields",
                      f"{desc_vs_regular['desc_avg']:.1f}%<br>{desc_vs_regular['desc_count']} fields"],
                textposition='outside',
                hovertemplate="<b>%{x}</b><br>Avg Fill Rate: %{y:.1f}%<extra></extra>"
            ),
            row=4, col=1
        )
        
    # 8. Missing Data by Estate Type
        estate_missing = self._get_estate_missing_data()
        
        fig.add_trace(
            go.Bar(
                x=list(estate_missing.keys()),
                y=list(estate_missing.values()),
                marker_color='#e74c3c',
                text=[f"{val:,.0f}" for val in estate_missing.values()],
                textposition='outside',
                hovertemplate="<b>%{x}</b><br>Missing Records: %{y:,.0f}<extra></extra>"
            ),
            row=4, col=2
        )
        
        # Update layout
        fig.update_layout(
            height=2000,
            showlegend=False,
            title={
                'text': f"🏠 Advanced Data Quality Analysis Dashboard<br><sub>Model: {self.model_name} | Table: {self.table_name}<br>{len(regular_fields)} Regular Fields + {len(desc_fields)} Description Variables</sub>",
                'x': 0.5,
                'font': {'size': 26}
            },
            template="plotly_white"
        )
        
        # Rotate x-axis labels
        fig.update_xaxes(tickangle=-45)
        
        self.fig = fig
        return fig
    
    def _calculate_field_statistics(self, all_fields):
        """Calculate comprehensive statistics for each field"""
        field_stats = {}
        
        for field_name in all_fields:
            total_missing = 0
            total_present = 0
            total_records = 0
            missing_pcts = []
            is_desc_var = field_name in self.description_vars
            desc_filled_total = 0
            manual_present_total = 0
            
            for estate in self.analysis_results:
                if field_name in estate['fields']:
                    field_data = estate['fields'][field_name]
                    total_missing += field_data['missing']
                    total_present += field_data['present']
                    total_records += field_data['total']
                    missing_pcts.append(field_data['missing_pct'])
                    desc_filled_total += field_data.get('desc_filled', 0)
                    manual_present_total += field_data.get('manual_present', 0)
            
            avg_missing_pct = np.mean(missing_pcts) if missing_pcts else 100
            avg_fill_rate = 100 - avg_missing_pct
            
            field_stats[field_name] = {
                'avg_fill_rate': avg_fill_rate,
                'avg_missing_pct': avg_missing_pct,
                'total_present': total_present,
                'total_missing': total_missing,
                'total_records': total_records,
                'status': self._get_status(avg_missing_pct),
                'priority': self._get_priority(avg_missing_pct),
                'is_description_var': is_desc_var,
                'desc_filled_total': desc_filled_total,
                'manual_present_total': manual_present_total,
                'desc_share_pct': (desc_filled_total / total_present * 100) if total_present else 0
            }
        
        return field_stats
    
    def _calculate_estate_performance(self):
        """Calculate performance metrics for each estate type"""
        estate_performance = {}
        
        for estate in self.analysis_results:
            estate_type = estate['estate_type']
            fields = estate['fields']
            
            if not fields:
                continue
            
            fill_rates = [100 - f['missing_pct'] for f in fields.values()]
            avg_fill_rate = np.mean(fill_rates)
            
            critical_failures = sum(1 for f in fields.values() if f['missing_pct'] > 90)
            good_fields = sum(1 for f in fields.values() if f['missing_pct'] < 20)
            
            estate_performance[estate_type] = {
                'avg_fill_rate': avg_fill_rate,
                'total_fields': len(fields),
                'critical_failures': critical_failures,
                'good_fields': good_fields,
                'total_listings': estate['total_listings'],
                'grade': self._get_grade(avg_fill_rate, critical_failures)
            }
        
        return estate_performance
    
    def _get_quality_by_field_type(self, field_stats):
        """Categorize fields by quality status"""
        quality_counts = {'Critical': 0, 'Poor': 0, 'Fair': 0, 'Good': 0, 'Excellent': 0}
        
        for stats in field_stats.values():
            quality_counts[stats['status']] += 1
        
        return quality_counts
    
    def _get_priority_counts(self, field_stats):
        """Count fields by priority level"""
        priority_counts = {
            'Fix Immediately': 0,
            'High Priority': 0,
            'Medium Priority': 0,
            'Low Priority': 0,
            'Monitor Only': 0
        }
        
        for stats in field_stats.values():
            priority = stats['priority'].split(' ', 1)[1]
            priority_counts[priority] += 1
        
        return priority_counts
    
    def _compare_field_types(self, field_stats):
        """Compare regular fields vs description variables"""
        regular_rates = []
        desc_rates = []
        
        for field, stats in field_stats.items():
            if stats['is_description_var']:
                desc_rates.append(stats['avg_fill_rate'])
            else:
                regular_rates.append(stats['avg_fill_rate'])
        
        return {
            'regular_avg': np.mean(regular_rates) if regular_rates else 0,
            'regular_count': len(regular_rates),
            'desc_avg': np.mean(desc_rates) if desc_rates else 0,
            'desc_count': len(desc_rates)
        }
    
    def _get_estate_missing_data(self):
        """Calculate total missing data points per estate type"""
        estate_missing = {}
        
        for estate in self.analysis_results:
            total_missing = sum(f['missing'] for f in estate['fields'].values())
            estate_missing[estate['estate_type']] = total_missing
        
        return estate_missing
    
    def _get_status(self, missing_pct):
        """Determine quality status"""
        if missing_pct > 90:
            return "Critical"
        elif missing_pct > 70:
            return "Poor"
        elif missing_pct > 50:
            return "Fair"
        elif missing_pct > 20:
            return "Good"
        else:
            return "Excellent"
    
    def _get_priority(self, missing_pct):
        """Determine priority level"""
        if missing_pct > 90:
            return "🔴 Fix Immediately"
        elif missing_pct > 70:
            return "🟡 High Priority"
        elif missing_pct > 50:
            return "🟠 Medium Priority"
        elif missing_pct > 20:
            return "🟢 Low Priority"
        else:
            return "✅ Monitor Only"
    
    def _get_grade(self, fill_rate, critical_failures):
        """Calculate letter grade"""
        if fill_rate >= 90 and critical_failures == 0:
            return 'A+'
        elif fill_rate >= 85 and critical_failures <= 1:
            return 'A'
        elif fill_rate >= 75:
            return 'B'
        elif fill_rate >= 60:
            return 'C'
        elif fill_rate >= 50:
            return 'D'
        else:
            return 'F'
    
    def generate_html_report(self, filename='advanced_data_quality_report.html'):
        """Generate comprehensive HTML report"""
        
        if not hasattr(self, 'fig'):
            print("⚠️ Creating visualizations first...")
            self.create_advanced_visualizations()
        
        field_stats = self._calculate_field_statistics(
            set(field for estate in self.analysis_results for field in estate['fields'].keys())
        )
        estate_performance = self._calculate_estate_performance()
        
        # Separate regular and description fields (still used for some stats)
        regular_fields = {k: v for k, v in field_stats.items() if not v['is_description_var']}
        desc_fields = {k: v for k, v in field_stats.items() if v['is_description_var']}
        total_desc_values = sum(v['desc_filled_total'] for v in field_stats.values())
        total_present_values = sum(v['total_present'] for v in field_stats.values())
        desc_overall_share = (total_desc_values / total_present_values * 100) if total_present_values else 0

        # Build raw records dataset for a pgAdmin-like view (SELECT * FROM table)
        record_columns = list(self.all_fields)
        # Ensure a stable order with id first if present
        if 'id' in record_columns:
            record_columns = ['id'] + [c for c in record_columns if c != 'id']
        records_qs = self.model.objects.all().values(*record_columns)
        records_data = []
        for row in records_qs:
            rec = {}
            for col in record_columns:
                val = row.get(col)
                # Convert all values to strings for safe HTML embedding; None -> empty string
                try:
                    rec[col] = "" if val is None else str(val)
                except Exception:
                    rec[col] = "" if val is None else json.dumps(val, ensure_ascii=False, default=str)
            records_data.append(rec)

        # Prepare explorer data
        explorer_data = [
            {
                'field_name': k,
                'is_description_var': v['is_description_var'],
                'fill_rate': round(v['avg_fill_rate'], 2),
                'missing_percent': round(v['avg_missing_pct'], 2),
                'present': v['total_present'],
                'missing': v['total_missing'],
                'total': v['total_records'],
                'desc_filled': v['desc_filled_total'],
                'manual_present': v['manual_present_total'],
                'desc_share_pct': round(v['desc_share_pct'], 2),
                'status': v['status'],
                'priority': v['priority']
            }
            for k, v in field_stats.items()
        ]
        
        html_content = f"""
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Advanced Data Quality Report - {self.model_name}</title>
    <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
    <style>
        * {{
            margin: 0;
            padding: 0;
            box-sizing: border-box;
        }}
        
        body {{
            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            min-height: 100vh;
            padding: 20px;
        }}
        
        .container {{
            max-width: 1800px;
            margin: 0 auto;
            background: white;
            border-radius: 20px;
            box-shadow: 0 25px 80px rgba(0,0,0,0.2);
            overflow: hidden;
        }}
        
        .header {{
            background: linear-gradient(135deg, #2c3e50 0%, #34495e 100%);
            color: white;
            padding: 50px;
            text-align: center;
        }}
        
        .header h1 {{
            font-size: 3.5em;
            font-weight: 300;
            margin-bottom: 15px;
        }}
        
        .header .subtitle {{
            font-size: 1.4em;
            opacity: 0.9;
            margin: 10px 0;
        }}
        
        .header .meta {{
            font-size: 1.1em;
            opacity: 0.8;
            margin-top: 15px;
        }}
        
        .content {{
            padding: 50px;
        }}
        
        .dashboard-viz {{
            margin: 40px 0;
            background: #f8f9fa;
            padding: 30px;
            border-radius: 15px;
            box-shadow: 0 10px 30px rgba(0,0,0,0.1);
        }}
        
        .section {{
            margin: 50px 0;
        }}
        
        .section-title {{
            font-size: 2.5em;
            color: #2c3e50;
            margin-bottom: 30px;
            padding-bottom: 15px;
            border-bottom: 4px solid #3498db;
        }}
        
        .field-category {{
            margin: 40px 0;
        }}
        
        .category-header {{
            background: linear-gradient(135deg, #9b59b6 0%, #8e44ad 100%);
            color: white;
            padding: 20px 30px;
            border-radius: 10px;
            font-size: 1.8em;
            margin-bottom: 20px;
            display: flex;
            align-items: center;
            gap: 15px;
        }}
        
        .category-header.description {{
            background: linear-gradient(135deg, #e67e22 0%, #d35400 100%);
        }}
        
        .table-container {{
            overflow-x: auto;
            border-radius: 10px;
            box-shadow: 0 5px 15px rgba(0,0,0,0.1);
        }}
        
        table {{
            width: 100%;
            border-collapse: collapse;
            background: white;
        }}
        
        th {{
            background: linear-gradient(135deg, #34495e 0%, #2c3e50 100%);
            color: white;
            padding: 20px 15px;
            text-align: left;
            font-weight: 600;
            font-size: 1.1em;
            position: sticky;
            top: 0;
            z-index: 10;
        }}
        
        td {{
            padding: 18px 15px;
            border-bottom: 1px solid #ecf0f1;
        }}
        
        tr:hover {{
            background: #f8f9fa;
        }}
        
        .status-badge {{
            display: inline-block;
            padding: 8px 16px;
            border-radius: 20px;
            font-weight: 600;
            color: white;
            text-transform: uppercase;
            font-size: 0.9em;
        }}
        
        .status-critical {{ background: linear-gradient(135deg, #e74c3c 0%, #c0392b 100%); }}
        .status-poor {{ background: linear-gradient(135deg, #e67e22 0%, #d35400 100%); }}
        .status-fair {{ background: linear-gradient(135deg, #f39c12 0%, #e67e22 100%); }}
        .status-good {{ background: linear-gradient(135deg, #27ae60 0%, #229954 100%); }}
        .status-excellent {{ background: linear-gradient(135deg, #2ecc71 0%, #27ae60 100%); }}
        
        .description-badge {{
            display: inline-block;
            padding: 5px 12px;
            background: linear-gradient(135deg, #9b59b6 0%, #8e44ad 100%);
            color: white;
            border-radius: 15px;
            font-size: 0.85em;
            font-weight: 600;
            margin-left: 10px;
        }}

        /* Explorer Styles */
        .explorer-controls {{ display: flex; gap: 15px; align-items: center; margin-bottom: 15px; }}
        .explorer-controls input {{ padding: 8px 12px; border: 1px solid #ddd; border-radius: 6px; }}
        .pagination {{ display: flex; gap: 8px; margin-top: 12px; flex-wrap: wrap; }}
        .pagination button {{ padding: 6px 12px; border: none; background: #ecf0f1; border-radius: 6px; cursor: pointer; }}
        .pagination button.active {{ background: #3498db; color: white; }}
        .tag {{ display: inline-block; padding: 3px 8px; border-radius: 10px; font-size: 0.85em; margin-left: 8px; }}
        .tag.desc {{ background: #9b59b6; color: white; }}
        .tag.regular {{ background: #95a5a6; color: white; }}
        
        /* Records table clamping and controls */
    #recordsTable td {{ position: relative; vertical-align: top; }}
        #recordsTable th {{ position: sticky; top: 0; z-index: 5; }}
        .cell-content {{ white-space: normal; }}
        .cell-content.clamped {{
            display: -webkit-box;
            -webkit-line-clamp: 2; /* clamp to 2 lines */
            -webkit-box-orient: vertical;
            overflow: hidden;
        }}
        .toggle-btn {{
            display: inline-block;
            margin-top: 6px;
            padding: 3px 8px;
            font-size: 0.8em;
            color: #2c3e50;
            background: #ecf0f1;
            border: 1px solid #d0d7de;
            border-radius: 6px;
            cursor: pointer;
        }}
        .toggle-btn:hover {{ background: #e1e8ed; }}
        .desc-counter {{
            position: absolute;
            top: 6px;
            right: 8px;
            font-size: 0.75em;
            color: rgba(0,0,0,0.45);
        }}

        /* High-tech styling for Records section */
        .records-card {{
            background: rgba(255,255,255,0.75);
            backdrop-filter: blur(8px);
            border-radius: 16px;
            border: 1px solid rgba(255,255,255,0.4);
            box-shadow: 0 12px 40px rgba(0,0,0,0.12);
            padding: 20px;
        }}
        #recordsTable {{
            border-collapse: separate;
            border-spacing: 0;
        }}
        #recordsTable thead th {{
            background: linear-gradient(135deg, #0f2027 0%, #203a43 50%, #2c5364 100%);
            color: #ecf0f1;
            letter-spacing: 0.3px;
            user-select: none;
        }}
        #recordsTable tbody tr:nth-child(even) {{ background: #fafbfc; }}
        #recordsTable tbody tr:hover {{ background: #eef6ff; box-shadow: inset 0 0 0 9999px rgba(52, 152, 219, 0.06); }}
        .th-sortable {{ cursor: pointer; position: relative; }}
        .th-sortable::after {{ content: '\25B4'; position: absolute; right: 8px; opacity: 0.35; transform: translateY(-2px); }}
        .th-sortable.desc::after {{ content: '\25BE'; }}
        .col-wide-xxl {{ min-width: 640px; }}
        .col-wide-xl {{ min-width: 420px; }}
        .col-wide-lg {{ min-width: 320px; }}
        .col-wide-md {{ min-width: 240px; }}
        .col-wide-sm {{ min-width: 180px; }}
        .chip {{ display: inline-block; padding: 6px 10px; background: #ecf0f1; border-radius: 999px; font-size: 0.85em; color: #2c3e50; }}
        .toolbar {{ display: flex; gap: 12px; align-items: center; flex-wrap: wrap; }}
        .toolbar .spacer {{ flex: 1; }}
        .density-compact #recordsTable td {{ padding: 8px 10px; }}
        .density-comfortable #recordsTable td {{ padding: 14px 16px; }}
        .density-cozy #recordsTable td {{ padding: 18px 18px; }}
        
        .stats-grid {{
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
            gap: 25px;
            margin: 30px 0;
        }}
        
        .stat-card {{
            background: linear-gradient(135deg, #f8f9fa 0%, #e9ecef 100%);
            padding: 30px;
            border-radius: 15px;
            border-left: 6px solid;
            transition: transform 0.3s ease, box-shadow 0.3s ease;
        }}
        
        .stat-card:hover {{
            transform: translateY(-5px);
            box-shadow: 0 15px 40px rgba(0,0,0,0.15);
        }}
        
        .stat-card.primary {{ border-color: #3498db; }}
        .stat-card.success {{ border-color: #27ae60; }}
        .stat-card.danger {{ border-color: #e74c3c; }}
        .stat-card.warning {{ border-color: #f39c12; }}
        .stat-card.info {{ border-color: #9b59b6; }}
        
        .stat-number {{
            font-size: 3.5em;
            font-weight: bold;
            background: linear-gradient(45deg, #667eea, #764ba2);
            -webkit-background-clip: text;
            -webkit-text-fill-color: transparent;
            margin-bottom: 10px;
        }}
        
        .stat-label {{
            font-size: 1.2em;
            color: #2c3e50;
            font-weight: 500;
        }}
        
        .stat-description {{
            font-size: 0.95em;
            color: #7f8c8d;
            margin-top: 8px;
        }}
        
        .progress-bar {{
            height: 8px;
            background: #ecf0f1;
            border-radius: 4px;
            overflow: hidden;
            margin: 10px 0;
        }}
        
        .progress-fill {{
            height: 100%;
            background: linear-gradient(90deg, #27ae60, #2ecc71);
            transition: width 0.3s ease;
        }}
        
        .progress-fill.low {{
            background: linear-gradient(90deg, #e74c3c, #c0392b);
        }}
        
        .progress-fill.medium {{
            background: linear-gradient(90deg, #f39c12, #e67e22);
        }}
        
        @media print {{
            body {{ background: white; }}
            .container {{ box-shadow: none; }}
        }}
    </style>
</head>
<body>
    <div class="container">
        <div class="header">
            <h1>🏠 Advanced Data Quality Analysis</h1>
            <div class="subtitle">Django Model: {self.model_name}</div>
            <div class="meta">
                Database Table: {self.table_name}<br>
                Analysis Date: {datetime.now().strftime('%B %d, %Y at %I:%M %p')}<br>
                Total Records: {sum(e['total_listings'] for e in self.analysis_results):,}
                <br>Description-filled Values: {total_desc_values:,} ({desc_overall_share:.1f}% of all present values)
            </div>
        </div>
        
        <div class="content">
            <!-- Key Statistics -->
            <section class="section">
                <h2 class="section-title">📊 Key Statistics</h2>
                <div class="stats-grid">
                    <div class="stat-card primary">
                        <div class="stat-number">{len(self.all_fields)}</div>
                        <div class="stat-label">Total Fields</div>
                        <div class="stat-description">All database columns analyzed</div>
                    </div>
                    <div class="stat-card success">
                        <div class="stat-number">{len(regular_fields)}</div>
                        <div class="stat-label">Regular Fields</div>
                        <div class="stat-description">Standard scraped fields</div>
                    </div>
                    <div class="stat-card info">
                        <div class="stat-number">{total_desc_values:,}</div>
                        <div class="stat-label">Used Description (values)</div>
                        <div class="stat-description">Across all fields ({desc_overall_share:.1f}%)</div>
                    </div>
                    <div class="stat-card danger">
                        <div class="stat-number">{len([f for f in field_stats.values() if f['status'] == 'Critical'])}</div>
                        <div class="stat-label">Critical Issues</div>
                        <div class="stat-description">Fields >90% missing</div>
                    </div>
                    <div class="stat-card warning">
                        <div class="stat-number">{sum(f['total_missing'] for f in field_stats.values()):,}</div>
                        <div class="stat-label">Total Missing Data</div>
                        <div class="stat-description">Across all fields</div>
                    </div>
                    <div class="stat-card success">
                        <div class="stat-number">{sum(f['total_present'] for f in field_stats.values()):,}</div>
                        <div class="stat-label">Total Present Data</div>
                        <div class="stat-description">Available data points</div>
                    </div>
                </div>
            </section>
            
            <!-- Interactive Dashboard -->
            <section class="section">
                <h2 class="section-title">📈 Interactive Visualizations</h2>
                <div class="dashboard-viz">
                    <div id="plotly-dashboard"></div>
                </div>
            </section>
            
            <!-- Unified Fields Analysis -->
            <section class="section">
                <h2 class="section-title">📋 Field-by-Field Analysis (Unified)</h2>
                <div class="field-category">
                    <div class="category-header">📊 All Fields ({len(field_stats)} fields)</div>
                    <div class="table-container">
                        <table>
                            <thead>
                                <tr>
                                    <th>Field Name</th>
                                    <th>Fill Rate</th>
                                    <th>Status</th>
                                    <th>Priority</th>
                                    <th>Present</th>
                                    <th>Missing</th>
                                    <th>Total</th>
                                    <th>Used Description</th>
                                    <th>Filled By</th>
                                </tr>
                            </thead>
                            <tbody>
                                {self._generate_field_rows(field_stats)}
                            </tbody>
                        </table>
                    </div>
                </div>
            </section>

            <!-- Variables Explorer -->
            <section class="section">
                <h2 class="section-title">🧭 Variables Explorer</h2>
                <div class="field-category">
                    <div class="category-header">Browse All Variables</div>
                    <div class="explorer-controls">
                        <input type="text" id="searchInput" placeholder="Search field..." />
                        <span id="resultCount"></span>
                    </div>
                    <div class="table-container">
                        <table id="explorerTable">
                            <thead>
                                <tr>
                                    <th>Field</th>
                                    <th>Type</th>
                                    <th>Fill Rate</th>
                                    <th>Present</th>
                                    <th>Missing</th>
                                    <th>Desc Filled</th>
                                    <th>Manual Filled</th>
                                    <th>Status</th>
                                </tr>
                            </thead>
                            <tbody></tbody>
                        </table>
                    </div>
                    <div class="pagination" id="pagination"></div>
                </div>
            </section>

            <!-- Records (pgAdmin-like) Explorer -->
            <section class="section">
                <h2 class="section-title">📄 Records (pgAdmin-like)</h2>
                <div class="field-category records-card">
                    <div class="toolbar">
                        <div class="category-header" style="flex: 1;">Browse Individual Ads</div>
                        <span class="chip" id="recordsCount"></span>
                        <input type="text" id="recordsSearch" placeholder="Search records..." />
                        <label for="recordsPageSize">Page size:</label>
                        <select id="recordsPageSize">
                            <option value="10">10</option>
                            <option value="25" selected>25</option>
                            <option value="50">50</option>
                            <option value="100">100</option>
                        </select>
                        <label for="densitySelect">Density:</label>
                        <select id="densitySelect">
                            <option value="density-compact">Compact</option>
                            <option value="density-comfortable" selected>Comfortable</option>
                            <option value="density-cozy">Cozy</option>
                        </select>
                    </div>
                    <div class="table-container">
                        <table id="recordsTable">
                            <thead>
                                <tr id="recordsHeaderRow"></tr>
                            </thead>
                            <tbody></tbody>
                        </table>
                    </div>
                    <div class="pagination" id="recordsPagination"></div>
                </div>
            </section>
            
            <!-- Estate Type Performance -->
            <section class="section">
                <h2 class="section-title">🏠 Estate Type Performance</h2>
                <div class="table-container">
                    <table>
                        <thead>
                            <tr>
                                <th>Estate Type</th>
                                <th>Total Records</th>
                                <th>Avg Fill Rate</th>
                                <th>Grade</th>
                                <th>Critical Failures</th>
                                <th>Good Fields</th>
                            </tr>
                        </thead>
                        <tbody>
                            {self._generate_estate_rows(estate_performance)}
                        </tbody>
                    </table>
                </div>
            </section>
        </div>
    </div>
    
    <script>
        // Render Plotly visualization
        var plotData = {self.fig.to_json()};
        Plotly.newPlot('plotly-dashboard', plotData.data, plotData.layout, {{responsive: true}});

        // Explorer data and pagination
        const explorerData = {json.dumps(explorer_data)};
        let currentPage = 1;
        const pageSize = 25;
        const tbody = document.querySelector('#explorerTable tbody');
        const pagination = document.getElementById('pagination');
        const searchInput = document.getElementById('searchInput');
        const resultCount = document.getElementById('resultCount');

        function renderTable(data, page=1) {{
            tbody.innerHTML = '';
            const start = (page - 1) * pageSize;
            const end = start + pageSize;
            const slice = data.slice(start, end);
            slice.forEach(row => {{
                const tr = document.createElement('tr');
                const typeTag = row.is_description_var ? '<span class="tag desc">Desc</span>' : '<span class="tag regular">Regular</span>';
                tr.innerHTML = `
                    <td><strong>${{row.field_name}}</strong></td>
                    <td>${{typeTag}}</td>
                    <td>${{row.fill_rate.toFixed(1)}}%</td>
                    <td>${{row.present.toLocaleString()}}</td>
                    <td>${{row.missing.toLocaleString()}}</td>
                    <td>${{row.desc_filled.toLocaleString()}} (${{row.desc_share_pct.toFixed(1)}}%)</td>
                    <td>${{row.manual_present.toLocaleString()}}</td>
                    <td>${{row.status}} — ${{row.priority}}</td>
                `;
                tbody.appendChild(tr);
            }});
            resultCount.textContent = `${{data.length}} fields`;
        }}

        function renderPagination(dataLength) {{
            pagination.innerHTML = '';
            const totalPages = Math.ceil(dataLength / pageSize);
            for (let i = 1; i <= totalPages; i++) {{
                const btn = document.createElement('button');
                btn.textContent = i;
                btn.className = i === currentPage ? 'active' : '';
                btn.addEventListener('click', () => {{
                    currentPage = i;
                    renderTable(filteredData, currentPage);
                    renderPagination(filteredData.length);
                }});
                pagination.appendChild(btn);
            }}
        }}

        let filteredData = explorerData;
        searchInput.addEventListener('input', () => {{
            const q = searchInput.value.trim().toLowerCase();
            filteredData = explorerData.filter(r => r.field_name.toLowerCase().includes(q));
            currentPage = 1;
            renderTable(filteredData, currentPage);
            renderPagination(filteredData.length);
        }});

        renderTable(filteredData, currentPage);
        renderPagination(filteredData.length);

        // Records (pgAdmin-like) Explorer logic
        const recordColumns = {json.dumps(record_columns)};
        const recordsData = {json.dumps(records_data)};
        const recordsHeaderRow = document.getElementById('recordsHeaderRow');
        const recordsTbody = document.querySelector('#recordsTable tbody');
        const recordsPagination = document.getElementById('recordsPagination');
        const recordsSearch = document.getElementById('recordsSearch');
        const recordsCount = document.getElementById('recordsCount');
        const recordsPageSizeSelect = document.getElementById('recordsPageSize');
        const densitySelect = document.getElementById('densitySelect');
        const recordsTable = document.getElementById('recordsTable');
        let recordsCurrentPage = 1;
        let recordsSort = {{ column: null, dir: 'asc' }};

        // Column width preferences for wide content
        const wideCols = {{
            'description': 'col-wide-xxl',
            'title': 'col-wide-xl',
            'name': 'col-wide-xl',
            'address': 'col-wide-lg',
            'location': 'col-wide-lg',
            'street': 'col-wide-lg',
            'city': 'col-wide-md',
            'url': 'col-wide-xl',
            'image_url': 'col-wide-xl',
            'images': 'col-wide-xl',
            'image_urls': 'col-wide-xl',
        }};

        function getColClass(col) {{
            const key = (col || '').toString().toLowerCase();
            return wideCols[key] || 'col-wide-sm';
        }}

        // Build header
        function buildRecordsHeader() {{
            recordsHeaderRow.innerHTML = '';
            recordColumns.forEach(col => {{
                const th = document.createElement('th');
                th.textContent = col;
                th.className = 'th-sortable ' + getColClass(col);
                th.addEventListener('click', () => {{
                    if (recordsSort.column === col) {{
                        recordsSort.dir = recordsSort.dir === 'asc' ? 'desc' : 'asc';
                    }} else {{
                        recordsSort.column = col;
                        recordsSort.dir = 'asc';
                    }}
                    // Update header sort state
                    Array.from(recordsHeaderRow.children).forEach(h => h.classList.remove('desc'));
                    if (recordsSort.dir === 'desc') {{ th.classList.add('desc'); }}
                    sortRecords();
                    recordsCurrentPage = 1;
                    renderRecordsTable(filteredRecordsData, recordsCurrentPage);
                    renderRecordsPagination(filteredRecordsData.length);
                }});
                recordsHeaderRow.appendChild(th);
            }});
        }}

        function renderRecordsTable(data, page=1) {{
            recordsTbody.innerHTML = '';
            const size = parseInt(recordsPageSizeSelect.value) || 25;
            const start = (page - 1) * size;
            const end = start + size;
            const slice = data.slice(start, end);
            slice.forEach(row => {{
                const tr = document.createElement('tr');
                recordColumns.forEach(col => {{
                    const td = document.createElement('td');
                    const val = (row[col] ?? '').toString();
                    const isDescription = (col || '').toString().toLowerCase() === 'description';
                    td.className = getColClass(col);

                    // Content block with clamp by default
                    const content = document.createElement('div');
                    content.className = 'cell-content clamped';
                    content.textContent = val;
                    td.appendChild(content);

                    // Add description counter in top-right corner
                    if (isDescription) {{
                        const words = val ? val.trim().split(/\s+/).filter(Boolean).length : 0;
                        const chars = val.length;
                        const counter = document.createElement('span');
                        counter.className = 'desc-counter';
                        counter.textContent = `${{words}}w | ${{chars}}c`;
                        td.appendChild(counter);
                    }}

                    // Add show more/less toggle for long content
                    const longText = val.length > 160; // slightly earlier for 2-line clamp
                    if (longText) {{
                        const btn = document.createElement('button');
                        btn.type = 'button';
                        btn.className = 'toggle-btn';
                        btn.textContent = 'Show more';
                        btn.addEventListener('click', () => {{
                            const clamped = content.classList.toggle('clamped');
                            btn.textContent = clamped ? 'Show more' : 'Show less';
                        }});
                        td.appendChild(btn);
                    }}

                    tr.appendChild(td);
                }});
                recordsTbody.appendChild(tr);
            }});
            recordsCount.textContent = `${{data.length}} records`;
        }}

        function renderRecordsPagination(dataLength) {{
            recordsPagination.innerHTML = '';
            const size = parseInt(recordsPageSizeSelect.value) || 25;
            const totalPages = Math.max(1, Math.ceil(dataLength / size));
            for (let i = 1; i <= totalPages; i++) {{
                const btn = document.createElement('button');
                btn.textContent = i;
                btn.className = i === recordsCurrentPage ? 'active' : '';
                btn.addEventListener('click', () => {{
                    recordsCurrentPage = i;
                    renderRecordsTable(filteredRecordsData, recordsCurrentPage);
                    renderRecordsPagination(filteredRecordsData.length);
                }});
                recordsPagination.appendChild(btn);
            }}
        }}

        function filterRecords() {{
            const q = recordsSearch.value.trim().toLowerCase();
            if (!q) return recordsData;
            return recordsData.filter(r => recordColumns.some(c => ((r[c] || '') + '').toLowerCase().includes(q)));
        }}

        let filteredRecordsData = recordsData;
        recordsSearch.addEventListener('input', () => {{
            filteredRecordsData = filterRecords();
            recordsCurrentPage = 1;
            renderRecordsTable(filteredRecordsData, recordsCurrentPage);
            renderRecordsPagination(filteredRecordsData.length);
        }});

        recordsPageSizeSelect.addEventListener('change', () => {{
            recordsCurrentPage = 1;
            renderRecordsTable(filteredRecordsData, recordsCurrentPage);
            renderRecordsPagination(filteredRecordsData.length);
        }});

        densitySelect.addEventListener('change', () => {{
            const cls = densitySelect.value;
            document.body.classList.remove('density-compact', 'density-comfortable', 'density-cozy');
            document.body.classList.add(cls);
        }});

        function sortRecords() {{
            if (!recordsSort.column) return;
            const col = recordsSort.column;
            const dir = recordsSort.dir === 'asc' ? 1 : -1;
            filteredRecordsData.sort((a, b) => {{
                const av = (a[col] ?? '').toString().toLowerCase();
                const bv = (b[col] ?? '').toString().toLowerCase();
                if (av < bv) return -1 * dir;
                if (av > bv) return 1 * dir;
                return 0;
            }});
        }}

        // Initialize records view
        buildRecordsHeader();
        // Apply initial density
        document.body.classList.add('density-comfortable');
        sortRecords();
        renderRecordsTable(filteredRecordsData, recordsCurrentPage);
        renderRecordsPagination(filteredRecordsData.length);
    </script>
</body>
</html>
        """
        
        # Write to file
        with open(filename, 'w', encoding='utf-8') as f:
            f.write(html_content)
        
        print(f"\n✅ Report generated successfully: {filename}")
        print(f"📊 Summary:")
        print(f"   • Total Fields: {len(self.all_fields)}")
        print(f"   • Regular Fields: {len(regular_fields)}")
        print(f"   • Description Variables: {len(desc_fields)}")
        print(f"   • Critical Issues: {len([f for f in field_stats.values() if f['status'] == 'Critical'])}")
        
        return filename
    
    def _generate_field_rows(self, field_dict):
        """Generate HTML table rows for fields"""
        sorted_fields = sorted(field_dict.items(), key=lambda x: x[1]['avg_missing_pct'], reverse=True)

        html = ""
        for field_name, stats in sorted_fields:
            status_class = f"status-{stats['status'].lower()}"

            # Progress bar color
            fill_rate = stats['avg_fill_rate']
            bar_class = 'low' if fill_rate < 50 else 'medium' if fill_rate < 80 else ''

            html += f"""
            <tr>
                <td><strong>{field_name}</strong></td>
                <td>
                    {stats['avg_fill_rate']:.1f}%
                    <div class=\"progress-bar\">
                        <div class=\"progress-fill {bar_class}\" style=\"width: {stats['avg_fill_rate']:.1f}%\"></div>
                    </div>
                </td>
                <td><span class=\"status-badge {status_class}\">{stats['status']}</span></td>
                <td>{stats['priority']}</td>
                <td>{stats['total_present']:,}</td>
                <td>{stats['total_missing']:,}</td>
                <td>{stats['total_records']:,}</td>
                <td>📝 {stats.get('desc_filled_total', 0):,} ({stats.get('desc_share_pct', 0):.1f}%)</td>
                <td>
                    🛠️ Manual: {stats.get('manual_present_total', 0):,}
                    {('<span class=\'description-badge\'>Description Var</span>') if stats.get('is_description_var') else ''}
                </td>
            </tr>
            """
        return html
    
    def _generate_estate_rows(self, estate_dict):
        """Generate HTML table rows for estate types"""
        sorted_estates = sorted(estate_dict.items(), key=lambda x: x[1]['avg_fill_rate'], reverse=True)
        
        html = ""
        for estate_type, stats in sorted_estates:
            fill_rate = stats['avg_fill_rate']
            bar_class = 'low' if fill_rate < 50 else 'medium' if fill_rate < 80 else ''
            
            html += f"""
            <tr>
                <td><strong>{estate_type}</strong></td>
                <td>{stats['total_listings']:,}</td>
                <td>
                    {stats['avg_fill_rate']:.1f}%
                    <div class="progress-bar">
                        <div class="progress-fill {bar_class}" style="width: {stats['avg_fill_rate']:.1f}%"></div>
                    </div>
                </td>
                <td><strong>{stats['grade']}</strong></td>
                <td>{stats['critical_failures']}</td>
                <td>{stats['good_fields']}</td>
            </tr>
            """
        
        return html
    
    def export_to_csv(self, filename='data_quality_export.csv'):
        """Export analysis results to CSV"""
        field_stats = self._calculate_field_statistics(
            set(field for estate in self.analysis_results for field in estate['fields'].keys())
        )
        
        # Create DataFrame
        data = []
        for field_name, stats in field_stats.items():
            data.append({
                'field_name': field_name,
                'is_description_variable': stats['is_description_var'],
                'fill_rate_percent': round(stats['avg_fill_rate'], 2),
                'missing_percent': round(stats['avg_missing_pct'], 2),
                'total_present': stats['total_present'],
                'total_missing': stats['total_missing'],
                'total_records': stats['total_records'],
                'status': stats['status'],
                'priority': stats['priority']
            })
        
        df = pd.DataFrame(data)
        df.to_csv(filename, index=False)
        
        print(f"✅ CSV export saved: {filename}")
        return filename


# ============================================
# USAGE EXAMPLE - ADD THIS TO YOUR DJANGO APP
# ============================================

"""
# In your Django management command or view:

from extractly.models import AdsManual
from your_app.data_analyzer import AdvancedDjangoDataAnalyzer

# Initialize analyzer
analyzer = AdvancedDjangoDataAnalyzer(
    model_class=AdsManual,
    estate_type_field='estate_type'  # or whatever field you group by
)

# Run analysis
analyzer.analyze_data_quality()

# Create visualizations
analyzer.create_advanced_visualizations()

# Generate HTML report
analyzer.generate_html_report('data_quality_report.html')

# Optional: Export to CSV
analyzer.export_to_csv('data_quality_export.csv')

print("✅ Analysis complete! Open the HTML report in your browser.")
"""