
    ,/i                     t    d dl Zd dlZd dlmZ d dlmZ d dl	Z	d dl
m
Z
 d dlmZ d dlmZ d dlZ G d d      Zy)    N)make_subplots)datetime)
connection)appsc                   ~    e Zd ZddZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd ZddZd Zd ZddZy)AdvancedDjangoDataAnalyzerc                 p   || _         || _        |j                  j                  | _        |j
                  | _        | j                         | _        t               | _
        i | _        t        d| j                          t        d| j                          t        dt        | j                                y)z
        Initialize analyzer with Django model
        
        Args:
            model_class: Django model class (e.g., AdsManual)
            estate_type_field: Field name for grouping (default: 'estate_type')
        u   🔍 Analyzing Django Model: u   📊 Table: u   📝 Total Fields: N)modelestate_type_field_metadb_table
table_name__name__
model_name_get_all_fields
all_fieldssetdescription_varsdesc_vars_by_pkprintlen)selfmodel_classr   s      @/var/www/extractly/manual_agregator/visual_viewer/data_visual.py__init__z#AdvancedDjangoDataAnalyzer.__init__   s     !
!2%++44%.. ..0 #!-doo->?@T__-./#C$8#9:;    c                     g }| j                   j                  j                         D ]*  }|j                  s|j	                  |j
                         , |S )z)Extract all field names from Django model)r
   r   
get_fieldsconcreteappendname)r   fieldsfields      r   r   z*AdvancedDjangoDataAnalyzer._get_all_fields#   sB    ZZ%%002E~~ejj) 3 r   c           
      ^   t        d       t               }g }d| j                  v r|j                  d       d| j                  v r|j                  d       |r|D ]  }t               }|D ]S  }t	        ||d      }|s	 t        |t              rt        j                  |      }nt        |t              r|}nd}t        |t              r)|j                  t        |j                                      	 	 t        j                  t        |            }t        |t              r)|j                  t        |j                                      nrt        |t         t        t"        f      rQ|j                  |D 	ch c]5  }	t        |	      j%                         st        |	      j%                         7 c}	       nt&        V |sk|j                  |       || j,                  |j.                  <    || _        t        dt3        |       d|        |S # t        j                  t        f$ r Y =w xY wc c}	w # t(        $ rh t        |t              rT|j+                  d      D 
cg c]#  }
|
j%                         s|
j%                         % nc c}
w }}
|j                  |       Y @w xY w)z;Extract variables from description_scraped_variables columnu6   🔎 Extracting description-based scraped variables...description_based_scraperdescription_scraped_variablesN,u
   ✅ Found z description-based variables: )r   r   r   r    getattr
isinstancestrjsonloadsdictupdatekeysJSONDecodeErrorAttributeErrorastliteral_evallisttuplestrip
ValueError	Exceptionsplitr   pkr   r   )r   querysetr   fields_to_checkobjcombined_keyscolvar_datavar_dictxppartss               r   _extract_description_variablesz9AdvancedDjangoDataAnalyzer._extract_description_variables,   sD   FG5 &$//9""#>?*doo=""#BC #*C&sC6H# %h4'+zz(';H'$7'/H'+H%h5)00X]]_1EF$ 68#&#3#3CM#B%h5)00X]]_1EF'4e2DE)00(1e(QVYZ[V\VbVbVd#a&,,.(1ef #-,9 +D !$++M:3@D((0M  P !1
3/011OP`Oabc1 !00.A  2f % 8%h48@s8K$Y8K1qwwyQWWY8K$YE$Y)0078sP    A3H6BH;7H6H61H;H32H36H;;,J,'J
=J
J,+J,c                 
    t        d       t        d       t        d        j                  j                  j                         }|j	                         }t        d|d       |dk(  rt        d       y j                  |       |j                   j                  d	
      j                         }t        dt        t        |                    g }|D ]  }|d}t        d|         |j                  di  j                  |i}|j	                         }t        d|d       t        |j                  dd	
            }i }	 j                  D ]   j                  k(  r |j                  di  dd	ij	                         }
	  j                  j                  j                         t        dfd             }h d}||v r$ |j                  di  dij	                         }nd}|
|z   }|dkD  r||z  dz  nd}||z
  } j"                  v }t%         fd|D              }t'        ||z
  d      }|||
||||||d	|	<    |j)                  |||	d        | _        |S # t         $ r d}Y w xY w)z1Main analysis method - replaces SQL null analysiszQ
================================================================================u1   🚀 Starting Comprehensive Data Quality AnalysiszQ================================================================================
u   📊 Total Records: r'   r   u!   ❌ No records found in database!NT)flatu   🏠 Estate Types Found: unknownu   
📋 Analyzing Estate Type: z   Records: r:   __isnullget_internal_typec                  ,    t        t                     S )N)r*   type)	field_objs   r   <lambda>zAAdvancedDjangoDataAnalyzer.analyze_data_quality.<locals>.<lambda>   s    QTUYZcUdQer   >   URLField	CharField	SlugField	TextField
EmailField d   c              3   p   K   | ]-  }j                   j                  |t                     v s*d  / yw)   N)r   getr   ).0r:   
field_namer   s     r   	<genexpr>zBAdvancedDjangoDataAnalyzer.analyze_data_quality.<locals>.<genexpr>   s1     !njJ$J^J^JbJbcegjglJm<m!js   +66)	missingmissing_pct
null_countempty_countpresentdesc_filledmanual_presenttotalis_description_var)estate_typetotal_listingsr"    )r   r
   objectsallcountrE   values_listr   distinctr   r4   filterr   r   	get_fieldr(   r8   r   summaxr    analysis_results)r   r;   total_recordsestate_typesrq   re   	estate_qsestate_count
estate_pksfield_analysisr^   
field_typestring_like_typesr_   total_missingr]   present_countrd   ra   rb   rZ   rM   s   `                   @@r   analyze_data_qualityz/AdvancedDjangoDataAnalyzer.analyze_data_qualityf   s   mABm ::%%))+ ($]1$567A56 	++H5  ++D,B,B+NWWY)#d<.@*A)BCD 'K"'2;-@A (P4+A+A;*OPI$??,LLa 012 i33Dt3DEJ  N"oo
!7!77 .Y--PJ<x1H$0OPVVX

$ $

 0 0 : :: FI!f4GIe!f!hJ(i%!%66&6i&6&6&OZL29N&O&U&U&W&' !+[ 8FRUVFV}|;cA\] ,} < &043H3H%H"!!nj!nn!$][%@!!D  -#.",#.,#.&4)*<
.z*A .X ##*".(% } (H !1E ! $"#K$s   A(I44JJc                 T   t        | d      r| j                  st        d       yt               }| j                  D ]$  }|j	                  |d   j                                & |D cg c]  }|| j                  vs| }}|D cg c]  }|| j                  v s| }}t        d       t        dt        |              t        dt        |              | j                  |      }| j                         }t        dd	g d
ddiddigddiddigddiddigddiddiggdd      }| j                  |      }	|j                  t        j                  t        |	j                               t        |	j!                               g ddd      dd       t        |j                               }
|
D cg c]
  }||   d    }}|j                  t        j"                  |
||D cg c]  }|dk\  rdn|dk\  rdnd c}t%        |
|      D cg c]  \  }}|dd||   d    d!d" c}}d#d$%      dd	       t'        |j)                         D cg c]  \  }}||d&   |d'   f c}}d( d)*      dd+ }|rt%        | \  }}}|D cg c]  }|rd,nd
 }}|j                  t        j"                  t        |      t        |      |t%        ||      D cg c]  \  }}|dd|rd-nd.  c}}d#d/%      d	d       t'        |j)                         D cg c]  \  }}||d   |d'   f c}}d0 d)*      dd+ }|rt%        | \  }}}|D cg c]  }|rd1nd
 }}|j                  t        j"                  t        |      t        |      |t%        ||      D cg c]  \  }}|dd|rd-nd.  c}}d#d$%      d	d	       |j!                         D cg c]  }|d   	 }}|j                  t        j*                  |d2d3d4d56      d7d       | j-                  |      }|j                  t        j"                  t        |j                               t        |j!                               g d8t        |j!                               d#9      d7d	       | j/                  |      }|j                  t        j"                  d:d;g|d<   |d=   gd3d>g|d<   dd|d?    d@|d=   dd|dA    d@gd#dB%      dd       | j1                         }|j                  t        j"                  t        |j                               t        |j!                               d|j!                         D cg c]  }|dC c}d#dD%      dd	       |j3                  dEdFdG| j4                   dH| j6                   dIt        |       dJt        |       dK	dLdMdNidOdPQ       |j9                  dRS       || _        |S c c}w c c}w c c}w c c}w c c}}w c c}}w c c}w c c}}w c c}}w c c}w c c}}w c c}w c c}w )TzJCreate comprehensive visualizations with description variable highlightingrq   uD   ❌ No analysis results available. Run analyze_data_quality() first.Nr"   u   
📊 Field Categories:z   Regular Fields: z   Description-based Fields:       )u'   📊 Overall Data Quality by Field Typeu'   🏠 Estate Type Performance Comparisonu*   🔥 Top 10 Critical Fields (Missing Data)u)   ⭐ Top 10 Complete Fields (Best Quality)u(   📈 Fill Rate Distribution (All Fields)u!   🎯 Priority Actions by Severityu,   🔍 Description Variables vs Regular Fieldsu    📉 Missing Data by Estate TyperL   piebar	histogramg?gQ?)rowscolssubplot_titlesspecsvertical_spacinghorizontal_spacing)#e74c3c#e67e22#f39c12#27ae60z#2ecc71zlabel+percentzL<b>%{label}</b><br>Fields: %{value}<br>Percentage: %{percent}<extra></extra>)labelsvaluesmarker_colorstextinfohovertemplaterW   )rowr?   avg_fill_rateF   r   2   r   r   .1fz%<br>rf   r'   z recordsoutsidez2<b>%{x}</b><br>Fill Rate: %{y:.1f}%<extra></extra>)rB   ymarker_colortexttextpositionr   avg_missing_pctrd   c                     | d   S NrW   rg   rB   s    r   rN   zKAdvancedDjangoDataAnalyzer.create_advanced_visualizations.<locals>.<lambda>      !A$r   Tkeyreverse
   z#ff6b6bu	   📝 Descu   📊 Regularz0<b>%{x}</b><br>Missing: %{y:.1f}%<extra></extra>c                     | d   S r   rg   r   s    r   rN   zKAdvancedDjangoDataAnalyzer.create_advanced_visualizations.<locals>.<lambda>1  r   r   z#66ff66   z#3498dbg      ?z3Fill Rate: %{x:.1f}%<br>Fields: %{y}<extra></extra>)rB   nbinsxr   opacityr      )r   r   r   z#f1c40fr   )rB   r   r   r   r   zRegular FieldszDescription Variablesregular_avgdesc_avgz#9b59b6regular_countz fields
desc_countz6<b>%{x}</b><br>Avg Fill Rate: %{y:.1f}%<extra></extra>z,.0fz8<b>%{x}</b><br>Missing Records: %{y:,.0f}<extra></extra>i  Fu=   🏠 Advanced Data Quality Analysis Dashboard<br><sub>Model: z
 | Table: z<br>z Regular Fields + z Description Variables</sub>g      ?size   )r   rB   fontplotly_white)height
showlegendtitletemplatei)	tickangle)hasattrrq   r   r   r.   r/   r   r   _calculate_field_statistics_calculate_estate_performancer   _get_quality_by_field_type	add_tracegoPier4   r   Barzipsorteditems	Histogram_get_priority_counts_compare_field_types_get_estate_missing_dataupdate_layoutr   r   update_xaxesfig)r   r   estatefregular_fieldsdesc_fieldsfield_statsestate_performancer   quality_by_typeestate_namesr!   estate_scoressscorecritical_fields
crit_names	crit_pctsis_descdesccolorspctbest_fields
best_names
best_ratesrateall_fill_ratespriority_countsdesc_vs_regularestate_missingvals                                  r   create_advanced_visualizationsz9AdvancedDjangoDataAnalyzer.create_advanced_visualizations   s    t/08M8MXY U
++FfX.3356 , &0RZ1D<Q<Q3Q!ZR",K*QT5J5J0Jq*K(*#C$7#89:-c+.>-?@A 66zB!??A 	 %65/2%65/2+&8%65/2	 "#'
. 99+FFFO0023O2245U(l q 	 		
 .3356O[\|t+D1/B|\FFerser`a17iQ"W	R[[ers),\=)IK)I+$ s5);D)ABR)STU(VV^_)IK&R q 	 	
 !LWL]L]L_`L_DAqa$%q)='>?L_`
 2	 -0/-B*J	7CJK744iY67FKMM:&9o!'+.y'+BD+Bic4 !IU$;N*ST+BD!*"T 1   JUJ[J[J]^J]$!Qa?#Q';%<=J]^
 2	 .1;.?+J
GCJK744iY67FKMM:&:&!',/
G,DF,DjdD "#Je4K^+TU,DF!*"V 1   7B6H6H6JK6J!O,6JKLL &S q 	 		
 33K@FF++-.--/0T/0023& q 	 		
 33K@FF#%<="=1?:3NO'3(7<E/RaBbAccjk(4S9|?\>]]deg&V q 	 	
 668FF~**,-~,,./&/=/D/D/FG/FT
m/FG&X q 	 
	
 	WX\XgXgWhhrsw  tC  tC  sD  DH  IL  M[  I\  H]  ]o  ps  t  p@  oA  A]  ^
 $ 	 		
 	3'
y SKb ] tK a LD _ LF Lf HsZ   &W$:W$W)W)1W."W3
W8W>
XX	X
XXX X%c                    i }|D ]  }d}d}d}g }|| j                   v }d}	d}
| j                  D ]h  }||d   v s|d   |   }||d   z  }||d   z  }||d   z  }|j                  |d          |	|j                  dd      z  }	|
|j                  dd      z  }
j |rt	        j
                  |      nd	}d	|z
  }|||||| j                  |      | j                  |      ||	|
|r|	|z  d	z  ndd
||<    |S )z1Calculate comprehensive statistics for each fieldr   r"   r\   r`   rc   r]   ra   rb   rU   )r   r   total_presentrz   rr   statuspriorityrd   desc_filled_totalmanual_present_totaldesc_share_pct)r   rq   r    rX   npmean_get_status_get_priority)r   r   r   rZ   rz   r   rr   missing_pctsis_desc_varr   r   r   
field_datar   r   s                  r   r   z6AdvancedDjangoDataAnalyzer._calculate_field_statistics  sW   $JMMML$(=(==K !#$ //!11!'!1*!=J!Z	%::M!Z	%::M!Z%88M ''
=(AB%q)II%(JNN;KQ,OO( 0 8Dbggl3O/1M "/#2!.!.!.**?; ..?&1%6(<O\#4}#Ds#Jbc'K
#- %H r   c           
         i }| j                   D ]  }|d   }|d   }|s|j                         D cg c]
  }d|d   z
   }}t        j                  |      }t	        d |j                         D              }t	        d |j                         D              }	|t        |      ||	|d   | j                  ||      d||<    |S c c}w )	z2Calculate performance metrics for each estate typere   r"   rU   r]   c              3   2   K   | ]  }|d    dkD  sd  yw)r]   Z   rW   Nrg   rY   r   s     r   r[   zKAdvancedDjangoDataAnalyzer._calculate_estate_performance.<locals>.<genexpr>  s     #X!!MBRUWBWA   c              3   2   K   | ]  }|d    dk  sd  yw)r]   r   rW   Nrg   r   s     r   r[   zKAdvancedDjangoDataAnalyzer._calculate_estate_performance.<locals>.<genexpr>  s     RAAm<Lr<Qar   rf   )r   total_fieldscritical_failuresgood_fieldsrf   grade)rq   r   r   r   ro   r   
_get_grade)
r   r   r   re   r"   r   
fill_ratesr   r   r   s
             r   r   z8AdvancedDjangoDataAnalyzer._calculate_estate_performance  s    ++F /KH%F:@--/J/Q#- 00/JJGGJ/M ##Xv}}#X XRRRK "/ #F%6*"()9":8IJ/{+ ,, "! Ks   Cc                 `    dddddd}|j                         D ]  }||d   xx   dz  cc<    |S )z#Categorize fields by quality statusr   )CriticalPoorFairGood	Excellentr   rW   )r   )r   r   quality_countsstatss       r   r   z5AdvancedDjangoDataAnalyzer._get_quality_by_field_type  sB    &'AqWXY '')E5?+q0+ * r   c                     dddddd}|j                         D ]'  }|d   j                  dd      d   }||xx   dz  cc<   ) |S )zCount fields by priority levelr   )zFix ImmediatelyzHigh PriorityzMedium PriorityzLow PriorityzMonitor Onlyr    rW   )r   r9   )r   r   r   r  r   s        r   r   z/AdvancedDjangoDataAnalyzer._get_priority_counts  sc      ! 
 !'')EZ(..sA6q9HH%*% * r   c                 $   g }g }|j                         D ]3  \  }}|d   r|j                  |d           |j                  |d          5 |rt        j                  |      ndt	        |      |rt        j                  |      ndt	        |      dS )z/Compare regular fields vs description variablesrd   r   r   )r   r   r   r   )r   r    r   r   r   )r   r   regular_rates
desc_ratesr#   r  s         r   r   z/AdvancedDjangoDataAnalyzer._compare_field_types  s    
'--/LE5)*!!%"89$$U?%;<	 0 6C277=1 //9
+qj/	
 	
r   c                     i }| j                   D ]-  }t        d |d   j                         D              }|||d   <   / |S )z3Calculate total missing data points per estate typec              3   &   K   | ]	  }|d      yw)r\   Nrg   r   s     r   r[   zFAdvancedDjangoDataAnalyzer._get_estate_missing_data.<locals>.<genexpr>	  s     P6O)6O   r"   re   )rq   ro   r   )r   r   r   rz   s       r   r   z3AdvancedDjangoDataAnalyzer._get_estate_missing_data  sL    ++FPfX6F6M6M6OPPM4AN6-01 , r   c                 4    |dkD  ry|dkD  ry|dkD  ry|dkD  ryy	)
zDetermine quality statusr   r   r   r  r   r  r   r  r  rg   r   r]   s     r   r   z&AdvancedDjangoDataAnalyzer._get_status  s1    222r   c                 4    |dkD  ry|dkD  ry|dkD  ry|dkD  ryy	)
zDetermine priority levelr   u   🔴 Fix Immediatelyr   u   🟡 High Priorityr   u   🟠 Medium Priorityr   u   🟢 Low Priorityu   ✅ Monitor Onlyrg   r  s     r   r   z(AdvancedDjangoDataAnalyzer._get_priority  s1    )2'2)2&%r   c                 T    |dk\  r|dk(  ry|dk\  r|dk  ry|dk\  ry|d	k\  ry
|dk\  ryy)zCalculate letter grader   r   zA+U   rW   AK   B<   Cr   DFrg   )r   	fill_rater   s      r   r   z%AdvancedDjangoDataAnalyzer._get_grade(  sG    ?0A5"_!2a!7"_"_"_r   c                 D
   t        | d      st        d       | j                          | j                  t	        d | j
                  D                    }| j                         }|j                         D ci c]  \  }}|d   r|| }}}|j                         D ci c]  \  }}|d   s|| }}}t        d |j                         D              }t        d |j                         D              }	|	r||	z  dz  nd}
t        | j                        }d	|v rd	g|D cg c]
  }|d	k7  s	| c}z   } | j                  j                  j                         j                  | }g }|D ]@  }i }|D ]&  }|j                  |      }	 |dn
t!        |      ||<   ( |j)                  |       B |j                         D cg c]S  \  }}||d   t+        |d   d      t+        |d   d      |d   |d   |d   |d   |d   t+        |d   d      |d   |d   dU }}}dj-                  g d| j.                   d| j.                   d| j0                   dt3        j4                         j7                  d       dt        d  | j
                  D              d!d"|d!d#|
d$d%t9        | j                         d&t9        |       d'|d!d(|
d$d)t9        |j                         D cg c]  }|d   d*k(  s| c}       d+t        d, |j                         D              d!d-t        d. |j                         D              d!d/t9        |       d0| j;                  |       d1| j=                  |       d2| j>                  jA                          d3t%        j&                  |       d4t%        j&                  |       d5t%        j&                  |       d6      }tC        |d7d89      5 }|jE                  |       d
d
d
       t        d:|        t        d;       t        d<t9        | j                                t        d=t9        |              t        d>t9        |              t        d?t9        |j                         D cg c]  }|d   d*k(  s| c}              |S c c}}w c c}}w c c}w # t"        $ r' |dnt%        j&                  |dt               ||<   Y w xY wc c}}w c c}w # 1 sw Y   xY wc c}w )@z"Generate comprehensive HTML reportr   u'   ⚠️ Creating visualizations first...c              3   P   K   | ]  }|d    j                         D ]  }|    ywr"   Nr/   rY   r   r#   s      r   r[   zBAdvancedDjangoDataAnalyzer.generate_html_report.<locals>.<genexpr>?  *     _$9&vhGWG\G\G^eG^$9   $&rd   c              3   &   K   | ]	  }|d      yw)r   Nrg   rY   vs     r   r[   zBAdvancedDjangoDataAnalyzer.generate_html_report.<locals>.<genexpr>F  s     U@T1"5 6@Tr  c              3   &   K   | ]	  }|d      ywr   Nrg   r$  s     r   r[   zBAdvancedDjangoDataAnalyzer.generate_html_report.<locals>.<genexpr>G  s     "T?S!1_#5?Sr  rU   r   idNrT   F)ensure_asciidefaultr   r   r   r   rz   rr   r   r   r   r   r   )rZ   rd   r  missing_percentr`   r\   rc   ra   rb   r   r   r   z
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Advanced Data Quality Report - u'  </title>
    <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
    <style>
        * {
            margin: 0;
            padding: 0;
            box-sizing: border-box;
        }
        
        body {
            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            min-height: 100vh;
            padding: 20px;
        }
        
        .container {
            max-width: 1800px;
            margin: 0 auto;
            background: white;
            border-radius: 20px;
            box-shadow: 0 25px 80px rgba(0,0,0,0.2);
            overflow: hidden;
        }
        
        .header {
            background: linear-gradient(135deg, #2c3e50 0%, #34495e 100%);
            color: white;
            padding: 50px;
            text-align: center;
        }
        
        .header h1 {
            font-size: 3.5em;
            font-weight: 300;
            margin-bottom: 15px;
        }
        
        .header .subtitle {
            font-size: 1.4em;
            opacity: 0.9;
            margin: 10px 0;
        }
        
        .header .meta {
            font-size: 1.1em;
            opacity: 0.8;
            margin-top: 15px;
        }
        
        .content {
            padding: 50px;
        }
        
        .dashboard-viz {
            margin: 40px 0;
            background: #f8f9fa;
            padding: 30px;
            border-radius: 15px;
            box-shadow: 0 10px 30px rgba(0,0,0,0.1);
        }
        
        .section {
            margin: 50px 0;
        }
        
        .section-title {
            font-size: 2.5em;
            color: #2c3e50;
            margin-bottom: 30px;
            padding-bottom: 15px;
            border-bottom: 4px solid #3498db;
        }
        
        .field-category {
            margin: 40px 0;
        }
        
        .category-header {
            background: linear-gradient(135deg, #9b59b6 0%, #8e44ad 100%);
            color: white;
            padding: 20px 30px;
            border-radius: 10px;
            font-size: 1.8em;
            margin-bottom: 20px;
            display: flex;
            align-items: center;
            gap: 15px;
        }
        
        .category-header.description {
            background: linear-gradient(135deg, #e67e22 0%, #d35400 100%);
        }
        
        .table-container {
            overflow-x: auto;
            border-radius: 10px;
            box-shadow: 0 5px 15px rgba(0,0,0,0.1);
        }
        
        table {
            width: 100%;
            border-collapse: collapse;
            background: white;
        }
        
        th {
            background: linear-gradient(135deg, #34495e 0%, #2c3e50 100%);
            color: white;
            padding: 20px 15px;
            text-align: left;
            font-weight: 600;
            font-size: 1.1em;
            position: sticky;
            top: 0;
            z-index: 10;
        }
        
        td {
            padding: 18px 15px;
            border-bottom: 1px solid #ecf0f1;
        }
        
        tr:hover {
            background: #f8f9fa;
        }
        
        .status-badge {
            display: inline-block;
            padding: 8px 16px;
            border-radius: 20px;
            font-weight: 600;
            color: white;
            text-transform: uppercase;
            font-size: 0.9em;
        }
        
        .status-critical { background: linear-gradient(135deg, #e74c3c 0%, #c0392b 100%); }
        .status-poor { background: linear-gradient(135deg, #e67e22 0%, #d35400 100%); }
        .status-fair { background: linear-gradient(135deg, #f39c12 0%, #e67e22 100%); }
        .status-good { background: linear-gradient(135deg, #27ae60 0%, #229954 100%); }
        .status-excellent { background: linear-gradient(135deg, #2ecc71 0%, #27ae60 100%); }
        
        .description-badge {
            display: inline-block;
            padding: 5px 12px;
            background: linear-gradient(135deg, #9b59b6 0%, #8e44ad 100%);
            color: white;
            border-radius: 15px;
            font-size: 0.85em;
            font-weight: 600;
            margin-left: 10px;
        }

        /* Explorer Styles */
        .explorer-controls { display: flex; gap: 15px; align-items: center; margin-bottom: 15px; }
        .explorer-controls input { padding: 8px 12px; border: 1px solid #ddd; border-radius: 6px; }
        .pagination { display: flex; gap: 8px; margin-top: 12px; flex-wrap: wrap; }
        .pagination button { padding: 6px 12px; border: none; background: #ecf0f1; border-radius: 6px; cursor: pointer; }
        .pagination button.active { background: #3498db; color: white; }
        .tag { display: inline-block; padding: 3px 8px; border-radius: 10px; font-size: 0.85em; margin-left: 8px; }
        .tag.desc { background: #9b59b6; color: white; }
        .tag.regular { background: #95a5a6; color: white; }
        
        /* Records table clamping and controls */
    #recordsTable td { position: relative; vertical-align: top; }
        #recordsTable th { position: sticky; top: 0; z-index: 5; }
        .cell-content { white-space: normal; }
        .cell-content.clamped {
            display: -webkit-box;
            -webkit-line-clamp: 2; /* clamp to 2 lines */
            -webkit-box-orient: vertical;
            overflow: hidden;
        }
        .toggle-btn {
            display: inline-block;
            margin-top: 6px;
            padding: 3px 8px;
            font-size: 0.8em;
            color: #2c3e50;
            background: #ecf0f1;
            border: 1px solid #d0d7de;
            border-radius: 6px;
            cursor: pointer;
        }
        .toggle-btn:hover { background: #e1e8ed; }
        .desc-counter {
            position: absolute;
            top: 6px;
            right: 8px;
            font-size: 0.75em;
            color: rgba(0,0,0,0.45);
        }

        /* High-tech styling for Records section */
        .records-card {
            background: rgba(255,255,255,0.75);
            backdrop-filter: blur(8px);
            border-radius: 16px;
            border: 1px solid rgba(255,255,255,0.4);
            box-shadow: 0 12px 40px rgba(0,0,0,0.12);
            padding: 20px;
        }
        #recordsTable {
            border-collapse: separate;
            border-spacing: 0;
        }
        #recordsTable thead th {
            background: linear-gradient(135deg, #0f2027 0%, #203a43 50%, #2c5364 100%);
            color: #ecf0f1;
            letter-spacing: 0.3px;
            user-select: none;
        }
        #recordsTable tbody tr:nth-child(even) { background: #fafbfc; }
        #recordsTable tbody tr:hover { background: #eef6ff; box-shadow: inset 0 0 0 9999px rgba(52, 152, 219, 0.06); }
        .th-sortable { cursor: pointer; position: relative; }
        .th-sortable::after { content: 'B4'; position: absolute; right: 8px; opacity: 0.35; transform: translateY(-2px); }
        .th-sortable.desc::after { content: 'BE'; }
        .col-wide-xxl { min-width: 640px; }
        .col-wide-xl { min-width: 420px; }
        .col-wide-lg { min-width: 320px; }
        .col-wide-md { min-width: 240px; }
        .col-wide-sm { min-width: 180px; }
        .chip { display: inline-block; padding: 6px 10px; background: #ecf0f1; border-radius: 999px; font-size: 0.85em; color: #2c3e50; }
        .toolbar { display: flex; gap: 12px; align-items: center; flex-wrap: wrap; }
        .toolbar .spacer { flex: 1; }
        .density-compact #recordsTable td { padding: 8px 10px; }
        .density-comfortable #recordsTable td { padding: 14px 16px; }
        .density-cozy #recordsTable td { padding: 18px 18px; }
        
        .stats-grid {
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
            gap: 25px;
            margin: 30px 0;
        }
        
        .stat-card {
            background: linear-gradient(135deg, #f8f9fa 0%, #e9ecef 100%);
            padding: 30px;
            border-radius: 15px;
            border-left: 6px solid;
            transition: transform 0.3s ease, box-shadow 0.3s ease;
        }
        
        .stat-card:hover {
            transform: translateY(-5px);
            box-shadow: 0 15px 40px rgba(0,0,0,0.15);
        }
        
        .stat-card.primary { border-color: #3498db; }
        .stat-card.success { border-color: #27ae60; }
        .stat-card.danger { border-color: #e74c3c; }
        .stat-card.warning { border-color: #f39c12; }
        .stat-card.info { border-color: #9b59b6; }
        
        .stat-number {
            font-size: 3.5em;
            font-weight: bold;
            background: linear-gradient(45deg, #667eea, #764ba2);
            -webkit-background-clip: text;
            -webkit-text-fill-color: transparent;
            margin-bottom: 10px;
        }
        
        .stat-label {
            font-size: 1.2em;
            color: #2c3e50;
            font-weight: 500;
        }
        
        .stat-description {
            font-size: 0.95em;
            color: #7f8c8d;
            margin-top: 8px;
        }
        
        .progress-bar {
            height: 8px;
            background: #ecf0f1;
            border-radius: 4px;
            overflow: hidden;
            margin: 10px 0;
        }
        
        .progress-fill {
            height: 100%;
            background: linear-gradient(90deg, #27ae60, #2ecc71);
            transition: width 0.3s ease;
        }
        
        .progress-fill.low {
            background: linear-gradient(90deg, #e74c3c, #c0392b);
        }
        
        .progress-fill.medium {
            background: linear-gradient(90deg, #f39c12, #e67e22);
        }
        
        @media print {
            body { background: white; }
            .container { box-shadow: none; }
        }
    </style>
</head>
<body>
    <div class="container">
        <div class="header">
            <h1>🏠 Advanced Data Quality Analysis</h1>
            <div class="subtitle">Django Model: zF</div>
            <div class="meta">
                Database Table: z$<br>
                Analysis Date: z%B %d, %Y at %I:%M %pz$<br>
                Total Records: c              3   &   K   | ]	  }|d      yw)rf   Nrg   )rY   es     r   r[   zBAdvancedDjangoDataAnalyzer.generate_html_report.<locals>.<genexpr>  s     #WAVAA&6$7AVr  r'   z0
                <br>Description-filled Values:  (r   u}  % of all present values)
            </div>
        </div>
        
        <div class="content">
            <!-- Key Statistics -->
            <section class="section">
                <h2 class="section-title">📊 Key Statistics</h2>
                <div class="stats-grid">
                    <div class="stat-card primary">
                        <div class="stat-number">a$  </div>
                        <div class="stat-label">Total Fields</div>
                        <div class="stat-description">All database columns analyzed</div>
                    </div>
                    <div class="stat-card success">
                        <div class="stat-number">a  </div>
                        <div class="stat-label">Regular Fields</div>
                        <div class="stat-description">Standard scraped fields</div>
                    </div>
                    <div class="stat-card info">
                        <div class="stat-number">z</div>
                        <div class="stat-label">Used Description (values)</div>
                        <div class="stat-description">Across all fields (z%)</div>
                    </div>
                    <div class="stat-card danger">
                        <div class="stat-number">r   a  </div>
                        <div class="stat-label">Critical Issues</div>
                        <div class="stat-description">Fields >90% missing</div>
                    </div>
                    <div class="stat-card warning">
                        <div class="stat-number">c              3   &   K   | ]	  }|d      yw)rz   Nrg   r   s     r   r[   zBAdvancedDjangoDataAnalyzer.generate_html_report.<locals>.<genexpr>       5gRfQa6HRfr  a  </div>
                        <div class="stat-label">Total Missing Data</div>
                        <div class="stat-description">Across all fields</div>
                    </div>
                    <div class="stat-card success">
                        <div class="stat-number">c              3   &   K   | ]	  }|d      ywr'  rg   r   s     r   r[   zBAdvancedDjangoDataAnalyzer.generate_html_report.<locals>.<genexpr>  r0  r  uM  </div>
                        <div class="stat-label">Total Present Data</div>
                        <div class="stat-description">Available data points</div>
                    </div>
                </div>
            </section>
            
            <!-- Interactive Dashboard -->
            <section class="section">
                <h2 class="section-title">📈 Interactive Visualizations</h2>
                <div class="dashboard-viz">
                    <div id="plotly-dashboard"></div>
                </div>
            </section>
            
            <!-- Unified Fields Analysis -->
            <section class="section">
                <h2 class="section-title">📋 Field-by-Field Analysis (Unified)</h2>
                <div class="field-category">
                    <div class="category-header">📊 All Fields (a$   fields)</div>
                    <div class="table-container">
                        <table>
                            <thead>
                                <tr>
                                    <th>Field Name</th>
                                    <th>Fill Rate</th>
                                    <th>Status</th>
                                    <th>Priority</th>
                                    <th>Present</th>
                                    <th>Missing</th>
                                    <th>Total</th>
                                    <th>Used Description</th>
                                    <th>Filled By</th>
                                </tr>
                            </thead>
                            <tbody>
                                u  
                            </tbody>
                        </table>
                    </div>
                </div>
            </section>

            <!-- Variables Explorer -->
            <section class="section">
                <h2 class="section-title">🧭 Variables Explorer</h2>
                <div class="field-category">
                    <div class="category-header">Browse All Variables</div>
                    <div class="explorer-controls">
                        <input type="text" id="searchInput" placeholder="Search field..." />
                        <span id="resultCount"></span>
                    </div>
                    <div class="table-container">
                        <table id="explorerTable">
                            <thead>
                                <tr>
                                    <th>Field</th>
                                    <th>Type</th>
                                    <th>Fill Rate</th>
                                    <th>Present</th>
                                    <th>Missing</th>
                                    <th>Desc Filled</th>
                                    <th>Manual Filled</th>
                                    <th>Status</th>
                                </tr>
                            </thead>
                            <tbody></tbody>
                        </table>
                    </div>
                    <div class="pagination" id="pagination"></div>
                </div>
            </section>

            <!-- Records (pgAdmin-like) Explorer -->
            <section class="section">
                <h2 class="section-title">📄 Records (pgAdmin-like)</h2>
                <div class="field-category records-card">
                    <div class="toolbar">
                        <div class="category-header" style="flex: 1;">Browse Individual Ads</div>
                        <span class="chip" id="recordsCount"></span>
                        <input type="text" id="recordsSearch" placeholder="Search records..." />
                        <label for="recordsPageSize">Page size:</label>
                        <select id="recordsPageSize">
                            <option value="10">10</option>
                            <option value="25" selected>25</option>
                            <option value="50">50</option>
                            <option value="100">100</option>
                        </select>
                        <label for="densitySelect">Density:</label>
                        <select id="densitySelect">
                            <option value="density-compact">Compact</option>
                            <option value="density-comfortable" selected>Comfortable</option>
                            <option value="density-cozy">Cozy</option>
                        </select>
                    </div>
                    <div class="table-container">
                        <table id="recordsTable">
                            <thead>
                                <tr id="recordsHeaderRow"></tr>
                            </thead>
                            <tbody></tbody>
                        </table>
                    </div>
                    <div class="pagination" id="recordsPagination"></div>
                </div>
            </section>
            
            <!-- Estate Type Performance -->
            <section class="section">
                <h2 class="section-title">🏠 Estate Type Performance</h2>
                <div class="table-container">
                    <table>
                        <thead>
                            <tr>
                                <th>Estate Type</th>
                                <th>Total Records</th>
                                <th>Avg Fill Rate</th>
                                <th>Grade</th>
                                <th>Critical Failures</th>
                                <th>Good Fields</th>
                            </tr>
                        </thead>
                        <tbody>
                            z
                        </tbody>
                    </table>
                </div>
            </section>
        </div>
    </div>
    
    <script>
        // Render Plotly visualization
        var plotData = z;
        Plotly.newPlot('plotly-dashboard', plotData.data, plotData.layout, {responsive: true});

        // Explorer data and pagination
        const explorerData = u
  ;
        let currentPage = 1;
        const pageSize = 25;
        const tbody = document.querySelector('#explorerTable tbody');
        const pagination = document.getElementById('pagination');
        const searchInput = document.getElementById('searchInput');
        const resultCount = document.getElementById('resultCount');

        function renderTable(data, page=1) {
            tbody.innerHTML = '';
            const start = (page - 1) * pageSize;
            const end = start + pageSize;
            const slice = data.slice(start, end);
            slice.forEach(row => {
                const tr = document.createElement('tr');
                const typeTag = row.is_description_var ? '<span class="tag desc">Desc</span>' : '<span class="tag regular">Regular</span>';
                tr.innerHTML = `
                    <td><strong>${row.field_name}</strong></td>
                    <td>${typeTag}</td>
                    <td>${row.fill_rate.toFixed(1)}%</td>
                    <td>${row.present.toLocaleString()}</td>
                    <td>${row.missing.toLocaleString()}</td>
                    <td>${row.desc_filled.toLocaleString()} (${row.desc_share_pct.toFixed(1)}%)</td>
                    <td>${row.manual_present.toLocaleString()}</td>
                    <td>${row.status} — ${row.priority}</td>
                `;
                tbody.appendChild(tr);
            });
            resultCount.textContent = `${data.length} fields`;
        }

        function renderPagination(dataLength) {
            pagination.innerHTML = '';
            const totalPages = Math.ceil(dataLength / pageSize);
            for (let i = 1; i <= totalPages; i++) {
                const btn = document.createElement('button');
                btn.textContent = i;
                btn.className = i === currentPage ? 'active' : '';
                btn.addEventListener('click', () => {
                    currentPage = i;
                    renderTable(filteredData, currentPage);
                    renderPagination(filteredData.length);
                });
                pagination.appendChild(btn);
            }
        }

        let filteredData = explorerData;
        searchInput.addEventListener('input', () => {
            const q = searchInput.value.trim().toLowerCase();
            filteredData = explorerData.filter(r => r.field_name.toLowerCase().includes(q));
            currentPage = 1;
            renderTable(filteredData, currentPage);
            renderPagination(filteredData.length);
        });

        renderTable(filteredData, currentPage);
        renderPagination(filteredData.length);

        // Records (pgAdmin-like) Explorer logic
        const recordColumns = z;
        const recordsData = a  ;
        const recordsHeaderRow = document.getElementById('recordsHeaderRow');
        const recordsTbody = document.querySelector('#recordsTable tbody');
        const recordsPagination = document.getElementById('recordsPagination');
        const recordsSearch = document.getElementById('recordsSearch');
        const recordsCount = document.getElementById('recordsCount');
        const recordsPageSizeSelect = document.getElementById('recordsPageSize');
        const densitySelect = document.getElementById('densitySelect');
        const recordsTable = document.getElementById('recordsTable');
        let recordsCurrentPage = 1;
        let recordsSort = { column: null, dir: 'asc' };

        // Column width preferences for wide content
        const wideCols = {
            'description': 'col-wide-xxl',
            'title': 'col-wide-xl',
            'name': 'col-wide-xl',
            'address': 'col-wide-lg',
            'location': 'col-wide-lg',
            'street': 'col-wide-lg',
            'city': 'col-wide-md',
            'url': 'col-wide-xl',
            'image_url': 'col-wide-xl',
            'images': 'col-wide-xl',
            'image_urls': 'col-wide-xl',
        };

        function getColClass(col) {
            const key = (col || '').toString().toLowerCase();
            return wideCols[key] || 'col-wide-sm';
        }

        // Build header
        function buildRecordsHeader() {
            recordsHeaderRow.innerHTML = '';
            recordColumns.forEach(col => {
                const th = document.createElement('th');
                th.textContent = col;
                th.className = 'th-sortable ' + getColClass(col);
                th.addEventListener('click', () => {
                    if (recordsSort.column === col) {
                        recordsSort.dir = recordsSort.dir === 'asc' ? 'desc' : 'asc';
                    } else {
                        recordsSort.column = col;
                        recordsSort.dir = 'asc';
                    }
                    // Update header sort state
                    Array.from(recordsHeaderRow.children).forEach(h => h.classList.remove('desc'));
                    if (recordsSort.dir === 'desc') { th.classList.add('desc'); }
                    sortRecords();
                    recordsCurrentPage = 1;
                    renderRecordsTable(filteredRecordsData, recordsCurrentPage);
                    renderRecordsPagination(filteredRecordsData.length);
                });
                recordsHeaderRow.appendChild(th);
            });
        }

        function renderRecordsTable(data, page=1) {
            recordsTbody.innerHTML = '';
            const size = parseInt(recordsPageSizeSelect.value) || 25;
            const start = (page - 1) * size;
            const end = start + size;
            const slice = data.slice(start, end);
            slice.forEach(row => {
                const tr = document.createElement('tr');
                recordColumns.forEach(col => {
                    const td = document.createElement('td');
                    const val = (row[col] ?? '').toString();
                    const isDescription = (col || '').toString().toLowerCase() === 'description';
                    td.className = getColClass(col);

                    // Content block with clamp by default
                    const content = document.createElement('div');
                    content.className = 'cell-content clamped';
                    content.textContent = val;
                    td.appendChild(content);

                    // Add description counter in top-right corner
                    if (isDescription) {
                        const words = val ? val.trim().split(/\s+/).filter(Boolean).length : 0;
                        const chars = val.length;
                        const counter = document.createElement('span');
                        counter.className = 'desc-counter';
                        counter.textContent = `${words}w | ${chars}c`;
                        td.appendChild(counter);
                    }

                    // Add show more/less toggle for long content
                    const longText = val.length > 160; // slightly earlier for 2-line clamp
                    if (longText) {
                        const btn = document.createElement('button');
                        btn.type = 'button';
                        btn.className = 'toggle-btn';
                        btn.textContent = 'Show more';
                        btn.addEventListener('click', () => {
                            const clamped = content.classList.toggle('clamped');
                            btn.textContent = clamped ? 'Show more' : 'Show less';
                        });
                        td.appendChild(btn);
                    }

                    tr.appendChild(td);
                });
                recordsTbody.appendChild(tr);
            });
            recordsCount.textContent = `${data.length} records`;
        }

        function renderRecordsPagination(dataLength) {
            recordsPagination.innerHTML = '';
            const size = parseInt(recordsPageSizeSelect.value) || 25;
            const totalPages = Math.max(1, Math.ceil(dataLength / size));
            for (let i = 1; i <= totalPages; i++) {
                const btn = document.createElement('button');
                btn.textContent = i;
                btn.className = i === recordsCurrentPage ? 'active' : '';
                btn.addEventListener('click', () => {
                    recordsCurrentPage = i;
                    renderRecordsTable(filteredRecordsData, recordsCurrentPage);
                    renderRecordsPagination(filteredRecordsData.length);
                });
                recordsPagination.appendChild(btn);
            }
        }

        function filterRecords() {
            const q = recordsSearch.value.trim().toLowerCase();
            if (!q) return recordsData;
            return recordsData.filter(r => recordColumns.some(c => ((r[c] || '') + '').toLowerCase().includes(q)));
        }

        let filteredRecordsData = recordsData;
        recordsSearch.addEventListener('input', () => {
            filteredRecordsData = filterRecords();
            recordsCurrentPage = 1;
            renderRecordsTable(filteredRecordsData, recordsCurrentPage);
            renderRecordsPagination(filteredRecordsData.length);
        });

        recordsPageSizeSelect.addEventListener('change', () => {
            recordsCurrentPage = 1;
            renderRecordsTable(filteredRecordsData, recordsCurrentPage);
            renderRecordsPagination(filteredRecordsData.length);
        });

        densitySelect.addEventListener('change', () => {
            const cls = densitySelect.value;
            document.body.classList.remove('density-compact', 'density-comfortable', 'density-cozy');
            document.body.classList.add(cls);
        });

        function sortRecords() {
            if (!recordsSort.column) return;
            const col = recordsSort.column;
            const dir = recordsSort.dir === 'asc' ? 1 : -1;
            filteredRecordsData.sort((a, b) => {
                const av = (a[col] ?? '').toString().toLowerCase();
                const bv = (b[col] ?? '').toString().toLowerCase();
                if (av < bv) return -1 * dir;
                if (av > bv) return 1 * dir;
                return 0;
            });
        }

        // Initialize records view
        buildRecordsHeader();
        // Apply initial density
        document.body.classList.add('density-comfortable');
        sortRecords();
        renderRecordsTable(filteredRecordsData, recordsCurrentPage);
        renderRecordsPagination(filteredRecordsData.length);
    </script>
</body>
</html>
        wzutf-8)encodingu$   
✅ Report generated successfully: u   📊 Summary:u      • Total Fields: u      • Regular Fields: u      • Description Variables: u      • Critical Issues: )#r   r   r   r   r   rq   r   r   ro   r   r4   r   r
   rh   ri   rX   r*   r8   r+   dumpsr    roundjoinr   r   r   nowstrftimer   _generate_field_rows_generate_estate_rowsr   to_jsonopenwrite)r   filenamer   r   kr%  r   r   total_desc_valuestotal_present_valuesdesc_overall_sharerecord_columnsc
records_qsrecords_datar   recr?   r   explorer_datar   html_contents                         r   generate_html_reportz/AdvancedDjangoDataAnalyzer.generate_html_report7  s    tU#;<//166_D$9$9__
 "??A ,7+<+<+>^+>41aaH\F]!Q$+>^(3(9(9(;W(;1qAU?Vq!t(;WU@R@R@TUU""T{?Q?Q?S"TTQe/2FFLkl doo.>!"V.&N.QAIq.&NNN4TZZ''++-44nE
CC%ggclg%([rc#hCH	 & $ 6 $))+
 ,1  &'(<&="1_#5q9#(+<)=q#A_-_-?+ !45"#$:";"'*:(;Q"?H+jM , 	 
$X X + X ,0??*;Xu<1Xv	 260Aw	Xv	B!w	Xz	 "& 1{	Xz	2 {	X|	 !) 7 78O PQ}	X|	R }	X~	 !$#WAVAV#W WXYZ	X~	[0	X@
 1B!/DA
X@
 EGA
X@
 HZZ]F^A
X@

_2A
XT
 36doo2F1GU
XT
H2U
X^
 36n2E1F_
X^
G2_
Xh
 3DA1Fi
Xh
GJi
Xl
 K]]`Iam
Xl
b2m
Xr
 36+BTBTBV6tBVQZ[\dZeisZsqBV6t2u1vs
Xr
w2s
X|
 365gR]RdRdRf5g2ghi1j}
X|
k2}
XF 365gR]RdRdRf5g2ghi1jGXFkCGXl DG{CSBTmXlU!mXN "&!:!:;!G HOXNWIOX| "778JKL}X|
M}XP ((*+QXP,QXX #jj78YXX<9YXP  $zz.9:QXP;QXR "ZZ56SXRo7	SXv (C'2aGGL! 3 	5hZ@A%c$//&:%;<='N(;'<=>.s;/?.@AB(9K9K9M-k9MAQRS[Q\`jQja9M-k)l(mnoa _W 'O ! g%([rdjjSXbe6fCHg

V 7uD 32 .lsa   6SSS-S 
S+SS?AT0T>TT!T/T,TTTc                     t        |j                         d d      }d}|D ]  \  }}d|d   j                          }|d   }|dk  rd	n|d
k  rdnd}|d| d|d   dd| d|d   dd| d|d    d|d    d|d   dd|d   dd|d   dd|j                  dd      dd|j                  dd      dd|j                  d d      dd!|j                  d"      rd#nd d$z  } |S )%z#Generate HTML table rows for fieldsc                     | d   d   S )NrW   r   rg   r   s    r   rN   zAAdvancedDjangoDataAnalyzer._generate_field_rows.<locals>.<lambda>X  s    1FWAXr   Tr   rT   zstatus-r   r   r   lowP   medium.
            <tr>
                <td><strong>z8</strong></td>
                <td>
                    r   c%
                    <div class="progress-bar">
                        <div class="progress-fill " style="width: zi%"></div>
                    </div>
                </td>
                <td><span class="status-badge z">z!</span></td>
                <td>r   </td>
                <td>r   r'   rz   rr   u   </td>
                <td>📝 r   r   r.  r   uA   %)</td>
                <td>
                    🛠️ Manual: r   z
                    rd   z6<span class='description-badge'>Description Var</span>z5
                </td>
            </tr>
            )r   r   lowerrX   )	r   
field_dictsorted_fieldshtmlrZ   r  status_classr  	bar_classs	            r   r9  z/AdvancedDjangoDataAnalyzer._generate_field_rowsV  s   z//17Xbfg!.J$U8_%:%:%<$=>L o.I!*RRXUWI 'L )?+C0 144=;>PQVWfQghkPl m0 1=~Sx@Q R:&' (?+A. /?+A. /?+A. /))$7;A>bK[]^A_`c@d e%%*YY/Eq%I!$L MUZU^U^_sUtPz|} ~# D "/8 r   c                     t        |j                         d d      }d}|D ]M  \  }}|d   }|dk  rdn|dk  rd	nd}|d
| d|d   dd|d   dd| d|d   dd|d    d|d    d|d    dz  }O |S )z)Generate HTML table rows for estate typesc                     | d   d   S )NrW   r   rg   r   s    r   rN   zBAdvancedDjangoDataAnalyzer._generate_estate_rows.<locals>.<lambda>{  s    1Q4CXr   Tr   rT   r   r   rM  rN  rO  rP  z#</strong></td>
                <td>rf   r'   z/</td>
                <td>
                    r   rQ  rR  zW%"></div>
                    </div>
                </td>
                <td><strong>r   r   rS  r   z$</td>
            </tr>
            )r   r   )r   estate_dictsorted_estatesrW  re   r  r  rY  s           r   r:  z0AdvancedDjangoDataAnalyzer._generate_estate_rowsy  s     1 1 39Xbfg"0Ko.I!*RRXUWI (M *+,Q/ 0?+C0 133<+=MeTcNdehMi j #7^, -./0 1=)* + D	 #1( r   c                    | j                  t        d | j                  D                    }g }|j                         D ]L  \  }}|j	                  ||d   t        |d   d      t        |d   d      |d   |d   |d   |d	   |d
   d	       N t        j                  |      }|j                  |d       t        d|        |S )zExport analysis results to CSVc              3   P   K   | ]  }|d    j                         D ]  }|    ywr  r  r   s      r   r[   z;AdvancedDjangoDataAnalyzer.export_to_csv.<locals>.<genexpr>  r!  r"  rd   r   r   r   r   rz   rr   r   r   )	rZ   is_description_variablefill_rate_percentr+  r   rz   rr   r   r   F)indexu   ✅ CSV export saved: )
r   r   rq   r   r    r5  pd	DataFrameto_csvr   )r   r>  r   datarZ   r  dfs          r   export_to_csvz(AdvancedDjangoDataAnalyzer.export_to_csv  s    66_D$9$9__

 !,!2!2!4JKK(+01E+F%*5+A1%E#(/@)A1#E!&!7!&!7!&!7/!*-
 
 "5 \\$
		(%	(&xj12r   N)re   )z!advanced_data_quality_report.html)zdata_quality_export.csv)r   
__module____qualname__r   r   rE   r|   r   r   r   r   r   r   r   r   r   r   rJ  r9  r:  rh  rg   r   r   r   r      se    <.8 t` DIV(T"8 
$&]~!F6r   r   )pandasrc  numpyr   plotly.graph_objectsgraph_objectsr   plotly.subplotsr   r+   r   	django.dbr   django.appsr   r2   r   rg   r   r   <module>rr     s2      ! )      
b bR-r   