.$hN ddlmZddlmZmZddlmZddlmZm Z ddl m Z ddl m Z ddlZddlZddlmZd ed efd Zd ed efd Zd ed eeeffdZd ed eeeffdZd&deeded efdZdedededeededzf dZdeed efdZd ed eeeffdZd'deededzd eefdZ d'deededzd efdZ!defdZ"de defdZ#de defd Z$de defd!Z%de d"efd#Z&Gd$d%eZ'y)() BaseCommand)QCount)now)FieldDoesNotExist FieldError)models)PathN)NetworkMonitoredPage field_namereturnc |jj|}t|tj tj fS#t$rYywxYw)uDSprawdza czy pole (liść, bez '__') jest Char/Text na danym modelu.F)_meta get_fieldr isinstancer CharField TextField)modelr fs A/var/www/extractly/extractly/management/commands/parser_health.py_is_text_fieldrsO KK ! !* - a&**F,<,<= >> sA AA field_pathcd|vr!tdi|dditdi|ddizStt|r!tdi|dditdi|ddizStdi|ddiS)u Q wykrywające 'pustą' wartość w polu/ścieżce: - zawsze IS NULL - dla pól tekstowych także whitespace-only przez __regex ____isnullT__regexz^\s*$)r )rrr )rs r_empty_qrs  z3zl(+T23a6]j\;QS[:\6]]]*zB3zl(+T23a6]j\;QS[:\6]]] /:,h'. //ct|} tjj|j dfS#t $r7tjjdi|ddij dfcYSwxYw)z#Zwraca (count, null_only_fallback).FrTrrr objectsfiltercountrrqs r_count_total_empty_for_fieldr'$s}Ad#++2215;;=uDD d#++22U H6Mt5TU[[]_cccds.<=A<;A<c t|} tjj|dj dfS#t $r;tjjdi|ddiddij dfcYSwxYw)zJZwraca (count_inactive, null_only_fallback) dla pustych z is_active=False.F is_activerTr*rr!r%s r%_count_total_empty_inactive_for_fieldr+-sAu#++2212FLLNPUUU u#++22f H6Mt5Tf`efllnptttus0>ABBfields use_null_onlyci}|D]w}d|jddz}d|jddz}|rtd i|ddi}n t|}td|||<td|td z||<y|r%t}|D]}|td i|ddiz}nt}|D]}|t|z}td||d <td|td z|d <|S)u Adnotacje do per-name; dodaje: - empty__{fp} dla każdego pola - inactive__{fp} dla pustych + is_active=False - empty_any = OR(po wszystkich _empty_q) - inactive_empty_any = OR(po wszystkich _empty_q) + is_active=False empty__r_ inactive__rTid)r#Fr) empty_anyinactive_empty_anyr)replacerrr)r,r-annfp empty_aliasinactive_aliasq_emptyany_qs r_build_annotations_for_fieldsr<8s C"**T3"77 % 4(== 2bT?D12GrlG g6K#D1u;M1MNNB Q1RD/401 1EB Xb\ !ET%0C  %d51u;M3M NC Jrgrouplimitfield selected_namec tjj}|xs|gDcgc]&}|s|js|j(}}i}i}g} g} |D]O} t | \} } | || <| r| j | t | \}}||| <|s?| j | Q||d}||d}d}d}d}|r$ t|d}tjjdjddtdi|jdd}i}|D]}|dxsd }|d|jd d|jd diid }|D]\} d| jddz}d| jddz}|j|d|d| <|j|d|d| <^|||<tt!|j#dt%d|}|||vr d|i||}ntjj'|}|j}ii}}|D]]} t)|| \} } | || <| r| | vr| j |  t+| }!|j'|!dj|| <_ t-|}"|j'|"j}#|j'|"dj}$|||#|$||d}t1j3||d|||||| | d }%| ||%d<||%d<|||%d<|%Scc}w#t$rZt|d }tjjdjddtdi|jdd}YowxYw#t$r0|j&di| dd iddij|| <YwxYw#t$rht/}"|D]} |"t/di| dd iz}"|j'|"j}#|j'|"dj}$Y[wxYw)NrF)r-nametotalr2z -empty_anyz-totalT(null)r3r4)rCr3r4empty_by_fieldinactive_by_fieldr/rr0r1rErFrBr)rr*)rBrCr3r4rErF) tsr, primary_field total_pages empty_totalempty_total_by_fieldinactive_total_primaryinactive_total_by_fieldnull_only_fallback_fields"null_only_fallback_fields_inactiveby_name _by_name_topr@r)r r"r$stripr'appendr+r<valuesannotaterorder_byrgetr5dictlistitemsmaxr#"_count_total_empty_for_field_for_qr_or_q_for_fieldsrr isoformat)&r=r>r?r,r@rCrrMrOrPrQr7cnt null_only cnt_inactnull_only_inactrLrN by_name_toprRselected annotationsrowsrkeyentrye_aliasi_aliasbase total_sel empty_map inactive_mapnlfr&r; empty_any_cntinactive_any_cntsnaps& r_count_snapshotrv]s ( ( . . 0E"("3UG"3 H"3Aaggiaggi"3F H+-.0+-46&5b9Y#&R  % , ,R 0%J2%N" ?&/#  . 5 5b 9'vay1K4VAY?KGH  7eTK$,,33F;< %d >./eeGQ.?&'+12w1B)*2.  !GCL 4 03q%=AB  $'"MLW]5KL+33:: :N JJL *,b<  BA$KHC$'IbMr)BB188<k$RL+/;;qE;+J+P+P+R R(! S,V4E$(KK$6$<$<$>M'+{{5E{'J'P'P'R$*&!.*:&/)5 eoo" 4#9#:%>.P D !Y*^ (_ KwIB 7dSK$,,33F;< %d M'+{{5E{'J'P'P'R$ SsJL)L)L) AL.(/NA O.ANN5O  O A-QQcDt}|D]}|t|z}|SN)rr)r,r&r7s rr_r_s& A Xb\ Hrct|} |j|jdfS#t$r*|jdi|ddijdfcYSwxYw)NFrTr)rr#r$r)qsrr&s rr^r^smAJyy|!!#U** Jryy;zl(3T:;AACTIIJs .0A! A!c tjj}|r|j|} t |jt |j dd}|S#t$rSt}|D]}|tdi|ddiz}t |j|j dd}Y|SwxYw)uTZwraca listę ID pustych w *którymkolwiek* z pól (opcjonalnie zawężone do name).rHr2T)flatrr) r r"allr#r[r_ values_listrr)r,r@rzidsr;r7s r_collect_empty_idsrs  % % ) ) +B YYMY *B299-f56BB4dBST J BB Q1RD/401 1E299U#//4/@A J Bs4A**ACCc tjj}|r|j|} |jt |j dd}i}|D]A\}}|xsd} |j| dgd} | d j|| d xxd z cc<C|S#t $rHt}|D]}|td i|ddiz}|j|j dd}YwxYw) u Zwraca słownik: name -> {"count": int, "ids": [int, ...]}. Jeśli selected_name podane, zwraca tylko tę jedną grupę. rHr2rBrTrDr)r$rrr$rGr) r r"r}r#r_r~rr setdefaultrU) r,r@rzpairsr;r7grouped_idnmrjbuckets r_collect_empty_ids_groupedrs  % % ) ) +B YYMY *; *623??fM "GRnH##C1R)@Au S!w1  N ;B Q1RD/401 1E % ,,T6: ;s*B((AC98C9ructdtd|dtddj|dtd|dtd |d td |d d |jddd|jdr|td|dD]i}|dj|d}|jdij|d}||jdgvrdnd}td|dd|d|d|kd|vr|drtd|djD]\}}td|dd|dd d!|d"d d#|jd$dd d |jd%i}|jd&i}|dD]:}td'|d(d)|j|dd d*|j|dd d<yyy)+Nu# Parser health — current snapshotzTime: rIzFields: , r,zPrimary field: rJzTotal pages: rKzEmpty (primary): rLz (inactive: rNr)rMz By field (totals):rOrPz (NULL-only)- 40z empty= (inactive: rSz By name (top):30z total=rC6dz empty_any=r3 (inactive_any: r4rErFu · 36 z (inactive: )printjoinrYr\) rur7rainactmarkerrBvalsebfibfs r_print_snapshotrsC 01 d4j\ *+ diiX78 9: d?34 56 d=12 34 d=12.JbdeAf@ggh ij xx&' $%x.B-.222q9CHH6;??AFE(*dhh7RTV.W(W_]_F Br"gWSEeWAfXF G ! $~"6  !~.446JD$ BtBiwtG}R&8 T+EVWYDZZjkoksksuIJKlLMOkPPQR S((+R0C((.3C8n2waAr':-PRTUWYGZZ[\]% 7#7rpathcg}|jd|d|jddj|d|jd|d|jd|d |jd |d d |jd dd|jdr|jd|dD]n}|dj|d}|jdij|d}||jdgvrdnd}|jd|d|d |d|p|jdr|jd|djD]\}}|jd|d|dd|dd|jddd |dD]]}|jd |d|jd!ij|dd |jd"ij|dd_|jd#|jj d$d$%|j d&d'(jd)j|d)zy)*Nz SNAPSHOT rIzfields: rr,zprimary_field: rJztotal_pages: rKzempty_primary: rLrrNrrrMzempty_total_by_field:rOrPz (NULL-only)rz - z: rRzby_name:z: total=rCz , empty_any=r3rr4u · rErFz---Tparentsexist_okautf-8encoding )rUrrYr\parentmkdiropenwrite) rrulinesr7rarrrBrs r _append_txtr6s E LL9T$ZL)* LL' $x.(A'BCD LL'_(='>?@ LL'](;'<=> LL'](;'>7>3DXXmTFF &41!'')QE4D56]\\^A.446 :,'(# G H#!5 0 )!A.//0sc-IA8H>:I=BIAII1I5 I I I  I I5I00I5ceZdZdZdZdZy)Commanda*Counts empties for given field(s) in NetworkMonitoredPage, grouped by 'name', and writes a snapshot to a TXT/JSON/JSONL file (with history). Use --print to output JSON list of IDs that are empty in ANY of the selected fields. Use --print-by-name to output JSON grouped by 'name' with IDs per group.c|jdddd|jddd |jd td d |jdtdd |jddd |jddd|jdd|jdd|jdddd|jdd dd!y)"Nz--outputz-ozparser_health.jsonlz0Output file path (.jsonl default), or .json/.txt)defaulthelpz--group store_truez6Include per-name breakdown in snapshot (console/file).)actionrz--limitz3How many 'name' groups to print in console (top-N).)typerrz--lastrz=After writing, display last N snapshots from the output file.z --no-printz)Do not print current snapshot to console.z--fieldnetwork_ad_manualz0Primary field (used when --fields not provided).z--fieldszWComma-separated fields (e.g. 'network_ad_manual__html,network_ad_manual__sliced_html').)rz--namezIOptional exact name filter for print modes and snapshot selected section.z--print print_idsz;Print JSON with IDs empty in ANY of the fields (flat list).)destrrz--print-by-name print_groupedz0Print JSON grouped by 'name' with IDs per group.) add_argumentint)selfparsers r add_argumentszCommand.add_argumentssJ6K!S  UIl!Y  [IC!V  XH3!`  bL!L  NI/B!S  UJ!z  |H!l  nIK !^  `-OL!S  UrcNt|d}t|jd}t|jdxsd}t |jdxsd}|jd}|r-|j dDcgc]}|j c}ng} |jd } | xs|gD cgc]} | s|  } } |jd rg|jd sVt| | } tj| | d t| | d} ttj| dy|jd rt| | }td|j!D}t#|j%dDcgc]}|||d||dd}}tj| | d ||d} ttj| dyt'|||| | }|jds t)||j*j-}|dvrEt/|||j0j3|j4j7d|dn|dk(rEt9|||j0j3|j4j7d|dn|dk(rEt;|||j0j3|j4j7d|d nFt/|||j0j3|j4j=d!|d"|t|jd#xsd$}|d$kDr t?||yycc}wcc} wcc}w)%Noutputr=r>rr?rr,rrBrr)r@any)rIr,rBmatchr$rFrc3&K|] }|d yw)r$Nr).0rs r z!Command.handle..sK:JF7O:Jsc|du|fSrxr)rs rz Command.handle..s !t)Qr)rjr$r)rBr$r)rIr, name_filterr total_countgroups)r=r>r?r,r@no_printrz Saved to z (JSONL append)rz (JSON array)rz (TXT append)z Unknown extension 'z', saved as JSONL to rr) r boolrYrstrrrTrrr`lenrrrrsumrVsortedkeysrvrrrrstdoutrstyleSUCCESSrrWARNINGr)rargsoptsout_pathr=r>rJ fields_rawsr,r@rrpayloadr total_idsrB groups_listrurlast_ns rhandlezCommand.handlesDX'TXXg&'DHHW%+,DHHW-D1DE XXh' ?IZ%5%5c%:;%:!'')%:;r( $77>7A!7> 88K /)B$V=ICeoo' %S G $**W59 :  88O $0}UGK'..:JKKI#7<<>7OPPD g(>wt}UZG[\P  eoo' ,(% G $**W59 : '  xx # D !oo##% . (D ) KK  djj00;xj1XY Z G^ x . KK  djj00;xj 1VW X F] $ ' KK  djj00;xj 1VW X (D ) KK  djj003HMbckbl1mn oTXXf%*+ A: 6 2 C<>*s NN N#N"N)__name__ __module__ __qualname__rrr rrrrrs S U,H3rr)Frx)(django.core.management.baserdjango.db.modelsrrdjango.utils.timezonerdjango.core.exceptionsrr django.dbr pathlibr rrextractly.modelsr rrrrtuplerr'r+r[rZr<rvr_r^rrrrrrrrrrrrs3%%@ 1?c?d? 0 0 0dSdU395EducueCI>Nu $s) D UY J~d~3~s~DI~^adh^h~J T#Y 1 JsJuS$Y?OJ tCy t tTWy tCytW[4^$^8DdD$D4XXDX UT U U&0t&03&0Rf3kf3r