
    h+                         d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZm	Z	m
Z
 d dlmZ ddlmZmZmZmZmZ ddlmZ d d	lmZ d d
lmZmZ d dlmZ dedefdZdde	defdZdedefdZ y)    N)BeautifulSoup)now)NetworkSourceErrorNetworkMonitoredPageSourceManual)send_alert_notification   )check_inactivevalue_is_emptynormalize_rulesnormalize_selectorsmap_data_to_manual_model)get_handler)apply_dynamic_rules)apply_transferredcompute_stats)is_inactive	selectorsstrictc           	         i i }}|xs i j                         D ]  \  }}t        |t              s|j                  d      s)d|v rdn|j                  dd      }t	        |      }	 |j                  ||| |||      }	|	||<   |	||<   |r|	t        d|       	 t        |j                         D 
cg c]  }
|
 c}
      }t        d| d|	dnd	 d
|         |xs i j                         D ]  \  }}t        |t              r|j                  d      r(d|v rdn|j                  dd      }t	        |      }	 |j                  ||| |||      }	|	||<   |r|	t        d|       	 t        d| d|	dnd	         ||fS # t        $ r d }	Y w xY wc c}
w # t        $ r Y w xY w# t        $ r d }	Y gw xY w# t        $ r Y w xY w)NisMainfromMain	fieldTypetextzValue not found (isMain): z	[isMain] z -> Noneokz; main_keys=zValue not found: z[field] )items
isinstancedictgetr   parse	Exception
ValueErrorsortedkeysprint)soupr   r   	extractedmain_values
field_nameconfig
field_typehandlervalkmv_keyss               4/var/www/extractly/manual_agregator/parser/engine.py_parse_with_selectorsr2      s#   {I  )B557
F&$'::h'1V';KY_A`J!*-GmmJk9V_` '*K
#$'Ij!#+ #=j\!JKK [-=-=-?!@-?!-?!@A	*TS[6d1TT`ah`ijk# 8,  )B557
F&$'6::h+?#-#7ZVZZU[=\
j)	--
FD+yR[\C !$	*ck0=>>	HZLf,OPQ 8" k!!?   "A   	C	  		sZ   &FF0	F
9F:F*(F;FFF	F'&F'*F87F8;	GGpagereturnc           
         | j                   }	 |j                  }	 t        d       t        d| j                          t        d| j                  xs d	d d
  d       | j                  xs d	}t        | dd       xs d	}|r| d| n|}t        |d      }|rt        |d      nd }|r5t        |j                               t        |j                               kD  r|n|}	i }
t        t        t        |dd             |	| |
d      \  }}|r|
j!                  |       t#        t        |dd             }|}t%        |t&              r|r||v rt%        ||   t&              r||   }nt)        |      }d|v rt%        |d   t&              r|d   }|xs d}nY|r	|}|xs d}nNt        |      dk(  r<t+        t-        |j/                                     \  }}t%        |t&              r|ni }|}ni }d }|r||vrt        d| d|xs d d       n|}|st        d       yt        d|xs d dt1        |j3                                       t5        |	||      \  }}|
j!                  |       t        t        t        |dd             |	| |
d      \  }}|r|
j!                  |       |r?||k7  r:t%        |t&              r*||v r&t        d| d       |}||   }t5        |	||      \  }
}t7        | ||
dd       }t9        ||
|t;        |      !      |
d"<   t        d#|
d"           t        |d$g       xs g }t=        ||	|| |
%      \  }}| |
d&<   |rVd| _        tA               | _!        | jD                  xs i | _"        || jD                  d'<   t        d(| j                   d)|        n	 tG        jH                  | j                  d*d+d,i-      }|jJ                  xs d	}|rt        |d      }t=        |||| |
%      \  }}|rid|
d&<   d| _        tA               | _!        | jD                  xs i | _"        || jD                  d'<   d.| jD                  d/<   t        d0| j                   d)|        tO        jP                  |
d1      | _)        |
| _*        d| _+        | jY                          t        d2| j                   dd        y# t        j                  $ r3 t        j
                  j                  |dd       t        d|        Y yw xY w# tL        $ r Y w xY w# tL        $ r t        d3| j                   dt[        j\                                 t        j
                  j                  |t[        j\                         d4       t_        d5| j                   d6|j`                   d7t[        j\                          d8|j`                   9       Y yw xY w):Nu)   Brak ManualDataSource dla tego źródła.MissingManualDataSource)sourceerror_message
error_typez3[ERROR] ManualDataSource does not exist for source FzP================================================================================z[INFO] Parsing page: z%[INFO] First 500 characters of HTML:
 i  
sliced_htmlzhtml.parserrulespre)phaser   defaultflatr	   z[WARN] Type 'z$' not in selectors; using fallback 'unknownz'.z/[ERROR] No usable selectors resolved. Skipping.z[INFO] Using selectors (type: z) -> postz*[INFO] Post-rule matched -> switching to 'z' and re-parsingT)skip_if_present	overwrite)r   r(   selected_typeextra_exclude_statsz[INFO] Stats: inactive)htmlr'   r=   r3   r(   	is_activeinactive_reasonz[INACTIVE] Ad is not active     z
User-AgentzkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0 Safari/537.36)timeoutheaders
live_checkinactive_viaz#[INACTIVE-LIVE] Ad became inactive )ensure_asciiz*[SUCCESS] Data parsed and saved for page: z&[EXCEPTION] Error while parsing page: ManualParsingErroru   Błąd parsowania ogłoszenia z (z)

u   ❌ Błąd parsowania: )subject)1r7   manual_data_source_fetcherr   DoesNotExistr   objectscreater&   urlrJ   getattrr   lenget_textr   r   updater   r   r   _looks_like_fields_mapnextiterr   listr%   r2   r   r   setr   rK   r   inactive_datemetarequestsr    r   r"   jsondumpsraw_data
parse_datais_completesave	traceback
format_excr   title) r3   r   r7   manual_config	full_htmlr<   merged_html	soup_fullsoup_slicedr'   r(   pre_typepre_setall_sel_rootrF   r   
looks_flatr/   v
parsed_nowr)   	post_typepost_settransferred_keysinactive_rulesrI   reasonresp	live_html	soup_liveinactive_livereason_lives                                    r1   parse_manual_datar   E   s   [[F	&,&G&Gch%dhhZ016		R#7N6OrRS IIO	dM48>B7B2k]3	!)];	CNmK?TX*s;3G3G3I/JSQZQcQcQeMf/f{mv 	/GM7DAB$	
' W%*7=+t+TU !lD),!>:l[hNikoCp(7	 4LA
,L<SUY1Z ,Y 7I$1$>YM ,I$1$;VM <(A-#D););)=$>?1)3At)<A"	()$&	(, <M(3WXeXrirWssuvw$ICD.}/I	.J%PTU^UcUcUePfOghi #8i"P
K$ 2GM7DAB$	
	8 X& m3
<QU8V[dht[t>ykIYZ[%M$Y/I%:4F%S"I{ -- 
 ,'./	
	( 	y2345 !
B?E2& 
& &.	+"DN!$D		RDI+1DII'(0
!F8DE||DHHb   #PC  !IIO	 -i GI1<&&,!"+2.M; %16	+.).-0U*$(IIO	7B		"344@		.1 CDHH:Q{m\] 

95A#		:488*BxjQR $$ ""))E0 	* 	

 	CF8LMf    6txxj9CWCWCYBZ[\""))#..0+ 	* 	

 	 ,TXXJbeIL`L`LbKcd-fll^<	
 sR   R3 G:T FT C S< AT 3AS98S9<	TT TT B3W Wdc                     t        | t              sy| j                         D ]%  }t        |t              sd|v sd|v s	d|v sd|v s% y y)NFr   selectorr   r   T)r   r   values)r   ry   s     r1   r_   r_      sN    aXXZaK1$4
a8WX=\fjk\k      )T)!rg   rm   rf   bs4r   django.utils.timezoner   extractly.modelsr   r   r   manual_agregator.notificationsr   utilsr
   r   r   r   r   handlersr   manual_agregator.parser.rulesr   #manual_agregator.parser.transferredr   r    manual_agregator.parser.inactiver   r   boolr2   r   r_    r   r1   <module>r      sq       % S S B  " = P 8+"4 +" +"^p0 p$ phd t r   