, /i^ 8 d dl Z d dlZd dlmZ d dlmZmZ d Zd Zd Z d Z
d dlmZm Z d dlZd Z d dlZd dlmZm Z h d
Z d e
d e
fd
ZdedefdZd dlmZ d dlmZ de
d e
fdZde
fdZd%de
fdZd Zded efdZd ed efd Z d Z d Z d Z d! Z d"ed#ed efd$Z!y)& N
BeautifulSoup)NetworkMonitoredPage AdsManualc @ | j dd }|dk( ry|dk( ryy )N ifMissingnulltrueTfalseFget)configopts 3/var/www/extractly/manual_agregator/parser/utils.pyresolve_missing_boolr s'
**[&
)C
f}T
g~e c z | j d dk( r| j dd S | j d dk( ry y )Nr default defaultValue r r )r s r resolve_missing_textr
s;
zz+ )+zz."--
zz+ &( r c | sy t | j j j dd }ddddddddddddddd}|j D ]
\ }}||v s |c S y )N r PLNEURUSDGBP) złplnzlu zł.eureuro €usddolar$zus$gbpfunt £)strstriplowerreplaceitems)
label_or_texttext CURRENCY_MAPkvs r parse_currencyr4 s } # # % + + - 5 5c2 >D%U%5uEe%5e L ""$1 9H % r c | syt | j dd j dd }t j d| }d x}x}}|ru|j d j dd } d|v r t | n
t
| }||j d j }t | xs t | }|||fS # t $ r d }Y Kw xY w) N)NNN r ,.z([\d\s]+(?:\.\d+)?) r ) r* r- researchgroupfloatint Exceptionendr+ r4 )r0 tmnumberlabelcurrency
number_strs r extract_number_and_labelrG & s D &#&..sC8A
(!,A $$F$UXWWQZ''R0
*-*;U:&Z F !%%'( !!#!%(=N1,= 5( "" F s /C CC)r Tag)r r r# r! r& r$ r) r' > r& r! r r r# sreturnc | sy| j dd } t j dd| } | j } | j d r| d d j } | j S )Nr r6 r z\s+:)r- r: subr+ endswithr, )rI s r _norm_labelrP C s] &# A
vsA A Azz#
crFLLN 779r soupr c
'()* | r|r|j d sy|j d }|j dg xs g }|j dd '|j dd (t |j d xs d }|j d
g D ch c] }|s| c})|j d d }|j d
}|j dd }dt dt f'fd } t | h|D
ch c]
}
t |
c}
z *dt dt f'(*fd } |r| j | n| g} | D ]y }
|
j d }|D ]a }|j
dd }t | dk r%|d j dd } | | sD|d j dd }|s]|c c S { dt dt fd }g }| D ]H }
|
j
d D ]2 }|j dd }|s | | s"|j | 4 J |syd t dt f)fd }|D ] }|d v r|j }t |t r|j |v r || r|j dd }|r|c S |j d }|r^|j d! xs d"j r;|j d! j }d#|v sd$|v sd%|v r y&d'|v sd(|v sd)|v r y*|j }t |t r|r|} t# d+ D ] } t | t r | j$ nd} t | t s ng } |D ] } | j
| d D ]u } || s | |u r| j dd }|s' || rdnd}!t' | d,d xs d}"t' |d,d xs d}#|"|#k\ r |"s|#r|!dz
}!| j |!|f w | s| j) d- d. | d d c c S |d/k( rG|j+ }$t |$t r'|$j |v r|$j dd }|r|c S |j }%d}&t |%t r|&d+k r|%j d }|r^|j d! xs d"j r;|j d! j }d#|v sd$|v sd%|v r y&d'|v sd(|v sd)|v r y*|%j }%|&dz
}&t |%t r|&d+k r yc c}w c c}
w # t $ r Y Ew xY w# t $ r Y w xY w)0u
Kompatybilny ekstraktor etykieta→wartość.
Obsługuje:
- OtoDom:
ETYKIETA
WARTOŚĆ
- fallbacki: 'sibling'/'parent'/'next_tag' + valueTags
Klucze config (tak jak dotychczas, wszystkie opcjonalne poza 'label'):
- label: str
- selector: str (zawęź obszar szukania)
- altLabels: list[str]
- caseInsensitive: bool (domyślnie True)
- match: "equals" | "contains" (domyślnie "equals")
- valueTags: list[str] (domyślnie ["p","span","div"])
- labelPosition: "sibling" | "parent" | "next_tag" (domyślnie "sibling")
rD N altLabelscaseInsensitiveTmatchequals valueTags)pspandiv valueClasses
labelPositionsiblingselectorparentFallbackrI rJ c T | xs dj } r| j S | S )Nr )r+ r, )rI case_insensitives r normz$extract_value_by_label..normi s&
W"OO ,qwwy3!3r r0 c r t | n | j j d dk( rt fdD S v S )NrL containsc 3 & K | ] }|v
y wN ).0wrA s r z;extract_value_by_label..is_match..q s .v!qAvv )rP r+ rstripany)r0 rA ra
match_typewanteds @r is_matchz(extract_value_by_label..is_matchn sE !1K tzz|7J7J37O #.v.. .F{r z=div[data-sentry-element="ItemGridContainer"], div.css-1xw0jqprX ) recursive r r r+ r9 c | xs dj t t j d xs t fdt
D S )Nr z\dc 3 B K | ] }|j v y wrf r, )rh hrA s r rj z>extract_value_by_label.._good_value.. s /X1QWWY )r+ boolr: r; rm _CURRENCY_HINTS)r0 rA s @r _good_valuez+extract_value_by_label.._good_value s;
ZR BIIeQ'(XC/X/X,XXr tagc sy| j d syt t rj t fdD S )NTclassFc 3 & K | ] }|v
y wrf rg )rh clsclassess r rj zGextract_value_by_label.._matches_value_class.. s ;]c3'>]rk )r
isinstancer* splitrm )r| r
value_classess @r _matches_value_classz4extract_value_by_label.._matches_value_class sC '''" gs # mmoG;];;;r )r] parentimgsrcr ikona_takicon_yesyestak ikona_nieicon_nononie
sourcelinec | d S )Nr rg )xs r z(extract_value_by_label.. s !A$r )keyreversenext_tag)r
tupler* rP ry selectfind_alllenget_textappendrH find_next_siblingr namefindr, r? ranger getattrsort find_next)+rQ r rD
alt_labels
value_tagscstrategy
root_selectorallow_parent_fallbackrb r rp rootsrootrowsrowpslabval_textr{ label_nodeseltxtr sibvalr r up_
candidatestag_namecandscorec_linee_linenxt
value_cellhopsra rn r ro s+ @@@@r extract_value_by_labelr M sg vVZZ%8 zz'*E zz+r28bJ zz"3T: zz'84J VZZ 4 L8LMJ#)::nb#AG#AaQ#AGM zz/9=H zz*-M "JJ'7 >4 4 4 %
!Z$HZ[^Z$H
HFs t +8DKK
&dVE {{Z[C cT 2B2w{Q%..D.1CC=!u~~c~6H Y# Y$ Y
K--%B++c+.Cx}""2& & <# <$ < , ,&&(C S#&88z).B3.G ,,s$,7C "
!hhuo CGGEN$8b#?#?#A"%''%."6"6"8C *c 1Z35F%SV,', *c 1Y#5EQT', ++-! S#&& !B1X",R"5RYY4!"c*
*H " H E 3D 9 $ 2: $ "mmCtm < " $&1#&6A!(|Q!?!D1!(\1!=!B!V+6 !QJE"))5#,7 !F !+ OOOE%a=++/ 4 z !,,.C#s#J(>ll3dl3 J --/JD Z-$( ooe,CGGEN 0b779 ''%...0C"c)Z3->%3, $"c)Y#-= $ ' 9 9 ;
Z-$( X M H %IF % n sJ T* T*=T/
A!T4- T4#BU- U;*U4 U U UU) OrderedDict)urljoinuc X | sy| j } | j d rd| z S | S )Nr z//zhttps:)r+
startswith)r s r _normalize_urlr s. A||D!| Hr srcsetc g }| s|S | j d D ]F }|j j d d j }|s6|j | H |S )uQ
Parsuje atrybut srcset i zwraca listę URL-i (bez descriptorów w/h/x).
r7 r r )r r+ r )r urlspartr s r
_split_srcsetr s^ D S!zz|!!#&q)//1 KK " Kr base_urlc t |r| j | n| g}dt ffd }|D ] }|j d D ] } ||j d ||j d ||j d ||j d |j d xs |j d }|st | D ]
} || |j d
D ]A } | j d xs | j d }|s*t | D ]
} || C |j d D ] }
d d
lm} | |
j d } | j d D ]o } ||j d ||j d |j d xs |j d }|sXt | D ]
} || q | j dddi }
|
r |
j d r ||
d | j dddi }|r |j d r ||d t j S )u
Zwraca listę absolutnych URL-i obrazów, bez duplikatów, z zachowaniem kolejności.
Wspiera: , lazy atrybuty, , meta og:image/twitter:image.
Jeśli podasz `selector`, przeszukuje tylko wybrany fragment DOM-u.
urlc t | } | sy | j d sC| j d s2r/| j d s| j d s
t | } ny | vrd| < y y )Nzhttp://zhttps:///zdata:T)r r r )r r seens r addz extract_image_links..add sg S! y)S^^J-GS^^C 0w8O h, d? DI r r r zdata-srcz
data-originalz data-lazyr z data-srcsetsourcenoscriptr r z html.parsermetapropertyzog:image)attrscontentr z
twitter:image) r r r* r r
r bs4r decode_contentsr listkeys)rQ r^ r scopesr scoper r r r nsr innerogtwr s ` @r extract_image_linksr s =D&.T[[
"TFF >>%(C
# $ ( ) $ %WWX&@#''-*@F&v.AF / ) >>(+CWWX&@#''-*@F&v.AF / , ..,B
*!""4"4"6
FE ~~e,CGGEN#CGGJ'( *Dcggm.D *6 2A 3 - -/ L
6*j!9 :B bffY ByM
6&/!: ;B bffY ByM r c | yt | t r| j dk( ryt | t t f r| syy)NTr F)r r* r+ r dict)r3 s r value_is_emptyr X s6 y!Saggi2od!dD\"1T r inactive_rulesc 6 |xs g D ] }|j d }|dk( rI|j d xs dj }|s@|| j xs dj v sa y|dk( rr|j d }|j d xs dj }|r|j | nd }|s|j d j |k( s y|d k( rt|j d }|j d xs dj }|r|j | nd }|s,||j d
d j v sQ y|d k( r*|j d }|sm|j | r y|d k( s|j d
} |j d xs dj }
t | | d } t
| t rt j | } | s|
t | j v s y y)Ntype
text_containsr0 r T
selector_textr^ rs selector_containsr selector_missingsource_field_matchfieldrU F) r
r, html
select_oner r r r jsondumpsr* ) pagerQ r rulerA r selexpectedr r rU values r check_inactiver ^ s $"$ HHV 88F#)r002CstyyB 5 5 77
/
!((:&C (.B557H), %$Bbkkk-335A
%
%((:&C88F#)r002C), %$BcR[[D[ 9 ? ? AA
$
$((:&C4??3/
&
&HHW%EXXg&,"335ED%,E%&
5)#e*"2"2"44; %< r dc b t d | j D S # t $ r Y yw xY w)Nc 3 ` K | ]& }t |t xr d |v xs
d|v xs d|v ( yw) fieldType selectorsfromMainN)r r )rh r3 s r rj z'is_field_config_dict.. s@
q$ ][A%5%\9I%\Z[\_ ] s ,.F)rm valuesr? )r s r is_field_config_dictr s;
XXZ
s " ..c | sg S t | t rt j | } t | t
r| gS t | t t f r t
| S g S # t $ r g cY S w xY w)u
Zwraca listę reguł (list[dict]) albo pustą listę.
Akceptuje: None, str(JSON), dict, list/tuple.
Inne typy -> [].
)r r* r loadsr? r r r )ruless r normalize_rulesr so eS !JJu%E % w%$ 'E{
I
s %A! ! A/.A/c | si S t | t r* t j | }t |t r|S i S t | t r| S i S # t
$ r i cY S w xY w)u
Zwraca dict z selektorami (gałęzie lub płaska mapa pól).
Akceptuje: None -> {}, str(JSON) -> dict lub {},
dict -> dict, inne -> {}.
)r r* r r r r? )r r s r normalize_selectorsr
si )S! **Y'C$S$/3 7R 7 #9d39;; I s &A A A! A!c d}t t | dd }t t | dd }|D ] }t |t sd|v rd|v r|j d }|r|j
| nd}|sE|j d j |j dg }t |t r|g}t fd |D s|j d } n d
|v sd|v s |r ||v rt || t r|| |fS d |v rt |d t r|d d fS t | r|d fS t | d
k( r9t t |j \ } }
t |
t r|
| fS i dfS )u
Zwraca ZAWSZE krotkę: (selectors_dict, selected_type|None).
Obsługuje:
- reguły starego typu: {'selector','match','type'} / {'source','match','type'}
- 'default' gałąź
- płaską mapę pól
- jedyną gałąź
Nr r r^ r Trs rU c 3 B K | ] }|j v y wrf rv )rh r3 r s r rj z$resolve_selectors.. s 8RqwwyG+Rrx r r flatr9 )r r r
r r r
r r r, r* rm r r nextiterr. )
manual_configrQ
selected_typer all_selr r r mvr2 r3 r s @r resolve_selectorsr s M GM7D A BE!'-d"KLG $% &D.((:&C), %$B ++D+1779XXgr* b#&bT8R88$(HHV$4Me
&D. '1j AWY]6^}%}44G
79+=t Dy!9,,G$
7|qD )*1 a a4K t8Or c dt dt ffd t | }|D ] }t |t s|j d }t |t s6d|v r$t fd|j dg D nd}|j dg }|rt
fd |D nd}|s|s|j d
c S y ) u
Nowy styl:
{ "when": { "all": [...], "any": [...] }, "type": "otodom_v2" }
Zwraca nazwę gałęzi (type) lub None.
condrJ c R d| v r t j | d S d| v rj | d d u S d| v r/| j d }t |xr j | S d| v r1| j d }t | xr j | d u S d| v r| d j d }| d j d xs dj }|rj | nd }t |xr# ||j d d
j v S y )
N field_empty
field_missingselector_existsr r r^ r0 r r Trs F)r r
ry r r, r )r r r r extractedrQ s r cond_okz$apply_dynamic_rules..cond_ok s+ D !)--]0C"DE E d " ==o!674? ? $((,-C4 45 5 %((-.C9?$//#"6$"> ? $ &*+//
;C+,008>BEEGC), %$BJsbkk#Tk&B&H&H&J JK Kr whenallc 3 . K | ] } | y wrf rg rh r r s r rj z&apply_dynamic_rules.. s =). s 2AWQZr! r N)r ry r r r
r rm )
r rQ r
rules_listr r all_okany_listany_okr s
`` @r apply_dynamic_rulesr' s d t $ !'J$% xx $% AF$ =%)<==TX88E2&6>222D f88F# # r datar c | t j j D ch c] }|j dvr |j }}| xs i j D ci c]
\ }}||v s || }}}d|v r#| xs i j d|j |d< d|v r>| xs i j d }|"t |di xs i }|j d }||d< dD ]) } | |v s|j | t || d || < + d|v rd|vr|j |d< d|v rd| xs i v r| d |d< |j dd }
|
d u xs# t |
t xr |
j } | r|j d | xs i j d } |j d
| xs i j d
}
d } || } ||
}|(|&|d kD r!t t ||z }d|v r||d< n
d|v r d|vrd |d< d
|v rF|j d
| xs i j d
}dt dt fd} || }|d}||d
< d dlmm fd}dD ]! }||v s ||j | ||< # |S c c}w c c}}w # t $ r Y w xY w# t $ r Y ]w xY w# t $ r Y |S w xY w)N)id
created_at is_activeinactive_reasonr ) image_linksr r
inactive_date date_fetchedr+ r estate_type
offer_typer original_image_urls price_per_m2pricesquare_footagec | y t | t t f r t | S t | j dd j dd j dd }t | S # t
$ r Y y w xY w)Nr6 r r r7 r8 )r r> r= r* r- r? )r3 rI s r _to_floatz+map_data_to_manual_model.._to_floatA sp 9 a#u . 8O Avs3;;CDLLSRUVA 8O s AA, , A87A8r area_unitr rJ c ~ | y t | j j }|j dd j dd j dd j dd j dd }dd l} |j
d| }|r|j
d S d
|v r d |v sd |v ryd
|v ryd|v ryy )Nu m²m2zm^2zm kwmkwzm.2r z\b(m2|ha|ar|ft2|sqft)\br9 rB 2 ²hektarhaar)r* r+ r, r- r: r; r< )r rI r: rB s r _normalize_unitz1map_data_to_manual_model.._normalize_unit^ s ; HNN$**,IIeT*225$?GGPTU]]^ceijrrsxz~ BII 8!< 771:%!8TQY q= 19 r r; )datetimedatec | y t | rt | s| S t | j }|sy |j dd j dd j dd j }h d}||v ry g d}|D ]$ } j || j
c S y # t $ r Y 3w xY w)Nu „"u ”u ’'> —n/an/d
nie podano nie dotyczybrak informacji-brak)z%Y-%m-%dz%d.%m.%Yz%d/%m/%Yz%d-%m-%Y)r r* r+ r- r, strptimerD r? )r rI s_norm placeholdersfmtsfmtrD rC s r _clean_date_valz1map_data_to_manual_model.._clean_date_val s {#t$ZX-F
C AYYx-55hDLLXWZ[aacFmL %D #,,Q499;; ! s B22 B>=B>)available_from listing_date)r _metafieldsr r. r
r, r r r r* r+ r> roundr? rm rC rD )r( r fallowedr2 r3 mappedreasonr r pp2is_missing_pp2 price_valarea_valr8 rX acomputedraw_urB rb rU dfrD rC s @@r map_data_to_manual_modelrg
s/ '''A 66- -
' !%
1 1 3
D 3tq!qG|ad 3F
D g #zr . .{DNN K{G#*"!!"34 >4,2DXX /0F$* ! '>vzz# 6!$T2F3K
E / u
',Adjb,Q(,-B(C$%
jj . +T:c3+?+S O
7TZR,<,*!W,.2N-1F>*
' !JJ{TZR,<,<[,IJE
S
S
& #5)D|"&F; &
+ < 5BV| ,VZZ^ <r
5 Ms EJ
@
T
M
sO !J J
&J
1CJ A
J J. & J. J J J+*J+. J;:J;)NN)"r r: r r extractly.modelsr r r r r4 rG rH rz r* rP r r collectionsr urllib.parser r r r r r ry r r r r
r r' rg rg r r rk s <
#& # G "33 3 ] ] ]P $
c
c
# D s D T t BD T &<$-b$TZ4 Z/C Z Zr