:hUddlZddlZddlZddlmZddlmZddlmZm Z m Z ddl m Z dZ dZdZd Zd Zd ed efd ZddZdde d efdZy)N) BeautifulSoup)now)NetworkSourceErrorNetworkMonitoredPageManualDataSource)send_alert_notificationc@|jdd}|dk(ry|dk(ryy)N ifMissingnulltrueTfalseFget)configoptions -/var/www/extractly/manual_agregator/parser.py_resolve_missing_boolr s+ ZZ V ,F  7  cP|jddk(r|jddSy)Nr default defaultValuer)rs r_resolve_missing_textrs' zz+)+zz."-- rc|sy|jjjdd}ddddddddddddddd}|jD] \}}||vs |cSy) zAStandardizes and detects currency code (PLN, EUR, USD, GBP, etc).N rPLNEURUSDGBP)uzłplnzluzł.eureurou€usddolar$zus$gbpfunt£)striplowerreplaceitems) label_or_texttext CURRENCY_MAPkvs rparse_currencyr3s     & & ( 0 0b 9DU%eEu5e5 L""$1 9H% rc|sy|jddjdd}tjd|}d}d}d}|ru|jdjdd } d|vr t |n t |}||jdj}t|xs t|}|||fS#t $rd}YKwxYw) z?Extracts number, label (suffix), and currency code from a text.)NNN r,.z([\d\s]+(?:\.\d+)?)Nr) r,researchgroupfloatint Exceptionendr*r3)r/matchnumberlabelcurrency number_strs rextract_number_and_labelrE,s  << $ , ,S# 6D II,d 3E F EH [[^++C4  *-*;U:&ZFUYY[\"((*!%(@N4,@ 5( ""  F s&B>> C  C c|jdddiD]R}|jd}t|dk\s#||djvs5|djjcSy) Ndivzdata-sentry-elementItemGridContainer)attrsprr8)find_alllenr/r*)souprBrGpss rextract_value_by_labelrPAse}}U+@BU*V}W \\#  r7a rct}|r|j|n|g}|D]}|jdD]9}|jd}|s|j ds)|j |;|jdD]t}|jd}|s|j dD]J}|jj dd}|s(|j ds:|j |Lvt|S) u Zwraca listę unikalnych linków do zdjęć (src z , srcset z ) wg selektora lub domyślnie cała strona. imgsrchttpsourcesrcsetr6rr) setselectrLr startswithaddsplitr*list) rNrWurlscontextscopernrorrurls rextract_image_linksr}ns 5D'/dkk(#dVG>>%(C''%.Cs~~f- ) >>(+CWWX&F!<<,C))+++C03Cs~~f5 - , :rrfc 012|j} |j} tdtd|jtd|jdd d t|jd }i}i}d}|jxsg}|D]>} d | vr|j| d } | s| jd j0| jdg} t!| t"r| g} t%0fd| Dsy| d}td| d d|nd| vst'|| dd2t!2t(rt+j,22n2t#2j2| jdg} t!| t"r| g} t%2fd| Ds&| d}td| dd|n|j.} t!| t(r |r || vr| |} n| } td|xsddt1| j3| j5D]\}}|jds|jdd}d}td|d||d k(rA|jd!}t7||}|jd"d}|d#k(r]t9|\}}}|jd$r |r|||d$<|jd%r |r|||d%<|}td&|d'|d(|n|d)k(r|xsdj;j1|jd*gd+}|jd,gd-}t%1fd.|Drd }n"t%1fd/|Drd}n t=|}td0|d'|n|}td1|d'|n|d#k(r|jd }|j|} | r| jd n t?|}t9|\}}}|jd$r |r|||d$<|jd%r |r|||d%<|}td2|d'|d(|nY|d)k(r|jd }|j|} | r| jd n t?|}|xsdj;j1|jd*gd+}|jd,gd-}t%1fd3|Drd }n"t%1fd4|Drd}n t=|}td5|d'|n~|dk(ry|jd }|j|} | r9|jd6dr| jd d 7}n| jd }n t?|}td8|d'||||<|||<| j5D]4\}}|jdr|jdd}d}td9|d||d:k(r>|jd }tA||}td;|d|Drd }n"t%1fd?|Drd}n t=|}td0|d'|nV|}td1|d'|nA|d#k(r|jd }|j|} | r| jd n t?|}t9|\}}}|jd$r |r|||d$<|jd%r |r|||d%<|}td2|d'|d(|n|d)k(r|jd }|j|} | r| jd n t?|}|xsdj;j1|jd*gd+}|jd,gd-}t%1fd@|Drd }n"t%1fdA|Drd}n t=|}td5|d'|ndB|vr|jdB}|j|}tdC|d'||jdDxs"| j|ijdDdE}|jdF}|rv|t|jE|Dcgc]}|j;} }dG|cxkrtC| krnn| |}tdH|dI|d'|nd}tdH|dI|dJn|r|}nt?|}n|dk(r{|jd }|j|} | r9|jd6dr| jd d 7}n| jd }n t?|}td8|d'|n9|dKk(r|jd }|j|} | r| jd j0t%0fdL|jd*gDrd }n2t%0fdM|jd,gDrd}n t=|}tdN|d'|nt=|}tdN|dO|dPne|dQk(r|jd }|j|} | rx| jd j0|jdRi}!d}"|!j5D]\}#}$t%0fdS|$Ds|#}"n|"}tdT|d'|nt?|}tdT|dO|dPn|dUk(r|jdVg}%d}&|%D]/}'|j|'} | s|&dW| jdWd 7zz }&1|jdXgD]}(|&jG|(d}&|&j;}&|&r|&n t?|}tdY|%d'||r%|#tdZ|d[|dPtId\|d]||||<7td^|jd_|j5D]\})}*td`|)da|*t'|dbixsij5D]!\}+},tK||,st'||,||+<#|jLxsg}-tO|||-rd|_(tS|_*tC| }.tWdc|jYD}/|/|.|j5D)*cgc]\})}*|)ddk7s |*ddgifvs|)c}*})|j5D)*cgc]\})}*|)ddk7s |*ddgifvs|)c}*})|xsdde|dd<tdf|ddt+j,|dg|_-||_.d |_/|jatdh|jd dy #tj$r3tj j |ddtd|YywxYwcc}wcc}*})wcc}*})w#tb$rtdi|jd tejftj j |tejfdjtidk|jdl|jjdmtejfdn|jjoYywxYw)pNu)Brak ManualDataSource dla tego źródła.MissingManualDataSource)rq error_message error_typez3[ERROR] ManualDataSource does not exist for source FzP================================================================================z[INFO] Parsing page: z%[INFO] First 500 characters of HTML: i z html.parserrWTrXr@c3BK|]}|jvywNr+).0valcontents r z$parse_manual_data..sJ\c399;'1\rTz"[DEBUG] Rule matched by selector: z, type: rqrc3BK|]}|jvywrr)rrrks rrz$parse_manual_data..sD|syy{e+|rz [DEBUG] Rule matched by source: z[INFO] Using selectors (type: rz): isMain fieldTyper/z! [FIELD] (isMain) Parsing field: z , fieldType: labelPairrB valueTyperA currencyField labelFieldz [DEBUG] labelPair-number: label=z, value=z , currency=bool trueOptions)takyesr 1 falseOptions)nienor 0c3(K|] }|k( ywrroptrs rrz$parse_manual_data..s?YcsczYc3(K|] }|k( ywrrrs rrz$parse_manual_data..s Bzzrz[DEBUG] labelPair-bool: label=z[DEBUG] labelPair-text: label=z[DEBUG] number: selector=c3(K|] }|k( ywrrrs rrz$parse_manual_data..;#3#:rc3(K|] }|k( ywrrrs rrz$parse_manual_data..>:CSCZ:rz[DEBUG] bool: selector= paragraphs) separatorr*z[DEBUG] text: selector=z [FIELD] Parsing field: imagesz[DEBUG] images: selector=z, found=z linksc3(K|] }|k( ywrrrs rrz$parse_manual_data..9rrc3(K|] }|k( ywrrrs rrz$parse_manual_data..;rrc3(K|] }|k( ywrrrs rrz$parse_manual_data..Ws7YcsczYrc3(K|] }|k( ywrrrs rrz$parse_manual_data..Ys:zzrfromMainz[DEBUG] fromMain: base_field=splitByr6 splitIndexrz![DEBUG] fromMain split: split_by=z, split_index=z, value=None (out of range)boolKeyc3BK|]}|jvywrrrrrs rrz$parse_manual_data..s[=Zc399;'1=Zrc3BK|]}|jvywrrrs rrz$parse_manual_data..s^?]SYY[G3?]rz[DEBUG] boolKey: selector=z , value=None (missing, fallback=)keykeyMapc3BK|]}|jvywrr)rvariantrs rrz$parse_manual_data..sRgw}}'9rz[DEBUG] key: selector=many selectorsr cleanOptionsz[DEBUG] many: selectors=z%[WARNING] Value not found for field: z (config: zValue not found: z -> z [INFO] Extracted fields for :z z: trasferredc32K|]}|ddgifvs dyw)Nrr8r)rrs rrz$parse_manual_data..s&\(:c$PRTVXZI[>[1(:s _stats) found_fields total_fields found_keys missing_keys type_usedz[INFO] Stats: ) ensure_asciiz*[SUCCESS] Data parsed and saved for page: z&[EXCEPTION] Error while parsing page: ManualParsingErroruBłąd parsowania ogłoszenia z (z) u❌ Błąd parsowania: )subject)6rqmanual_data_source_fetcherr DoesNotExistrobjectscreateprintr|r]rrulesr^r_r+rrareanyr`rbrcrdrrxkeysr-rPrEr*rrr}rMrwr, ValueErrorhasattrinactiverl is_activer inactive_datesumvaluesraw_data parse_data is_completesaver> traceback format_excrtitle)3rfstrictrq manual_configrN extracted main_values selected_typerrgrj match_values all_selectorsr field_namer field_type text_valuerBraw value_typerA label_valrC true_opts false_optsrW base_field base_valuesplit_by split_indexrJpartskey_mapmatched final_valuevariantsselectors_listcombinedsel clean_itemr1r2r source_fieldrQrrrrrks3 @@@rparse_manual_datars+ [[F *0*K*K K f  %dhhZ01 6tyy#6GrJKTYY 6   ##)rDT!//$z*:;%..T.:@@BG#'88GR#8L!,4(4~ J\JJ(,V  B4 CSBTT\]j\klmT!d8nb9eT* JJu-E]E ((*#xx4 lC0$0>LD|DD$(LM!::l3 >GIf\&:;%+  @xPZ|[fgofpqr#v-"yb//1779$*JJ}>Y$Z %+ZZ@[%\ ?Y??)-J Bz BB).J)>v)FJ >ugXj\Z[%(  >ugXj\Z[8+%zz*5H"ooh7G:A'***6G\]cGdC2J32O/FIxzz/2x=E &"9:zz,/I:C &"67!'J5hZx |S^_g^hij6)%zz*5H"ooh7G:A'***6G\]cGdC9"++-335C & =:U VI!'N:>>%* %:6%B 3H:Xj\RS6)%zz*5H"ooh7G!::lE:)0)9)9DPT)9)UJ)0)9)9)9)EJ%:6%B 3H:Xj\RS+5 J'(2 *%Y#4^#,//"3 Jzz(#  K8JJ -j\zlS TX%!::j10x@ 1(8C OCTTZ[\(2 *%[( 7+,T59#ZZ V< )2J32O/FIxzz/2x=E &"9:zz,/I:C &"67!'J:>>%* %:6%B :5'*VW!$J:5'*VWx'!::j1//(36=g&&T&2CXY_C`.Fs.K+ 8::o.89AIf_56::l+ 6?If\23# 1(8J<{[cZdefv%!::j1//(36=g&&T&2CXY_C`yb'')//1"JJ}6QR #ZZ8ST 7Y77!%J:z::!&J!6v!>J/z*NOv%#ZZ 3 (__Z8 5j\*VW!::i0eIMM*b4Q4U4UV_ad4e$jj6 +"90:0@0@0JK0J1QWWY0JEKK4#e*4%*;%7  A(>ZeYffnoynz{|%)  A(>ZeYfgBCD!+J!6v!>Jv%!::j1//(3zz,6%,%5%5D%5%Q %,%5%5D%5%A !6v!>J/z*NOy(!::j1//(3%..T.:@@BG[VZZ WY=Z[[%) ^vzz.Z\?]^^%* %:6%B 6xj UV!6v!>J6xj@`ak`llmnou$!::j1//(3%..T.:@@BG$jj26G"G18- XRRR&1G!2A")J28*HZLQR!6v!>J28*<\]g\hhijkv%!'K!<)C"ooc2G C'*:*:SPT*:*U$UU*#)**^R"@J'// B?H#A#>>+)1X7LV7T 00@ UV*,=j\TZS[[\]^ #4ZLVH!MNN$.Ij !u#4x .txxj:;OO%DAq D2aS/ "&#*-r"J"Pb!W!W!Y Ct\*!(|!< #"Z '//52 4~ 6"DN!$D 9~ \ (8(8(:\\ (()2):l):Aa8mPQZ^`bdfhjYkPk1):l+4??+<j+<41aX RSX\^`bdfhWiRiQ+<j&3)  ( y2345 95A #  :488*BvhOPO  ( (""))E0 * CF8LMnLDmj   6txxj9CWCWCYBZ[\""))#..0+ * ,TXXJbeIL`L`LbKc d-fll^<  s {B |*A||$B |1B| ]!|-| H8|=A(|&D|=B| | | | |2 | | | A4|A|| |B3r)T)rcrr9bs4rdjango.utils.timezonerextractly.modelsrrrmanual_agregator.notificationsrrrr3rErPrxrrlr}rrrrrsf  %WWB ,#*   L8Y0Y$Yr