
    Rh                     z   d dl Z d dlmZmZ d dlmZ d dlmZ d dlm	Z	 d dl
mZmZ d dlmZ g dZ ed	d
ddddddd      ZdZ edd      ddd       Z edd      dddddd       Z edd      	 	 	 	 	 d)d edz  d!ed"edz  d#efd$       Z ed%d      	 	 	 	 	 	 d*d&ed!ed"edz  d'edz  d#ef
d(       Zy)+    N)shared_taskchain)scrape_all_sources)html_agregator_task)clear_cache_task)run_status_only_checksrun_status_pipeline)task_send_inactive_ads_to_cloud)z---disable-blink-features=AutomationControlledz--no-sandboxz--disable-dev-shm-usagezoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36zpl-PLzEurope/WarsawiV  i   )widthheightzpl-PL,pl;q=0.9,en;q=0.81)zAccept-LanguagezUpgrade-Insecure-Requests)
user_agentlocaletimezone_idviewportextra_http_headersa  
Object.defineProperty(Navigator.prototype, 'webdriver', { get: () => undefined });
window.chrome = window.chrome || { runtime: {} };
Object.defineProperty(navigator, 'languages', { get: () => ['pl-PL','pl'] });
Object.defineProperty(navigator, 'plugins',  { get: () => [1,2,3,4,5] });
z'link_agregator.fetch_all_and_queue_htmllink)namequeueT)auto_queue_nextc           	      0   d }t        j                   |              | rjt        t        j                  g ddgdd      j                  d      t        j                  d	d dd
      j                  d            j                          dt        |       dS )Nc                     K   	 t        d       t        d d dd       d {    y 7 # t        $ r} t        d|         Y d } ~ y d } ~ ww xY ww)Nz*Uruchamiam scrape_all_sources (mode=fetch)fetchT	source_idparamsmodeheadlessu   Błąd w link_agregator_task: printr   	Exceptiones    */var/www/extractly/link_agregator/tasks.pyrunnerz#link_agregator_task.<locals>.runner   sK     	8>?$tDwY]^^^ 	821#677	86   A) ') A) 	A
A AA

Azmanual_pipeline_lock:*zimg_upload_lock:*Fz[link->html] keyspatterns	clear_all
log_prefixdefault)r   trueT)enablenamesinclude_fetchedr   htmlok)statusqueued_html)	asynciorunr   r   sisetr   apply_asyncbool)r   r%   s     r$   link_agregator_taskr;      s    8 KK24GH*	
 c	c"""T5RVcc
	
 +-4+@AA    zlink_agregator.check_activecheckF)clear_cache_beforer*   
clear_keysclear_patternsc           	          | r$t        j                  t        |||d      d       d }t        j                   |              t
        j                  ddd d d ddd	       d
ddS )Nz[is_active_check] r'   r,   kwargsr   c                     K   	 t        d       t        d d dd       d {    y 7 # t        $ r} t        d|         Y d } ~ y d } ~ ww xY ww)Nz*Uruchamiam scrape_all_sources (mode=check)r=   Tr   u   Błąd w is_active_check_task: r   r"   s    r$   r%   z$is_active_check_task.<locals>.runnerG   sS     	9>?$t'D    	93A3788	9r&      T
batch_sizer   r   source_namemax_batchesrun_actionsr=   r2   check_flagged_ads_pipeline_task)r3   queued)r   r9   dictr5   r6   rK   )r>   r*   r?   r@   r%   s        r$   is_active_check_taskrN   ;   s     $$D#+	-

 	9 KK $//
  0 
 &GHHr<   z link_agregator.check_flagged_adslimitr   rH   rJ   c           	          t        j                  t        | ||||            }t        j                  dd d dd       d|j                  dd      |j                  d	d      d
dS )N)rO   r   r   rH   rJ   pipeliner   rO   timeoutinactiverB   r2   checkedr   errorsT)r3   rU   rV   queued_inactive_pipeline)r5   r6   r   r
   r9   get)rO   r   r   rH   rJ   outs         r$   check_flagged_ads_taskrZ   d   st     ++, C $//"TdC 779a(''(A&$(	 r<   z)link_agregator.check_flagged_ads_pipelinerG   rI   c           
          t        j                  t        | |||||            }t        j                  dd d dd       dd|j                  dd      |j                  d	d      |j                  d
d      ddS )NrF   rQ   rR   rT   rB   r2   batchesr   rU   rV   T)r3   r   r\   rU   rV   rW   )r5   r6   r	   r
   r9   rX   )rG   r   r   rH   rI   rJ   rY   s          r$   rK   rK      s     ++) C $//"TdC 779a(779a(''(A&$( r<   )i,  TNNT)rE   TNNNT)r5   celeryr   r   link_agregator.source_managmentr   html_agregator.tasksr   abstractclass.tasksr   %link_agregator.check_active.run_checkr   r	   houslyspace.tasksr
   PLAYWRIGHT_ARGSrM   CTX_KWSTEALTH_INIT_SCRIPTr;   rN   intr:   strrZ   rK    r<   r$   <module>ri      s\    % > 4 0 ] = m	Es++Dbef
  ;6J+/ B KB: /w?5TRV$I @$IP 4GD": t	
  E< =WM"" t	
 t  Nr<   