
    =h              	           d dl Z d dlmZ d dlmZ d dlmZ d dlmZmZ d dlm	Z	 d dl
mZ d dlmZ d d	lmZ g d
Z eddddddddd      ZdZdZdZ edd      d dddddd       Zy)!    N)shared_task)sync_to_async)async_playwright)r   chain)clear_cache_task)fetch_and_save_html_for_pages)task_process_manual)get_pages_to_process)z---disable-blink-features=AutomationControlledz--no-sandboxz--disable-dev-shm-usagezoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36zpl-PLzEurope/WarsawiV  i   )widthheightzpl-PL,pl;q=0.9,en;q=0.81)zAccept-LanguagezUpgrade-Insecure-Requests)
user_agentlocaletimezone_idviewportextra_http_headersa  
Object.defineProperty(Navigator.prototype, 'webdriver', { get: () => undefined });
window.chrome = window.chrome || { runtime: {} };
Object.defineProperty(navigator, 'languages', { get: () => ['pl-PL','pl'] });
Object.defineProperty(navigator, 'plugins',  { get: () => [1,2,3,4,5] });
html_aggr_locki  zhtml_agregator.fetch_pageshtml)namequeuetrueFT)enablenamesinclude_fetchedauto_queue_nextc                F   fd}t        j                   |              |rnt        t        j                  dgdgdd      j                  d      t        j                  d	d
d d ddd d       j                  d            j                          dt        |      dS )Nc                    K   	 t        	       d {   }  t        t              |        d {   }t        dt	        |       d       t               4 d {   }|j                  j                  dt               d {   } |j                  di t         d {   }|j                  t               d {    |j                          d {   }t        ||       d {    |j                          d {    |j                          d {    d d d       d {    y 7 7 7 7 7 7 ~7 h7 U7 ?7 )7 # 1 d {  7  sw Y   y xY w# t         $ r}t        d|        Y d }~y d }~ww xY ww)	N)r   r   r   z#[html_agregator_task] Do pobrania: z stronT)headlessargs)pagesu   Błąd w html_agregator_task:  )r
   r   listprintlenr   chromiumlaunchPLAYWRIGHT_ARGSnew_contextCTX_KWadd_init_scriptSTEALTH_INIT_SCRIPTnew_pager   close	Exception)
pages_qsr    pbrowsercontextpageer   r   r   s
          */var/www/extractly/html_agregator/tasks.pyrunnerz#html_agregator_task.<locals>.runner/   sF    	81e]lmmH---h77E7E
|6JK'))Q !

 1 14o 1 VV 3 3 3 =f ==--.ABBB$--//3DFFFmmo%%mmo%% *))	 n7 *V=B/F%% *)))  	821#677	8s   FE" D5E" D8*E" D;E" "%ED=E%D?&EEEEE/E0EEEE	 E$E" /E0E" 4F5E" 8E" ;E" =E?EEEEE	EE" EEEE" FE" "	F+E>9F>FFr   zmanual_pipeline_lock:*Fz[html->manual] )keyspatterns	clear_all
log_prefixdefault)r   pipelinei  )modelimitr   only_iddry_runforce	manual_idforce_namesmanualok)statusqueued_manual)	asynciorunr   r   ssetr	   apply_asyncbool)previous_task_resultr   r   r   r   r6   s    ```  r5   html_agregator_taskrO   -   s    8" KK&'23,	
 c	c"!! 	 cc!!	
" +-T/-BCC    )N)rH   celeryr   asgiref.syncr   playwright.async_apir   r   abstractclass.tasksr   html_agregator.html_fetcherr   manual_agregator.tasksr	   html_agregator.utils.processr
   r'   dictr)   r+   LOCK_KEYLOCK_TTLrO   r!   rP   r5   <module>r[      s      & 1 % 0 E 6 =
 
	= s+4%(
   .f=(DV4afx| (D >(DrP   