
    h>S                        d dl mZ d dlZd dlmZ d dlZd dlmZmZ d dlZd dlZd dlmZ d dl	 d dl
mZ  ej                         ad Zd Zd d	lmZ d d
lmZ d dl mZ ddlmZ ddlmZ d dlmZ d dlmZ  edg      d        Z edg      d        Z edg      d        Zd d	lmZ d d
lmZ d dl mZ d dlZd dlmZ  edg      d        Z  edg      d        Z! edg      d        Z" edg      d        Z#d dl$m%Z% d dl&Z&d dlZd dlmZ d dl'm(Z( d d	lmZ d d
lmZ d dl mZ  edg      d        Z)d d	lmZ d d
lmZ d dl mZ  edg      d        Z* edg      d         Z+d dlZd! Z, edg      d"        Z-d dlZd d	lmZ d d
lmZ d dl mZ d d#l	m.Z. d d$l/mZ d% Z0d*d&Z1 edg      d'        Z2d d	lmZ d d
lmZ d d(l	m3Z3  eddg      d)        Z4y)+    )viewsetsN)JsonResponse)datetime	timedelta)*)	AdsManualc                    	 t        j                  g ddd      }|j                  j                         }t        j                  g ddd      }|j                  j                         j	                  d      }t        j                  d      }|D cg c]  }|j                  |      s| }}|rt        j                         a
t        j                         t        z
  }|t        d      kD  rd	}	d
}
n"t        |      t        |      k(  r|rd}	d}
nd}	d}
t        ||	|
|d      S c c}w # t        $ r"}t        dt        |      id      cY d}~S d}~ww xY w)u@   Zwraca status i logi usługi otodom.service z analizą błędów)sudo	systemctl	is-activeotodom.serviceTcapture_outputtext)r
   
journalctl-ur   -n50
--no-pagerz--output=short
z\b(403|404|500|502|503|504)\b   )minutesu   ⚠️u'   Brak nowych logów przez ponad 3 minutyu   ❌u&   Wszystkie logi zawierają błędy HTTPu   ✅u   Usługa działa poprawnie)statusstatus_emojistatus_descriptionlogserror  r   N)
subprocessrunstdoutstripsplitrecompilesearchr   nowlast_log_timer   lenr   	Exceptionstr)requeststatus_output	is_activelogs_outputr   error_patternlog
error_logstime_since_last_logr   r   es               '/var/www/extractly/houslyspace/views.pyotodom_service_statusr7      sO   .;"@

 "((..0	 !nnf

 !!'')//5 

#CD%)GTc]-A-A#-FcT
G $LLNM 'lln}<1!55#L!J_D	)d L!I L!<("4	
  	' H4  ;Wc!f-c::;s7   BD+ D&)D&-A8D+ &D+ +	E4EEEc                    	 t         j                  j                  |      j                         }|st	        ddid      S |j
                  }t        j                  dddd	d
| gdd      }|j                  }d|v rd|v rd}nd}t	        ||d      S # t        $ r"}t	        dt        |      id      cY d}~S d}~ww xY w)u:   API do sprawdzenia, czy konkretne ogłoszenie jest aktywne)idr   u   Ogłoszenie nie istniejei  r   scrapycrawlnonline_single_ad_is_activez-azurl=Tr   zAd idzis not activeF)r9   r/   r   N)r   objectsfilterfirstr   urlr    r!   r"   r+   r,   )r-   
listing_idlistingr@   processoutputr/   r5   s           r6   check_listing_statusrE   H   s    ;##**j*9??A*D EcRRkk ..w =ttC5\R
 fF!:II :IFGG ;Wc!f-c::;s$   >B AB 	B?B:4B?:B?)api_view)Responser      )DataSetSnapshot)DataSetSnapshotSerializer)Q)r   POSTc                    i }g }|j                         D ]'  \  }}|j                  j                         }|j                  j                  t	        d      t	        d      z        j                         }||z
  }|j                  j                  t	        d      t	        d      z        j                         }||z
  }	|j                  j                  t	        d      t	        d      z        j                         }
||
z
  }t
        j                  j                  |||||	|||
	      }|j                  |       * t        |d
      }t        |j                  t        j                        S )NT)title__isnull )title)original_image_urls__isnull)original_image_urls)images__isnull)images)
model_nametotaltitle_filledtitle_emptyoriginal_filledoriginal_emptyimages_filledimages_empty)manyr   )itemsr=   countr>   rK   rI   createappendrJ   rG   datar   HTTP_201_CREATED)r-   models	snapshotsnamemodelrV   rX   rW   rZ   rY   r\   r[   snap
serializers                 r6   refresh_statsrj   t   s@   F I||~e##%mm**14+@12;+NOUUW{*--aD.QTUjlTm.mnttv.0}}++AT,BQb\,QRXXZ,&&--%#+)'% . 	
 	' &* +94@JJOOF,C,CDD    GETc                    t         j                  j                         j                  dd      }i }|D ]3  }|j                  |vst        |      j                  ||j                  <   5 t        |j                               S )NrU   -created_at)	rI   r=   allorder_byrU   rJ   rb   rG   values)r-   latestresultrh   s       r6   latest_statsrt      sj    $$((*33L-PFF??&(&?&E&J&JF4??#  FMMO$$rk   c                 |   ddl m} t        j                  j	                         j                  dd      } |t              }|D ]<  }t        ||j                           dk  s||j                     j                  |       > g }|j                         D ]  \  }}t        |      dk  r|d   |d   }	}|t        |j                  |	j                  z
        |j                  |	j                  z
  |j                  |	j                  z
  |j                  |	j                  z
  d}
|j                  |
        t!        |      S )Nr   defaultdictrU   rn      rH   )rU   	time_difftitle_deltaoriginal_deltaimages_delta)collectionsrw   rI   r=   ro   rp   listr*   rU   ra   r^   r,   
created_atrW   rY   r[   rG   )r-   rw   re   rr   rh   rs   rU   snapsnewolddiffs              r6   compare_last_twor      s   '''++-66|]SIFvdoo&'!+4??#**40  F#\\^
Eu:>8U1XS$S^^cnn<=++c.>.>>!11C4G4GG--0A0AA
 	d , Frk   move_to_archivec                    i }d}d}g }|j                         D ]  \  }}d}|j                  j                         j                  ddd      D ]N  }d}	d}
|j                  ra	 t        |j                  t              rt        j                  |j                        n|j                  }t        |t              rt        |      }	|j                  ra	 t        |j                  t              rt        j                  |j                        n|j                  }t        |t              rt        |      }
|	|
k(  }|j                  |k7  r||_        |j                  dg       |dz  }t        j                  j!                  ||j                  j#                         |       Q ||j                  j#                         z  }||z  }|j%                  ||j                  j#                         |d        t'        |||d	t(        j*                  
      S # t        j                  $ r Y w xY w# t        j                  $ r Y w xY w)Nr   rR   rT   haveAllPhotos)update_fieldsrH   )rU   checkedupdated)rg   r   r   )total_checkedtotal_updateddetailsr   )r^   r=   ro   onlyrR   
isinstancer,   jsonloadsr~   r*   JSONDecodeErrorrT   r   savePhotoConsistencyLogr`   r_   ra   rG   r   HTTP_200_OK)r-   rd   r   r   r   rU   rg   updated_adsadoriginal_countprocessed_countoriginal_listimages_listhave_all_photoss                 r6   check_photos_consistencyr      s   F MMG#\\^
E--##%**+@(O\BNO %%JTUWUkUkmpJqDJJr/E/E$Fwy  xN  xNM!-6),]);
 yy;EbiiQT;U$**RYY"7[][d[dK!+t4*-k*:
 -?O?2#2 &78q ''..%++-# / = ]H 	,,..$8K8K8MZefgS ,V &&   	" "= ++  ++ s&   "A HA H*H'&H'*I Ic                     t         j                  j                  d      }d}|D ]  }t        |j                         |dz  } t        |d| ddt        j                        S )	NF)r/   r   rH   zZarchiwizowano u    ogłoszeń.)archived_countmessager   )r   r=   r>   r   r9   rG   r   r   )r-   listingsr_   rB   s       r6   archive_inactive_listingsr     sm      ''%'8HE

#
  $UG<8   " "rk   c                    i }d}d}d}d}d}g }|j                         D ]  \  }}	|	j                  j                         }
||
z  }d}d}d}d}|	j                  j                         j	                  ddd      D ]  }t        |j                  xr |j                  j                               }|j                  xr  t        |j                  t        t        f      }|j                  xr  t        |j                  t        t        f      }|r|r|dz  }|r	|r|r|dz  }|j                  rd	 t        |j                  t              rt        j                  |j                        n|j                  }t        |t              r|t        |      z  }|j                  s#	 t        |j                  t              rt        j                  |j                        n|j                  }t        |t              r|t        |      z  } ||z  }||z  }||z  }||z  }|j#                  ||
||||d        |r||z  nd}|r||z  nd}|r||z  nd}|r||z  nd}t%        |||||t'        |d      t'        |d      t'        |d      t'        |d      |d
t(        j*                  	      S # t        j                   $ r Y Aw xY w# t        j                   $ r Y [w xY w)
Nr   rR   rT   rP   rH   )rg   adswith_title_and_originalwith_full_datalinks_in_originallinks_in_imagesrx   )
	total_adsdetailed_adsfully_processed_adstotal_links_originaltotal_links_processedavg_links_originalavg_links_processedavg_links_original_filledavg_links_processed_filledr   r   )r^   r=   r_   ro   r   boolrP   r#   rR   r   r~   r,   rT   r   r   r*   r   ra   rG   roundr   r   )r-   rd   r   total_detailed_adstotal_processed_adsr   r   r   rU   rg   	ads_countlinks_originallinks_processeddetailed_ads_countprocessed_ads_countr   	has_titlehas_original_imageshas_processed_imagesr   r   r   r   avg_links_original_processedavg_links_processed_processeds                            r6   image_link_statisticsr     s   F IG#\\^
EMM'')	Y	--##%**+@(GTBRXX:"((..*:;I"$"8"8"lZH^H^aegj`k=l#%99#SBIIc{1S 0"a'"05I#q(#%%JTUWUkUkmpJqDJJr/E/E$Fwy  xN  xNM!-6&#m*<< yy;EbiiQT;U$**RYY"7[][d[dK!+t4'3{+;;- U4 	.00022'91!/.
 	Q ,b GY-0BB^_HZ/2DD`aQd#7:M#Mjk Sf$9<O$Olm!*2 4!6#$6:$%8!<%*+G%K&+,I1&M   " "= ++  ++ s&   *A#J#A#J=#J:9J:=KKc                   	
 ddl m ddlm g 		fd
i }| j                  j                  d      }|r||vrt        ddid	      S ||   }|d
   |d   |d   }|d   }
fd}
fd}	  
d|         |         
d|        t        j                  dd|g|d        |         |         
d       t        |	dd	      S # t        $ r} 
d|        Y d }~)d }~ww xY w)Nr   r   r(   c                 >    t        |        j                  |        y )N)printra   )msg
action_logs    r6   r2   z!update_source_status.<locals>.log~  s    c
#rk   sourcer   u&   Nieprawidłowy lub brakujący 'source'i  r   r   r   spiderpathc                      j                   j                         } j                   j                         j                           d|  dj                          y )Nu   🧹 Usunięto u    wpisów z )r=   r_   ro   delete__name__)r_   r2   model_statuss    r6   cleanz#update_source_status.<locals>.clean  sN    $$**,  "))+oeWK0E0E/FGHrk   c                  0   t        j                  j                  dd            } t        	j                  j                  dd            }|| z
  }|rL dt        |       d       j                  j	                  |D cg c]  } |d               c}       | |z
  }|D ]1  }	 j                  j                  |      } 
|j                         3  d
t        |       d       y c c}w # j                  $ r  d| d	       Y lw xY w)Nr@   Tflatu   🆕 Dodano u    nowych ogłoszeń)r@   r/   r   )r@   u   ⚠️ Ogłoszenie u    już nie istniejeu   ✅ Przeniesiono u%    nieaktualnych ogłoszeń do archiwum)setr=   values_listr*   bulk_creategetr@   DoesNotExist)listings_urlsupdate_status_urlsr   uinactiver@   objr2   model_listingsr   r   r(   s          r6   checkz#update_source_status.<locals>.check  s*   N22>>u4>PQ !5!5!A!A%d!A!ST =0,s3xj(:;<""..QT0QTA1GQT0  !#55CC$,,00S09(  	H.STU0 ".. C)#.@ABCs   C2,.C77DDu,   🔁 Uruchamianie pełnej aktualizacji dla: u   🚀 Uruchamiam scrapy r:   r;   T)cwdr   u   ✅ Aktualizacja zakończona!u   ❌ Błąd: )r   r2      )
archive.utilsr   django.utils.timezoner(   rb   r   rG   r    r!   r+   )r-   SOURCESr   configr   spider_pathr   r   r5   r   r2   r   r   r   r(   s            @@@@@@r6   update_source_statusr   v  s    .)J
G< \\h'FV7*"JKTWXXV_FJ'N(#LHF.KI
V V*	 :6(CD%fX./'624P+, vj9#FF   l1# s   5AC 	C1C,,C1rv   )r   )
FieldErrorc                 "   i }t        t              }|j                         D ]  \  }}|D ]   }|s	 |j                  j	                  |d      j                         }|D ]  }t        |t              r|j                         }|j                  d      ra|j                  d      rP	 t        j                  |      }	t        |	t              r)|	D ]$  }
||   j                  |
j                                & ||   j                  |       |||   j                  t        |                |j                         D ci c]  \  }}|t!        |       }}}dt#        j$                         j'                          d}t)        |dd	      5 }t+        j,                  ||d
d       d d d        t/        d||dt0        j2                        S #  ||   j                  |       Y xY w# t        $ r Y w xY wc c}}w # 1 sw Y   ZxY w)NTr   []unique_values_z.jsonwzutf-8)encoding   F)indentensure_asciisuccess)r   unique_values
saved_filer   )rw   r   r^   r=   r   distinctr   r,   r#   
startswithendswithastliteral_evalr~   addr   sortedr   todaydateopenr   dumprG   r   r   )r-   tables_and_columnsvalues_dictrg   columnscolumnr   valuecleanedr^   itemkeyrq   rb   filenamefs                   r6   get_unique_listing_valuesr	    s    c"K,224wF % 9 9&t 9 L U U W*E!%-"'++-"--c2w7G7G7LA(+(8(8(A#-eT#:05(3F(;(?(?

(M 16
 (/33G<*#F+//E
; +  52 4?3D3D3FG3FKCC3FDG 0 5 5 78>H	hg	.!		$!%8 
/    	" "A +F 3 7 7 @
   H	.	.sC   A2G/&AG4G/G/G?HG,	)G//	G<;G<Hc                     	 t        ddit        j                        S # t        $ r3}t        ddt	        |       it        j
                        cY d }~S d }~ww xY w)Nr   u/   ✅ Ogłoszenia zostały pomyślnie zmergowane.r   r   u   ❌ Wystąpił błąd: )rG   r   r   r+   r,   HTTP_500_INTERNAL_SERVER_ERROR)r-   r5   s     r6   merge_listings_to_monitoringr  !  sX    u$UV_e_q_qrr u$;CF8"DEfNsNsttus    	A(AAAc                     t         j                  j                         j                  dddd      }t	        dt        |      i      S )Nrf   last_runr   r   scrapers)ScraperStatusr=   ro   rq   rG   r~   )r-   statusess     r6   scraper_healthr  ,  s<    $$((*11&*iQZ[HZh011rk   c                 *   	 t        j                  dd| gt         j                        j                         j	                         }d|iS # t         j
                  $ r5}d|j                  j                         j	                         dcY d }~S d }~ww xY w)Nr   r   stderrr   unknown)r   r   r    check_outputSTDOUTdecoder#   CalledProcessErrorrD   )service_namerD   r5   s      r6   get_scraper_statusr  4  s    I((+|4$$
 &(557 	
 &!!(( I#ahhoo.?.E.E.GHHIs   AA
 
B*BBBc                 f    | j                   j                  dd      }t        |      }t        |      S )Nservicescraper-gethome.service)rl   r   r  rG   )r-   r  rs   s      r6   scraper_statusr!  @  s,    kkooi)BCG(FFrk   )r  r   c                 &   	 t        j                  dd| gt         j                        j                         j	                         }|S # t         j
                  $ r5}d|j                  j                         j	                          cY d }~S d }~ww xY w)Nr   r   r  error: r  )r  r   r5   s      r6   check_systemctlr$  Q  s{    5((+|4$$
 &(557 	
 (( 5*0023445s   AA B*BBBc           	      >   	 t        j                  dd| dt        |      dgt         j                        j	                         j                         }|S # t         j                  $ r5}d|j                  j	                         j                          cY d }~S d }~ww xY w)Nr   r   r   r   r  r#  )r    r  r,   r  r  r#   r  rD   )r  limitr   r5   s       r6   get_recent_logsr'  ]  s    5&&4tSZN$$
 &(557 	
 (( 5*0023445s   AA B'*BBBc                    | j                   j                  dd      }|j                  dd      j                  dd      }t        |      }t	        |      }	 t
        j                  j                  |      }|j                  }|j                  }|j                  }t        |||||||t               d	t        j                  
      S # t
        j                  $ r	 d }d}d}Y Gw xY w)Nr  r   z.servicerO   zscraper-)rf   FzBrak wpisu w bazie)r  logical_namesystemctl_statusr  r   r   recent_logs
checked_atr   )rl   r   replacer$  r'  r  r=   r  r   r   r   rG   r(   r   r   )	r-   r  r)  status_systemctlr   scraperr  r   r   s	            r6   scraper_diagnosticsr0  i  s    ;;??9.GHL''
B7??
BOL&|4<(D'''+++>##//// $,e	   	" 	" %% '&'s   AC C"!C")ScraperConfigc                    | j                   dk(  rKt        j                  j                         D ci c]  }|j                  |j
                   }}t        |      S | j                   dk(  r^| j                  }|j                         D ]1  \  }}t        j                  j                  |dt        |      i       3 t        d|d      S y c c}w )Nrl   rL   r  )rf   defaultsr   )r   rb   )methodr1  r=   ro   rf   r  rG   rb   r^   update_or_creater,   )r-   r   configsrb   r  r  s         r6   scraper_config_viewr7    s    ~~;H;P;P;T;T;VW;V6;;,;VW  	6	!||**,JC!!22wPSTYPZF[2\ '9d;<<	 
" Xs   C	)
   )5rest_frameworkr   r    django.httpr   r%   r   r   r   houslyspace.modelsextractly.modelsr   r(   r)   r7   rE   rest_framework.decoratorsrF   rest_framework.responserG   r   rd   rI   serializersrJ   django.db.modelsrK   rj   rt   r   r   r   r   r   r   r   r}   rw   r   django.core.exceptionsr   r	  r  r  r  r!  r  r   r$  r'  r0  r1  r7   rk   r6   <module>rC     s*   #  $ 	 (   $   & 2;p;H / , ! # 2  	6(E E< 
5'% % 
5' L / , !  ) 
6(7" 7"~ 
6(
" 
"& 
5'L" L"l 
6(YG YGL $ 
   - . , ! 
5')" )"j / , !	6(u u 
5'2 2
 	I 
5'   . , ! , %	5	5 
5'" "N / , ,	5&/	= 	=rk   