
    2h              	         d dl mZ d dlZd dlZd dlmZmZmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZ  ej(                  e      Zdd
ZddZ e	d      dddddd       Z e	defddddidd      ddd       Zy)    )annotationsN)IterableListOptional)shared_task)cache)Q)	AdsManual)store_main_image)task_send_ads_to_cloudc                   g }| s|S t        | t        t        f      rt        |       }nt        | t              r| j	                         }|j                  d      r:|j                  d      r)	 t        j                  |      }t        |t              r|}nG|j                  d      D cg c]  }|j	                          }}nt        |       j	                         g}|D cg c]'  }|s|j                         j                  d      s&|) }}|S # t        $ r5 |j                  d      D cg c]  }|j	                          nc c}w }}Y qw xY wc c}w c c}w )zSNormalize candidates from str(list/CSV)/list/tuple to a clean list of http(s) URLs.[],)zhttp://zhttps://)
isinstancelisttuplestrstrip
startswithendswithjsonloads	Exceptionsplitlower)valueurlssloadedus        +/var/www/extractly/image_agregator/tasks.py_normalize_candidatesr#      s   D%$'E{	E3	KKM<<C9Afd+!D ()wws|4|!AGGI|D4 E
  "# St!qQWWY%9%9:Q%RAtDSK  9+,773<8<a	<889
 5 Ts6   ,'D 'EE'EEE+EEEc                    	 t        j                  dddd       y # t        $ r"}t        j	                  d| ||       Y d }~y d }~ww xY w)Npipeline   )modelimitdefault)kwargsqueuez.send_to_cloud enqueue failed (%s) ad_id=%s: %s)r   apply_asyncr   loggerwarning)
log_suffixad_ides      r"   _enqueue_send_to_cloud_safelyr2   2   sM    _**&5	
  _GUZ\]^^_s    	AAAimages)r+   i  F)r(   ad_idsinclude_inactivec           
     z   |r*t         j                  j                  t        |            }nt         j                  j                  t	        d            j                  t	        d            j                  t	        d      t	        d      z  t	        i       z        j                  d	      }|s|j                  t	        d
            }|st        |d|        n
t        |      }d}|D ]  }	 t        |      r|dz  } |S # t        $ r,}t        j                  dt        |dd      |       Y d}~Hd}~ww xY w)zy
    If ad_ids are provided -> process exactly those records.
    Otherwise backfill missing images (light filters).
    )pk__inF)original_image_urls__isnullTprocessed_without_image)images__isnull )r3   id	is_activeNr      z)backfill store_main_image error id=%s: %spk?)r
   objectsfilterr   r	   excludeorder_byr   r   r-   r.   getattr)r(   r4   r5   base_qsqsokobjr1   s           r"   backfill_main_imagesrL   =   s    ##**$v,*?
 VA%89WQt45V&
 Xd^ 	  ooa%&89G&,gfuo	$w-B	
B	$a  I  	NNFPSUY[^H_abc	s   1D	D:"D55D:T<   max_retries   )bindautoretry_forretry_backoffretry_jitterretry_kwargs	acks_later+   c                   t         j                  j                  |      j                  t	        d      t	        d      z        j                         }|s 	 | j                  dt        d|             d| }t        j                  |dd      sd|d
S 	 |s|j                  }d}t        |t              rt        |j                  d            }n)t        |t               rt        |j#                               }|r&t%        d|       d|d
t        j&                  |       S t)        |j*                        }|sqt-        |dd      }|rbt-        |dd      xs i }	t)        |	j                  d            }
|
st)        t-        |dd            }
|
r|
}	 |
|_        |j/                  dg       |s 	 | j                  dt        d|             t7        |      }|r't%        d|       d||dt        j&                  |       S d|d
t        j&                  |       S # | j                  $ r d	|d
cY S w xY w# t0        $ r!}t2        j5                  d||       Y d}~d}~ww xY w# | j                  $ r d|d
cY t        j&                  |       S w xY w# t0        $ rE}t2        j9                  d||       d|t!        |      d cY d}~t        j&                  |       S d}~ww xY w# t        j&                  |       w xY w)!z
    More robust single-ID image store:
    - Consistent filters vs backfill (do not flip check_active logic)
    - Soft retry on early-miss conditions (record not yet visible, candidates not yet saved)
    - Lock to avoid races
    )rA   Fr>   Tr9      zad_missing:)	countdownexc	not_found)statusr0   zimg_upload_lock:1i,  )timeoutlockedmainskip_existsnetworkmonitoredpageN
parse_dataoriginal_image_urlsimage_links)update_fieldsz6Could not persist original_image_urls for ad_id=%s: %szno_candidates:no_candidatesrJ   )r[   r0   urlupload_nonez$Image upload failed for ad_id=%s: %serror)r[   r0   ri   )r
   rC   rD   rE   r	   firstretryRuntimeErrorMaxRetriesExceededErrorr   addr3   r   dictboolgetr   r   r2   deleter#   rc   rG   saver   r-   debugr   r.   )selfr0   	overwriteadlock_keyimgshas_main
candidatesnmppdcand2r1   rg   s                r"   task_store_main_imager   d   s   $ 		5		U#a&EE	F		  	;**ql[;P.Q*RR "%)H99XsC0"U33599DH$% 01D#&

--mUC"/%@T 	XO +2+A+AB
"4d;CS,5;-bff5J.KL1'#}d2STE!&Ji16./D.EF Cjj1,PUw?W2XjYY
 r")$6"U3? 	X (%8 	Xy ++ 	;)E::	;L % i%]_dfghhi // C"1EBB 	XC  D=uaH!ECFCCX	D 	Xs   H A6J &A-J H8 .J 1I% J J H54H58	I"IJ I""J %J9J JJ 	K"%KK"K% K""K% %K<)returnz	List[str])r/   r   r0   int)r(   r   r4   zOptional[Iterable[int]]r5   rp   )F)r0   r   rv   rp   r   ro   )
__future__r   r   loggingtypingr   r   r   celeryr   django.core.cacher   django.db.modelsr	   extractly.modelsr
   image_agregator.imagesr   houslyspace.tasksr   	getLogger__name__r-   r#   r2   rL   r   r        r"   <module>r      s    "   + +  #  & 3 4			8	$>_ 8)-QUpu " "L 	,#
NNr   