
    :jy                        d Z ddlZddlZddlZddlZddlZddlZddlZddlm	Z	 ddl
mZmZmZmZmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlm Z  ddl!m"Z"m#Z#m$Z$  ed      Z% G d d      Z& G d d      Z' G d d      Z(e)dk(  r ejT                         Z+e%jY                  de+         G d d      Z-	  e-       Z. e'e.e+dd      Z/e/ja                  ejb                  jd                  ddg       e/jg                  ddg       e/ja                  ejb                  jd                  d        ejh                  d        ejj                  jm                  e+d!d"d#      Z7ejj                  jq                  e7      r5 e9e7      5 Z:e:jw                         Z<e%jY                  d$e<        ddd       e/j{                         Z>e%jY                  d%e>j~                   d&e>j                  j                          e/j                          e%jY                  d'        ej                  e+       yy# 1 sw Y   xY w#  ej                  e+       w xY w)(z
Result management system with real-time persistence.
Supports batch saving for keys, links, and other results with atomic file operations.
    N)deque)AnyDictListOptionalUnion)RESULT_MAPPINGS)
ResultType)AllRecoveredTasksRecoveredTasksResultStorageService)	IProvider)PersistenceMetrics)
get_logger   )AtomicFileWriter)ShardStrategySimpleFileStrategySnapshotManagerstoragec                       e Zd ZdZddededefdZdede	fdZ
dee   fd	Zdeeeeeef   f   fd
ZdefdZde	fdZy)ResultBufferz=Optimized buffer for batching results before writing to filesresult_type
batch_sizeflush_intervalc                     || _         || _        || _        t               | _        t        j
                         | _        t        j                         | _	        d| _
        d| _        y )Nr   )r   r   r   r   buffertime
last_flush	threadingLocklock_total_items_total_flushes)selfr   r   r   s       :/root/.openclaw/workspace/harvester/storage/persistence.py__init__zResultBuffer.__init__"   sM    &$,"W))+NN$	    itemreturnc                     | j                   5  | j                  j                  |       | xj                  dz  c_        t	        | j                        | j
                  k\  cddd       S # 1 sw Y   yxY w)z@Add item to buffer. Returns True if buffer is full after adding.r   N)r#   r   appendr$   lenr   )r&   r*   s     r'   addzResultBuffer.add-   sV    YY 	7KKt$"t{{#t6	7 	7 	7s   AA))A2c                 2   | j                   5  | j                  sg cddd       S t        | j                        }| j                  j                          t	        j                         | _        | xj                  dz  c_        |cddd       S # 1 sw Y   yxY w)zFlush buffer and return itemsNr   )r#   r   listclearr   r    r%   )r&   itemss     r'   flushzResultBuffer.flush4   ss    YY 		;;		 		
 %EKK"iikDO1$		 		 		s   BABBc                     | j                   5  | j                  t        | j                        | j                  | j
                  | j                  | j                  dcddd       S # 1 sw Y   yxY w)zGet buffer statistics)r   current_sizer   total_itemstotal_flushesr    N)r#   r   r.   r   r   r$   r%   r    r&   s    r'   	get_statszResultBuffer.get_statsA   sX    YY 	#// #DKK 0"oo#00!%!4!4"oo	 	 	s   AA%%A.c                 p    | j                   5  t        | j                        cddd       S # 1 sw Y   yxY w)zGet current buffer sizeN)r#   r.   r   r9   s    r'   sizezResultBuffer.sizeM   s*    YY 	$t{{#	$ 	$ 	$s   ,5c                     | j                   5  t        j                         | j                  z
  | j                  k\  cddd       S # 1 sw Y   yxY w)z8Check if buffer should be flushed based on time intervalN)r#   r   r    r   r9   s    r'   should_flushzResultBuffer.should_flushR   s?    YY 	JIIK$//1d6I6II	J 	J 	Js   .AAN)d         >@)__name__
__module____qualname____doc__strintfloatr(   r   boolr/   r   r4   r   r   r:   r<   r>    r)   r'   r   r      s~    G	 C 	 S 	 PU 	 7 7 7tCy 
4U3U?%; ;< 
$c $
Jd Jr)   r   c                      e Zd ZdZ	 	 	 	 d.dedededededefdZ	d	ed
e
fdZd	edefdZdee   fdZdedee   fdZd ZdefdZd/dZdeee
f   dee   fdZdeee
f   dee   fdZ	 d0dededee   defdZd	ededefdZd ed	ededefd!Zdefd"Zd	edefd#Zdeeef   fd$Z d%edee   fd&Z!d' Z"d1d(eddfd)Z#d/d*Z$d+ Z%d	efd,Z&d- Z'y)2ResultManagerz@Manages results for a single provider with real-time persistenceprovider	workspacer   save_intervalsimpleshutdown_timeoutc           	         |j                   | _         || _        || _        || _        || _        t        t        d|            | _        t        j                  j                  |d| j                  j                  j                        | _        t        j                  | j                  d       t               | _        t#        j$                         D ]y  \  }}|j                  j&                  j)                  |j*                  d      }	|	s9t        j                  j                  | j                  |	      | j                   |j,                  <   { |r-t/        | j                  | j                         | _        d | _        nxt5        | j                  | j                         | _        | j                   j7                         D 
cg c]
  }
|
dk7  s	|
 }}
t9        | j                  || j                         | _        | j                   j7                         D ci c]  }|dk7  r|t;        |||       c}| _        i | _        tA               | _!        tE        jF                         | _$        d| _%        tE        jL                  | jN                  d      | _(        | jP                  jS                          tT        jW                  d| j                    d	|rd
nd        y c c}
w c c}w )N      ?	providersTexist_ok summary)targetdaemonz)Initialized result manager for provider: z, mode: rO   shard),namerL   rM   r   rN   rG   maxrP   ospathjoinresultfolder	directorymakedirsdictfilesr	   r3   	filenamesgetfilenamevaluer   strategysnapshot_managerr   keysr   r   buffersmodels_datar   statsr!   r"   r#   runningThread_periodic_flushflush_threadstartloggerinfo)r&   rL   rM   r   rN   rO   rP   r   mappingrh   rtresult_typess               r'   r(   zResultManager.__init__[   s:    MM	 "$* %c#/?&@ A idmm>R>R>Y>YZ
DNNT2 V
$3$9$9$; 	S K0044W5E5ErJH,.GGLL,RDJJ{(()		S .t~~tzzJDM$(D!)$..$**EDM)-):N2bIoBNLN$3DNNLRVR[R[$\D!
  $zz0
i' k:}MM
 24 ()
 NN$	 %,,D4H4HQUV!?		{(_eS[krRstu1 O
s   <
KKKr   datac                    || j                   vrt        j                  d|        yg }t        |t              r|}n|g}| j                   |   }d}|D ]  }|j                  |      sd} | j                  |t        |             |r| j                  |       t        j                  dt        |       d| d| j                          y)zAdd result to appropriate buffer

        Args:
            result_type: Result type string (enum value)
            data: Data to add (single item or list)
        z[persist] unknown result type: NFT[persist] added   for )rm   ru   error
isinstancer1   r/   _update_statisticsr.   _flush_bufferdebugr[   )r&   r   rz   r3   r   needs_flushr*   s          r'   
add_resultzResultManager.add_result   s     dll*LL:;-HI dD!EFE k* 	#Dzz$"	#
 	SZ8 {+'E
|1[MtyykRSr)   countc                 l   | j                   5  t        j                         D ]|  \  }}|j                  |k(  s|j                  s#t        | j                  j                  |j                  d      }t        | j                  j                  |j                  ||z           n ddd       y# 1 sw Y   yxY w)zCUpdate statistics for given result type using configuration mappingr   N)r#   r	   r3   ri   ro   getattrresourcesetattr)r&   r   r   rx   rw   currents         r'   r   z ResultManager._update_statistics   s    YY 	.446 G88{*w}}%djj&9&97==!LGDJJ//%P	 	 	s   *B*B*AB**B3linksc                 4   |sy|D cg c]*  }|st        |t              s|j                  d      s)|, }}|rY| j                  t        j
                  j                  |       t        j                  dt        |       d| j                          yyc c}w )z3Convenience method for adding links with validationNhttpr|   z links for )r   rE   
startswithr   r
   LINKSri   ru   r   r.   r[   )r&   r   linkvalid_linkss       r'   	add_linkszResultManager.add_links   s     ).l*T3:OTXTcTcdjTktllOOJ,,22K@LL+C,<+=[TU  ms   BBBBkeymodelsc                 \   | j                   5  |t        j                         d| j                  |<   | j                  j                  xj
                  dz  c_        ddd       | j                          t        j                  dt        |       d| j                          y# 1 sw Y   HxY w)z:Add model list for a key (not buffered, saved immediately))r   	timestampr   Nr|   z models for key in )r#   r   rn   ro   r   r   _save_modelsru   r   r.   r[   )r&   r   r   s      r'   
add_modelszResultManager.add_models   s    YY 	,/5DIIK$PDS!JJ&&!+&	,
 	'F}4G		{ST	, 	,s   AB""B+c                     | j                   j                         D ]  }| j                  |        | j                          t        j                  d| j                          y)zFlush all buffers immediatelyzFlushed all buffers for N)rm   rl   r   r   ru   rv   r[   r&   r   s     r'   	flush_allzResultManager.flush_all   sR    <<,,. 	,K{+	, 	.tyyk:;r)   r+   c                 ^    | j                   5  | j                  cddd       S # 1 sw Y   yxY w)zGet current statisticsN)r#   ro   r9   s    r'   r:   zResultManager.get_stats   s%    YY 	::	 	 	s   #,Nc           
         g }| j                   j                         D ]8  \  }}t        j                  j	                  |      s&|j                  ||f       : |s#t        j                  d| j                          yt        j                  j                         j                  d      }t        j                  j                  | j                  d|       }t        j                  |d       |D ]~  \  }}	 t        j                  j                  |t        j                  j                  |            }t        j                   ||       t        j                  d| d| j                           t        j'                  dt)        |       d| j                   d|        y# t"        $ r2}t        j%                  d	| d
| j                   d|        Y d}~d}~ww xY w)z2Backup existing result files to timestamped folderz No existing files to backup for Nz%Y%m%d-%H%M%Szbackup-TrT   z
Backed up z
 file for zFailed to backup r~   : z files for z to )re   r3   r]   r^   existsr-   ru   r   r[   datetimenowstrftimer_   rb   rc   basenamerename	Exceptionr   rv   r.   )r&   existing_files	file_typefilepathr   
backup_dirbackup_pathes           r'   backup_existing_filesz#ResultManager.backup_existing_files   s    #'::#3#3#5 	=Ixww~~h'%%y(&;<	= LL;DII;GH %%))+44_E	WW\\$..GI;2GH

J. $2 	SIxS ggll:rww7G7G7QR		(K0z)JtyykJK		S 	j^!4 5[4PZ|\]  S052aSQRRSs   7A8F##	G,(GGobjc                     |j                  d      xs |j                  d      }t        |t              r|j                  d      r|S y)zProcess links data from NDJSON object.

        Args:
            obj: Parsed JSON object from shard file

        Returns:
            Valid URL string or None
        urlri   r   N)rg   r   rE   r   )r&   r   r   s      r'   _process_links_dataz!ResultManager._process_links_data  s<     ggen0 0c3CNN6$:Jr)   c                     	 d|v rt        j                  |d         S t        j                  |      S # t        $ r Y yw xY w)zProcess service data from NDJSON object.

        Args:
            obj: Parsed JSON object from shard file

        Returns:
            Valid Service object or None
        ri   N)r   deserialize	from_dictr   )r&   r   s     r'   _process_service_dataz#ResultManager._process_service_data  sH    		#~**3w<88 ((-- 		s   3 3 	??r   target_listestimated_linesc                    t               }|rt        j                  d| d| d       	 t        |d      5 }|D ][  }|j	                         }|s	 t        j                  |      } ||      }	|	&|	|vr"|j                  |	       |j                  |	       ] 	 ddd       t        |      S # t        $ r Y }w xY w# 1 sw Y   #xY w# t        $ r/}
t        j                  d| d|
        Y d}
~
t        |      S d}
~
ww xY w)	a  Generic shard file processor with custom data handler and deduplication.

        Args:
            filepath: Path to the shard file to process
            processor_func: Function to process each JSON object
            target_list: List to append processed items to
            estimated_lines: Optional estimated line count for debug logging

        Returns:
            Number of unique items processed from this file
        zProcessing shard z (estimated z lines)utf-8encodingNzFailed to process shard file r   )setru   r   openstripjsonloadsr/   r-   r   r   r.   )r&   r   processor_funcr   r   
seen_itemsfliner   processed_itemr   s              r'   _process_shard_genericz$ResultManager._process_shard_generic+  s    U
LL,XJl?BSSZ[\	Jh1 !Q !D::<D !"jj.)7)<)5.PZ:Z&NN>:'..~>!!  : % ! !! !  	JLL8
"QCHII:	JsS   C B;AB,B;C ,	B85B;7B88B;;C C 	C?C::C?c           	         d}t         j                  j                  | j                  d|j                        }t         j                  j                  |      rt         j                  j                  |      r	 g }g }t        t        j                  |            D ]  }|j                  d      st         j                  j                  ||      }	t         j                  j                  |	      d   dz   }
	 t        |
d      5 }t        j                  |      }t        |j                  dd            }|dkD  r|j!                  |	|f       ddd        |j%                  d	 
       |D ]9  \  }	}t        |j                  dd            }| j'                  |	|||      }||z  }; |D ]  }	| j'                  |	||      }||z  } |dkD  r4t(        j+                  d| d|j                   d| j,                          |S 	 | j0                  j                  |j                        }|rqt         j                  j                  |      rR	 | j3                  |||      }|dkD  r7t(        j+                  d| d|j                   d| j,                          ||z  }|S |S # 1 sw Y   PxY w# t"        $ r |j!                  |	       Y )w xY w# t"        $ r=}t(        j/                  d|j                   d| j,                   d|        Y d}~	d}~ww xY w# t"        $ r=}t(        j/                  d|j                   d| j,                   d|        Y d}~|S d}~ww xY w)a0  Unified recovery flow for different result types.

        Args:
            result_type: Type of result to recover
            target_list: List to append recovered items to
            processor_func: Function to process each JSON object

        Returns:
            Number of items recovered
        r   shardsz.ndjsonz.index.jsonr   r   linesNc                 ,    | d   j                  dd      S )Nr   first_tsrV   )rg   )xs    r'   <lambda>z4ResultManager._recover_result_type.<locals>.<lambda>v  s    !A$((:r2J r)   )r   
Recovered z unique z items from shards for zFailed to read z shards for r   z items from legacy file for z legacy file for )r]   r^   r_   rb   ri   r   isdirsortedlistdirendswithsplitextr   r   loadrF   rg   r-   r   sortr   ru   rv   r[   r   re   _recover_from_legacy_file)r&   r   r   r   total
shards_dirindexed_shardsunindexed_shardsrh   
shard_path
index_pathr   
index_datar   r   r   r   	file_pathfallback_counts                      r'   _recover_result_typez"ResultManager._recover_result_typeQ  s9     WW\\$..(K<M<MN
77>>*%"''--
*C%`!##%  &rzz*'= > <H#,,Y7 !#j(!CJ!#!1!1*!=a!@=!PJ<!*w? P1)-1J$'
w(B$CE$qy . 5 5z:6N O	P<  ##(J#K.< #*J
&)*..!*D&EO 77
NT_apqEUNE# #3 #J 77
NT_`EUNE# 19KK*UG8K<M<M;NNefjfofoep qr L  JJNN;#4#45		2e!%!?!?	;Xc!d!A%KK$^$4H[=N=N<OOklplulukvw ^+E uQP P
 % <(//
;<&  `{/@/@.Adii[XZ[\Z]^__`  e{/@/@.AARSWS\S\R]]_`a_bcddesp   9A<K 6J'A	JJ'B*K AL J$	J''KK KK 	L2L		L	M2MMr   c                 2   t               }	 t        |d      5 }|D ]  }|j                         }|sd}|t        j                  k(  r|j                  d      r:|}n7|t        j                  t        j                  fv r| j                  |      }|r|}|y||vs~|j                  |       |j                  |        	 ddd       t        |      S # 1 sw Y   xY w# t        $ r/}	t        j                  d| d|	        Y d}	~	t        |      S d}	~	ww xY w)a0  Recover data from legacy text files with deduplication.

        Args:
            file_path: Path to the legacy file
            result_type: Type of result being recovered
            target_list: List to append recovered items to

        Returns:
            Number of unique items recovered
        r   r   Nr   zFailed to process legacy file r   )r   r   r   r
   r   r   MATERIALINVALID_deserialize_servicer/   r-   r   ru   r   r.   )
r&   r   r   r   r   r   r   r   servicer   s
             r'   r   z'ResultManager._recover_from_legacy_file  s    U
	Li'2 ;a ;D::<D %)N"j&6&66??62-1N$)<)<j>P>P(QQ"&";";D"A"-4N%1nJ6V"~6#**>:%;;. :/; ;(  	LLL9)BqcJKK:	Ls;   C A;CC$C?C CC 	D'DDc                 @   t               }| j                  t        j                  |j                  | j
                        }| j                  t        j                  |j                  | j                        }g }| j                  t        j                  || j                        }|r|j                  j                  |       |dkD  s
|dkD  s|dkD  r.t        j                  d| j                   d| d| d| d	       |S t        j                  d| j                          |S )a&  Recover tasks from existing result files and NDJSON shards.

        Supports recovery of:
        - acquisition_tasks from LINKS data
        - check_tasks from MATERIAL data
        - invalid_keys from INVALID data

        Returns:
            RecoveredTasks with all recovered data
        r   zRecovery completed for r   z acquisition tasks,  check tasks, z invalid keyszNo tasks recovered for )r   r   r
   r   acquisitionr   r   checkr   r   invalidupdateru   rv   r[   r   )r&   	recoveredlinks_countmaterial_countinvalid_listinvalid_counts         r'   recover_taskszResultManager.recover_tasks  s    #$	 //
0@0@)BWBWY]YqYqr 22:3F3F	Y]YsYst 11*2D2DlTXTnTno $$\2 ?nq0MA4EKK)$))B-3!". /0  LL2499+>?r)   c                 R    | j                   sy| j                   j                  |      S )z(Build snapshot for specific result type.r   )rk   build_snapshotr   s     r'   r   zResultManager.build_snapshot  s%    $$$$33K@@r)   c                 R    | j                   si S | j                   j                         S )z%Build snapshots for all result types.)rk   build_all_snapshotsr9   s    r'   r   z!ResultManager.build_all_snapshots  s%    $$I$$88::r)   r   c                     	 t        j                  |      S # t        $ r"}t        j	                  d|        Y d}~yd}~ww xY w)z&Deserialize service object from stringzFailed to deserialize service: N)r   r   r   ru   warning)r&   r   r   s      r'   r   z"ResultManager._deserialize_service  s?    	&&t,, 	NN<QC@A	s    	A=Ac                    d| _         	 | j                          | j                  j                         r&| j                  j                  | j                         | j                          	 | j                          	 | j                  j                          t        j                  d	| j
                          y# t        $ r/}t        j	                  d| j
                   d|        Y d}~d}~ww xY w# t        $ r/}t        j	                  d| j
                   d|        Y d}~d}~ww xY w# t        $ r/}t        j	                  d| j
                   d|        Y d}~d}~ww xY w)
zAStop the result manager and flush all data, then build snapshots.Fz/[persist] failed to stop periodic snapshot for r   N)timeoutz([persist] failed to build snapshots for z
 on stop: z)[persist] failed to cleanup strategy for zStopped result manager for )rp   stop_periodic_snapshotr   ru   r   r[   rs   is_aliver_   rP   r   r   rj   cleanuprv   )r&   r   s     r'   stopzResultManager.stop  s>   	]'')
 %%'""4+@+@"A 		^$$&
	WMM!!# 	1$))=>-  	]LLJ499+UWXYWZ[\\	]  	^LLCDII;jYZX[\]]	^  	WLLDTYYKrRSQTUVV	WsG   B8 *C3 ;D. 8	C0%C++C03	D+<%D&&D+.	E&7%E!!E&interval_secc                 T    | j                   sy| j                   j                  |       y)z!Start periodic snapshot building.N)rk   start_periodic)r&   r   s     r'   start_periodic_snapshotz%ResultManager.start_periodic_snapshot  s"    $$,,\:r)   c                 R    | j                   sy| j                   j                          y)z Stop periodic snapshot building.N)rk   r   r9   s    r'   r   z$ResultManager.stop_periodic_snapshot  s     $$""$r)   c                    | j                   r	 t        j                  | j                         | j                  j                         D ]Y  \  }}|j                         dkD  st        j                         |j                  z
  | j                  k\  sI| j                  |       [ 	 | j                   ryy# t        $ r/}t        j                  d| j                   d|        Y d}~Bd}~ww xY w)zPeriodic flush threadr   z&[persist] error in periodic flush for r   N)rp   r   sleeprN   rm   r3   r<   r    r   r   ru   r   r[   )r&   r   r   r   s       r'   rr   zResultManager._periodic_flush%  s    ll	X

4--. ,0<<+=+=+? 8'K{{}q(TYY[6;L;L-LPTPbPb-b**;78 ll  XEdii[PRSTRUVWWXs$   AB3 !.B3 B3 3	C+<%C&&C+c           	         | j                   j                  |      }|sy|j                         }|sy	 | j                  j	                  ||| j
                         | j                  5  t        j                         | j
                  _        ddd       y# 1 sw Y   yxY w# t        $ r2}t        j                  d| d| j                   d|        Y d}~yd}~ww xY w)z2Flush a specific buffer using persistence strategyNz[persist] failed to save r~   r   )rm   rg   r4   rj   
write_dataro   r#   r   	last_saver   ru   r   r[   )r&   r   r   r3   r   s        r'   r   zResultManager._flush_buffer3  s    !!+.	YMM$$[%D 3'+yy{

$3 3 3  	YLL4[MtyykQSTUSVWXX	Ys5   3B &$B
B BB B 	C((CCc                    | j                   sy	 | j                  j                  t        j                  j
                        }|s$t        j                  d| j                   d       yt               }| j                   j                         D ]#  }|j                  |j                  dg              % t        |      }~| j                  t        j                         | j                   t        | j                         |dd}t        j                  |dd	      }t!        j"                  ||       t        j                  d
| j                          y# t$        $ r/}t        j'                  d| j                   d|        Y d}~yd}~ww xY w)zSave models data to JSON fileNz*[persist] summary file not configured for z, skip models saver   )
total_keystotal_models)rL   
updated_atr   ro      F)indentensure_asciiz#[persist] saved models summary for z$[persist] failed to save models for r   )rn   re   rg   r
   SUMMARYri   ru   r   r[   r   valuesr   r.   r   r   dumpsr   write_atomicr   r   )r&   r   unique_modelsrz   r
  rW   contentr   s           r'   r   zResultManager._save_modelsG  sI    	Rzz~~j&8&8&>&>?HI$))Tfgh  EM((//1 =$$TXXh%;<= }-L !II"iik**"%d&6&6"7$0	G jjGG))(G<LL>tyykJK 	RLL?		{"QCPQQ	Rs   AE (C)E 	F
%FF
)2   r@   F      @r+   NNi,  )(rA   rB   rC   rD   r   rE   rF   rG   rH   r(   r   r   r   r   r   r   r   r   r:   r   r   r   r   r   r   r   r
   r   r   r   r   r   r   r   r   r  r   rr   r   r   rI   r)   r'   rK   rK   X   s   J #"%;v;v ;v 	;v
 ;v ;v  ;vz!Tc !T !TF	c 	# 	
VtCy 
VUc U49 U<- 
^:tCH~ (3- c3h HW<M * bf$$:>$QYZ]Q^$	$LD
 D Dbe DL%3 %Z %^b %gj %N&~ &PA# A# A;T#s(^ ; '1B ?>;C ;$ ;%XY Y(%Rr)   rK   c                   
   e Zd ZdZ	 	 	 	 	 ddedeeef   dedede	defd	Z
d
edefdZdededefdZdedee   fdZdededee   fdZd Zdeeef   fdZdefdZd dZd!deddfdZd dZdeeeeef   f   fdZd Zy)"MultiResultManagerz&Manages results for multiple providersNrM   rS   r   rN   rO   rP   c                 ^   || _         |xs i | _        || _        || _        || _        t        t        d|            | _        i | _        t        j                         | _        t        j                  |d       t        j                  t        j                  j                  |d      d       y )NrR   TrT   rS   )rM   rS   r   rN   rO   rG   r\   rP   managersr!   r"   r#   r]   rc   r^   r_   )r&   rM   rS   r   rN   rO   rP   s          r'   r(   zMultiResultManager.__init__r  s     #"b$* %c#/?&@ A24NN$	 	I-
BGGLLK84Hr)   r[   r+   c           	      v   | j                   5  || j                  vr{| j                  j                  |      }|st	        d|       t        || j                  | j                  | j                  | j                  | j                        | j                  |<   | j                  |   cddd       S # 1 sw Y   yxY w)z)Get or create result manager for providerzProvider instance not found: )rO   rP   N)r#   r  rS   rg   
ValueErrorrK   rM   r   rN   rO   rP   )r&   r[   rL   s      r'   get_managerzMultiResultManager.get_manager  s    YY 	'4==(>>--d3$'DTF%KLL&3NNOO&&;;%)%:%:'d# ==&	' 	' 	's   BB//B8rL   r   rz   c                 J    | j                  |      }|j                  ||       y)z"Add result for a specific providerN)r   r   )r&   rL   r   rz   managers        r'   r   zMultiResultManager.add_result  s"    ""8,;-r)   r   c                 H    | j                  |      }|j                  |       y)z!Add links for a specific providerN)r   r   )r&   rL   r   r"  s       r'   r   zMultiResultManager.add_links  s     ""8,% r)   r   r   c                 J    | j                  |      }|j                  ||       y)z"Add models for a specific providerN)r   r   )r&   rL   r   r   r"  s        r'   r   zMultiResultManager.add_models  s"    ""8,3'r)   c                     | j                   5  | j                  j                         D ]  }|j                           	 ddd       y# 1 sw Y   yxY w)zFlush all providersN)r#   r  r  r   r&   r"  s     r'   r   zMultiResultManager.flush_all  sF    YY 	$==//1 $!!#$	$ 	$ 	$s   0AAc                     i }| j                   5  | j                  j                         D ]  \  }}|j                         ||<    	 ddd       |S # 1 sw Y   |S xY w)z Get statistics for all providersN)r#   r  r3   r:   )r&   ro   rL   r"  s       r'   get_all_statsz MultiResultManager.get_all_stats  s_    YY 	6%)]]%8%8%: 6!'")"3"3"5h6	6 	6 s   6AAc           	         t               }| j                  j                         D ]6  }	 | j                  |      }|j	                         }|j                  ||       8 |j                         rIt        j                  d|j                          d|j                          d|j                          d       |S # t        $ r%}t        j                  d| d|        Y d}~d}~ww xY w)z.Recover tasks from all providers' result fileszFailed to recover tasks for r   Nr   r   z acquisition tasks, and z  invalid keys from all providers)r   rS   rl   r   r   add_providerr   ru   r   has_providersrv   total_check_taskstotal_acquisition_taskstotal_invalid_keys)r&   all_recoveredr[   r"  r   r   s         r'   recover_all_tasksz$MultiResultManager.recover_all_tasks  s    )+NN'') 	IDI**40#113	**4;		I &&(KK]<<>?~ 88:;;S 33566VX   I;D6A3GHHIs   3B99	C'C""C'c                     | j                   j                         D ]$  }	 | j                  |      }|j                          & y# t        $ r%}t
        j                  d| d|        Y d}~Qd}~ww xY w)z'Backup existing files for all providerszFailed to backup files for r   N)rS   rl   r   r   r   ru   r   )r&   r[   r"  r   s       r'   backup_all_existing_filesz,MultiResultManager.backup_all_existing_files  ss    NN'') 	HDH**40--/	H  H:4&1#FGGHs   !A	A1A,,A1r   c           	         d}| j                   5  | j                  j                         D ]%  }	 |j                  r|j	                  |       |dz  }' 	 ddd       |dkD  rt        j                  d| d| d       yt        j                  d	       y# t
        $ r/}t        j                  d|j                   d|        Y d}~d}~ww xY w# 1 sw Y   |xY w)
z+Start periodic snapshots for all providers.r   r   z&Failed to start periodic snapshot for r   NzStarted periodic snapshots for z providers, interval: sz;No periodic snapshots started (simple mode or no providers))r#   r  r  rk   r  r   ru   r   r[   rv   r   )r&   r   started_countr"  r   s        r'   start_periodic_snapshotsz+MultiResultManager.start_periodic_snapshots  s    YY 	_==//1 __//77E%*_	_ 1KK9-H^_k^llmnoLLVW ! _LL#I',,WYZ[Y\!]^^_	_ 	_s4   C"BC	C%CCCCCc           	      H   | j                   5  | j                  j                         D ]  }	 |j                           	 ddd       t
        j                  d       y# t        $ r/}t
        j                  d|j                   d|        Y d}~hd}~ww xY w# 1 sw Y   ZxY w)z*Stop periodic snapshots for all providers.z%Failed to stop periodic snapshot for r   Nz,Stopped periodic snapshots for all providers)	r#   r  r  r   r   ru   r   r[   rv   )r&   r"  r   s      r'   stop_periodic_snapshotsz*MultiResultManager.stop_periodic_snapshots  s    YY 	^==//1 ^^224^	^ 	BC ! ^LL#HVXYZX[!\]]^		^ 	^s3   BAB	B&%BBBBB!c           	         i }| j                   5  | j                  j                         D ]  \  }}	 |j                         ||<    	 ddd       t        d |j                         D              }t
        j                  d| dt        |       d       |S # t        $ r*}t
        j                  d| d|        i ||<   Y d}~d}~ww xY w# 1 sw Y   xY w)z:Build snapshots for all result types across all providers.zFailed to build snapshots for r   Nc              3   2   K   | ]  }t        |        y wr  )r.   ).0provider_resultss     r'   	<genexpr>z=MultiResultManager.build_all_snapshots_all.<locals>.<genexpr>  s     ]8Hc"23]s   zBuilt z snapshots across z
 providers)r#   r  r3   r   r   ru   r   sumr  rv   r.   )r&   resultsprovider_namer"  r   total_snapshotss         r'   build_all_snapshots_allz*MultiResultManager.build_all_snapshots_all  s    -/YY 	0*.--*=*=*? 0&w0-4-H-H-JGM*0	0 ]GNNL\]]f_--?G~ZXY ! 0LL#A-PRSTRU!VW-/GM*0		0 	0s4   !CBC	C	 C?CC		CCc                     | j                   5  | j                  j                         D ]  }|j                           	 ddd       t        j                  d       y# 1 sw Y   xY w)zStop all result managersNzStopped all result managers)r#   r  r  r   ru   rv   r&  s     r'   stop_allzMultiResultManager.stop_all  sR    YY 	==//1 	 	12		 	s   0AA%)Nr  r@   Fr  r  r  )rA   rB   rC   rD   rE   r   r   rF   rG   rH   r(   rK   r   r   r   r   r   r   r   r   r(  r   r0  r2  r6  r8  rB  rD  rI   r)   r'   r  r  o  s5   0
 +/#"%II Y'I 	I
 I I  I,' ' '".3 .S . .
!# !d3i !
(3 (S ($s) (
$tC);$;< #4 *HXS X4 X$Dc4S>.A)B 3r)   r  __main__zTesting in workspace: c                       e Zd Zd Zy)MockProviderc                 F    d| _         t        ddddddddd	
      | _        y )Ntest_providerzvalid-keys.txtzno-quota-keys.txtzwait-check-keys.txtzinvalid-keys.txtzmaterial.txtzsummary.json	links.txt)validno_quota
wait_checkr   materialrW   r   )ra   rf   )r[   r   r`   r9   s    r'   r(   zMockProvider.__init__  s2    'DI'&- 3"71 .-(DKr)   N)rA   rB   rC   r(   rI   r)   r'   rG  rG    s    	r)   rG     )r   rN   key1key2zhttp://example.com/1zhttp://example.com/2key3r  rS   rI  rJ  zLinks file content:
zStats: valid_keys=z, links=zResult manager test completed!)DrD   r   r   r]   shutiltempfiler!   r   collectionsr   typingr   r   r   r   r   constant.runtimer	   
core.enumsr
   core.modelsr   r   r   r   
core.typesr   state.modelsr   tools.loggerr   atomicr   
strategiesr   r   r   ru   r   rK   r  rA   mkdtemprM   rv   rG  mock_providerr"  r   VALIDri   r   r  r^   r_   
links_filer   r   r   readr  r:   ro   rK  r   r   r   rmtreerI   r)   r'   <module>re     s  
   	      3 3 , ! Q Q   + # $ J J	I	6J 6JrTR TRnU3 U3p z !  "I
KK(45  !$yQVWX 	:++11FF3CD13IJK:++116: 	

1 WW\\)[/;W
77>>*%j! ?Q&&(3G9=>?
 !!#(Xenn>R>R=STU 	45 	i m N? ? 	i s&   	CI %H75A.I 7I <I I