
    :jI                        d Z ddlZddlZddlZddlZddlmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZmZmZ ddlmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3m4Z4 ddl5m6Z6 ddl7m8Z8 ddl9m:Z:  e0d      Z; G d d      Z< G d d      Z= G d de6e+      Z>ej~                  fde@de>fd ZAeBd!k(  r	  eA       ZCe;j                  d" eEeCj                  j                                       eCj                  j                         D ]D  \  ZIZJe;j                  d#eI d$eJj                  j                   d% eLeJj                         d&       F eCj                         ZOe;j                  d'eOj                          e;j                  d(       yy# eP$ r/ZQe;j                  d)eQ         ej                          Y dZQ[QydZQ[Qww xY w)*z
Task manager for coordinating multi-provider pipeline processing.
Creates provider instances from configuration and manages task distribution.
    N)CallableDictListOptionalSet)load_config)	ApiConfigConfig
TaskConfig)	ConditionPatternsProviderTask
SearchTaskTaskRecoveryInfo)	IProvider)client)AIBaseProvider)ProviderRegistry)
StageUtils)TaskFactory)StatusBuilder)ProviderStatusSystemStateSystemStatus)TaskDataProvider)get_session	get_token)
get_logger)get_service_namehandle_exceptionstrim   )LifecycleManager)Pipeline)TaskRecoveryManagermanagerc                       e Zd ZdZd Zdeg df   ddfdZdeg df   ddfdZ edd	      dd
       Z	e
defd       Zy)CompletionEventManagerzASimple completion event manager for task completion notificationsc                 b    t               | _        t        j                         | _        d| _        y )NF)set
_listeners	threadingLock_lock_completion_notifiedselfs    3/root/.openclaw/workspace/harvester/manager/task.py__init__zCompletionEventManager.__init__)   s!    365^^%
$)!    callbackNreturnc                 |    | j                   5  | j                  j                  |       ddd       y# 1 sw Y   yxY wzAdd completion event listenerN)r.   r+   addr1   r5   s     r2   add_listenerz#CompletionEventManager.add_listener.   s0    ZZ 	*OO)	* 	* 	*   2;c                 |    | j                   5  | j                  j                  |       ddd       y# 1 sw Y   yxY wz Remove completion event listenerN)r.   r+   discardr:   s     r2   remove_listenerz&CompletionEventManager.remove_listener3   s0    ZZ 	.OO##H-	. 	. 	.r<   error)default_result	log_levelc                    | j                   5  | j                  r
	 ddd       yd}| j                  D ]
  }	  |         || _        ddd       y# t        $ r$}d}t        j                  d|        Y d}~Ed}~ww xY w# 1 sw Y   yxY w)z"Notify all listeners of completionNTFzError in completion callback: )r.   r/   r+   	ExceptionloggerrA   )r1   successr5   es       r2   notify_completionz(CompletionEventManager.notify_completion8   s     ZZ 	0((	0 	0 G OO GGJG )0D%	0 	0 ! G#GLL#A!!EFFG	0 	0s8   A?A?A	A?	A<A72A?7A<<A??Bc                 ^    | j                   5  | j                  cddd       S # 1 sw Y   yxY w)z%Check if completion has been notifiedN)r.   r/   r0   s    r2   is_notifiedz"CompletionEventManager.is_notifiedI   s)     ZZ 	-,,	- 	- 	-s   #,r6   N)__name__
__module____qualname____doc__r3   r   r;   r@   r    rI   propertyboolrK    r4   r2   r(   r(   &   sx    K*
*Xb$h%7 *D *
.T(: .t .
 dg>0 ?0  -T - -r4   r(   c                   4    e Zd ZdZededee   defd       Z	y)ProviderFactoryz:Factory for creating provider instances from configurationtask_config
conditionsr6   c                 N   | j                   }| j                  }| j                  }| j                  xs i }|j	                         }| j
                  }|rFt        |j                        st        |j                        r|j                  |j                  d|d<   ||j                  |j                  |j                  |j                  d}|j                         D ]  \  }	}
t        |
      s|
||	<    t        |j                        r|j                  |d<   t        |j                   t"              rC|j                   r7|j%                  di       }t        |t"              si }i ||j                   |d<   t        |j&                        r|j&                  |d<   t        |j(                  t*        t,        f      r|j(                  dkD  r|j(                  |d<   t/        |j0                        t*        k(  r|j0                  dkD  r|j0                  |d	<   t3        j4                  |fd
|i|S )z.Create provider instance using global registry)	directoryplanstorage)namebase_urlcompletion_path
model_pathdefault_modelauth_keyextra_headersapi_versionr   timeoutretriesrW   )provider_typer\   apiextrascopyr[   r!   rY   rZ   r]   r^   r_   r`   itemsra   
isinstancerb   dictgetrc   rd   intfloattypere   r   create)rV   rW   rf   r\   
api_configrh   kwargsr[   paramskeyvaluerb   s               r2   create_providerzProviderFactory.create_providerS   s    $11 __
##)r %%W../43E$..!F9 "++)99$//'55
 !,,. 	$JCE{#s	$ 
##$!+!4!4F:j..5*:R:R"JJ;MmT2 "&S&S*:R:R&SF?#
&&'$.$:$:F=!j((3,7J<N<NQR<R * 2 2F9
""#s*z/A/AA/E * 2 2F9&&}VVvVVr4   N)
rM   rN   rO   rP   staticmethodr   r   r   r   rw   rS   r4   r2   rU   rU   P   s6    D1WZ 1WT)_ 1WQ_ 1W 1Wr4   rU   c                   &    e Zd ZdZdeddf fdZdee   fdZddZ	ddZ
dd	Zdd
Zdeg df   ddfdZdeg df   ddfdZdef fdZdefdZdee   fdZdeddfdZdedefdZdeeee   f   deeee   f   fdZdedee    fdZ! xZ"S )TaskManagerzDMain task manager for multi-provider coordination and data provisionconfigr6   Nc                 b   t         |   d       || _        t               | _        d | _        t        j                         | _        d | _        d | _	        t               | _        | j                          | j                          t        j                  dt!        | j                         d       y )Nrz   zInitialized task manager with z
 providers)superr3   r{   rl   	providerspipelinetime
start_time_cached_provider_status_config_hashr(   completion_events_initialize_providers_create_pipelinerF   infolen)r1   r{   	__class__s     r2   r3   zTaskManager.__init__   s    '/3v,0))+ (,$  "8!9 	""$ 	4S5H4ITUr4   c                 P   t        | j                  j                  D cg c]n  }|j                  |j                  |j
                  j                  |j
                  j                  |j
                  j                  |j
                  j                  fp c}      }t        |      }| j                  | j                  |k(  r| j                  S g }| j                  j                  D ]  }|j                  s|j                  | j                  v s)t        |j                  |j                  |j
                  j                  |j
                  j                  |j
                  j                  |j
                  j                        }|j                  |        || _        || _        |S c c}w )zLGet provider status information with caching to avoid duplicate construction)r\   enabled
searchable
gatherable	checkableinspectable)strr{   tasksr\   r   stagessearchgathercheckinspecthashr   r   r~   r   append)r1   taskru   current_hashprovider_statusesprovider_statuss         r2   _get_provider_statusesz"TaskManager._get_provider_statuses   sY     !KK--
  IILLKK&&KK&&KK%%KK''

 Cy ''38I8I\8Y/// 35KK%% 
	:D||		T^^ ;"0 LL#{{11#{{11"kk// $ 3 3# "((9
	: (9$(  I
s   A3F#c                    | j                   j                  D ]  }|j                  s#t        j	                  d|j
                          3	 |j                  D cg c]  }|j                         s| }}|s$t        j                  d|j
                   d       t        j                  ||      }|| j                  |j
                  <   t        j                  |      }t        j                  d|j
                   d|j                   dt!        |       ddj#                  |       d		        | j                  st)        d      yc c}w # t$        $ r0}t        j'                  d
|j
                   d|        Y d}~jd}~ww xY w)z3Initialize all enabled providers from configurationzSkipping disabled provider: z!No valid conditions for provider z
, skippingzCreated provider: z (z) with z conditions, stages: [, ]zFailed to create provider : NzNo valid providers configured)r{   r   r   rF   debugr\   rW   is_validwarningrU   rw   r~   r   get_enabledr   rf   r   joinrE   rA   
ValueError)r1   rV   crW   providerenabled_stagesrH   s          r2   r   z!TaskManager._initialize_providers   sm   ;;,, 	K&&;K<L<L;MNO)4)?)?PA1::<aP
P!NN%F{GWGWFXXb#cd +::;
S3;{//0 ",!7!7!D()9)9(:"[=V=V<W X
O,,B499^C\B]]^`)	: ~~<== - Q$  9+:J:J9K2aSQRs7   EE/E3'EBEE	F%FFc                    | j                   j                  j                         }| j                   j                  D ]3  }|j                  st        |j                        }|j                  ||<   5 t        j                  | j                         }||_        t        || j                        | _        t        j                  d       y)z#Create pipeline with all componentsz#Created pipeline with all providersN)r{   
ratelimitsri   r   r   r   r\   
rate_limitdeepcopyr$   r~   r   rF   r   )r1   rate_limitsrV   service_nameruntime_configs        r2   r   zTaskManager._create_pipeline   s     kk,,113;;,, 	CK""/0@0@A,7,B,BL)	C t{{3$/! @9:r4   c           
      0   | j                   j                          | j                   j                  j                         }| j	                  |      }| j                   j
                  j                         }t        ||t        d |j                         D              |j                         |j                         z         }| j                  |       | j                   j
                  j                          | j                   j                  j                  | j                   j                         | j!                         }|r| j                   j#                  |       t$        j'                  d|j(                   d|j*                   dt-        |       d       y)z#Start the task manager and pipelinec              3   2   K   | ]  }t        |        y wN)r   ).0r   s     r2   	<genexpr>z(TaskManager._on_start.<locals>.<genexpr>  s     !N#e*!Ns   )queue_tasksresult_taskstotal_queue_taskstotal_result_taskszStarted task manager: z queue tasks, z result tasks, z initial tasksN)r   startqueue_managerload_all_queues_filter_recoveryresult_managerrecover_all_tasksr   sumvaluestotal_check_taskstotal_acquisition_tasks_add_recovered_tasksbackup_all_existing_filesstart_periodic_saver   _create_initial_tasksadd_initial_tasksrF   r   r   r   r   )r1   recoverd_tasks
undo_tasks	old_tasksrecovery_infoinitial_taskss         r2   	_on_startzTaskManager._on_start  sh    	 44DDF **>:
 MM00BBD	 )""!!N*:K:K:M!NN(::<y?`?`?bb	
 	!!-0 	$$>>@ 	##778L8LM 224MM++M: 	$]%D%D$E^TaTtTtSu  vE  FI  JW  FX  EY  Yg  h	
r4   c                 z    | j                   r| j                   j                          t        j                  d       y)z Stop the task manager gracefullyzStopped task managerN)r   stoprF   r   r0   s    r2   _on_stopzTaskManager._on_stop(  s&    ==MM *+r4   r5   c                 :    | j                   j                  |       yr8   )r   r;   r:   s     r2   add_completion_listenerz#TaskManager.add_completion_listener/  s    ++H5r4   c                 :    | j                   j                  |       yr>   )r   r@   r:   s     r2   remove_completion_listenerz&TaskManager.remove_completion_listener3  s    ..x8r4   c                    t         |          ry| j                  sy| j                  j                         }|rE| j                  j                  s/| j                  j                          t        j                  d       |S )z6Check if task manager is finished processing all tasksTz/TaskManager finished, notified other components)r}   is_finishedr   r   rK   rI   rF   r   )r1   finishedr   s     r2   r   zTaskManager.is_finished7  sf     7 }}==,,. D22>>""446KKIJr4   c                    t               }| j                  dkD  r!t        j                         | j                  z
  nd}| j                  rt        j
                  nt        j                  }|j                  ||       |j                  | j                         | j                  rf|j                  | j                         | j                  j                  r5| j                  j                  j                         }|j                  |       | j                         }|j!                  |       |j#                  t%        j&                                |j)                         S )zGet current task manager statistics using enhanced StatusBuilder

        Implements TaskDataProvider.stats() interface method.
        r   )github_stats)r   r   r   runningr   RUNNINGSTOPPEDwith_basic_infowith_providers_infor~   r   with_pipeline_statsr   get_all_statswith_result_statsr   with_provider_statuswith_additional_datar   get_github_statsbuild)r1   builderruntimestateresult_statsr   s         r2   statszTaskManager.statsI  s     / 48??Q3F$))+/A'+||##9L9L/ 	##DNN3 ==''6 }}++#}};;IIK)),7 557$$_5 	$$&2I2I2K$L}}r4   c                    g }| j                   j                  D ]  }|j                  s|j                  j                  s$t
        j                  d|j                   d       K	 t               du}t               du}|s|r|j                  r|r|j                  s&|s$t
        j                  d|j                   d       | j                  j                  |j                        }|s|j                  D ]  }t!        j"                  |j                  |j$                  xs |j&                  j(                  |j&                  j(                  d|j                  |j&                  j*                  |j&                  j,                  |j&                  j.                        }|j1                  |         |rSt3        d |D              }t
        j                  d	t5        |       d
t5        |       ddj7                  |              |S t
        j                  d       |S # t        $ r' t
        j                  d|j                   d       Y 3w xY w)z-Create initial search tasks for all providersz+Skipping initial search tasks for provider z due to search stage disabledNzSkipping search for provider z* as no github token or session is providedr"   )r   queryregexpageuse_apiaddress_patternendpoint_patternmodel_patternc              3   4   K   | ]  }|j                     y wr   )r   )r   r   s     r2   r   z4TaskManager._create_initial_tasks.<locals>.<genexpr>  s     &Gt}}&Gs   zCreated z initial search tasks for z providers: r   zaNo initial search tasks created - all providers have search stage disabled or missing credentials)r{   r   r   r   r   rF   r   r\   r   r   r   r   rE   r~   rm   rW   r   create_search_taskr   patternskey_patternr   r   r   r   r*   r   r   )	r1   r   rV   	has_tokenhas_sessionr   	conditionr   providers_with_taskss	            r2   r   z!TaskManager._create_initial_tasksk  s6   ;;,, /	#K&&%%,,A+BRBRASSpq %Kt3	)m47!'#++I'//NN78H8H7IIst  ~~))+*:*:;H%00 #	"55(--#//KY-?-?-K-K#,,88'//$-$6$6$F$F%.%7%7%H%H"+"4"4"B"B	 T"#G/	#d #&&G&G#G KK3u:,&@EYAZ@[[ghlhqhq  sG  iH  hI  J 	 KKs G  3K4D4D3EEop 	s   %AH,IIr   c                     t        | j                  | j                        }|j                  |j                         |j                  |j                         y)z7Add recovered tasks using enhanced TaskRecoveryStrategyN)r%   r   r~   recover_queue_tasksr   recover_result_tasksr   )r1   r   recovery_strategys      r2   r   z TaskManager._add_recovered_tasks  sE     0t~~N 	--m.G.GH 	..}/I/IJr4   r   c                 `    |j                   r|j                   d   j                  S t               S )z)Extract patterns from provider conditionsr   )rW   r   r   r1   r   s     r2   _get_provider_patternsz"TaskManager._get_provider_patterns  s,     &&q)222zr4   	recoveredc           	      h   i }|j                         D ]  \  }}g }|D ]  }|r|j                  | j                  vr| j                  |j                        }|r(t	        j
                  ||      r|j                  |       ct        j                  d| d|j                   d        |s|||<    |S )z3Filter recovered tasks based on stage configurationzSkipping recovery of z task for provider z - stage disabled)	rj   r   r~   _get_configr   r   r   rF   r   )r1   r  filteredstager   valid_tasksr   r{   s           r2   r   zTaskManager._filter_recovery  s    %OO- 	.LE5K ut}}DNNB))$--8j..vu=&&t,LL#8?RSWS`S`Raar!stu "-	. r4   c                 V    t        fd| j                  j                  D        d      S )zGet task config for providerc              3   B   K   | ]  }|j                   k(  s|  y wr   )r\   )r   tr   s     r2   r   z*TaskManager._get_config.<locals>.<genexpr>  s     H1QVVx5GQHs   N)nextr{   r   r  s    `r2   r  zTaskManager._get_config  s     H 1 1H$OOr4   rL   )#rM   rN   rO   rP   r
   r3   r   r   r   r   r   r   r   r   r   r   rR   r   r   r   r   r   r   r   r   r   r  r   r   r   r   r   r   r  __classcell__)r   s   @r2   rz   rz      s"   NVv V$ V0(!^(< (!T >D;$%
N,6T0B 6t 698BH3E 9$ 9T $ |  D@tJ'7 @D
K2B 
Kt 
K~ ( $sD4F/F*G DQTVZ[gVhQhLi *PC PHZ,@ Pr4   rz   config_filer6   c                 4    t        |       }|syt        |      S )z:Factory function to create task manager from configurationN)r   rz   )r  r{   s     r2   create_task_managerr    s    %Fvr4   __main__z%Created task manager with providers: z  r   z with z conditionszManager stats: zTask manager test completed!zTask manager test failed: )TrP   ri   r,   r   	tracebacktypingr   r   r   r   r   constantr{   r   config.schemasr	   r
   r   core.modelsr   r   r   r   r   
core.typesr   r   r   search.provider.baser   search.provider.registryr   
stage.baser   stage.factoryr   state.builderr   state.modelsr   r   r   state.typesr   tools.coordinatorr   r   tools.loggerr   tools.utilsr   r    r!   baser#   r   r$   recoveryr%   rF   r(   rU   rz   DEFAULT_CONFIG_FILEr   r  rM   r&   r   listr~   keysrj   r\   r   r   r   rW   	get_statsr   rE   rH   rA   	print_excrS   r4   r2   <module>r*     s  
     6 6   8 8 W W    / 5 ! % ' B B ( 4 # A A "  )	I	'- '-T5W 5WpPP"$4 PPf
 ,4+G+G S K  z%';DARARAWAWAY<Z;[\] &//557 	oND(KK"TF"X%7%7%@%@$AHL_L_H`Gaalmn	o !!#oeoo%67823! $  1!56	s   )CG G8	%G33G8