:jgNdZddlZddlZddlZddlZddlmZmZddlm Z m Z ddl m Z m Z mZmZmZddlmZe d Gd d Ze Gd d eZe GddeZe GddeZe GddeZe GddeZe GddZe GddZe GddZe GddZe GddZe Gd d!Ze Gd"d#Ze Gd$d%Z e Gd&d'Z!e Gd(d)Z"e Gd*d+Z#e Gd,d-Z$e Gd.d/Z%d0e!d1e"d2dfd3Z&e Gd4d5Z'e Gd6d7Z(e Gd8d9Z)e Gd:d;Z*y)zProviderTask.'sTZZ\1Brrtask_idprovider created_atrattemptsreturnc|jj|j|j|j|j |j dS)z,Serialize task to dictionary for persistence)typer&r(r)r*data) __class__rr&r(r)r*_serialize_dataselfs rto_dictzProviderTask.to_dict,sCNN++|| // ((*   rr.c|j|}|d|_|d|_|d|_|d|_|j |d|S)z Deserialize task from dictionaryr&r(r)r*r.)__new__r&r(r)r*_deserialize_data)clsr.instances r from_dictzProviderTask.from_dict7s\;;s# ? ,"<0 ,""4<0rcy)zSerialize task-specific dataNrr1s rr0zProviderTask._serialize_dataB rNcy)zDeserialize task-specific dataNrr2r.s rr6zProviderTask._deserialize_dataGr;rc.|xjdz c_y)zIncrement attempt counterr N)r*r1s rincrement_attemptszProviderTask.increment_attemptsLs  rcDtj|jz S)zGet task age in seconds)timer)r1s rget_age_secondszProviderTask.get_age_secondsPsyy{T__,,rmax_age_secondsc(|j|kDS)z&Check if task has exceeded maximum age)rB)r2rCs r is_expiredzProviderTask.is_expiredTs##%77rr+N)rrrrrr&rrr(rAr)floatr*intrrr3 classmethodr9rr0r6r?rBboolrErrrr r s)BCGSCHcdii8J8Hc  c3h  T#s(^ c3h   d38n   --8%8D8rr ceZdZUdZdZeed<dZeed<dZe ed<dZ e ed<dZ eed <dZ eed <dZeed <d eeeffd Zdeeefd dfdZd efdZy) SearchTaskz0Task for searching GitHub for potential API keysr'queryregexr pageFuse_apiaddress_patternendpoint_pattern model_patternr+c|j|j|j|j|j|j |j dS)NrMrNrOrPrQrRrSrUr1s rr0zSearchTask._serialize_dataesCZZZZII||#33 $ 5 5!//  rr.Nc|d|_|jdd|_|d|_|jdd|_|jdd|_|jdd|_|jd d|_y) NrMrNr'rOrPFrQrRrS)rMgetrNrOrPrQrRrSr=s rr6zSearchTask._deserialize_datapsw'] XXgr* L xx 51 #xx(92> $);R @!XXor:rc6|jxs |jS)Get the primary search term)rMrNr1s rget_search_termzSearchTask.get_search_termyszz'TZZ'r)rrrrrMrrrNrOrHrPrJrQrRrSrrr0r6rZrrrrLrLYs:E3OE3OD#MGTOScM3  c3h  ;d38n;;((rrLceZdZUdZdZeed<dZeed<dZe ed<dZ eed<dZ eed<dZ eed <d e eeffd Zd e eefd d fdZy )AcquisitionTaskz0Task for acquiring API keys from discovered URLsr'url key_patternretriesrQrRrSr+c|j|j|j|j|j|j dS)Nr]r^r`rQrRrSrbr1s rr0zAcquisitionTask._serialize_datas>88++||#33 $ 5 5!//   rr.Nc|d|_|d|_|jdd|_|jdd|_|jdd|_|jdd|_y) Nr]r^r`r_rQr'rRrS)r]r^rWr`rQrRrSr=s rr6z!AcquisitionTask._deserialize_datasg; .xx 1- #xx(92> $);R @!XXor:r)rrrrr]rrr^r`rHrQrRrSrrr0r6rrrr\r\~sp:CMKGSOScM3 c3h ;d38n;;rr\c|eZdZUdZedZded<dZeed<dZ e ed <d e ee ffd Z d e ee fd d fdZy ) CheckTaskzTask for validating API keysctSr"Servicerrrr%zCheckTask.wyrrrhservicer' custom_urlr_r`r+cf|jj|j|jdSN)rjrkr`rjr3rkr`r1s rr0zCheckTask._serialize_data*||++-//||  rr.Nctj|d|_|jdd|_|jdd|_yNrjrkr'r`r_rhr9rjrWrkr`r=s rr6zCheckTask._deserialize_data<((i9 ((<4xx 1- rrrrrrrjrrkrr`rHrrr0r6rrrreresX&/@AGYAJGS c3h .d38n..rrec|eZdZUdZedZded<dZeed<dZ e ed <d e ee ffd Z d e ee fd d fdZy ) InspectTaskz$Task for inspecting API capabilitiesctSr"rgrrrr%zInspectTask.rirrrhrjr'rkr_r`r+cf|jj|j|jdSrmrnr1s rr0zInspectTask._serialize_datarorr.Nctj|d|_|jdd|_|jdd|_yrqrrr=s rr6zInspectTask._deserialize_datarsrrtrrrrvrvsX./@AGYAJGS c3h .d38n..rrvceZdZUdZdZeed<dZeed<dZ e ed<dZ eed <d Z eed <d Z eed <dZeed<dZdedefdZy)RateLimitConfigz#Unified rate limiting configuration? base_rate burst_limitTadaptive?backoff_factor皙?recovery_factorg@max_rate_multiplier皙?min_rate_multiplierc|jdkr td|jdkr tdd|jcxkrdkstdtd|jdkr tdy)z!Validate rate limit configurationrzbase_rate must be positivezburst_limit must be positiver z&backoff_factor must be between 0 and 1zrecovery_factor must be > 1N)r} ValueErrorrrrr1s r __post_init__zRateLimitConfig.__post_init__s >>Q 9: :   q ;< <D''+!+EF F,EF F   1 $:; ; %r success_ratior+c|js |jS|dkDr!t|j|j}n(|dkr!t |j |j}nd}|j|zS)z.Calculate adjusted rate based on success ratiog?rr|)rr}minrrmaxrr)r2r multipliers rcalculate_adjusted_ratez'RateLimitConfig.calculate_adjusted_ratesg}}>> ! 3 T55t7K7KLJ S T55t7J7JKJJ~~ **rN)rrrrr}rGrrrHrrJrrrrrrrrrr{r{se-IuKHdNE OU !$$!$$ < +U +u +rr{cNeZdZUdZeed<eed<eed<eed<dZeeed<y) LogFileInfozInformation about a log filefilenamesizemodifiedpathNerror)rrrrrrrr rrrrrs'&M IM IE8C=rrcVeZdZUdZeed<eeZe e e fed<dZ e e ed<y) LoggingStatszLogging system statisticsactive_loggersr log_filesNlogs_directory)rrrrrHrrrrrrrrr rrrrrs2#(-d(CItC$%C$(NHSM(rrceZdZUdZeeZeded<eeZ ee ed<ee Z e ded<defdZdefd Zdefd Zdefd Zdedfd Zde fd Zy)RecoveredTasksz#Recovery data for a single providerrrhcheck acquisitioninvalidr+cdt|jxs|jxs |jS)z Check if any tasks need recovery)rJrrrr1s r has_taskszRecoveredTasks.has_tasks s%DJJB$"2"2BdllCCrc,t|jS)zGet number of check tasks)lenrr1s r check_countzRecoveredTasks.check_count s4::rc,t|jS)zGet number of acquisition tasks)rrr1s racquisition_countz RecoveredTasks.acquisition_counts4##$$rc,t|jS)zGet number of invalid keys)rrr1s r invalid_countzRecoveredTasks.invalid_counts4<<  rc^|jDcgc]}||jvs|c}Scc}w)z(Get check tasks filtered by invalid keys)rr)r2tasks rvalid_check_tasksz RecoveredTasks.valid_check_taskss%!%Ht4<>""rcVtd|jjDS)z*Get total check tasks across all providersc3<K|]}|jywr")r.0rs r z6AllRecoveredTasks.total_check_tasks..;sL55$$&Lsumrvaluesr1s rtotal_check_tasksz#AllRecoveredTasks.total_check_tasks9s LDNN4I4I4KLLLrcVtd|jjDS)z0Get total acquisition tasks across all providersc3<K|]}|jywr")rrs rrz.?sR5**,Rrrr1s rtotal_acquisition_tasksz)AllRecoveredTasks.total_acquisition_tasks=s R$..:O:O:QRRRrcVtd|jjDS)z+Get total invalid keys across all providersc3<K|]}|jywr")rrs rrz7AllRecoveredTasks.total_invalid_keys..CsNU5&&(Nrrr1s rtotal_invalid_keysz$AllRecoveredTasks.total_invalid_keysAs Ndnn6K6K6MNNNrc |jsyd|jd|jd|jd|j S)z"Get summary of all recovered taskszNo tasks recoveredz Providers: z , Check: z, Acquisition: z , Invalid: )rrrrrr1s rrzAllRecoveredTasks.summaryEsj!!#'$--/01,,./0 88:;<//12 4 r)rrrrrrrrrrrrrrJrrHrrrrrrrrrr"s)+0+FItC'(F))^)) :::$t$##M3MSSOCO    rrcneZdZUdZeeZeee e fe d<dZ e ee d<dZee d<dZee d<y) TaskRecoveryInfozInformation for task recoveryr queue_tasksN result_tasksrtotal_queue_taskstotal_result_tasks)rrrrrrrrrr r rrr rrrHrrrrrrRsI'16t1LKc4 --.L04L(,-4srrceZdZUdZdZeed<dZeed<dZeed<dZ eed<de fdZ d e de fd Zde fd Zdefd Zdeeeffd ZedeeefddfdZdefdZedededfdZy)rhzCore service data model representing an API service endpoint This is the fundamental data structure representing a discovered API service with its authentication and endpoint information. r'addressendpointkeymodelr+cpt|j|j|j|jfS)z2Hash based on all fields for use in sets and dicts)hashrrrrr1s r__hash__zService.__hash__ps&T\\4==$((DJJGHHrotherct|tsy|j|jk(xrO|j|jk(xr4|j|jk(xr|j |j k(S)z'Equality comparison based on all fieldsF) isinstancerhrrrrr2rs r__eq__zService.__eq__tsg%) LLEMM ) * / *EII% * ekk)  rcdt|jxr|jxs |jS)z1Check if service has minimum required information)rJrrrr1s ris_validzService.is_valids#DHH@$,,"?$--AArcZ|jd|jd|jdddS)z&Get unique identifier for this service:Nz...)rrrr1s rget_identifierzService.get_identifiers-,,qq"1cBBrc`|j|j|j|jdS)'Convert to dictionary for serializationrrrrrr1s rr3zService.to_dicts*|| 88ZZ   rr.c ||jdd|jdd|jdd|jddS)zCreate Service from dictionaryrr'rrrr)rW)r7r.s rr9zService.from_dictsIHHY+XXj"-#((7B'   rcp|js$|js|js |jSi}|jr|j|d<|jr|j|d<|jr|j|d<|jr|j|d<|sdSt j |S)Nrrrrr')rrrrjsondumpsr=s r serializezService.serializes||DMM$**88O <<"llDO ==#}}D  88((DK :: JJDMr34::d#33rtextc |sy tj|}||jdd|jdd|jdd|jddS#t$r ||cYSwxYw)Nrr'rrrr)r)rloadsrW Exception)r7ritems r deserializezService.deserializes{ !::d#DB/*b1HHUB'hhw+    !4=  !sA A&&A;:A;N)rrrrrrrrrrrHrobjectrJrrrrr3rIr9rr rrrrrhrh\sGSHcCME3OI#I  F  t  B$BCC c3h  T#s(^   434  !s !x ': ! !rrhc eZdZUdZdZeed<ejZ eed<dZ e ed<dZ e ed<d Zeeed <ed efd Zed efd Zedde de d dfdZe ddede de d eed df dZd efdZd ee effdZy ) CheckResultzResult of API token validation check Represents the outcome of validating an API token against a service, including success status, error information, and additional metadata. F available error_reasonr'message response_timeN status_coder+c|jS)z.Alias for available for backward compatibility)rr1s rokzCheckResult.oks~~rc|jS)z1Alias for error_reason for backward compatibility)rr1s rreasonzCheckResult.reasons   rc6|dtj||S)z Create a successful check resultT)rrrr)r UNKNOWN)r7rrs rsuccesszCheckResult.successs#$,,'   rrc8|d||xs |j||S)zCreate a failed check resultFrrrrr)value)r7rrrrs rfailzCheckResult.fails) +v||'#   rc6|jjS)zCheck if the error is retryable)r is_retryabler1s rr zCheckResult.is_retryables  --//rc|j|jj|j|j|j dS)rr)rrr rrrr1s rr3zCheckResult.to_dicts= --33||!//++   r)zToken is validr)r'rN)rrrrrrJrr rrrrrrGrr rHpropertyrrrIrr r rrr3rrrrrs It + 3 3L+3GSM5!%K#% D! !! c U Ub  nr     +.  DI  ^fgj^k      0d0 c3h rrc`eZdZUdZdZeed<dZeed<dZeed<dZ eed<de eeffdZ y ) Patternsz)Extraction patterns for keys and metadatar'r^rQrRrSr+c`|j|j|j|jdS)zConvert to dictionaryr^rQrRrSrr1s rr3zPatterns.to_dicts2 ++#33 $ 5 5!//   rN) rrrrr^rrrQrRrSrr3rrrrrsB3KOScM3 c3h rrceZdZUdZdZeeed<ee Z e ed<dZ eed<dZ e ed <d Zd efd Zd e fd Zd efdZded e fdZd eeeffdZedeeefd dfdZy) ConditionzSearch condition with complete pattern configuration Defines search parameters and extraction patterns used to discover API keys and services for a specific provider. NrMrpatternsr' descriptionTenabledc`|js"|jjs tdyy)z'Validate condition after initializationz/Condition must have either query or key_patternN)rMrr^rr1s rrzCondition.__post_init__s(zz$--";";NO O# > >>A ;< < <s7$**,ws4::.?xDLLHYGZZbccrN)rrrrrrrlr rrrmrnrHrprGrrrrcrrrrkrk,sy*"48E4 ?8$T:GT)_: %d ;Hd9o;"'"=JY=_s_=%= dSdrrkcLeZdZUdZeeZeee d<de fdZ defdZ y)InspectTaskResultz"Result from inspect task executionrmodelsr+c2t|jdkDS)z&Check if models listing was successfulrrrvr1s rr_zInspectTaskResult.is_successfulIs4;;!##rc4dt|jdS)rarbz modelsrxr1s rrczInspectTaskResult.get_summaryMsDKK()11rN) rrrrrrrvr rrrJr_rcrrrruruCs0,d3FDI3$t$2S2rru)+rrr(rAr#abcrr dataclassesrrtypingrrr r r enumsr rr rLr\rervr{rrrrrrhrrrr"rBrLrYr[rerkrurrrr~s  #(11 $<<< 78378 78t !(!( !(H ;l; ;: . . .( .,. .( "+"+ "+L      )) ) xx x@ , ,  , ^      \!\! \!~ : :  : z     $ 7 7  7 t ~~ ~B FF F6 00 0, @X @) @ @   b b  b  : :  : dd d,  2 2  2r