
    :j                        d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZmZ dd	lmZ d
dddddZdadeddfdZ	  ej:                         dk(  rd ee	j>                  d      rQe	j>                  jA                         r7ejB                  jD                  Z"e"jG                  e"jI                  d      d        G d dejL                        Z' G d dejL                        Z( G d dejL                        Z)defdZ* G d d ejV                        Z, e'd!      Z- e)       Z.d"a/dejL                  fd#Z0d$e1ddfd%Z2 G d& d'ejL                        Z3 G d( d)ejh                  jj                        Z6 G d* d+ejL                        Z7 e7d,      Z8ejr                  ejt                  ejv                  ejx                  ejz                  d-Z> G d. d/      Z?d_d0Z@ G d1 d2ej                        ZBd3e1deBfd4ZCd5ej~                  deBfd6ZD G d7 d8      ZE eE       ZFd5ej~                  d9e1d:e1fd;ZGd`d3ee1   dej~                  fd<ZH	 	 	 	 	 dad=e1d>eId?e1d@eIdAeJf
dBZKdC ZLdD ZMdefdEZNdbdFeIfdGZOdH ZPdI ZQdJ ZRdK ZSeTdLk(  rD eKdM        eHdN      ZU eHdO      ZVeUj                  dP       eVj                  dQ       eUj                  dR        eHdS      ZZeZj                  dT       eZj                  dU       e?j                           eN       Z\ e]dV        e]dWe\j                           e]dXe\j                           e]dY       e\j                  j                         D ]`  \  ZbZWeWj                  6 e]dZeb         e]d[eWj                           e]d\eWj                          H e]dZeb d]eWj                   d^       b yy# e%$ r Y w xY w)ca  
Standardized logging system for the async pipeline project.
Based on gemini-balance logger with modular organization.

Features:
- Color-coded console output with ANSI support
- File logging with rotation (main log + error-only log)
- API key redaction for security
- Module-specific loggers
- Automatic log cleanup
- Fixed-width formatting for readability

Log files are saved to the 'logs' directory:
- pipeline_YYYYMMDD_HHMMSS.log: All log levels
- pipeline_errors_YYYYMMDD_HHMMSS.log: ERROR and CRITICAL only
    N)datetime)Path)AnyDictOptional)DEFAULT_LOG_CLEANUP_DELETE)LogFileInfoLoggingStats   )redact_api_keys_in_textz[34mz[32mz[33mz[31mz[1;31m)DEBUGINFOWARNINGERRORCRITICALTenabledreturnc                     t        |       ay)z-Enable or disable ANSI color in console logs.N)bool_COLOR_ENABLED)r   s    3/root/.openclaw/workspace/harvester/tools/logger.pyset_color_enabledr   5   s     ']N    Windowsisattyi   c                   @     e Zd ZdZdej
                  def fdZ xZS )ColoredFormatterz9Custom log formatter with color support and file locationrecordr   c                    t        j                  |j                  |j                  |j                  |j
                  |j                  |j                  |j                  |j                  t        |dd       	      }|j                  j                         D ]  \  }}t        ||      rt        |||       ! t        j!                  |j"                  d      }t$        r!t'               r| |j"                   d|_        nt)        |j"                        |_        d|j*                   d|j
                   d|_        t.        | a  |      S )N
stack_info)	namelevelpathnamelinenomsgargsexc_infofuncr!    z[0m[:])logging	LogRecordr"   levelnor$   r%   r&   r'   r(   funcNamegetattr__dict__itemshasattrsetattrCOLORSget	levelnamer   _is_ttystrfilenamefilelocsuperformat)selfr   record_copykeyvaluecolor	__class__s         r   r?   zColoredFormatter.formatH   s   ''..__==

__v|T:

 !////1 	1JC;,S%0	1
 

;00"5gi',gk.C.C-DG$LK!$'(=(=$>K! !"+"6"6!7q9K9K8LANw~k**r   	__name__
__module____qualname____doc__r.   r/   r;   r?   __classcell__rE   s   @r   r   r   E   s$    C+W.. +3 + +r   r   c                   h     e Zd ZdZdededdf fdZdej                  def fdZ	d	edefd
Z
 xZS )APIKeyRedactionFormatterz6Custom formatter that redacts API keys in log messagesr'   kwargsr   Nc                 $    t        |   |i | y Nr>   __init__r@   r'   rO   rE   s      r   rS   z!APIKeyRedactionFormatter.__init__k       $)&)r   r   c                 D    t         |   |      }| j                  |      S rQ   )r>   r?   _redact_api_keys_in_messager@   r   formatted_msgrE   s      r   r?   zAPIKeyRedactionFormatter.formatn   s"    v.//>>r   messagec                     	 t        |      S # t        $ r7}t        j                  t              }|j                  d|        Y d}~yd}~ww xY w)6Replace API keys in log message with redacted versionsz!Error redacting API keys in log: Nz[LOG_REDACTION_ERROR])r   	Exceptionr.   	getLoggerrG   error)r@   rZ   eloggers       r   rW   z4APIKeyRedactionFormatter._redact_api_keys_in_messaget   sI    	+*733 	+&&x0FLL<QC@A*		+s   
 	A-AArG   rH   rI   rJ   r   rS   r.   r/   r;   r?   rW   rK   rL   s   @r   rN   rN   h   sL    @*c *S *T *?W.. ?3 ?+3 +3 +r   rN   c                   4    e Zd ZdZdej
                  defdZy)JSONFormatterz6Simple JSON formatter for structured logging to files.r   r   c           	         	 | j                  |d      |j                  |j                  |j                  |j                  |j                         d}|j                  j                         D ]!  \  }}||vs|j                  d      r|||<   # t        j                  |d      S # t        $ r< |j                   d|j                   d|j                   d|j                          cY S w xY w)	N%Y-%m-%d %H:%M:%S)datefmt)tsr#   ra   filelinerZ   _F)ensure_ascii r,   )
formatTimer9   r"   r<   r%   
getMessager3   r4   
startswithjsondumpsr]   )r@   r   payloadrB   rC   s        r   r?   zJSONFormatter.format   s    	aoof6IoJ)) ++!,,.G %oo335 )
Ug%cnnS.A#(GCL) ::gE:: 	a&&'q(96==/6K\K\K^J_``	as   A4B' 7B' 	B' 'AC,+C,N)rG   rH   rI   rJ   r.   r/   r;   r?    r   r   rd   rd      s!    @aW.. a3 ar   rd   c                      	 t        t        j                  d      xr t        j                  j                         S # t        $ r Y yw xY w)z0Detect if stdout is a TTY for safe color output.r   F)r5   sysstdoutr   r]   rt   r   r   r:   r:      s<    szz8,D1B1B1DD s   9< 	AAc                   L     e Zd ZdZdZd fdZdej                  defdZ	 xZ
S )RedactionFilterz>Filter that redacts API keys in log records before formatting.Nr   c           	          t         |           t        j                  It	        j
                  ddddddd       }t        |j                  j                               t        _        y y )Nr*   r   rt   )r"   r#   r$   r%   r&   r'   r(   )	r>   rS   ry   _standard_attrsr.   r/   setr3   keys)r@   baseline_recordrE   s     r   rS   zRedactionFilter.__init__   s[    **2%//Rq2VW]_fhswxO.1/2J2J2O2O2Q.RO+ 3r   r   c                 t   	 |j                         }t        |      }t        |j                  t              r||_        |j
                  j                         D ]O  \  }}|j                  d      rt        |t              s)|| j                  vs8t        |      }t        |||       Q y# t        $ r Y yw xY w)Nrk   T)ro   r   
isinstancer&   r;   r3   r4   rp   r{   r6   r]   )r@   r   r&   rB   rC   redacteds         r   filterzRedactionFilter.filter   s    	##%C)#.C&**c* 
 %oo335 3
U~~c*z%/E#UYUiUiJi6u=HFC23  		s$   A-B+ 0B+ B+ B+ +	B76B7r   N)rG   rH   rI   rJ   r{   rS   r.   r/   r   r   rK   rL   s   @r   ry   ry      s,    H OSW.. 4 r   ry   z=%(asctime)s | %(levelname)-17s | %(fileloc)-30s | %(message)stextc                  ,    t         dk(  rt        S t        S )zEReturn the active formatter for file handlers based on configuration.rq   )_FILE_LOG_FORMATFILE_FORMATTER_JSONFILE_FORMATTERrt   r   r   get_file_formatterr      s    "2f"<P.Pr   fmtc                 &    | dvrt        d      | ay)z/Set file logging format mode: "text" or "json".r   rq   z(file log format must be 'text' or 'json'N)
ValueErrorr   )r   s    r   set_file_log_formatr      s     ""CDDr   c                   @     e Zd ZdZdej
                  def fdZ xZS )FileFormatterzFFile formatter that removes ANSI color codes and uses clean formattingr   r   c                 f    d|j                    d|j                   d|_        t        |   |      S Nr+   r,   r-   )r<   r%   r=   r>   r?   r@   r   rE   s     r   r?   zFileFormatter.format   s1    V__-Qv}}oQ?w~f%%r   rF   rL   s   @r   r   r      s$    P&W.. &3 & &r   r   c                   x     e Zd ZdZd fd	Zd Zdej                  f fdZd Z	d Z
d Zd	 Zd
 Zd Zd Z xZS )SafeRotatingFileHandlerzGCross-platform safe rotating file handler with hybrid rollover strategyc                 p    t         |   |||||d       d| _        d| _        d| _        dddd| _        y)	z7Initialize with delayed file creation and safe rolloverT)delayF   皙?r   )rename_successcopy_successfailuresN)r>   rS   _initialized_rollover_retries_rollover_delay_rollover_stats)r@   r<   mode	max_bytesbackup_countencodingrE   s         r   rS   z SafeRotatingFileHandler.__init__   sF    4L(RVW!!""23QTUVr   c                    | j                   st        j                          t        j                  j                  | j                        sjt        j                  t        j                  j                  | j                        d       t        | j                  d| j                        5  	 ddd       d| _         yy# 1 sw Y   xY w)zDEnsure file is created and handler is initialized before first writeTexist_okwr   N)r   Logger_ensure_logs_directoryospathexistsbaseFilenamemakedirsdirnameopenr   r@   s    r   _ensure_initializedz+SafeRotatingFileHandler._ensure_initialized   s      ))+77>>$"3"34BGGOOD,=,=>N$++S4==I  $D !
 s   *B==Cr   c                 \    | j                   s| j                          t        |   |       y)z/Ensure initialization before writing log recordN)r   r   r>   emitr   s     r   r   zSafeRotatingFileHandler.emit   s$      $$&Vr   c                 2   t        j                          }| j                  r!| j                  j                          d| _        | j                         }|r| j                  dxx   dz  cc<   n)| j                         }|r| j                  dxx   dz  cc<   |s'| j                          | j                  dxx   dz  cc<   | j                  s| j                         | _        t        j                          |z
  }|dkD  rt        d|dd	| j                          yy)
z*Perform safe rollover with hybrid strategyNr   r   r   r   g      ?zSlow rollover detected: .2fzs for )timestreamclose_try_rename_rolloverr   _try_copy_rollover_truncate_currentr   _openprintr   )r@   
start_timesuccessdurations       r   
doRolloverz"SafeRotatingFileHandler.doRollover   s    YY[
;;KKDK ++-  !12a72--/G$$^494""$  ,1, zz**,DK 99;+c>,XcN&ARAR@STU r   c                    t        | j                        D ]}  }	 | j                  d      }| j                  dkD  r| j	                          t
        j                  j                  | j                        r t        j                  | j                  |        y y# t        t        f$ rU}|| j                  dz
  k  r*t        j                  | j                  |dz   z         Y d}~t        d|        Y d}~ yd}~ww xY w)z1Try fast rename-based rollover (preferred method)r   r   TNzRename rollover failed: F)ranger   _get_backup_namebackupCount_rotate_backupsr   r   r   r   renameOSErrorPermissionErrorr   sleepr   r   )r@   ibackup_namer`   s       r   r   z,SafeRotatingFileHandler._try_rename_rollover  s    t--. 	A"33A6 ##a'((* 77>>$"3"34IId//=	.  _- t--11JJt33q1u=>045s   A9BC;&7C6"C66C;c                    	 | j                  d      }| j                  dkD  r| j                          t        j                  j                  | j                        rmt        j                  | j                  |       t        | j                  d| j                        5 }|j                  d       |j                          ddd       yy# 1 sw Y   yxY w# t        $ r}t        d|        Y d}~yd}~ww xY w)	z,Try copy+truncate rollover (fallback method)r   r   r   r   NTzCopy rollover failed: F)r   r   r   r   r   r   r   shutilcopy2r   r   truncateflushr]   r   )r@   r   fr`   s       r   r   z*SafeRotatingFileHandler._try_copy_rollover8  s    	//2K !#$$& ww~~d//0T..< $++S4==I QJJqMGGI 	  	*1#./	s6   BC "C	?C 	CC C 	C6C11C6c                 D   	 t        | j                  d| j                        5 }|j                  d       |j	                          t        j                  |j                                ddd       y# 1 sw Y   yxY w# t        $ r}t        d|        Y d}~yd}~ww xY w)z$Fallback: just truncate current filer   r   r   NzFile truncation failed: )
r   r   r   r   r   r   fsyncfilenor]   r   )r@   r   r`   s      r   r   z)SafeRotatingFileHandler._truncate_currentQ  s~    	2d''t}}E %

1	$% % %  	2,QC011	2s5   "A> AA2)A> 2A;7A> ;A> >	BBBc                 $    | j                    d| S )zGenerate backup filename.)r   )r@   indexs     r   r   z(SafeRotatingFileHandler._get_backup_name[  s    ##$AeW--r   c                    | j                  | j                        }t        j                  j	                  |      r	 t        j
                  |       t        | j                  dz
  dd      D ]^  }| j                  |      }| j                  |dz         }t        j                  j	                  |      sH	 t        j                  ||       ` y# t        $ r Y w xY w# t        $ r Y |w xY w)zRotate existing backup filesr   r   N)	r   r   r   r   r   remover   r   r   )r@   oldestr   old_namenew_names        r   r   z'SafeRotatingFileHandler._rotate_backups_  s     &&t'7'7877>>&!		&!
 t''!+Q3 	A,,Q/H,,QU3Hww~~h'IIh1		    s#   C 4C	CC	C('C(c                 |    | j                   j                         | j                  | j                  | j                  dS )zGet handler statistics)rollover_statsretriesr   ri   )r   copyr   r   r   r   s    r   	get_statsz!SafeRotatingFileHandler.get_statss  s;     #22779--))%%	
 	
r   )ar   r   N)rG   rH   rI   rJ   rS   r   r.   r/   r   r   r   r   r   r   r   r   rK   rL   s   @r   r   r      sG    QW	%7,, V>622.(
r   r   c                   h     e Zd ZdZdededdf fdZdej                  def fdZ	d	edefd
Z
 xZS )FileFormatterWithRedactionzDFile formatter that combines clean formatting with API key redactionr'   rO   r   Nc                 $    t        |   |i | y rQ   rR   rT   s      r   rS   z#FileFormatterWithRedaction.__init__  rU   r   r   c                     d|j                    d|j                   d|_        t        |   |      }| j                  |      S r   )r<   r%   r=   r>   r?   rW   rX   s      r   r?   z!FileFormatterWithRedaction.format  sA    V__-Qv}}oQ?v.//>>r   rZ   c                 <    	 t        |      S # t        $ r |cY S w xY w)r\   )r   r]   )r@   rZ   s     r   rW   z6FileFormatterWithRedaction._redact_api_keys_in_message  s&    	*733 	N	s   
 rb   rL   s   @r   r   r   }  sL    N*c *S *T *?W.. ?3 ?3 3 r   r   z<%(asctime)s | %(levelname)-8s | %(fileloc)-30s | %(message)s)debuginfowarningr_   criticalc                      e Zd ZU dZi Zeeej                  f   e	d<   ej                  ZdZdZi Zeeej                  f   e	d<    e       Zee	d<   ed        Zed        Zed        Zed	efd
       Zed	efd       Zed	efd       Zed        Zed	edej                  fd       Zed!dedee   dej                  fd       Zededeej                     fd       Zededdfd       Zededdfd       Zedee    fd       Z!ed"de"ddfd       Z#ed        Z$edeee%f   fd       Z&ede'fd       Z(ed#de"de)fd       Z*ed         Z+y)$r   z$Centralized logger management system_loggersN_module_handlers_archived_logsc                      t         j                  syt        t         d      s[t         j                  j                  d       t        rt         j                          nt         j                          dt         _        yy)zLEnsure logs directory exists and handle existing logs based on configurationN_directory_initializedTr   )r   	_logs_dirr5   mkdirr   _delete_existing_logs_archive_existing_logsr   rt   r   r   r   zLogger._ensure_logs_directory  s^      v78""D"1 *,,.--/,0F) 9r   c                     t         j                  rt         j                  j                         syt         j                  j                  d      D ]<  } 	 | j	                          t         j
                  j                  | j                         > y# t        $ r%}t        d| j                   d|        Y d}~id}~ww xY w)zDelete all existing log filesN*.logFailed to delete : )
r   r   r   globunlinkr   addr"   r]   r   )log_filer`   s     r   r   zLogger._delete_existing_logs  s     v'7'7'>'>'@ ((--g6 	@H@!%%))(--8	@  @)(--1#>??@s   9B	B=B88B=c                     t         j                  rt         j                  j                         syt        j                         j                  d      } t         j                  j                  d      D ]  }d|j                  v r-|j                  j                  d      d   j                         r>|j                  d|  dz   }t         j                  |z  }	 |j                  |       t        d|j                   d|        t         j                  j                  |j                          y# t        $ r%}t        d	|j                   d
|        Y d}~d}~ww xY w)zAArchive all existing log files by adding timestamp to their namesN%Y%m%d%H%M%Sr   -r   .logArchived existing log:  -> Failed to archive r   )r   r   r   r   nowstrftimer   stemsplitisdigitr   r   r"   r   r  r]   )	timestampr  archived_namearchived_pathr`   s        r   r   zLogger._archive_existing_logs  s%    v'7'7'>'>'@LLN++N;	 ((--g6 	AHhmm#(;(;C(@(D(L(L(N$MMa	{$,??M",,}<MA./d=/RS%%))(--8	A  A*8==/A3?@@As   AD//	E8EEmodule_namec                    t         j                  rt         j                  j                         sy|  d}|t         j                  v ryt         j                  |z  }|j                         r1	 |j	                          t         j                  j                  |       yy# t        $ r}t        d| d|        Y d}~yd}~ww xY w)z.Delete a specific module log file if it existsNr  r   r   )r   r   r   r   r   r  r]   r   )r  log_file_namer  r`   s       r   _delete_module_logzLogger._delete_module_log  s     v'7'7'>'>'@&-t, F111##m3??@!%%))-8   @)-1#>??@s   ,/B 	C&B<<Cc                 (   t         j                  rt         j                  j                         sy|  d}|t         j                  v ryt         j                  |z  }|j                         rt	        j
                         j                  d      }|  d| d}t         j                  |z  }	 |j                  |       t        d| d|        t         j                  j                  |       yy# t        $ r}t        d| d|        Y d}~yd}~ww xY w)	z/Archive a specific module log file if it existsNr  r  r  r  r  r	  r   )r   r   r   r   r   r
  r  r   r   r  r]   )r  r  r  r  r  r  r`   s          r   _archive_module_logzLogger._archive_module_log  s     v'7'7'>'>'@&-t, F111##m3?? //?I*m1YKt<M",,}<MA./d=/RS%%))-8   A*=/A3?@@As   *AC- -	D6DDc                 f    t         rt        j                  |        yt        j                  |        y)z6Handle existing module log file based on configurationN)r   r   r  r  )r  s    r   _handle_module_logzLogger._handle_module_log  s#     &%%k2&&{3r   c                  L   t         j                  yt        d      t         _        t         j                  dz  } t	        | ddd      t         _        t         j                  j                  t                      t         j                  j                  t        j                         y)z(Setup file handlers for logging to filesNlogszmain.logi   
   utf-8r   r   r   )
r   _file_handlerr   r   r   setFormatterr   setLevelr.   r   )main_log_files    r   _setup_file_handlerszLogger._setup_file_handlers  s}     +  < ((:56%5BQX 
 	))*<*>?%%gmm4r   r   c                 Z   | t         j                  v rt         j                  |    S t         j                  |        t         j                  |  dz  }t	        |ddd      }|j                  t                      |j                  t        j                         |t         j                  | <   |S )z2Get or create a file handler for a specific moduler  i  @   r  r  )
r   r   r  r   r   r   r   r!  r.   r   )r  module_log_filemodule_handlers      r   _get_or_create_module_handlerz$Logger._get_or_create_module_handler&  s     &111**;77 	!!+. !**}D-AA0'7aRY
 	##$6$89./=,r   r"   r#   c                 $   |r3t         j                  |j                         t        j                        }nt
        j                  }| t
        j                  v r5t
        j                  |    }|j                  |k7  r|j                  |       |S t        j                  |       }|j                  |       d|_        t
        j                          t        j                  t        j                        }|j!                  t"               |j                  |       |j%                  |       t
        j'                  |       }|j%                  |       t
        j(                  r|j%                  t
        j(                         |t
        j                  | <   |S )z
        Setup and get logger instance
        :param name: logger name
        :param level: log level (optional, uses default if not provided)
        :return: logger instance
        F)
LOG_LEVELSr8   lowerr.   r   r   _default_levelr   r#   r!  r^   	propagater#  StreamHandlerrv   rw   r   	FORMATTER
addHandlerr(  r  )r"   r#   	log_levelexisting_loggerlogger_instanceconsole_handlerr'  s          r   setup_loggerzLogger.setup_logger:  s/    "u{{}gllCI--I6??"$ood3O$$	1((3""!++D1  +$)! 	##% "//

;$$Y/  +""?3  ==dC"">2 &&v';';< /r   c                 @    t         j                  j                  |       S )zo
        Get existing logger
        :param name: logger name
        :return: logger instance or None
        )r   r   r8   )r"   s    r   
get_loggerzLogger.get_loggerg  s     ""4((r   r1  c                 r   | j                         }t        j                  |t        j                        }|t
        _        t
        j                  j                         D ]W  \  }}|j                  |k7  r|j                  |       |j                  D ]#  }|j                  |k7  s|j                  |       % Y y)z?Update all existing loggers and their handlers to new log levelN)r+  r*  r8   r.   r   r   r,  r   r4   r#   r!  handlers)r1  log_level_str	new_levelrk   r3  handlers         r   update_log_levelszLogger.update_log_levelsp  s     ")NN=',,?	 )"(//"7"7"9 	0A$$	1((3 +33 0==I-$$Y/0	0r   c                 z    t         j                  | j                         t        j                        t
        _        y)z%Set default log level for new loggersN)r*  r8   r+  r.   r   r   r,  )r#   s    r   set_default_levelzLogger.set_default_level  s!     !+u{{}gll Kr   c                  "    t         j                  S )zGet the logs directory path)r   r   rt   r   r   get_logs_directoryzLogger.get_logs_directory  s     r   daysc                    t         j                  rt         j                  j                         syt        j                         | dz  dz  dz  z
  }t         j                  j	                  d      D ]>  }	 |j                         j                  |k  r|j                          t        d|        @ y# t        $ r}t        d| d|        Y d}~ad}~ww xY w),Clean up log files older than specified daysN   <   z*.log*zCleaned up old log file: zFailed to clean up log file r   )
r   r   r   r   r   statst_mtimer   r   r]   )rB  cutoff_timer  r`   s       r   cleanup_old_logszLogger.cleanup_old_logs  s     v'7'7'>'>'@iikTBY^b%89((--h7 	FHF==?++k9OO%5hZ@A		F
  F4XJbDEEFs   3;B11	C:CCc                      t         j                  rt         j                  j                          t         j                  j	                         D ]  } | j                           y)z2Flush all file handlers to ensure logs are writtenN)r   r  r   r   valuesr<  s    r   flush_all_handlerszLogger.flush_all_handlers  sH       &&( ..557 	GMMO	r   c                  X   i } t         j                  rt         j                  j                         rt         j                  j                  d      D ]  }	 |j	                         }|j
                  dz  }t        j                  |j                        }t        |j                  |dd|j                  d      t        |j                                     | |j                  <    | S # t        $ r:}t        |j                  dddt        |      	      | |j                  <   Y d
}~d
}~ww xY w)0Get detailed information about current log filesr   i   r   z MBrf   )r<   sizemodifiedr   Unknownr*   )r<   rQ  rR  r   r_   N)r   r   r   r   rG  st_sizer   fromtimestamprH  r	   r"   r  r;   absoluter]   )r   r  rG  size_mbmodified_timer`   s         r   get_log_files_infozLogger.get_log_files_info  s      0 0 7 7 9",,11': #==?D"llk:G$,$:$:4==$IM*5!) '}C0!.!7!78K!L !2!2!45	+D'  	 ! *5!)YY[cfghci+D's   BC&&	D)/0D$$D)c                      t         j                         } t        t        t         j                        | t         j
                  rt        t         j
                              S d      S )zGet logging system statisticsN)active_loggers	log_fileslogs_directory)r   rY  r
   lenr   r   r;   )log_files_infos    r   get_log_statszLogger.get_log_stats  sU      224v/$4:4D4D3v//0
 	
 KO
 	
r   r   r   c                 8   t         j                  rHt        t         j                  t              r*| t         j                  _        |t         j                  _        t         j                  j                         D ]!  }t        |t              s| |_        ||_        # y)z,Configure rollover behavior for all handlersN)r   r  r   r   r   r   r   rL  )r   r   r<  s      r   configure_rolloverzLogger.configure_rollover  st    
 Jv/C/CE\$]5<F  238F  0 ..557 	0G'#:;,3)*/'	0r   c                  d   i } t         j                  rIt        t         j                  t              r+t         j                  j                  j                         | d<   t         j                  j                         D ]3  \  }}t        |t              s|j                  j                         | |<   5 | S )z#Get rollover performance statisticsmain)r   r  r   r   r   r   r   r4   )statsr"   r<  s      r   get_rollover_statszLogger.get_rollover_stats  s      Jv/C/CE\$]"00@@EEGE&M $44::< 	=MD''#:;%55::<d	= r   rQ   r   )r   r   ),rG   rH   rI   rJ   r   r   r;   r.   r   __annotations__r   r,  r   r  r   Handlerr|   r   staticmethodr   r   r   r  r  r  r#  r(  r   r5  r7  r=  r?  r   rA  intrJ  rN  r	   rY  r
   r`  floatrb  rf  rt   r   r   r   r     s   .*,Hd3&',\\NIM35d3/05%NC1 1" @ @ A A, @ @ @& A A A. 4 4 4 5 5  3 7??  & *3 *x} * * *X ) )'..!9 ) ) 0S 0T 0 0 L L L L       Fs F4 F F   S+%5 6  , 
< 
 
 0C 0E 0 0  r   r   c                     t        j                  d      } | r+| j                         dv rt        | j                                t        j                  d      }|,|j	                         j                         }t        |dv        yy)zConfigure file format and color from environment variables.

    Supported variables:
      - LOG_FILE_FORMAT: "text" or "json"
      - LOG_COLOR: "1"/"0" or "true"/"false" (case-insensitive)
    LOG_FILE_FORMATr   	LOG_COLORN)1trueyeson)r   getenvr+  r   stripr   )r   rD   
normalizeds      r   configure_logging_from_envrw    sm     ))%
&C
syy{..CIIK(IIk"E[[]((*
*(BBC r   c                       e Zd ZdZd Zy)ContextLoggerAdapterzBLoggerAdapter that injects common context fields into log records.c                 X    |j                  di       }i | j                  |}||d<   ||fS )Nextra)r8   r{  )r@   r&   rO   r{  mergeds        r   processzContextLoggerAdapter.process  s8    

7B'(DJJ(%( wF{r   N)rG   rH   rI   rJ   r}  rt   r   r   ry  ry    s
    Lr   ry  categoryc                 0    t        |       }t        ||      S )z.Get a logger with pre-attached context fields.)r7  ry  )r~  contextbases      r   get_context_loggerr    s    hDg..r   ra   c                     t        | |      S )z7Wrap an existing logger with additional context fields.)ry  )ra   r  s     r   attach_contextr  	  s    00r   c                   <    e Zd ZdZd	dedefdZdedeeef   fdZ	y)
ErrorAggregatorz.Aggregates similar errors to reduce log noise.
window_sec	max_countc                 .    || _         || _        i | _        y rQ   )r  r  _errors)r@   r  r  s      r   rS   zErrorAggregator.__init__  s    $"24r   	error_keyr   c                 N   t        j                          }|| j                  vrd||d| j                  |<   y| j                  |   }|dxx   dz  cc<   ||d<   ||d   z
  | j                  kD  rd||d| j                  |<   y|d   dk(  xs |d   | j                  z  dk(  }||d   fS )zhCheck if error should be logged and return count.

        Returns: (should_log, current_count)
        r   )count
first_seen	last_seen)Tr   r  r  r  r   )r   r  r  r  )r@   r  r
  
error_info
should_logs        r   r  zErrorAggregator.should_log  s    
 iikDLL(01SV&WDLL#\\),
7q "%
; L))DOO;01SV&WDLL#  (A-ZG1Dt~~1UYZ1Z
:g...r   N)rF  r  )
rG   rH   rI   rJ   rk  rS   r;   tupler   r  rt   r   r   r  r    s4    853 5 5
/C /E$),< /r   r  r  rZ   c                     t         j                  |      \  }}|r$|dkD  r| d| d} | j                  |g|i | yy)zLog error with aggregation to reduce noise.

    Args:
        logger: Logger instance
        error_key: Unique key for this type of error (e.g., "bad_line_parse")
        message: Log message
        *args, **kwargs: Standard logging arguments
    r   z (occurred z times)N)_error_aggregatorr  r_   )ra   r  rZ   r'   rO   r  r  s          r   log_aggregated_errorr  2  sR     *44Y?J19 	UG7;GW.t.v. r   c                 \    | xs dj                         xs d}t        j                  |      S )z2Get or create a category logger using unified API.rd  )ru  r   r5  )r~  r"   s     r   r7  r7  B  s+    %%'16Dt$$r   r#   cleanup_daysfile_formatrollover_retriesrollover_delayc                 0   t         j                  |        t        |       t         j                  |       t         j	                          t         j                  ||       t               }t        j                         j                  |       t                t        d      }|j                  d| j                                 |j                  dt         j                                 |j                  d| d       |j                  d| d| d       y	)
a  
    Initialize logging system with specified level.
    :param level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
    :param cleanup_days: Number of days to keep old log files (default: 7)
    :param file_format: File log format mode: "text" or "json"
    :param rollover_retries: Number of retries for rollover operations (default: 3)
    :param rollover_delay: Delay between rollover retries in seconds (default: 0.1)
    rd  z$Logging system initialized - Level: Logs directory: zLog cleanup: keeping files for z dayszRollover config: z
 retries, zs delayN)r   r=  r   rJ  r#  rb  ry   r.   r^   	addFilter_setup_exit_handlersr7  r   upperrA  )r#   r  r  r  r  redaction_filterra   s          r   init_loggingr  H  s     U# $ L) ! .? '(!!"23  F
KK6u{{}oFG
KK"6#<#<#>"?@A
KK1,uEF
KK#$4#5Z?OwWXr   c                  6    t        j                  t               y)z1Setup exit handlers for graceful logging shutdownN)atexitregistershutdown_loggingrt   r   r   r  r  t  s     OO$%r   c                  *    t         j                         S )rP  )r   rY  rt   r   r   get_current_log_filesr  |  s    $$&&r   c                  *    t         j                         S )z+Get comprehensive logging system statistics)r   r`  rt   r   r   get_logging_statsr    s    !!r   rB  c                 .    t         j                  |        y)rD  N)r   rJ  )rB  s    r   cleanup_logsr    s    
D!r   c                  ,    t         j                          y)z9Flush all log handlers to ensure data is written to filesN)r   rN  rt   r   r   
flush_logsr    s    
r   c                      t         j                         } i }| j                         D ]E  \  }}|d   |d   z   |d   z   }|dkD  s|d   |d   z   |z  dz  }|dd||d   |d   |d   d||<   G |S )	zGet rollover health summaryr   r   r   r   d   z.1f%)success_ratetotal_rolloversr   copy_fallbackr   )r   rf  r4   )re  healthr"   rG  totalr  s         r   get_rollover_healthr    s    %%'EFkkm 

d%&n)==Z@PP19 !12T.5IIURUXXL#/"4A 6#("&'7"8!%n!5 ,F4L	
 Mr   c                  |   t         j                          t         j                  rt         j                  j                          t         j                  j                         D ]  } | j                           t         j                  j                          dt         _        t         j                  j                          y)z9Gracefully shutdown logging system and close all handlersN)r   rN  r  r   r   rL  r   clearrM  s    r   r  r    s    
 ""$ **113  OO  F
!!#r   c                  f   t        j                  d      } | j                  dd D ]  }| j                  |        t        j                  t
        j                        }t        d      }|j                  |       | j                  |       | j                  t         j                         d| _        | S )am  
    Configure access logging with API key redaction

    This function sets up custom access log formatting that automatically
    redacts API keys in HTTP access logs. It works by:

    1. Intercepting access log messages
    2. Using regex patterns to find API keys in URLs
    3. Replacing them with redacted versions (first6...last6)

    Supported API key formats:
    - Google/Gemini API keys: AIza[35 chars]
    - OpenAI API keys: sk-[48 chars]
    - OpenAI project keys: sk-proj-[48 chars]
    - Anthropic keys: anthrop[20+ chars]
    - GooeyAI keys: gsk_[20+ chars]
    - StabilityAI keys: stab_[20+ chars]
    zuvicorn.accessNz+%(asctime)s | %(levelname)-8s | %(message)sF)r.   r^   r9  removeHandlerr.  rv   rw   rN   r   r0  r!  r   r-  )access_loggerr<  access_formatters      r   setup_access_loggingr    s    ( %%&67M !))!, -##G,- ##CJJ/G/0]^)* W%7<<(#Mr   __main__r   rd  configz9Main logger test - this should appear in console and filez Config logger test - debug levelz3Test error message - should appear in error log tooproviderzFTesting API key redaction: sk-1234567890abcdefghij1234567890abcdefghijz;Testing Gemini key: AIza1234567890abcdefghij1234567890abcdez"
=== Logging System Statistics ===zActive loggers: r  z
=== Log Files ===z  - z
    Size: z    Modified: z	 (Error: )r   rQ   )r   r   r   r   r   rg  )erJ   r  ctypesrq   r.   logging.handlersr   platformr   rv   r   r   pathlibr   typingr   r   r   constant.systemr   core.modelsr	   r
   patternsr   r7   r   r   r   systemr5   rw   r   windllkernel32SetConsoleModeGetStdHandler]   	Formatterr   rN   rd   r:   Filterry   r/  r   r   r   r;   r   r   r9  RotatingFileHandlerr   r   r   r   r   r   r   r   r*  r   rw  LoggerAdapterry  r  r  r  r  r  r7  rk  rl  r  r  r  r  r  r  r  r  r  rG   main_loggerconfig_loggerr   r   r_   test_loggerrN  re  r   r[  r]  r\  r4   r<   rQ  rR  rt   r   r   <module>r     sp  "      	   
    & & 6 1 - 
 #t # #	xI%'#**h*GCJJL]L]L_==)) 5 5c :A> +w((  +F+w00 +0aG%% a. gnn F \]	#o   QG-- Q
S T &G%% &X
g..BB X
v!2!2 . ,,jk ]]LL]]  
C CL
D$700 / /4H /17>> 19M 1
/ /B $% / /C /# / %# %'.. % )Y)Y)Y )Y 	)Y
 )YX&'
"< "
"s "
 
($($N z V$Kx(MPQ:; KL Z(K]^RS  E	
/0	U112
34	U112
34	
 ////1 ;$::D
#$Jtyyk*+N4==/23D
)DJJ<q9:;;   		s   (A8O$ $O-,O-