:jzQ(UdZddlZddlZddlmZmZddlmZmZm Z ddl m Z ddl m Z ddlmZdd lmZdd lmZdd lmZdd lmZdd lmZmZmZmZddlmZddlm Z m!Z!m"Z"m#Z#e dZ$da%ede&d<ejNZ(GddZ)y)zH Main regex engine interface for pattern analysis and query generation. N)ListOptional)ALLOWED_OPERATORSPOPULAR_LANGUAGES SIZE_RANGES) get_logger)extract_github_query_pattern)handle_exceptions)RefineEngineConfig)QueryGenerator)EnumerationOptimizer) RegexParser)CharClassSegment FixedSegment GroupSegmentOptionalSegment)SplittabilityAnalyzer)IEnumerationOptimizerIQueryGenerator IRegexParserISplittabilityAnalyzerrefine RefineEngine _instancec zeZdZdZ d!deedeedeedeedee f dZ e d"deed dfd Z e d#d Z d ed eefd Zd ed efdZd eded eefdZd eded eefdZd ed eefdZeddddded efdZd$d eded eeeffdZd%d eded efdZdedeed dfd Zy)&rz,Main interface for regex pattern processing.Nconfigparser optimizer generator splittabilityc| t}||_|xst|j|_|xst |j |_|xst|j|_ |xsMt|j|j|j|j|j |j"|_y)N)enable_recursion_limitenable_value_thresholdenable_resource_limitmax_recursion_depthmin_enumeration_valuemax_resource_cost)r rrmax_quantifier_lengthrr max_queriesrr max_depthr rr#r$r%r&r'r(r!)selfrrrr r!s 4/root/.openclaw/workspace/harvester/refine/engine.py__init__zRefineEngine.__init__&s >')F I F,H,H I "N&:6;M;M&N"FnV5E5E&F* .C#)#@#@#)#@#@"(">"> & : :"(">">$66 / returnct8t5t||atjddddtS|tj dtS#1swYtSxYw)z#Get thread-safe singleton instance.Nz'RefineEngine singleton instance createdz:RefineEngine already initialized, config parameter ignored)r_lockloggerdebugwarning)clsrs r- get_instancezRefineEngine.get_instance@sg   L$ #F ILL!JK L   NNW X  L s $AA*cht5datjddddy#1swYyxYw)z.Reset singleton instance (mainly for testing).Nz%RefineEngine singleton instance reset)r2rr3r4)r6s r-reset_instancezRefineEngine.reset_instanceMs2 BI LL@ A B B Bs(1queryc|sy|jdr|jdr|ddStjd|r|St tj d|}|rt |d}|jdSt tj d |}|rt |d }|jdSy) a Extract regex pattern from GitHub search format, handling escaped slashes. Args: query: Query string in format "/pattern/" or "/pattern/ other terms" or "terms AND /pattern/" Returns: str: Extracted pattern or None if not found N/r z [\[\]{}+*?\\]z/([^/]*[\[\]{}+*?\\][^/]*)/c6t|jdSNr lengroupms r-z5RefineEngine._extract_regex_pattern..p#aggaj/r/)keyz /([^/]+)/c6t|jdSr?r@rCs r-rEz5RefineEngine._extract_regex_pattern..wrFr/) startswithendswithresearchlistfinditermaxrB)r,r:patternslongests r-_extract_regex_patternz#RefineEngine._extract_regex_patternUs   C U^^C%82;  99%u -L $BEJK ((ABG==# # L%89 ((ABG==# #r/c V|sy|j|}|sy |jj|}fd|}t|dkD}tj d|d|dt|d|S#t $r%}tjd|d |Yd }~yd }~wwxYw) z2Check if query contains enumerable regex patterns.Fcg}|D]\}t|tr|j|%t|ttfs<|j |j ^|SN) isinstancerappendrrextendcontent)segs variablessegfind_variable_segmentss r-r]z8RefineEngine.has_pattern..find_variable_segmentssa NC!#'78!((-#C,)HI!(()? )LM N ! r/rzPattern check: 'z' -> z (z variables)zPattern check failed for '': N)rRrparserAr3r4 Exceptionr5)r,r:patternsegmentsvariable_segmentsresulter]s @r- has_patternzRefineEngine.has_pattern|s--e4 {{((1H !!7x @ *+a/F LL+G9E&CHYDZC[[fg hM  NN7yA3G H sAA:: B(B##B( partitionsc|dks|stjd|d|gS|j||}|stjdgSt||k\r&tjdt|d||St }|D]F}|j |}|stjd,|D]}|s|j |Ht|}tjdt|d |d ||S) z-Generate refined queries from a query string.rzInvalid partitions=z or query: z+No queries generated from divide with regexzAlready have enough queries: z >= z.No queries generated from divide with language Generated z queries from query: z, partitions: )r3r4_dividerAset_divide_with_languageaddrM) r,r:rgqueries candidatesitemresultsrd conditionss r-generate_querieszRefineEngine.generate_queriess ?% LL.zl+eWM NI,,uj1 LLF GI w<: % LL8Wd:,W XNU  +D006G MN! +NN6* +  +*%  z#j/!22Gwn]g\hijr/c|dks|sgS|j|}|s|gS |jj|}|jj ||\}}|st j d||gS|jj||\}}|r7t j d|d|jj|||} nBt j d|d|jd|jj||} g} | D]b} |jdr,|jdr|jd|dd| d} n|j|| } | j| dt j d t!| d |d || S#t"$r(} t j%d |d | |gcYd} ~ Sd} ~ wwxYw)at Split a broad regular expression query into multiple regular expressions that match smaller ranges Args: query: Query string in format "/pattern/" or "/pattern/ other terms" or "terms AND /pattern/" partitions: Number of partitions to divide the query into Returns: List[str]: List of refined queries rz!Pattern cannot be split further: zFound suitable strategy for z, partitions, using minimum enumeration depthzNo strategy found for z2 partitions, using strategy with maximum queries ()r<riz queries from pattern: 'z', requested partitions: zQuery generation failed for 'r^N)rRrr_r! can_splitr3infor"evaluate_strategies_for_partitionsr generaternrIrJreplacerWrAr`r5)r,r:rgrarbenabledreasonstrategyfoundrefined_patternsrnrptextres r-rjzRefineEngine._divides ?%I--e47N/ {{((1H#00::7HMOGV ?xHIw#nnOOPXZdeOHe ::,Fvwx#'>>#:#:8Xz#Z  ,ZL9<>#:#:8X#N G( %##C(U^^C-@ ==1WIQ1TF!ED!==$7Dt$ % KKS\N*B7)Kdeodpq N  NN:7)3qcJ K7N s%AF$8D+F$$ G-G GGc|r|jnd}|stjdgSt}t j d|tj s,tD]}|j|d|t|St j d|tj s,tD]}|j|d|t|Stjd|j|t|S) zAGenerate refined queries with adaptive refinement language level.z)No query provided for language refinementz language:[a-zA-Z0-9#]+ )flagsz language:z size:[a-zA-Z0-9#=<>.]+ z size:zQCannot refine with language or sie refinement due to existing refinement criteria) stripr3r4rkrKmatchIrrmrrM)r,r:basernlangsizes r-rlz"RefineEngine._divide_with_languages %u{{}2 LLD EI%xx3TF) 7 tfJtf56 7G}5t244H# 3 tfF4&12 3 G} LLl m KK G}r/FzAnalysis failed) parseableerrorr5)default_result log_levelrac N|jj|}|jj|}fdt ||t |t |t|tt |j|j|jdd }|S)z0Analyze pattern and return detailed information.cd}|D]A}t||r|dz }t|ttfs,||j|z }C|S)Nrr )rVrrrY)rZseg_typecountr\count_segments_recursives r-rz>RefineEngine.analyze_pattern..count_segments_recursive!sXE Mc8,QJElO%DE5ckk8LLE  M Lr/T) rbfixed_segmentsrcoptional_segmentsgroup_segmentsenumeration_segmentsenumeration_valueestimated_queriesr) rr_roptimizerArrrrrbvaluern)r,rarbr}analysisrs @r-analyze_patternzRefineEngine.analyze_patterns;;$$W->>**84 H 6xN!9(DT!U!9(O!T6xN$'(9(9$:!)!)!1!1  r/recursion_depthct|}|sy |jj|}|jj |||S#t $r5}t jd|d|ddt|fcYd}~Sd}~wwxYw)z>'* *]!NN%DWISQRPS#TUD!)88$94@!"4&{NN4( )s,CJ'!CK' K0KK6LLrbrc6g}d}|D]V}t|trI|js!|jdk(rdndk(rddvr|sC|j|d}W|z }]t|tr|r|j|d}t |jdk(rat|jdtrD|jdjd vrt fd d Ds|j|j|j|t|tr|r|j|d}@|sD|j|d}Y|r|j||j||r|j|y y ) z Recursively extract fixed strings from segments. Args: segments: List of segments to process fixed: List to append found fixed strings to rrr<rr) .*+?{}^$|(ru[]r rrc3&K|]}|v ywrU).0charrYs r- z6RefineEngine._extract_fixed_strings..s1\d$'/1\sz []{}*+?()\N) rVrrYrWrrAanyrrrX)r,rbrmergedcurrentr\rYs @r-rz#RefineEngine._extract_fixed_stringssy1 !C#|,;;!kkG%'"% F*"&"cc""MM'2&(G 7*C.MM'* Gs{{#q(Z A -U!kk!n44Gg~c1\m1\.\  g.// VDC1MM'* GMM'* Gc1 !h  MM' "  V  MM' " r/)NNNNNrU)r0N)r)r)__name__ __module__ __qualname____doc__rr rrrrr. classmethodr7r9rrRboolrfintrrsrjrlr dictrtuplerrrrr/r-rr#s604)-59/3:>  +, & 12  O,   67  4 (+="> .  BB%C%HSM%NB c s tCy DCSCcCd3iCJ349,EDU&Vbklstm: 7c 7C 7dTWiHX 7B+B+B+B+HG#tG#DIG#$G#r/)*rrK threadingtypingrrconstant.searchrrr tools.loggerrtools.patternsr tools.utilsr rr r r rrrrsegmentrrrrr!rtypesrrrrr3r__annotations__Lockr2rrr/r-rsx !MM#7)&%+RR0 H &* 8N #* n#n#r/