
    $j&                        U d Z ddlZddlZddlZddlmZ ddlmZmZ ddl	m
Z
m	Z	 ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ  ej        e          Ze G d d                      Ze G d d                      Ze G d d                      Z e            Zeed<   de	dz  de	dz  de	dz  fdZ e G d d                      Z!dZ" G d d          Z#dS )z
Entity extraction and resolution for memory system.

Uses spaCy for entity extraction and implements resolution logic
to disambiguate entities across memory units.
    N)defaultdict)	dataclassfield)UTCdatetime)SequenceMatcher)AnyFinal   )acquire_with_retry)fq_table)build_labels_lookupc                   8    e Zd ZU dZeed<   eed<   edz  ed<   dS )_EntityToCreatezBAn entity that needs to be inserted (no matching candidate found).idxnameN
event_date)__name__
__module____qualname____doc__int__annotations__strr        l/home/rurouni/.hermes/hermes-agent/venv/lib/python3.11/site-packages/hindsight_api/engine/entity_resolver.pyr   r      s:         LL	HHH
III4r   r   c                   .    e Zd ZU dZeed<   edz  ed<   dS )_EntityStatzHStat accumulation entry for a resolved entity (post-transaction update).	entity_idNr   )r   r   r   r   r   r   r   r   r   r   r   r   !   s1         RRNNN4r   r   c                   6    e Zd ZU dZdZeed<   dZedz  ed<   dS )_EntityStatAggz4Aggregated stats used when flushing pending updates.r   countNmax_date)	r   r   r   r   r#   r   r   r$   r   r   r   r   r"   r"   )   s9         >>E3NNN $Hho$$$$$r   r"   _SENTINEL_MISSINGabreturnc                 &    | |S || S | |k    r| n|S )a#  Return whichever of ``a`` / ``b`` is later (None loses to any datetime).

    Used to fold duplicate co-occurrence pairs across a retain batch: legacy
    two-tuple callers surface ``None``, which must not clobber a real
    datetime that arrived for the same pair from an aware caller.
    r   )r&   r'   s     r   _later_dater*   8   s*     	yyA111r   c                   <    e Zd ZU dZeed<   eed<   dZedz  ed<   dS )_CooccurrencePairzRA (entity_id_1, entity_id_2) pair observed in a retain batch (for post-txn flush).entity_id_1entity_id_2Nr   )r   r   r   r   r   r   r   r   r   r   r   r,   r,   F   sF         \\
 #'J4&&&&&r   r,   c                      e Zd ZdZd,dedefdZdefdZd-d	Z	d-d
Z
ededz  dee         fd            Z	 	 d.dedee         dededz  dee         f
dZ	 d/dedee         dedee         dz  dee         f
dZdedee         dee         fdZdedee         dee         fdZdededee         dedz  dee         f
dZdedee         deeef         deeee         f         dee         f
dZdedededee         def
dZdededefdZded efd!Zd"ed#efd$Z	 d/d%eeeef                  eeeeedz  f                  z  fd&Zd%eeeeedz  f                  fd'Zd0d ed)edee         fd*Zdedededz  fd+ZdS )1EntityResolverzA
    Resolves entities to canonical IDs with disambiguation.
    fullpoolentity_lookupc                 l    || _         || _        d| _        ||j        nd| _        i | _        i | _        dS )u]  
        Initialize entity resolver.

        Args:
            pool: asyncpg connection pool
            entity_lookup: Lookup strategy — "full" loads all bank entities then
                matches in Python; "trigram" uses pg_trgm GIN index to fetch only
                similar candidates per entity name (much faster for large banks).
        FN)r2   r3   _pg_trgm_checkedops_ops_pending_stats_pending_cooccurrences)selfr2   r3   s      r   __init__zEntityResolver.__init__\   sE     	* % $ 0DHHd	 =?JL###r   r(   c                 N    t          j                    }|t          |          ndS )zMReturn a unique key for the current asyncio task (or 0 for non-task context).Nr   )asynciocurrent_taskid)r:   tasks     r   	_task_keyzEntityResolver._task_keyp   s%    #%%+r$xxx2r   Nc                     |                                  }| j                            |d           | j                            |d           dS )a  
        Discard accumulated entity stats and co-occurrence counts for the current task.

        Call this on any exception path between resolve_entities_batch /
        link_units_to_entities_batch and flush_pending_stats() to prevent the
        per-task dicts from growing unbounded when tasks fail before flushing.
        Safe to call even if no entries exist for the current task.
        N)rA   r8   popr9   )r:   keys     r   discard_pending_statsz$EntityResolver.discard_pending_statsu   sK     nnT***#''T22222r   c                   K   | j         dS |                                 }| j                            |g           }| j                            |g           }|s|sdS t          | j                   4 d{V }|rt          t                    }|D ]S}||j                 }|xj	        dz  c_	        |j
        -|j        |j
        nt          |j        |j
                  |_        Tt          d |                                D                       }|                    dt!          d           d|           d{V  |ri }	|D ]F}
|
j        |
j        f}|	                    |d          \  }}|dz   t)          ||
j
                  f|	|<   Gt+          j        t.                    |                    dt!          d	           d
t!          d	           dt!          d	           dt          fd|	                                D                                  d{V  ddd          d{V  dS # 1 d{V swxY w Y   dS )a8  
        Flush accumulated entity stats and co-occurrence counts for the current task.

        Must be called AFTER the retain transaction commits.  Pops only the items
        accumulated by the calling asyncio task so concurrent retain batches never
        flush each other's uncommitted entity IDs.
        Nr   c              3   :   K   | ]\  }}||j         |j        fV  d S N)r#   r$   ).0eidr&   s      r   	<genexpr>z5EntityResolver.flush_pending_stats.<locals>.<genexpr>   s1      SSVS!sAGQZ8SSSSSSr   
                    UPDATE entitiesz SET
                        mention_count = mention_count + $2,
                        last_seen     = GREATEST(last_seen, $3)
                    WHERE id = $1::uuid
                    )r   Nz!
                    INSERT INTO entity_cooccurrencesa  
                        (entity_id_1, entity_id_2, cooccurrence_count, last_cooccurred)
                    VALUES ($1, $2, $3, $4)
                    ON CONFLICT (entity_id_1, entity_id_2)
                    DO UPDATE SET
                        cooccurrence_count = zi.cooccurrence_count + EXCLUDED.cooccurrence_count,
                        last_cooccurred    = GREATEST(z@.last_cooccurred, EXCLUDED.last_cooccurred)
                    c              3   :   K   | ]\  \  }}\  }}||||pfV  d S rH   r   )rI   e1e2r#   r   nows        r   rK   z5EntityResolver.flush_pending_stats.<locals>.<genexpr>   s@      ssB_(2rL_UT^BE:+<=ssssssr   )r2   rA   r8   rC   r9   r   r   r"   r    r#   r   r$   maxsorteditemsexecutemanyr   r-   r.   getr*   r   rR   r   )r:   rD   statscooccurrencesconnaggsentryrowscoo_aggcpair
prev_count	prev_daterR   s                 @r   flush_pending_statsz"EntityResolver.flush_pending_stats   sV      9Fnn#''R00377R@@ 	] 	F%di00 /	 /	 /	 /	 /	 /	 /	D 1<^1L1L w wA,EKK1$KK|/9>9OUXY^YgijiuUvUv SSsyy{{SSSSS&&$Z00             
 OQ& [ [AM1=9D,3KKi,H,H)J	%/!^[AL5Y5Y$ZGDMMl3''&&!)*@!A!A  /77M.N.N  8@@V7W7W   sssscjcpcpcrcrsssss        I/	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	s   7F#H--
H7:H7entity_labelsc                      t          |           S )zFBuild a set of valid 'key:value' entity label strings for fast lookup.) _build_labels_lookup_from_config)re   s    r   _build_labels_lookupz#EntityResolver._build_labels_lookup   s     0>>>r   bank_identities_datacontextc           	      <  K   |sg S |                      |          }|`t          | j                  4 d{V }|                     ||||||           d{V cddd          d{V  S # 1 d{V swxY w Y   dS |                     ||||||           d{V S )aF  
        Resolve multiple entities in batch (MUCH faster than sequential).

        Groups entities by type, queries candidates in bulk, and resolves
        all entities with minimal DB queries.

        Args:
            bank_id: bank ID
            entities_data: List of dicts with 'text', 'type', 'nearby_entities'
            context: Context where entities appear
            unit_event_date: When this unit was created
            conn: Optional connection to use (if None, acquires from pool)

        Returns:
            List of entity IDs in same order as input
        N)rh   r   r2   _resolve_entities_batch_impl)r:   ri   rj   rk   unit_event_daterZ   re   taxonomy_lookups           r   resolve_entities_batchz%EntityResolver.resolve_entities_batch   s{     2  	I33MBB<)$)44       !>>'='?O                                    
 ::g}g        s    A++
A58A5ro   c                   K   | j         dk    r| j                                        }|dk    r|                     ||||           d {V S | j        scd| _        |                    d           d {V }|s?t                              d           d| _         |                     ||||           d {V S | 	                    ||||           d {V S |                     ||||           d {V S )Ntrigramoracle_fuzzyTzCSELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pg_trgm')u   pg_trgm extension is not available — falling back to 'full' entity lookup strategy. Install pg_trgm for faster entity resolution on large banks. See: https://github.com/vectorize-io/hindsight/issues/626r1   )
r3   r7   get_entity_resolution_strategy$_resolve_entities_batch_oracle_fuzzyr5   fetchvalloggerwarning_resolve_entities_batch_full_resolve_entities_batch_trigram)	r:   rZ   ri   rj   rk   rn   ro   backend_strategyhas_trgms	            r   rm   z+EntityResolver._resolve_entities_batch_impl   sV      **  $yGGII>11!FFtWVcetuuuuuuuuu ( r(,%!%/t!u!uuuuuuu rNNO   *0D&!%!B!B4R_ap!q!qqqqqqqq==dG]\klllllllll66tWmUdeeeeeeeeer   c           
        K   |                     dt          d           d|           d{V }d |D             }|                     dt          d           dt          d           d	t          d           d
|           d{V }i }|D ]}	|	d         |	d         }}
|
|vrt                      ||
<   ||vrt                      ||<   ||v r!||
                             ||                    |
|v r!||                             ||
                    i }t	          t          d |D                                 }|D ]}g }|                                }|D ]f}	|	d         }|	d         }|	d         }|	d         }|	d         }|                                }||k    s||v s||v r|                    |||||f           g|||<   |                     ||||||           d{V S )z?Original strategy: load all bank entities then match in Python.z\
            SELECT canonical_name, id, metadata, last_seen, mention_count
            FROM rM   z,
            WHERE bank_id = $1
            Nc                 P    i | ]#}|d          |d                                          $S r?   canonical_namelowerrI   rows     r   
<dictcomp>z?EntityResolver._resolve_entities_batch_full.<locals>.<dictcomp>   s0    ^^^#SY,<(=(C(C(E(E^^^r   z[
            SELECT ec.entity_id_1, ec.entity_id_2, ec.cooccurrence_count
            FROM rN   z8 ec
            WHERE ec.entity_id_1 IN (SELECT id FROM zI WHERE bank_id = $1)
               OR ec.entity_id_2 IN (SELECT id FROM z! WHERE bank_id = $1)
            r-   r.   c              3   &   K   | ]}|d          V  dS textNr   rI   es     r   rK   z>EntityResolver._resolve_entities_batch_full.<locals>.<genexpr>?  &      AAa&	AAAAAAr   r   r?   metadata	last_seenmention_count)fetchr   setaddlistr   append_resolve_from_candidates)r:   rZ   ri   rj   rn   all_entitiesentity_id_to_nameall_cooccurrencescooccurrence_mapr   eid1eid2all_candidatesentity_textsentity_textmatchingentity_text_lowerr   ent_idr   r   r   canonical_lowers                          r   ry   z+EntityResolver._resolve_entities_batch_full  s     
 "ZZ:&&  
 
 
 
 
 
 
 
 
 _^Q]^^^ #'**122  6>j5I5I  6>j5I5I	   #
 #
 
 
 
 
 
 
 13$ 	D 	DC]+S-?$D+++), &+++), &((( &**+<T+BCCC((( &**+<T+BCCC CAA=AAAAABB' 	3 	3KH + 1 1 3 3# b b!$%5!6Tz?,	 #O 4"0"6"6"8"8 &88(O;;&*;;;OOV^XyR_$`aaa*2N;''22'=/>K[
 
 
 
 
 
 
 
 	
r   c           	      (  K   t          t          d |D                                 }|                    d           d{V  |                    dt	          d           d||           d{V }|                    d           d{V  d |D             }t                      }|D ]c}	|	d	         }
||
                             |	d
         |	d         |	d         |	d         |	d         f           |                    |	d
                    di }|rt          |          }|                    dt	          d           d|           d{V }d |                                D             }|D ]}	|	d         |	d         }}||vrt                      ||<   ||vrt                      ||<   ||v r!||                             ||                    ||v r!||                             ||                    |                     ||||||           d{V S )aZ  
        Trigram strategy: fetch only similar candidates per entity name using pg_trgm.

        Instead of loading all bank entities (O(N)), uses a GIN trigram index to fetch
        only the small set of candidates that are textually similar to each input name.
        Reduces DB data transfer from 165K rows to ~5-20 rows per entity.
        c              3   &   K   | ]}|d          V  dS r   r   r   s     r   rK   zAEntityResolver._resolve_entities_batch_trigram.<locals>.<genexpr>b  r   r   z'SET pg_trgm.similarity_threshold = 0.15Nz
            SELECT DISTINCT ON (e.id)
                e.id, e.canonical_name, e.metadata, e.last_seen, e.mention_count,
                q.query_text
            FROM unnest($2::text[]) AS q(query_text)
            JOIN rM   z e ON (
                e.bank_id = $1
                AND LOWER(e.canonical_name) % LOWER(q.query_text)
            )
            z"RESET pg_trgm.similarity_thresholdc                     i | ]}|g S r   r   rI   ts     r   r   zBEntityResolver._resolve_entities_batch_trigram.<locals>.<dictcomp>}      *G*G*GQ1b*G*G*Gr   
query_textr?   r   r   r   r   L
                SELECT ec.entity_id_1, ec.entity_id_2
                FROM rN    ec
                WHERE ec.entity_id_1 = ANY($1::uuid[])
                   OR ec.entity_id_2 = ANY($1::uuid[])
                c                 n    i | ]2}d  |D             D ]#}|d         |d                                          $3S )c                 0    g | ]}|d          |d         dS r   r   r   r   rI   r`   s     r   
<listcomp>zMEntityResolver._resolve_entities_batch_trigram.<locals>.<dictcomp>.<listcomp>  (    PPPQ1Q41Q4@@PPPr   r?   r   r   rI   candsr   s      r   r   zBEntityResolver._resolve_entities_batch_trigram.<locals>.<dictcomp>  e       PP%PPP   D	3/06688   r   r-   r.   )	r   r   executer   r   r   r   valuesr   )r:   rZ   ri   rj   rn   r   r^   r   candidate_idsr   r   r   candidate_id_list	cooc_rows
id_to_namer   r   s                    r   rz   z.EntityResolver._resolve_entities_batch_trigramX  s       CAA=AAAAABB llDEEEEEEEEEZZ	
 :&&	 	 	 
 
 
 
 
 
 
 
 ll?@@@@@@@@@ +H*G,*G*G*G UU 	) 	)C\*J:&--TC 013z?CDTVYZiVjk   c$i(((( 13 	A $] 3 3"jj566   "       I +2244  J
 ! 	A 	A /]1Cd///-0UU$T*///-0UU$T*:%%$T*..z$/?@@@:%%$T*..z$/?@@@22'=/>K[
 
 
 
 
 
 
 
 	
r   rZ   rn   c           	        K   t          t          d |D                                 }t          d          }	 t          j        |          }|                    d| d||           d{V }nW# t          $ rJ}	t                              d|	           d| _	        | 
                    ||||           d{V cY d}	~	S d}	~	ww xY wd |D             }
t                      }|D ]c}|d	         }|
|                             |d
         |d         |d         |d         |d         f           |                    |d
                    di }|rt          |          }|                    dt          d           d|           d{V }d |
                                D             }|D ]}|d         |d         }}||vrt                      ||<   ||vrt                      ||<   ||v r!||                             ||                    ||v r!||                             ||                    |                     |||||
|           d{V S )u  
        Oracle strategy: fetch similar candidates using UTL_MATCH.JARO_WINKLER_SIMILARITY.

        Replaces pg_trgm for Oracle backends. Uses JSON_TABLE to expand the
        entity text list into rows (Oracle equivalent of PG's unnest), then
        joins with a Jaro-Winkler threshold of 70/100 (≈ pg_trgm 0.15).
        Falls back to the "full" strategy if UTL_MATCH is unavailable.
        c              3   &   K   | ]}|d          V  dS r   r   r   s     r   rK   zFEntityResolver._resolve_entities_batch_oracle_fuzzy.<locals>.<genexpr>  r   r   rM   z
                SELECT e.id, e.canonical_name, e.metadata, e.last_seen, e.mention_count,
                       q.query_text
                FROM JSON_TABLE($2, '$[*]' COLUMNS (query_text VARCHAR2(4000) PATH '$')) q
                JOIN z e ON (
                    e.bank_id = $1
                    AND UTL_MATCH.JARO_WINKLER_SIMILARITY(LOWER(e.canonical_name), LOWER(q.query_text)) > 70
                )
                Nuv   UTL_MATCH.JARO_WINKLER_SIMILARITY not available on Oracle — falling back to 'full' entity lookup strategy. Error: %sr1   c                     i | ]}|g S r   r   r   s     r   r   zGEntityResolver._resolve_entities_batch_oracle_fuzzy.<locals>.<dictcomp>  r   r   r   r?   r   r   r   r   r   rN   r   c                 n    i | ]2}d  |D             D ]#}|d         |d                                          $3S )c                 0    g | ]}|d          |d         dS r   r   r   s     r   r   zREntityResolver._resolve_entities_batch_oracle_fuzzy.<locals>.<dictcomp>.<listcomp>  r   r   r?   r   r   r   s      r   r   zGEntityResolver._resolve_entities_batch_oracle_fuzzy.<locals>.<dictcomp>  r   r   r-   r.   )r   r   r   jsondumpsr   	Exceptionrw   rx   r3   ry   r   r   r   r   )r:   rZ   ri   rj   rn   r   entities_tableentity_texts_jsonr^   r   r   r   r   r   r   r   r   r   r   r   s                       r   ru   z3EntityResolver._resolve_entities_batch_oracle_fuzzy  s]      CAA=AAAAABB!*--	j !%
< 8 8 %	   !       DD  
	j 
	j 
	j NNK  
 "(D::4-Yhiiiiiiiiiiiiii
	j +H*G,*G*G*G UU 	) 	)C\*J:&--TC 013z?CDTVYZiVjk   c$i(((( 13 	A $] 3 3"jj566   "       I +2244  J
 ! 	A 	A /]1Cd///-0UU$T*///-0UU$T*:%%$T*..z$/?@@@:%%$T*..z$/?@@@22'=/>K[
 
 
 
 
 
 
 
 	
s   5A/ /
C9?B>8C>Cr   r   c                   12K   dgt          |          z  }g }g }	t          |          D ]7\  }
}|d         1|                    dg           }|                    d|          }|                    1g           }|s&|	                    t	          |
1|                     xd}d}1fd|D             }|D ]K\  }}}}}d}t          d1                                |                                                                          }||dz  z  }|rN|                    |t                                }t          ||z            }|t          |          z  }||d	z  z  }|r|r|j	        r|n|
                    t          
          }|j	        r|n|
                    t          
          }t          ||z
                                  dz            }|dk     rt          dd|dz  z
            }||dz  z  }||k    r|}|}Md} || k    r+|||
<   |                    t          ||                     |	                    t	          |
|d         |                     9t!          |          }!|	rt"           G d d                      }"i }#|	D ]^}$|$j                                        }%|%|#vr |"|$j        |$j                  |#|%<   |#|%         j                            |$j                   _t-          |#                                          }&d |&D             }'d |&D             }(t1          d          })| j                            ||)||'|(           d{V 22fd|&D             }*|*r`| j                            ||)||*           d{V }+|+D ]:},|,d         2|,d         <   d|,v r#|,d         2|,d                                         <   ;|&D ]T\  }%}-2                    |%          }.|.r8|-j        D ]0}/|.||/<   |!                    t          |.|-j                             1U|                                 }0| j                            |0g                               |!           |S )z=Shared scoring + upsert logic used by both lookup strategies.Nr   nearby_entitiesr   )r   r   r           c                 \    h | ](}|d          k    |d                                           )S r   r   rI   r   r   s     r   	<setcomp>z:EntityResolver._resolve_from_candidates.<locals>.<setcomp>!  7     h h hqqQWy\gOgOg6!2!2OgOgOgr         ?333333?tzinfoQ    r         ?皙?333333?)r    r   c                   X    e Zd ZU eed<   edz  ed<    ee          Zee	         ed<   dS );EntityResolver._resolve_from_candidates.<locals>._NameGroupr   Nr   )default_factoryindices)
r   r   r   r   r   r   r   r   r   r   r   r   r   
_NameGroupr   V  sL         			$tO+++%*U4%@%@%@c@@@@@r   r   )r   r   c                 "    g | ]\  }}|j         S r   r   rI   _gs      r   r   z;EntityResolver._resolve_from_candidates.<locals>.<listcomp>e  s    ===tq!AF===r   c                 "    g | ]\  }}|j         S r   )r   r   s      r   r   z;EntityResolver._resolve_from_candidates.<locals>.<listcomp>f  s    CCCTQALCCCr   rM   c                 ,    g | ]\  }}|v	|j         S r   r   )rI   
name_lowerr   
id_by_names      r   r   z;EntityResolver._resolve_from_candidates.<locals>.<listcomp>  s*    jjj=:qZ_iMiMiMiMiMir   r?   r   
input_name) len	enumeraterW   r   r   r   r   ratior   r   replacer   abstotal_secondsrS   r   r   r   r   r   r   r   rT   rU   r   r7   bulk_insert_entitiesfetch_missing_entity_idsrA   r8   
setdefaultextend)3r:   rZ   ri   rj   rn   r   r   
entity_idsentities_to_updateentities_to_creater   entity_datar   entity_event_date
candidatesbest_candidate
best_scorenearby_entity_setcandidate_idr   r   r   r   scorename_similarityco_entitiesoverlapco_entity_scoreevent_date_utclast_seen_utc	days_difftemporal_score	thresholdpendingr   groupsr   r   sorted_groupsentity_namesentity_datesr   missing_originalexisting_rowsr   r   r    original_idxrD   r   r   s3                                                    @@r   r   z'EntityResolver._resolve_from_candidates   s      Vc-000
0246 )- 8 8 :	 :	C%f-K)oo.?DDO +o N N'++K<<J "))/c`q*r*r*rsss "NJ h h h hO h h hT^ 2 2Pnh	= #2$8I8I8K8K^MaMaMcMc"d"d"j"j"l"l3.. % 3"2"6"6|SUU"K"KK!"3k"ABBG&-4E0F0F&FO_s22E  	6!2 	6 .?-Ep))K\KdKdloKdKpKp # 2;1A$dIIyGXGX`cGXGdGdM #^m%C$R$R$T$TW\$\ ] ]I 1}}),Qy1}0E)F)F#!55:%%!&J%1N II%%"0
3"))+[l*m*m*mnnnn"))#+f2ERcddd    &**<%=%=
  C	bA A A A A A A YA
 -/F' 9 9V\\^^
V++)3AL)Y)Y)YF:&z"*11!%8888 #6<<>>22M==}===LCC]CCCL
 &j11N#y==         J$  kjjjMjjj J&*i&H&H"$	' ' ! ! ! ! ! ! ) J JC47IJs<01 $s**@CD	
3|#4#:#:#<#<=
 "/ b b
A&NN:66	 b()	 b b3<
<0{YSTS_'`'`'`aaaa nn&&sB//66w???r   r   r   c           
      j  K   t          | j                  4 d{V }|                    dt          d           d|d d           d{V }|s0|                     |||           d{V cddd          d{V  S d}d}	d}
fd|D             }|D ]}|d         }|d	         }|d
         }|d         }d}t          d                                |                                                                          }||dz  z  }|                    dt          d           dt          d           d|           d{V }d |D             }t          ||z            }|r|t          |          z  }||dz  z  }|r|j	        r|n|
                    t                    }|j	        r|n|
                    t                    }t          ||z
                                  dz            }|dk     rt          dd|dz  z
            }||dz  z  }||	k    r|}	|}|}
d}|	|k    rB|                    dt          d           d||           d{V  |cddd          d{V  S |                     |||           d{V cddd          d{V  S # 1 d{V swxY w Y   dS )a  
        Resolve an entity to a canonical entity ID.

        Args:
            bank_id: bank ID (entities are scoped to agents)
            entity_text: Entity text ("Alice", "Google", etc.)
            context: Context where entity appears
            nearby_entities: Other entities in the same unit
            unit_event_date: When this unit was created

        Returns:
            Entity ID (creates new entity if needed)
        NzU
                SELECT id, canonical_name, metadata, last_seen
                FROM rM   a  
                WHERE bank_id = $1
                  AND (
                    canonical_name ILIKE $2
                    OR canonical_name ILIKE $3
                    OR $2 ILIKE canonical_name || '%%'
                  )
                ORDER BY mention_count DESC
                %r   c                 \    h | ](}|d          k    |d                                           )S r   r   r   s     r   r   z0EntityResolver.resolve_entity.<locals>.<setcomp>  r   r   r?   r   r   r   r   z]
                    SELECT e.canonical_name, ec.cooccurrence_count
                    FROM rN   z ec
                    JOIN aI   e ON (
                        CASE
                            WHEN ec.entity_id_1 = $1 THEN ec.entity_id_2
                            WHEN ec.entity_id_2 = $1 THEN ec.entity_id_1
                        END = e.id
                    )
                    WHERE ec.entity_id_1 = $1 OR ec.entity_id_2 = $1
                    c                 B    h | ]}|d                                           S )r   r   )rI   rs     r   r   z0EntityResolver.resolve_entity.<locals>.<setcomp>  s*    SSSqq!1288::SSSr   r   r   r   r   r   r   r   r   rL   z
                    SET mention_count = mention_count + 1,
                        last_seen = $1
                    WHERE id = $2
                    )r   r2   r   r   _create_entityr   r   r   r   r   r   r   r   r   rS   r   )r:   ri   r   rk   r   rn   rZ   r   r   r   best_name_similarityr   r   r   r   r   r   r   r   co_entity_rowsr   r   r   _evt_seenr   r   r  s     `                         r   resolve_entityzEntityResolver.resolve_entity  sM     * &di00 i	^ i	^ i	^ i	^ i	^ i	^ i	^D#zz
z**
 
 
 "K"""         J"  ^!00w_]]]]]]]]+i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^: "NJ#&  h h h hO h h h! 2; 2;"4y!$%5!6z?,	 #2$8I8I8K8K^MaMaMcMc"d"d"j"j"l"l3..
 (,zz
"#9::
 
 #:..
 
 
 !( ( " " " " " " TSNSSS /+=>>$ 3&-4E0F0F&FO_s22E  6 />.Dm??/JaJailJaJmJmD)2)9\IIy?P?PX[?P?\?\E #TE\$@$@$B$BU$J K KI 1}}),Qy1}0E)F)F#!55:%%!&J%1N+:( II%%ll$Z00   $"	 	 	 	 	 	 	 	 	 &Mi	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^R "00w_]]]]]]]]Si	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^ i	^s   AJ"GJ"2J""
J,/J,c                    K   |                     dt          d           dt          d           d||||           d{V }|S )a  
        Create a new entity or get existing one if it already exists.

        Uses INSERT ... ON CONFLICT to handle race conditions where
        two concurrent transactions try to create the same entity.

        Args:
            conn: Database connection
            bank_id: bank ID
            entity_text: Entity text
            event_date: When first seen

        Returns:
            Entity ID
        
            INSERT INTO rM   z (bank_id, canonical_name, first_seen, last_seen, mention_count)
            VALUES ($1, $2, COALESCE($3, now()), COALESCE($4, now()), 1)
            ON CONFLICT (bank_id, LOWER(canonical_name))
            DO UPDATE SET
                mention_count = zh.mention_count + 1,
                last_seen = EXCLUDED.last_seen
            RETURNING id
            N)rv   r   )r:   rZ   ri   r   r   r    s         r   r  zEntityResolver._create_entity   s      , --!*-- 
 "**!5!5   
 
 
 
 
 
 
 
	 r   unit_idr    c                   K   t          | j                  4 d{V }|                    dt          d           d||           d{V  |                    dt          d           d||           d{V }d |D             }|D ]}|                     |||           d{V   	 ddd          d{V  dS # 1 d{V swxY w Y   dS )z
        Link a memory unit to an entity.
        Also updates co-occurrence cache with other entities in the same unit.

        Args:
            unit_id: Memory unit ID
            entity_id: Entity ID
        Nz
                INSERT INTO unit_entitieszm (unit_id, entity_id)
                VALUES ($1, $2)
                ON CONFLICT DO NOTHING
                z7
                SELECT entity_id
                FROM zH
                WHERE unit_id = $1 AND entity_id != $2
                c                     g | ]
}|d          S )r    r   r   s     r   r   z6EntityResolver.link_unit_to_entity.<locals>.<listcomp>g  s    ???3c+.???r   )r   r2   r   r   r   _update_cooccurrence)r:   r  r    rZ   r^   other_entitiesother_entity_ids          r   link_unit_to_entityz"EntityResolver.link_unit_to_entityG  s	      &di00 	R 	R 	R 	R 	R 	R 	RD,,%o66  
          //  
        D @?$???N $2 R R//iQQQQQQQQQQR5	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	Rs   BB<<
C	Cr-   r.   c                    K   ||k    r||}}|                     dt          d           dt          d           d||           d{V  dS )a#  
        Update the co-occurrence cache for two entities.

        Uses CHECK constraint ordering (entity_id_1 < entity_id_2) to avoid duplicates.

        Args:
            conn: Database connection
            entity_id_1: First entity ID
            entity_id_2: Second entity ID
        r  rN   z (entity_id_1, entity_id_2, cooccurrence_count, last_cooccurred)
            VALUES ($1, $2, 1, NOW())
            ON CONFLICT (entity_id_1, entity_id_2)
            DO UPDATE SET
                cooccurrence_count = zM.cooccurrence_count + 1,
                last_cooccurred = NOW()
            N)r   r   )r:   rZ   r-   r.   s       r   r  z#EntityResolver._update_cooccurrencem  s       $$'2KKll!"899 
 '//E&F&F   
 
 	
 	
 	
 	
 	
 	
 	
 	
 	
r   unit_entity_pairsc                   K   |sdS d |D             }|\t          | j                  4 d{V }|                     ||           d{V cddd          d{V  S # 1 d{V swxY w Y   dS |                     ||           d{V S )a  
        Link multiple memory units to entities in batch (MUCH faster than sequential).

        Also updates co-occurrence cache for entities that appear in the same unit.

        Args:
            unit_entity_pairs: List of (unit_id, entity_id) or
                (unit_id, entity_id, event_date) tuples. When `event_date` is
                supplied, ``entity_cooccurrences.last_cooccurred`` for pairs
                observed in that unit advances to the event time instead of
                ``now()``, which matters for backfilled corpora where ingest
                time is a single spike unrelated to the underlying timeline.
                Legacy two-tuples remain accepted.
            conn: Optional connection to use (if None, acquires from pool)
        Nc                 f    g | ].}|d          |d         t          |          dk    r|d         ndf/S )r   r         N)r   r   s     r   r   z?EntityResolver.link_units_to_entities_batch.<locals>.<listcomp>  sM     >
 >
 >
 qT1Q4Q11$7>
 >
 >
r   )r   r2   "_link_units_to_entities_batch_impl)r:   r   rZ   
normalizeds       r   link_units_to_entities_batchz+EntityResolver.link_units_to_entities_batch  sy     ( ! 	F>
 >
&>
 >
 >


 <)$)44 W W W W W W W!DDT:VVVVVVVVW W W W W W W W W W W W W W W W W W W W W W W W W W W W W W @@zRRRRRRRRRs   A
A(+A(c                   K   t          |d           }d |D             }d |D             }| j                            |t          d          ||           d {V  i }i }|D ]a\  }}	}
|                    |t                                                    |	           |
|                    |          |
||<   X||vr|
||<   bi }|                                D ]\  }}t          |          }|                    |          }
t          |          D ]g\  }}||dz   d          D ]T}||k    r	||k    r||}}||f}|                    |t                    }|t          u r|
||<   At          ||
          ||<   Uh|r`|                                 }| j                            |g                               d |                                D                        d S d S )Nc                 "    | d         | d         fS )Nr   r   r   )r   s    r   <lambda>zCEntityResolver._link_units_to_entities_batch_impl.<locals>.<lambda>  s    !ad| r   )rD   c                     g | ]
}|d          S )r   r   rI   ps     r   r   zEEntityResolver._link_units_to_entities_batch_impl.<locals>.<listcomp>  s    ///QAaD///r   c                     g | ]
}|d          S )r   r   r,  s     r   r   zEEntityResolver._link_units_to_entities_batch_impl.<locals>.<listcomp>  s    111qad111r   r  r   c              3   F   K   | ]\  \  }}}t          |||           V  dS ))r-   r.   r   N)r,   )rI   rP   rQ   eds       r   rK   zDEntityResolver._link_units_to_entities_batch_impl.<locals>.<genexpr>  sU       C C HRb "bbRPPPC C C C C Cr   )rT   r7   bulk_insert_unit_entitiesr   r   r   r   rW   rU   r   r   r%   r*   rA   r9   r   )r:   rZ   r   sorted_pairsunit_idsr   unit_to_entitiesrn   r  r    r   cooccurrence_pairsentity_listir-   r.   rD   prevs                     r   r%  z1EntityResolver._link_units_to_entities_batch_impl  s      /5K5KLLL//,///11L111
i11_%%	
 
 	
 	
 	
 	
 	
 	
 	
 1368.? 	6 	6*GY
''77;;IFFF%/*=*=g*F*F*N+5((//+5(
 FH#3#9#9#;#; 	P 	PGZz**K(,,W55J"+K"8"8 P P;#.q1uww#7 P PK"k11  #[003>[&4C-11#7HIID0002<*3//2=dJ2O2O*3//PP&  	..""C'223;;BB C C$6$<$<$>$>C C C     	 	r   d   limitc                    K   t          | j                  4 d{V }|                    dt          d           d||           d{V }d |D             cddd          d{V  S # 1 d{V swxY w Y   dS )z
        Get all units that mention an entity.

        Args:
            entity_id: Entity ID
            limit: Max results

        Returns:
            List of unit IDs
        Nz5
                SELECT unit_id
                FROM r  zp
                WHERE entity_id = $1
                ORDER BY unit_id
                LIMIT $2
                c                     g | ]
}|d          S )r  r   r   s     r   r   z6EntityResolver.get_units_by_entity.<locals>.<listcomp>  s    333sC	N333r   )r   r2   r   r   )r:   r    r:  rZ   r^   s        r   get_units_by_entityz"EntityResolver.get_units_by_entity  s>      &di00 	4 	4 	4 	4 	4 	4 	4D//   
 
 
 
 
 
 
 
D 43d333	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4   :A**
A47A4c                    K   t          | j                  4 d{V }|                    dt          d           d||           d{V }|r|d         ndcddd          d{V  S # 1 d{V swxY w Y   dS )z
        Find an entity by text (for query resolution).

        Args:
            bank_id: bank ID
            entity_text: Entity text to search for

        Returns:
            Entity ID if found, None otherwise
        Nz 
                SELECT id FROM rM   z
                WHERE bank_id = $1
                  AND canonical_name ILIKE $2
                ORDER BY mention_count DESC
                LIMIT 1
                r?   )r   r2   fetchrowr   )r:   ri   r   rZ   r   s        r   get_entity_by_textz!EntityResolver.get_entity_by_text  s=      &di00 	. 	. 	. 	. 	. 	. 	.D ( 4 4   
 
 
 
 
 
 
 
C !$-3t99	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	.r>  )r1   )r(   N)NNrH   )r9  ) r   r   r   r   r	   r   r;   r   rA   rE   rd   staticmethodr   r   rh   dictrp   rm   ry   rz   r   ru   r   r  r  r  r  tupler'  r%  r=  rA  r   r   r   r0   r0   W   s        M MS M M M M M(33 3 3 3 3
3 3 3 3A A A AF ?D4K ?CH ? ? ? \? %)% %% Dz% 	% d{% 
c% % % %\ ,0f f f Dz	f
 f SDf 
cf f f f@E
 E
15dE
	cE
 E
 E
 E
NN
 N
15dN
	cN
 N
 N
 N
`V
V
"%V
6:4jV
S[^bSbV
	cV
 V
 V
 V
p^ ^ Dz	^ S$Y^ sCH}-^ 
c^ ^ ^ ^@~^~^ ~^ 	~^
 d~^ 
~^ ~^ ~^ ~^@% % 	% 
% % % %N$R $R $R $R $R $RL
C 
c 
 
 
 
> !S !Sc3h04c3SW>W8X3YY!S !S !S !SF<PTUZ[^`cemptet[tUuPv < < < <|4 43 4s 4TRUY 4 4 4 42.. . 
t	. . . . . .r   r0   )$r   r=   r   loggingcollectionsr   dataclassesr   r   r   r   difflibr   typingr	   r
   db_utilsr   memory_enginer   retain.entity_labelsr   rg   	getLoggerr   rw   r   r   r"   objectr%   r   r*   r,   _nlpr0   r   r   r   <module>rP     sF        # # # # # # ( ( ( ( ( ( ( ( " " " " " " " " # # # # # #         ( ( ( ( ( ( # # # # # # Y Y Y Y Y Y		8	$	$                                 % % % % % % % % "688 5 # # #8d? x$ 8d?     	' 	' 	' 	' 	' 	' 	' 	' H. H. H. H. H. H. H. H. H. H.r   