
    fjW                        S r SSKrSSKrSSKrSSKJr  \R
                  R                  \R
                  R                  \5      5      r	\	\R
                  ;  a  \R
                  R                  S\	5        SSKJr  \R
                  R                  S5      r\R
                  R                  S5      r\R
                  R                  S5      r\R
                  R                  S5      r\R
                  R%                  \S	5      r\R(                  " \S
S9  S\4S jrS\S-  4S jrS\S-  4S jrS\S\S-  4S jrS r\S:X  a	   \" 5         gg! \ a2  r\" S\ 35        \" S\ 35        \R@                  " S5         SrCgSrCff = f)u  
Model service health watchdog — runs periodically via Hermes cron (no_agent).
- Silent when healthy (no output → nothing delivered)
- On dead service: restarts it, outputs notification message

Designed for cron jobs with no_agent=True — stdout IS the delivery.
    N)datetimeMODELSz~/.hermes/config.yamlz~/.hermes/.last-model-statez~/.hermes/.model-switch-notifyz~/.hermes/logszmodel-health.logT)exist_okmsgc                     [         R                  " 5       R                  S5      n[        [        S5       nUR                  U SU  S35        S S S 5        g ! , (       d  f       g = f)Nz%Y-%m-%d %H:%M:%Sa 
)r   nowstrftimeopenLOG_FILEwrite)r   tsfs      6/home/rurouni/.hermes/scripts/model-health-watchdog.pylogr      sI    		 	 !4	5B	h		2$auB  
		s   A
A$returnc                     [         R                  R                  [        5      (       d  g[	        [        5       n U  Hm  nUR                  5       nUR                  S5      (       d  M+  SU;   d  M3  UR                  S5      S   R                  5       R                  S5      s  sSSS5        $    SSS5        g! , (       d  f       g= f)z@Read model.default from config.yaml (simple line-by-line parse).Nzdefault:z.ggufz'")ospathexistsCONFIGr   strip
startswithsplit)r   linestrippeds      r   get_config_modelr!   #   s    77>>&!!	fDzz|H"":..7h3F~~j1"5;;=CCEJJ	 
 

  

 s   *B6'B6/2B6+B66
Cc                      [          HE  n [        R                  " SSU S   /SSSS9nUR                  R	                  5       S:X  d  M@  U S   s  $    g	)
z9Find which model service is currently active via systemd.	systemctlz	is-activeserviceT
   capture_outputtexttimeoutactivenameN)r   
subprocessrunstdoutr   )mrs     r   get_active_service_namer1   /   sS    NN+q|4dB
 88>>x'V9      
model_namec                 8    [          H  nUS   U :X  d  M  Us  $    g)z"Find model entry by gguf filename.r3   Nr   )r3   r/   s     r   find_targetr5   ;   s"    \?j(H  r2   c                    ^ [         R                  R                  [        5      (       ax  [	        [        5       n U R                  5       R                  5       nS S S 5        [         R                  " [        5        W(       a!  [        U5        [        R                  " S5        [        5       nU(       d  [        R                  " S5        [        U5      nU(       d  [        R                  " S5        [        5       mTUS   :X  a  [        R                  " S5        [        SUS    ST S35        T(       a>  [        U4S j[          5       S 5      nU(       a  ["        R$                  " SSS	US
   /SSS9  ["        R$                  " SSSUS
   /SSSS9nUR&                  S:X  aJ  [        SUS    S35        [        SUS    S[(        R*                  " 5       R-                  S5       S35        g [        SUS    SUR.                  R                  5       S S  35        [        SUS    SUR.                  R                  5       S S  35        [        R                  " S5        g ! , (       d  f       GN= f)Nr   r+   zDEAD: u    — active=z, restarting...c              3   <   >#    U  H  oS    T:X  d  M  Uv   M     g7f)r+   N ).0r/   r*   s     r   	<genexpr>main.<locals>.<genexpr>f   s     ?AV9+>aas   	sudor#   stopr$   T   )r'   r)   start   r&   zRESTARTED: z successfullyu   🔄 **u"   ** was down — auto-restarted at z%I:%M %pz PDTzFAILED: restart z:    u   ❌ **z** restart failed:    )r   r   r   NOTIFY_FILEr   readr   removeprintsysexitr!   r5   r1   r   nextr   r,   r-   
returncoder   r   r   stderr)r   contentconfig_modeltargetotherr0   r*   s         @r   mainrP   C   s   
 
ww~~k""+!ffhnn&G 
		+'NHHQK#%L&F %&F & VHODE ??FNNfeI.>?#R 		gvi'89$	A
 	||qk&.)78v''I(,,.JaJablJmInnrstvf~.b1A$31G0HIJvf~&&9!((..:J4C:P9QRSc s   I!!
I0__main__zUNHANDLED: u   ❌ Health watchdog error: rB   )!__doc__r,   r   rG   r   r   dirnameabspath__file___script_dirinsertmodelsr   
expanduserr   
STATE_FILErC   LOG_DIRjoinr   makedirsstrr   r!   r1   dictr5   rP   __name__	ExceptionerF   rH   r8   r2   r   <module>rc      sq    	 
  ggoobggooh78chhHHOOA{# 			3	4WW =>
gg  !AB
''

-
.77<<!34 Gd #!S !	#* 		t 	C D4K 7t z   k!+A3/0s   E F(FF