
    cjH              	       t   S r SSKrSSKrSSKrSSKrSSKrSSKJr  \R                  R                  \R                  R                  \
5      5      r\\R                  ;  a  \R                  R                  S\5        SSKJr  \R                  R                  S5      r\R                  R                  S5      r\R                  R                  S5      r\R                  R                  S5      r\R                  R)                  \S	5      r\R,                  " \S
S9  \R.                  " \\R0                  SSS9  \R2                  " S5      rS r\R                  R                  S5      rS\S\4S jrS/S\S\S\S\4S jjr S\4S jr!S\"4S jr#S\S\4S jr$S\S\%4S jr&S\S-  4S jr'S\S\%4S  jr(S\S\%4S! jr)S0S"\S#\4S$ jjr*S1S%\+S&\+S\"\%\,4   4S' jjr-S"\S\%4S( jr.S) r/S* r0\1S+:X  a	   \0" 5         gg! \2 a8  r3\Ri                  S,5        \" S-\3 35        \Rj                  " S.5         Sr3C3gSr3C3ff = f)2ar  
Auto-switch llama-server service when Hermes config model changes.

Triggered by systemd path unit (hermes-config.path) on config.yaml changes.
Uses state file guard to prevent re-trigger loops.
Performs rollback if the target service fails to start.
Logs to ~/.hermes/logs/model-switch.log for troubleshooting.

Single source of truth for model definitions: models.py
    Ndatetime)MODELSz~/.hermes/config.yamlz~/.hermes/.last-model-statez~/.hermes/.model-switch-notifyz~/.hermes/logszmodel-switch.logTexist_okz'%(asctime)s [%(levelname)s] %(message)sz%Y-%m-%d %H:%M:%S)filenamelevelformatdatefmtzmodel-switchc                     [        U SS9$ )NT)flush)print)msgs    2/home/rurouni/.hermes/scripts/auto-model-switch.py<lambda>r   ,   s    E#T*    z(~/.hermes/cache/openrouter_override.jsonfilepathreturnc                    0 nSnSn[        U 5       nU GH  nUR                  5       nU(       a  UR                  S5      (       a  M3  [        U5      [        UR	                  5       5      -
  nSU;   a  UR                  S5      u  pn
UR                  5       nU
R                  5       R                  S5      nUS:X  a  US:X  a  UnSnM  US:  a  US:X  a  UnUnM  U(       a  Xs:  a  U(       a  UOSX SU 3'   M  U(       a  U(       a  UOSX'   M  UnM  UR                  5       X'   GM     SSS5        U$ ! , (       d  f       U$ = f)u   Ultra-simple YAML parser for ~/.hermes/config.yaml — returns flat-ish dict.
Only handles basic indented key-value pairs and leaf values. Good enough
for model/provider keys.Nr   #:z'" .)openrstrip
startswithlenlstrip	partitionstrip)r   resultcurrent_keycurrent_indentflinestrippedindentkey_part_val_partkeyvals                r   parse_yaml_simpler-   2   s)    FKN	h1D{{}Hx22377YT[[]!33Fh(0(:(:3(?%Xnn&nn&,,U3Q;3"9"%K%&NaZC2I"%K%+N V%<<?SRF]!C512),#"FK"%K&.nn&6#-  
0 M1 
0 Ms   DD55
Eprovidermodelbase_urlc           	         SSK nSSKn[        U SS9 nUR                  U5      =(       d    0 nSSS5        [	        W[
        5      (       d  0 nUR                  S0 5      n[	        U[
        5      (       d  0 nXS'   XS'   X(S'   U(       a  X8S'   [        R                  R                  U 5      =(       d    S	n	UR                  S
SU	SS9u  p [        R                  " U
SSS9 nUR                  XvSSSS9  UR                  5         [        R                  " UR                  5       5        SSS5        [        R                   " X5        [        R                  R#                  U5      (       a  [        R$                  " U5        gg! , (       d  f       GNZ= f! , (       d  f       Nr= f! [        R                  R#                  U5      (       a  [        R$                  " U5        f f = f)a  Atomically update only the top-level model section in config.yaml.

Older versions rewrote the file in-place line by line. That created a race:
the gateway/model picker could read config.yaml while this script had it
truncated, producing YAML parse errors and making local models disappear
from the picker. It also matched any nested ``provider:``, ``default:``, or
``base_url:`` key by text. Use YAML + os.replace so readers only ever see a
complete old or complete new config.
r   Nzutf-8)encodingr/   r.   defaultr0   r   z.config.yaml.z.tmpT)prefixsuffixdirtextwF)default_flow_style	sort_keysallow_unicode)tempfileyamlr   	safe_load
isinstancedict
setdefaultospathdirnamemkstempfdopen	safe_dumpr   fsyncfilenoreplaceexistsunlink)r   r.   r/   r0   r<   r=   r$   cfg	model_cfg	directoryfdtmp_paths               r   write_model_yamlrR   T   sn    	h	)QnnQ%2 
*c4  w+Ii&&	 G$j i (*)0SI##?6y_c#dLB YYr31QNN3eu\`NaGGIHHQXXZ  2 	

8&77>>(##IIh $1 
*	)$ 21 77>>(##IIh $s0   F
F$  AF'F$ 
F
F!F$ $=G!c                      [         R                  R                  [        5      (       d  [        R                  S[        5        SSS.$ [        [        5      n U R                  SS5      U R                  SS5      S.$ )z7Read model.provider and model.default from config.yaml.zConfig file not found: %sN)r.   r/   zmodel.providerr   zmodel.default)rB   rC   rK   CONFIGlogwarningr-   get)parseds    r   get_config_settingsrY   }   s`    77>>&!!/8 400v&FJJ/4OR0 r   c                  N   [         R                  R                  [        5      (       d  g[	        [        5       n U R                  5       R                  5       R                  S5      nSSS5        W(       a  US   OSn[        U5      S:  a  US   OSnX#4$ ! , (       d  f       N5= f)z[Read last-known-GOOD state from state file.
Returns (model_name, provider) or (None, None).)NN
Nr      )	rB   rC   rK   
STATE_FILEr   readr    splitr   )r$   linesr/   r.   s       r   get_last_good_statera      sz     77>>*%%	j	Q &&t, 
E!H4Eu:>uQxtH	 
	s   .B
B$
model_namec                     U  SU S3n[        [        S5       nUR                  U5        SSS5        [        R	                  SX5        g! , (       d  f       N%= f)zgPersist the last good model+provider.
Format: first line = model name, optional second line = provider.r[   r8   Nz&Good state saved: provider=%s model=%s)r   r]   writerU   debug)rb   r.   contentr$   s       r   save_good_staterg      sK     Bxj+G	j#	!	 
II6M 
	s   A


Amodel_idc                    [         R                  R                  [        5      (       d  [        R                  S5        g [        [        5       n[        R                  " U5      nSSS5        WS   S   S    Vs/ s H  o3S   PM	     nnX;   $ ! , (       d  f       N/= fs  snf ! [        R                  [        4 a   n[        R                  SU5         SnAgSnAff = f)	z=Check if an OpenRouter model is in the curated override list.u6   OpenRouter override file not found — cannot validateTN	providers
openroutermodelsidz'Failed to parse OpenRouter override: %s)rB   rC   rK   OPENROUTER_OVERRIDErU   rV   r   jsonloadJSONDecodeErrorKeyErrorerror)rh   r$   datamallowedes         r   is_allowed_openrouter_modelrx      s    77>>-..LM%&!99Q<D '$($5l$CH$MN$MqT7$MN"" '&N  (+ 		;Q?s<    B% B&B% ;B 	B% 
BB% %C?CCc                      [          HB  n [        R                  " SSU S   /SSSS9nUR                  R	                  5       S:X  d  M@  U s  $    g)	z7Find which model service is currently active (systemd).	systemctl	is-activeserviceT   capture_outputr7   timeoutactiveN)r   
subprocessrunstdoutr    ru   rs     r   get_active_servicer      sN    NN+q|4dA
 88>>x'H  r   c                 j   [         R                  SU S   U S   5        [        R                  " SSSU S   /SSSS	9nUR                  S
:X  a  [         R                  SU S   5        O@[         R                  SU S   UR                  UR                  R                  5       SS 5        UR                  S
:H  $ )z?Stop a single model's systemd service. Returns True on success.zStopping %s (%s)...namer|   sudorz   stopT   r~   r   u     ✅ Stopped %su!     ⚠️  Stop %s returned %d: %sN   )rU   infor   r   
returncoderV   stderrr    r/   r   s     r   stop_servicer      s    HH"E&M53CD	feI&67$	A 	||q#U6]376]ALL!((..2B4C2H	J<<1r   c                 j   [         R                  SU S   U S   5        [        R                  " SSSU S   /SSSS	9nUR                  S
:X  a  [         R                  SU S   5        O@[         R                  SU S   UR                  UR                  R                  5       SS 5        UR                  S
:H  $ )z9Start a model's systemd service. Returns True on success.zStarting %s (%s)...r   r|   r   rz   startTr   r~   r   u     ✅ Started %su     ❌ Start %s failed (%d): %sNi,  )rU   r   r   r   r   rs   r   r    r   s     r   start_servicer      s    HH"E&M53CD	guY'78$	A 	||q#U6]3		2=!,,0@#0F	H<<1r   targetactionc                    Sn [         R                  " / SQSSSS9nUR                  S:X  aF  UR                  R	                  5       R                  S5      n[        U5      S:X  a  US    S	US
    S3nSSKJn  UR                  " 5       R                  S5      nSU SU S    SU S    SU SU R                  SS5       SU S3n[        R                  " [        R                  R                  [         5      SS9  [#        [         S5       nUR%                  US-   5        SSS5        g! [         a     Nf = f! , (       d  f       g= f)zWrite a one-shot notification file for the watchdog to pick up and deliver.
action is one of: 'switched', 'restarted', 'rollback'.?)z
nvidia-smiz$--query-gpu=memory.used,memory.totalz--format=csv,noheader,nounitsTr}   r~   r   z,    z MB / r\   z MBr   z%I:%M:%S %puO   🔀 **Model Switch**
━━━━━━━━━━━━━━━
**Event:** z
**Model:** r   z
**Port:** portz
**VRAM:** z
**Context:** contextzK
**Time:** z PDTr   r8   r[   N)r   r   r   r   r    r_   r   	Exceptionr   nowstrftimerW   rB   makedirsrC   rD   NOTIFY_FILEr   rd   )	r   r   vramr   partsr   tsr   r$   s	            r   write_notificationr      sM    DNN.dA

 <<1HHNN$**40E5zQ(6%(37 "		 	 	/BX Vn% &F^$ %F 

9c23 4D	  KK,t<	k3	1	d
 
 	%  $ 
 	s   A/D( 
D8(
D54D58
Er   timeout_secc                    SSK nSSKJn  SSKJn  SU  S3nUR                  5       n[        U5       H\  n UR                  UR                  U5      SS9nUR                  S:X  a  SUR                  5       U-
  4s  $  UR                  S	5        M^     S
UR                  5       U-
  4$ ! UR                  [        [        4 a     NGf = f)zEWait for the model API to respond. Returns (success, seconds_waited).r   Nhttp://127.0.0.1:z
/v1/modelsr   )r   r   Tr\   F)timeurllib.requestrequesturllib.errorrs   rangeurlopenRequeststatusURLErrorConnectionErrorOSErrorsleep)	r   r   _time_req_errurlr   r)   resps	            r   wait_for_modelr      s    !dV:
.CJJLE;	<<S 11<=D{{c!UZZ\E111 " 	A   %**,&&& 8 		s   AB''CCc                    [        5       nU(       a&  US   U S   :X  a  [        R                  SU S   5        gUnU(       a  [        SUS    S35        [	        U5        O[        R                  S5        [        SU S    S35        [        U 5      nU(       a  [        SU S    S	35        [        U S
   SS9u  pEU(       a  [        SU S    SUS S35        O[        SU S    SUS S35        [        R                  SU(       a  US   OSU S   U(       a  SOSU5        [        [        U R                  SS5      U S   SU S
    S35        [        U S5        g[        SU S    35        [        R                  S 5        U(       a  [        S!US    S35        [        U5      nU(       ac  [        R                  S"US   5        [        [        UR                  SS5      US   SUS
    S35        [        US#5        [        S$US    35        g([        R                  S%US   5        [        S&US    S'35        g([        R                  S)5        [        S*5        g()+z
Stop the currently active service, start *target*.
If starting fails, roll back to the previous service.
Returns True on success, False on total failure.
r   u!   Already on %s — nothing to swapTu     ⏹  Stopping ...zNo active service to stopu     ▶️  Starting u     ⏳ Waiting for z to respond...r   <   )r   u     ✅ z responded in z.1fsu
     ⚠️  z API not responding after z.0fu   s — continuingu3   Switch complete: %s → %s (API ready: %s in %.1fs)z(none)yesnor.   custom:localrb   r   /v1switchedu     ❌ Failed to start u%   Switch FAILED — attempting rollbacku     ↩️  Rolling back to z!Rollback successful: restarted %srollbacku     ✅ Rolled back to u.   Rollback FAILED — %s could not be restarted!u     ❌ Rollback failed! z not running!Fz#No previous service to roll back tou-     ⚠️  No previous service to roll back to)r   rU   r   _echor   r   r   rR   rT   rW   r   rs   critical)r   r   previousokloadedelapsedrollback_oks          r   swap_tor     so     !F &.F6N24fVnEH   045V,- 
v/s
34	v	B	"6&>"2.AB(RHF6&>*.QGHJvf~..HQTUefgF%-&!8VF^ dG	5 	J!G-,VF^,<C@	B 	6:. 
"6&>"2
34II56*8F+;*<C@A#H-HH8(6:JKVX\\*n%M%l30&1A0B#FH x4)(6*:);<=
  LLI!&)++HV,<+=]KL		78=>r   c                     [          Hz  n [        R                  " SSU S   /SSSS9nUR                  R	                  5       S:X  d  M@  [
        R                  SU S	   U S   5        [        R                  " S
SSU S   /SSS9  M|     g)z4Stop any running llama-server services to free VRAM.rz   r{   r|   Tr}   r~   r   z Stopping %s (%s) to free VRAM...r   r   r   r   )r   r   N)r   r   r   r   r    rU   r   r   s     r   _stop_all_local_servicesr   S  s~    NN+q|4dA
 88>>x'HH76AiLQNNfa	l;#R r   c                    ^ [        5       n U S   mU S   nT(       d+  [        R                  S5        [        R                  " S5        US:X  Ga/  [        5       u  p#[        T5      (       d  [        ST S35        [        R                  ST5        U(       aE  UT:w  a?  [        S	U 35        [        R                  S
U5        [        [        U=(       d    SU5        O8SnSn[        SU 35        [        R                  SU5        [        [        XT5        [        5       n [        U S   U S   5        [        R                  " S5        [        TU5        [        R                  ST5        [        R                  " S5        S n[         H  nUS   T:X  d  M  Un  O   U(       dB  [        R                  ST5        [        5         [        TU5        [        R                  " S5        [        5       u  p(UT:X  Ga  [        U4S j[         5       S 5      nU(       Ga4  [        5       n	U	(       a:  U	S   US   :X  a.  [        R!                  ST5        [        R                  " S5        GOU	(       a  [        R                  SUS   U	S   5        O[        R                  SUS   5        [        SUS    S35        [#        U5      n
U
(       aV  [        [        UR%                  SS5      US   SUS    S35        [        TU5        ['        US5        [        SUS    35        O[        SUS    35        [        R                  " U
(       a  SOS5        O,[        R!                  S T5        [        R                  " S5        [        R                  S!US   5        [        S"US    S#35        [)        U5      nU(       a7  [        TU5        [        R                  S$US   5        [        S%US    35        g [        R                  S&5        g )'Nr/   r.   u   No model in config — exitingr   rk   u     ❌ 'z/' is NOT in your curated OpenRouter model list!z2BLOCKED: OpenRouter model '%s' not in allowed listu     ↩️  Reverting to z'Reverting config to last good model: %szQwen3.6-35B-A3B-UD-Q4_K_M.ggufr   u;     ↩️  No last good state — reverting to local model: z&Reverting config to local fallback: %sr\   zOpenRouter model validated: %srb   u4   Config set to cloud model: %s — no service to swapc              3   <   >#    U  H  oS    T:X  d  M  Uv   M     g7f)rb   N ).0ru   config_models     r   	<genexpr>main.<locals>.<genexpr>  s     L&QlO|,Kqq&s   	r   u2   State unchanged (%s) — service healthy, skippingu2   State says %s but %s is active — fixing mismatchu0   State says %s but service is dead — restartingu     🔄 u    is dead — restarting...r   r   r   	restartedu     ✅ Restarted u     ❌ Failed to restart u/   State unchanged (%s) — skipping (cloud model)z!Config changed to local model: %su     🔄 Auto-switching to r   u   Done — now on %su     ✅ Now using uB   Swap failed — state NOT saved (will retry on next config change))rY   rU   r   sysexitra   rx   r   rV   rR   rT   rg   r   r   nextr   re   r   rW   r   r   )settingsconfig_providerlast_good_modellast_good_providerfallback_modelfallback_providerr   ru   r)   r   r   r!   r   s               @r   mainr   c  sb   "$HG$Lz*O12 ,&.A.C+*<88GL>)XYZKKLl[?l#B//@ABBOT );)K|_] "B$2!STbScdeA>R ):K +,HHW-x
/CDHHQK 	o61<@ F\?l*F 
  	GV "o6 -.O,&L&LdS')F&.F6N:		NP\]P"6NF6N< N"6N,v//IJK"6*$VVZZ
N-S%+L%9'88H%LN $L/B&v{;,VF^,<=>4VF^4DEFba(IIGVHHQK HH0&.A	%fVn%5S
9:V_Fo6%vf~6  012XYr   __main__z$Unhandled error in auto-model-switchu     ❌ Error: r\   )r   )r   )r   )6__doc__r   rB   r   ro   loggingr   rC   rD   abspath__file___script_dirinsertrl   r   
expanduserrT   r]   r   LOG_DIRjoinLOG_FILEr   basicConfigINFO	getLoggerrU   r   rn   strr@   r-   rR   rY   tuplera   rg   boolrx   r   r   r   r   intfloatr   r   r   r   __name__r   rw   	exceptionr   r   r   r   <module>r      s  	  	 
    ggoobggooh78chhHHOOA{#  
		3	4WW =>
gg  !AB
''

-
.77<<!34 Gd #   
,,4	 ' 	+ gg(()ST   D& s & c & # &  & R	T 		U 	N Ns N# $ 	D4K 	   $ "t "S "J' '3 'dEk8J '&AD AT AH bZJ z   <=aS!"s   0G9 9H7?.H22H7