
    %jt              	           d dl Z d dlZ e j        dg d          Z G d de          Zd Zd ZdZd	Z	 eej
        e	d
z   e	z   dz   fej        dfej        dfej        edz   fej        efej        dfg          ZdS )    NToken)character_indextypevaluec                   *    e Zd ZdZdZdZdZdZdZdZ	dS )		TokenType
identifiersymbol
whitespacestringzunterminated stringintegerendN)
__name__
__module____qualname__
IDENTIFIERSYMBOL
WHITESPACESTRINGUNTERMINATED_STRINGINTEGEREND     g/home/rurouni/.hermes/hermes-agent/venv/lib/python3.11/site-packages/mammoth/styles/parser/tokeniser.pyr   r      s2        JFJF/G
CCCr   r   c                 |     d  D                                    dt          j        d          f            fd}|S )Nc                 6    g | ]\  }}|t          |          fS r   )	_to_regex).0
token_typeregexs      r   
<listcomp>z#regex_tokeniser.<locals>.<listcomp>   s*    KKK0A
Ej)E**+KKKr   unknown.c           
         g }d}|t          |           k     rD ]j\  }}|                    | |          }|M|                    t          |||                    d                               |                                } nkt          d| |d          z             |t          |           k     |                    t          |t          j        d                     |S )Nr   zRemaining:  )	lenmatchappendr   groupr   	Exceptionr   r   )r   tokensindexr    r!   r(   ruless         r   tokenisez!regex_tokeniser.<locals>.tokenise   s    c%jj  %* ? ?!
EE511$MM%z5;;q>>"J"JKKK!IIKKEE %  eff =>>> c%jj   	eE9="55666r   )r)   recompile)r.   r/   s   ` r   regex_tokeniserr2      sU    KKUKKKE	LL)RZ__-...    $ Or   c                 N    t          | d          r| S t          j        |           S )Nr(   )hasattrr0   r1   )r   s    r   r   r   ,   s)    ug !z%   r   z'(?:\\.|[^'])*z(?:[a-zA-Z\-_]|\\.)z(?:z|[0-9])*z :|>|=>|\^=|=|\(|\)|\[|\]|\||!|\.z\s+'z([0-9]+))collectionsr0   
namedtupler   objectr   r2   r   _string_prefix_identifier_characterr   r   r   r   r   r   r/   r   r   r   <module>r;      s        				 	w(L(L(LMM         2! ! ! #. ?058;PPS]]^:;6"~+,"N3
#  r   