
    -j!                       d Z ddlmZ ddlZddlZddlZddlmZ ddlm	Z
 g dZ eh d          ZdZd	Zd
ZdZdZdZdZ G d de          Zd6dZd7dZd6dZd8dZd6dZd9dZd6dZd6d Zd:d$Zd;d&Z d<d(Z!d=d*Z"d>d-Z#d?d2Z$d@d5Z%dS )AzStdlib document-to-text extraction for ``read_file``.

Supports Jupyter notebooks, DOCX, and XLSX without adding hard dependencies.
Malformed documents raise :class:`ExtractionError`; callers can then fall back to
normal text/binary handling.
    )annotationsN)Path)ElementTree)EXTRACTABLE_EXTENSIONSExtractionErrorextract_document_textis_extractable_document>   .docx.xlsx.ipynbi   i     z<http://schemas.openxmlformats.org/wordprocessingml/2006/mainz9http://schemas.openxmlformats.org/spreadsheetml/2006/mainzChttp://schemas.openxmlformats.org/officeDocument/2006/relationshipsz<http://schemas.openxmlformats.org/package/2006/relationshipsc                      e Zd ZdZdS )r   zDRaised when a supported-looking document cannot be rendered as text.N)__name__
__module____qualname____doc__     8/home/rurouni/.hermes/hermes-agent/tools/read_extract.pyr   r      s        NNNNr   r   pathstrreturnc                h    t          |           j                                        }|t          v r|ndS )N )r   suffixlowerr   r   exts     r   
_extensionr   !   s1    
t**

!
!
#
#C///33R7r   boolc                :    t          t          |                     S N)r    r   )r   s    r   r	   r	   &   s    
4  !!!r   c                    t          |           }|dk    rt          |           S |dk    rt          |           S |dk    rt          |           S t	          d|           )Nr   r
   r   zUnsupported document type: )r   _extract_notebook_extract_docx_extract_xlsxr   r   s     r   r   r   *   sk    
T

C
h &&&
g~~T"""
g~~T"""
@@@
A
AAr   c                    t          | t                    r| S t          | t                    rd                    d | D                       S dS )Nr   c              3  D   K   | ]}t          |t                    |V  d S r"   )
isinstancer   ).0items     r   	<genexpr>z_source_text.<locals>.<genexpr>9   s1      HH*T32G2GHtHHHHHHr   )r)   r   listjoin)sources    r   _source_textr0   5   sR    &# &$ IwwHHHHHHHH2r   c           
        	 t          | dd          5 }t          j        |          }d d d            n# 1 swxY w Y   n7# t          t          t          j        f$ r}t          d|           |d }~ww xY wt          |t                    st          d          |	                    d          }t          |t                    s d |	                    dg           D             }|st          d	          d
d
d
d}dddd}g }|D ]}t          |t                    s|	                    d          }	|	|vr2||	xx         dz  cc<   |	dk    rd||	          nd}
|                    d||	          d|
 dt          |	                    dd                                        d          df           |st          d          d                    |                              d          dz   S )Nzutf-8replace)encodingerrorszNot a valid notebook: zNotebook root is not an objectcellsc                n    g | ]2}t          |t                    |                    d g           D ]}|3S )r5   )r)   dictget)r*   wscells      r   
<listcomp>z%_extract_notebook.<locals>.<listcomp>H   sa     
 
 
"d##
 w++	
 
  
 
 
 
r   
worksheetszNotebook contains no cellsr   )markdowncoderawMarkdownCodeRaw	cell_type   r?    r   u	   # ── z cell    ──r/   
z#Notebook contains no readable cells)openjsonloadOSError
ValueErrorJSONDecodeErrorr   r)   r7   r8   r-   extendr0   rstripr.   )r   fhnbexcr5   countslabelsoutr:   typr   s              r   r$   r$   =   s   G$;;; 	r2B	 	 	 	 	 	 	 	 	 	 	 	 	 	 	Z!56 G G G<s<<==3FGb$ @>???FF7OOEeT"" 

 
ff\2..
 
 
  <:;;;Qq11F$fUCCFC { {$%% 	hh{##fsq&)Ull"VC["""

AsAA&AAA<PTPXPXYacePfPfCgCgCnCnosCtCtvxyzzzz ECDDD99S>>  &&--s2   A 5A 9A 9A A5A00A5zfzipfile.ZipFilename
ET.Elementc                    	 t          j        |                     |                    S # t          $ r}t	          d|           |d }~wt           j        $ r}t	          d| d|           |d }~ww xY w)NzMissing zMalformed XML in z: )ET
fromstringreadKeyErrorr   
ParseError)rW   rY   rR   s      r   _zip_xmlra   b   s    J}RWWT]]+++ : : :///00c9= J J J?$??#??@@cIJs    &) 
A3AA3A..A3c                   	 t          j        |           5 }t          |d          }d d d            n# 1 swxY w Y   nT# t           j        $ r}t	          d|           |d }~wt
          $ r"}t	          t          |                    |d }~ww xY wdt           d}g }|                    | d          D ]}g }|                                D ]w}|j	        | dk    r|
                    |j        pd           -|j	        | dk    r|
                    d	           Q|j	        | d
| dhv r|
                    d           x|                    d                    |                              d                     t          d |D                       st	          d          d                    |                              d          dz   S )Nzword/document.xmlzNot a valid DOCX: {}ptr   tab	brcrrG   c              3  >   K   | ]}|                                 V  d S r"   strip)r*   lines     r   r,   z _extract_docx.<locals>.<genexpr>   s*      ..tzz||......r   z!DOCX contains no extractable text)zipfileZipFilera   
BadZipFiler   rK   r   _NS_WitertagappendtextrN   r.   splitanyrO   )	r   rW   rootrR   wlinesparabufnodes	            r   r%   r%   k   s=   1_T"" 	5bB 344D	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 C C C838899sB 1 1 1c#hh''S01 	UAE		Q'''"" 	/ 	/IIKK 	! 	!Dxa777""

49?++++YYY&&

4    hhh1111

4   RWWS\\''--........... CABBB99U""4((4//s9   ? 3? 7? 7? BA!!B.BBc                   	 t          j        |           5 }t          |                                          }t	          ||          }t          |          }t          ||          }g }|D ]\  }}}	|dv rt          |                    |	d                    }
|
|vr3	 t          |
                    |
          |          }n# t          j        $ r Y iw xY w|                    d| d           |                    d |D                        |s|                    d           |                    d           	 d d d            n# 1 swxY w Y   nT# t           j        $ r}t!          d|           |d }~wt"          $ r"}t!          t%          |                    |d }~ww xY w|st!          d          d	                    |                              d	          d	z   S )
N>   hidden
veryHiddenr   u   # ── Sheet: rF   c              3  @   K   | ]}d                      |          V  dS )rh   N)r.   )r*   rows     r   r,   z _extract_xlsx.<locals>.<genexpr>   s,      ::c499S>>::::::r   z(empty)zNot a valid XLSX: z'XLSX has no visible sheets with contentrG   )ro   rp   setnamelist_shared_strings_workbook_sheets_workbook_rels_sheet_partr8   _sheet_rowsr^   r\   r`   ru   rN   rq   r   rK   r   r.   rO   )r   rW   namessharedsheetsrelsrU   rY   stateridpartrowsrR   s                r   r&   r&      sW   1_T"" 	b&&E$R//F%b))F!"e,,DC$*   eS444"488C#4#455u$$&rwwt}}f==DD}   H

;d;;;<<<

::T:::::: *JJy)))

2	 	 	 	 	 	 	 	 	 	 	 	 	 	 	*  C C C838899sB 1 1 1c#hh''S01  IGHHH99S>>  &&--sl   E BE
#CE
CE
CA(E
>E 
EE EE F'%E88F'F""F'r   set[str]	list[str]c                    d|vrg S 	 t          j        |                     d                    }n# t           j        $ r g cY S w xY wdt           dfd|                     d          D             S )Nzxl/sharedStrings.xmlrc   rd   c           	     z    g | ]7}d                      d |                     d          D                       8S )r   c              3  (   K   | ]}|j         pd V  dS r   Nrv   r*   rf   s     r   r,   z-_shared_strings.<locals>.<listcomp>.<genexpr>   s(      ==QAFLb======r   rf   )r.   rs   )r*   r+   ss     r   r;   z#_shared_strings.<locals>.<listcomp>   sF    ^^^$BGG==$))qGGG*<*<=====^^^r   si)r\   r]   r^   r`   _NS_Srs   )rW   r   ry   r   s      @r   r   r      s    U**	}RWW%;<<===   			UA^^^^$))WXT\T\T\J]J]^^^^s   '1 AAlist[tuple[str, str, str]]c                    t          | d          }dt           ddt           dc}fd|                    | d          D             S )Nzxl/workbook.xmlrc   rd   c                    g | ]F}|                     d d          |                     dd          |                      dd          fGS )rY   Sheetr   visibleidr   r8   )r*   sheetrs     r   r;   z$_workbook_sheets.<locals>.<listcomp>   sb        
67	#	#UYYw	%B%BEIIQRhhhXZD[D[\  r   r   )ra   r   _NS_RELrs   )rW   ry   r   r   s      @r   r   r      so    B)**D>>>++++DAq   YY!{{{++   r   dict[str, str]c                    d}||vri S 	 t          j        |                     |                    }n# t           j        $ r i cY S w xY wdt           d}d |                    |          D             S )Nzxl/_rels/workbook.xml.relsrc   z}Relationshipc                    i | ]B}|                     d           |                     d d          |                     dd          CS )Idr   Targetr   )r*   rels     r   
<dictcomp>z"_workbook_rels.<locals>.<dictcomp>   sK    dddVYV]V]^bVcVcdCGGD"swwx44dddr   )r\   r]   r^   r`   _NS_PKG_RELrs   )rW   r   	rels_pathry   rel_tags        r   r   r      s    ,I	}RWWY//00=   			.;...Gdd		'@R@Rdddds   '2 AAtargetc                    |                      d          } t          j        |                     d          r| nd|            S )N/zxl/)lstrip	posixpathnormpath
startswith)r   s    r   r   r      sA    ]]3F(9(9%(@(@TffnFnnUUUr   refintc                    d}| D ]R}|                                 s n;|dz  t          |                                          z   t          d          z
  dz   }St          |dz
  d          S )Nr      ArD   )isalphaorduppermax)r   idxchs      r   
_col_indexr      sk    
C 8 8zz|| 	EBhRXXZZ(3s883a7sQw??r   	xml_bytesbytesr   list[list[str]]c                  	 t          j        |           }dt           d}g }|                    | d          D ]}t	          |          t
          k    r ni 	d}|                    | d          D ]o}|                    d          r#t          |                    dd                    n|dz   }|t          k    rKt          |||          	|<   t          ||          }p|                    |d	k    r	fd
t          |dz             D             ng            |rTt          d |d         D                       s5|                                 |rt          d |d         D                       5|S )Nrc   rd   r   cr   r   rD   r   c                <    g | ]}                     |d           S )r   r   )r*   ir5   s     r   r;   z_sheet_rows.<locals>.<listcomp>   s'    BBB!UYYq"%%BBBr   c              3  >   K   | ]}|                                 V  d S r"   rl   )r*   values     r   r,   z_sheet_rows.<locals>.<genexpr>   s*      ??U5;;==??????r   )r\   r]   r   rs   len_MAX_XLSX_ROWS_PER_SHEETr8   r   _MAX_XLSX_COLS_cell_valuer   ru   rangerx   pop)
r   r   ry   r   r   r   max_colr:   colr5   s
            @r   r   r      s   =##DUADyyA## \ \t99000E "HHWWW%% 	( 	(D3788C==Q*TXXc2..///gPQkCn$$$T6155E#J'3''GGgQRllBBBBuWq[/A/ABBBBXZ[[[[
 s??d2h????? 


  s??d2h????? Kr   r:   r   c                   |                      | d          pd}|                     dd          }|dk    r.	 |t          |                   S # t          t          f$ r Y dS w xY w|dk    rQ|                     | d          }|dn4d                    d |                    | d          D                       S |dk    r|                                d	v rd
ndS |dk    r|pdS |S )Nvr   rf   r   	inlineStrisc              3  (   K   | ]}|j         pd V  dS r   r   r   s     r   r,   z_cell_value.<locals>.<genexpr>   s(      0\0\!20\0\0\0\0\0\r   b>   1TRUEtruer   FALSEez#ERROR)	findtextr8   r   rL   
IndexErrorfindr.   rs   rm   )r:   r   r   r   rV   inlines         r   r   r      s   MMQ'''""(bE
((3

C
czz	#e**%%J' 	 	 	22	
ka888$$^rr0\0\v{{VWSZSZSZG[G[0\0\0\)\)\\
czz*???vvWL
czz  Ls   A A"!A")r   r   r   r   )r   r   r   r    )r   r   )rW   rX   rY   r   r   rZ   )rW   rX   r   r   r   r   )rW   rX   r   r   )rW   rX   r   r   r   r   )r   r   r   r   )r   r   r   r   )r   r   r   r   r   r   )r:   rZ   r   r   r   r   r   r   )&r   
__future__r   rI   r   ro   pathlibr   	xml.etreer   r\   __all__	frozensetr   MAX_XLSX_BYTESr   r   rr   r   r   r   	Exceptionr   r   r	   r   r0   r$   ra   r%   r&   r   r   r   r   r   r   r   r   r   r   <module>r      s)    # " " " " "             ' ' ' ' ' '
k
k
k"#?#?#?@@ ! FC
OLO O O O Oi O O O8 8 8 8
" " " "B B B B   ". ". ". ".JJ J J J0 0 0 04. . . .B_ _ _ _   	e 	e 	e 	eV V V V
      *     r   