
    %jw                        U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZmZmZmZmZmZ d dlmZ d dlmZ d dlmZ d dlZd dlZd dlZd dlZdd	lmZ dd
lmZm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7 ddl8m9Z9m:Z:m;Z; dZ<dZ=da>edee         f         e?d<   dedee         f         fdZ@ edd           G d d                      ZA G d d          ZBdS )    N)	dataclass)entry_points)AnyListDictOptionalUnionBinaryIO)Path)urlparse)warn   )
StreamInfo)parse_data_urifile_uri_to_path)PlainTextConverterHtmlConverterRssConverterWikipediaConverterYouTubeConverterIpynbConverterBingSerpConverterPdfConverterDocxConverterXlsxConverterXlsConverterPptxConverterImageConverterAudioConverterOutlookMsgConverterZipConverterEpubConverterDocumentIntelligenceConverterCsvConverter)DocumentConverterDocumentConverterResult)FileConversionExceptionUnsupportedFormatExceptionFailedConversionAttemptg        g      $@_pluginsreturnc                  *   t           t           S g a t          d          D ]l} 	 t                               |                                            0# t          $ r0 t          j                    }t          d| j         d|            Y iw xY wt           S )z3Lazy load plugins, exiting early if already loaded.Nzmarkitdown.plugin)groupPlugin 'z' failed to load ... skipping:
)	r*   r   appendload	Exception	traceback
format_excr   name)entry_pointtbs     ^/home/rurouni/.hermes/hermes-agent/venv/lib/python3.11/site-packages/markitdown/_markitdown.py_load_pluginsr8   A   s    
  H#*=>>> T T	TOOK,,..//// 	T 	T 	T%''BRK,RRbRRSSSSS	T Os   ,A7B
BT)kw_onlyfrozenc                   (    e Zd ZU dZeed<   eed<   dS )ConverterRegistrationzCA registration of a converter with its priority and other metadata.	converterpriorityN)__name__
__module____qualname____doc__r%   __annotations__float     r7   r<   r<   U   s+         MM    OOOOOrF   r<   c                      e Zd ZdZddddedef         dedef         fdZd)dZd)d	Zdd
dee	e
j        eef         dee         dedefdZdddddee	ef         dee         dee	         dee	         dedefdZdddddedee         dee	         dee	         dedefdZddddde	dee         dee	         dee	         dedefdZddddde	dee         dee	         dee	         dedefdZddddde
j        dee         dee	         dee	         dedefdZdedee         defdZd eddfd!Zed"d ed#eddfd$Zded%edee         fd&Zd'e	dz  de	dz  fd(Z dS )*
MarkItDownz(In preview) An extremely simple text-based document reader, suitable for LLM use.
    This reader will convert common file-types or webpages to Markdown.N)enable_builtinsenable_pluginsrI   rJ   c                   d| _         d| _        |                    d          }|:t          j                    | _        | j        j                            ddi           n|| _        t          j	                    | _
        d | _        d | _        d | _        d | _        d | _        g | _        ||r | j        di | |r | j        di | d S d S )NFrequests_sessionAcceptz;text/markdown, text/html;q=0.9, text/plain;q=0.8, */*;q=0.1rE   )_builtins_enabled_plugins_enabledgetrequestsSession_requests_sessionheadersupdatemagikaMagika_magika_llm_client
_llm_model_llm_prompt_exiftool_path
_style_map_convertersrI   rJ   )selfrI   rJ   kwargsrL   s        r7   __init__zMarkItDown.__init__a   s    "' %!::&899#%-%5%7%7D" "*11[    &6D"} !%-1.215-1 9; ## D **6*** 	*D))&)))))	* 	*rF   r+   c                 H   | j         s|                    d          | _        |                    d          | _        |                    d          | _        |                    d          | _        |                    d          | _        | j        t          j        d          | _        | j        Wt          j
        d          rAt          j                                      t          fd	d
D                       r| _        |                     t                      t                      |                     t#          |           t                      |                     t%                      t                      |                     t'                                 |                     t)                                 |                     t+                                 |                     t-                                 |                     t/                                 |                     t1                                 |                     t3                                 |                     t5                                 |                     t7                                 |                     t9                                 |                     t;                                 |                     t=                                 |                     t?                                 |                     tA                                 |                     tC                                 |                    d          }|zi }||d<   |                    d          }|||d<   |                    d          }|||d<   |                    d          }|||d<   |                     tE          di |           d| _         dS tG          dtH                     dS )z
        Enable and register built-in converters.
        Built-in converters are enabled by default.
        This method should only be called once, if built-ins were initially disabled.
        
llm_client	llm_model
llm_promptexiftool_path	style_mapNEXIFTOOL_PATHexiftoolc              3   ^   K   | ]'}|t           j                                      k    V  (d S N)ospathdirname).0d	candidates     r7   	<genexpr>z-MarkItDown.enable_builtins.<locals>.<genexpr>   sJ         RW__Y777     rF   )	z/usr/binz/usr/local/binz/optz/opt/binz/opt/local/binz/opt/homebrew/binzC:\Windows\System32zC:\Program FileszC:\Program Files (x86)r>   )
markitdowndocintel_endpointendpointdocintel_credential
credentialdocintel_file_types
file_typesdocintel_api_versionapi_versionTz(Built-in converters are already enabled.rE   )%rN   rP   rY   rZ   r[   r\   r]   rl   getenvshutilwhichrm   abspathanyregister_converterr   PRIORITY_GENERIC_FILE_FORMATr!   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r"   r$   r#   r   RuntimeWarning)r_   r`   ru   docintel_argsrw   docintel_typesdocintel_versionrq   s          @r7   rI   zMarkItDown.enable_builtins   s    % T	M%zz,77D$jj55DO%zz,77D"(**_"="=D$jj55DO"*&(i&@&@# "*"L44	 8 "	 : :I    
"     8 /8+
 ##"$$/K $    ##---8T $    ##*F $    ##LNN333##$6$8$8999##$4$6$6777##$5$7$7888##MOO444##MOO444##LNN333##MOO444##N$4$4555##N$4$4555##N$4$4555##LNN333##$7$9$9:::##MOO444##LNN333 !'

+> ? ? ,02,=j)&,jj1F&G&G#&22EM,/!',A!B!B!-2@M,/#)::.D#E#E #/3CM-0''1BBMBB   &*D""";^LLLLLrF   c                    | j         sgt                      }|J |D ]I}	  |j        | fi | # t          $ r+ t	          j                    }t          d| d|            Y Fw xY wd| _         dS t          dt                     dS )z
        Enable and register converters provided by plugins.
        Plugins are disabled by default.
        This method should only be called once, if plugins were initially disabled.
        Nr.   z!' failed to register converters:
Tz'Plugins converters are already enabled.)rO   r8   register_convertersr1   r2   r3   r   r   )r_   r`   pluginspluginr6   s        r7   rJ   zMarkItDown.enable_plugins   s     $ 	L#ooG&&&! T TT.F.t>>v>>>>  T T T"-//BRFRRbRRSSSSST %)D!!!:NKKKKKs   .2A#"A#)stream_infosourcer   r`   c                   t          |t                    r|                    d          s?|                    d          s*|                    d          s|                    d          r@d |                                D             }d|v r|d         |d<   |d=  | j        |fd|i|S  | j        |fd|i|S t          |t                    r | j        |fd|i|S t          |t          j                  r | j	        |fd|i|S t          |d	          r>t          |j                  r*t          |t          j                  s | j        |fd|i|S t!          d
t#          |           d          )a  
        Args:
            - source: can be a path (str or Path), url, or a requests.response object
            - stream_info: optional stream info to use for the conversion. If None, infer from source
            - kwargs: additional arguments to pass to the converter
        http:https:file:data:c                     i | ]\  }}||	S rE   rE   ro   kvs      r7   
<dictcomp>z&MarkItDown.convert.<locals>.<dictcomp>      ;;;DAq1a;;;rF   urlmock_urlr   readzInvalid source type: z,. Expected str, requests.Response, BinaryIO.)
isinstancestr
startswithitemsconvert_uriconvert_localr   rQ   Responseconvert_responsehasattrcallabler   io
TextIOBaseconvert_stream	TypeErrortype)r_   r   r   r`   _kwargss        r7   convertzMarkItDown.convert   s    fc"" !	!!'**U$$X..U $$W--U $$W--	U <;FLLNN;;;G##*1%.GJ''t'SSKS7SSS)t)&TTkTVTTT%% 	%4%fPP+PPPP 122 	(4(SS[SFSSS FF##		%%		 vr}55		
 '4&vQQ;Q&QQQbVbbb  rF   r   file_extensionr   rm   r   r   c                   t          |t                    rt          |          }t          |t          j                            |          d         t          j                            |                    }||                    |          }||                    |          }||                    |          }t          |d          5 }| 
                    ||          } | j        d||d|cd d d            S # 1 swxY w Y   d S )	Nr   )
local_path	extensionfilenamer   r   rbfile_stream
base_guessr   stream_info_guessesrE   )r   r   r   r   rl   rm   splitextbasenamecopy_and_updateopen_get_stream_info_guesses_convert)	r_   rm   r   r   r   r`   r   fhguessess	            r7   r   zMarkItDown.convert_local.  sy    dD!! 	t99D  g&&t,,Q/W%%d++
 
 

 "#33K@@J%#33n3MMJ?#333<<J$ 	X33: 4  G !4=WRWWWPVWW		X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	Xs   'DD	Dstreamc                   g }d }|||K|t                      }n|}||J |                    |          }||J |                    |          }|                                sYt          j                    }	 |                    d          }	|	sn|                    |	           .|                    d           |}|                     ||pt                                } | j	        d||d|S )	Nr   r   T   r   r   r   rE   )
r   r   seekabler   BytesIOr   writeseekr   r   )
r_   r   r   r   r   r`   r   r   bufferchunks
             r7   r   zMarkItDown.convert_streamS  s=    %' 
"n&@CO"'\\

(
)!---'77.7QQ
!---'77C7@@
    	Z\\F$D)) U###	$
 KKNNNF //:+E 0 
 
 t}WWWWPVWWWrF   r   r   r   r   c                &     | j         |f|||d|S )zAlias for convert_uri()r   )r   )r_   r   r   r   r   r`   s         r7   convert_urlzMarkItDown.convert_url  s;      t
#)	
 

 
 
 	
rF   uric                   |                                 }|                    d          r?t          |          \  }}|r|dk    rt          d| d           | j        |f|||d|S |                    d          rrt          |          \  }}	}
t          ||	                    d                    }||                    |          } | j	        t          j        |
          f|||d|S |                    d	          s|                    d
          rB| j                            |d          }|                                  | j        |f|||d|S t          d|                    d          d          d          )Nr   	localhostzUnsupported file URI: z$. Netloc must be empty or localhost.r   r   charset)mimetyper   r   r   T)r   zUnsupported URI scheme: :r   z4. Supported schemes are: file:, data:, http:, https:)stripr   r   
ValueErrorr   r   r   rP   r   r   r   r   rS   raise_for_statusr   split)r_   r   r   r   r   r`   netlocrm   r   
attributesdatar   responses                r7   r   zMarkItDown.convert_uri  s    iikk >>'"" -	+C00LFD &K// VSVVV   &4%'-	 
    ^^G$$ 	)7)<)<&Hj$#!"y11  J &'77DD
&4&
4  &-	 
    ^^G$$ 	x(@(@ 	-11#d1CCH%%'''(4('-	 
    r399S>>!+<rrr  rF   r   c                H   d }d }d|j         v r|j         d                             d          }|                    d                                          }|D ]k}	|	                                                    d          rB|	                    d          d                                         }
t          |
          dk    r|
}ld }d }d|j         v rt          j        d|j         d                   }|r_|                    d                              d	          }t          j
                            |          \  }}t          |          dk    r|}|tt          |j                  }t          j
                            |j
                  \  }}t          |          dk    r&t          j
                            |j
                  }|}t          |||||j        
          }||                    |          }||                    |          }||                    |          }t#          j                    }|                    d          D ]}|                    |           |                    d           |                     ||          } | j        d||d|S )Nzcontent-type;r   zcharset==r   zcontent-dispositionzfilename=([^;]+)z"')r   r   r   r   r   r   r   i   )
chunk_sizer   r   rE   )rT   r   popr   r   lenresearchr-   rl   rm   r   r   r   r   r   r   r   r   iter_contentr   r   r   r   )r_   r   r   r   r   r`   r   r   partspart_charsetr   r   m_
_extension
parsed_urlr   r   r   r   s                        r7   r   zMarkItDown.convert_response  s    #'!%X---$^4::3??Eyy||))++H + +::<<**:66 +#zz#q17799H8}}q(("* #'#'	 H$444	-x/?@U/VWWA +771::++E22 " 0 0 : ::z??Q&& *I !(,//JG,,Z_==MAz:""7++JO<<&	  
 
 

 "#33K@@J%#33n3MMJ?#333<<J **c*:: 	  	 ELLA //: 0 
 
 t}WWWWPVWWWrF   r   r   c          
      H   d }g }t          | j        d           }|                                }|t                      gz   D ].}|D ]'}	|	j        }
||                                k    s
J d            d |                                D             }d|vr| j        
| j        |d<   d|vr| j        
| j        |d<   d|vr| j        
| j        |d<   d|vr| j	        
| j	        |d<   d	|vr| j
        
| j
        |d	<   | j        |d
<   |"|j        
|j        |d<   |j        
|j        |d<   d}	  |
j        ||fi |}n# t          $ r Y nw xY w||                                k    sJ t          |
          j         d            |r	  |
j        ||fi |}nE# t$          $ r8 |                    t)          |
t+          j                                         Y nw xY w|                    |           n# |                    |           w xY w|bd                    d t3          j        d|j                  D                       |_        t3          j        dd|j                  |_        |c c S )0t;          |          dk    rt=          |          t?          d          )Nc                     | j         S rk   rs   )xs    r7   <lambda>z%MarkItDown._convert.<locals>.<lambda>%  s    aj rF   )keyz?File stream position should NOT change between guess iterationsc                     i | ]\  }}||	S rE   rE   r   s      r7   r   z'MarkItDown._convert.<locals>.<dictcomp>2  r   rF   rc   rd   re   rg   rf   _parent_convertersr   r   Fz4.accept() should NOT change the file_stream position)r=   exc_info
c                 6    g | ]}|                                 S rE   )rstrip)ro   lines     r7   
<listcomp>z'MarkItDown._convert.<locals>.<listcomp>k  s     XXX4XXXrF   z\r?\nz\n{3,}z

r   )attemptszCould not convert stream to Markdown. No converter attempted a conversion, suggesting that the filetype is simply not supported.) sortedr^   tellr   r=   r   rY   rZ   r[   r]   r\   r   r   acceptsNotImplementedErrorr   r?   r   r1   r/   r)   sysr   r   joinr   r   text_contentsubr   r'   r(   )r_   r   r   r`   resfailed_attemptssorted_registrationscur_posr   converter_registrationr=   r   _acceptss                r7   r   zMarkItDown._convert  s    59 :<
  &d&6<P<PQQQ ""$$.*,,? D	 D	K*> C C&2<	 {//11111T 211 <;FLLNN;;;  w..43C3O,0,<GL)g--$/2M+/?GK(w..43C3O,0,<GL)g--$/2M+/?GK("'11d6I6U/3/BGO, 150@,- *",84?4I 01"2)4 !0y0kUUWUUHH*   D
 {//111119oo.ddd 211  
2	2/i/[TTGTT$   '..3*3clnn       $((1111((1111?'+yyXX28HcFV3W3WXXX( (C$ (*viAQ'R'RC$JJJJJ #{CL !##)?CCCC ) O
 
 	
s<   +D;;
EEFG0?GG0GG00Hr=   c                 Z    t          dt                     |                     |           dS )z,DEPRECATED: User register_converter instead.zFregister_page_converter is deprecated. Use register_converter instead.N)r   DeprecationWarningr   )r_   r=   s     r7   register_page_converterz"MarkItDown.register_page_convertery  s6    T	
 	
 	
 		*****rF   rs   r>   c                Z    | j                             dt          ||                     dS )aw  
        Register a DocumentConverter with a given priority.

        Priorities work as follows: By default, most converters get priority
        DocumentConverter.PRIORITY_SPECIFIC_FILE_FORMAT (== 0). The exception
        is the PlainTextConverter, HtmlConverter, and ZipConverter, which get
        priority PRIORITY_SPECIFIC_FILE_FORMAT (== 10), with lower values
        being tried first (i.e., higher priority).

        Just prior to conversion, the converters are sorted by priority, using
        a stable sort. This means that converters with the same priority will
        remain in the same order, with the most recently registered converters
        appearing first.

        We have tight control over the order of built-in converters, but
        plugins can register converters in any order. The registration's priority
        field reasserts some control over the order of converters.

        Plugins can register converters with any priority, to appear before or
        after the built-ins. For example, a plugin with priority 9 will run
        before the PlainTextConverter, but after the built-in converters.
        r   )r=   r>   N)r^   insertr<   )r_   r=   r>   s      r7   r   zMarkItDown.register_converter  s<    8 	$y8LLL	
 	
 	
 	
 	
rF   r   c           
         g }|                                 }|j        @|j        9t          j        d|j        z   d          \  }}||                     |          }|j        Q|j        Jt          j        |j        d          }t          |          dk    r|                     |d                   }|                                }	 | j        	                    |          }	|	j
        dk    r|	j        j        j        d	k    rd}
|	j        j        j        rl|                    |           |                    d
          }t#          j        |                                          }||                     |j                  }
d}t          |	j        j        j                  dk    rd|	j        j        j        d         z   }d}|j        |j        |	j        j        j        k    rd}|j        -|j                            d          |	j        j        j        vrd}|j         |                     |j                  |
k    rd}|r\|                    t7          |j        p|	j        j        j        |j        p||j        p|
|j        |j        |j                             nq|                    |           |                    t7          |	j        j        j        ||
|j        |j        |j                             n|                    |           |                    |           n# |                    |           w xY w|S )zz
        Given a base guess, attempt to guess or expand on the stream info using the stream content (via magika).
        NplaceholderF)strict)r   r   r   okunknownr   .T)r   r   r   r   r   r   )r   r   r   	mimetypes
guess_typeguess_all_extensionsr   r   rX   identify_streamstatus
predictionoutputlabelis_textr   r   charset_normalizer
from_bytesbest_normalize_charsetencoding
extensions	mime_typelstripr   r/   r   r   r   r   )r_   r   r   r   enhanced_guess_mr   _er  resultr   stream_pagecharset_resultguessed_extension
compatibles                  r7   r   z#MarkItDown._get_stream_info_guesses  s    %' $3355 &:+?+K(
 44U  EB ~!/!?!?!?!L!L *z/C/K/
0CERRRB2ww{{!/!?!?"Q%!?!P!P ""$$F	&\11+>>F}$$):)A)G9)T)T$+3 S$$W---"-"2"24"8"8K%7%B;%O%O%T%T%V%VN%1"&"9"9.:Q"R"R %)!v(/:;;a??(+f.?.F.QRS.T(T% "
'3"+v/@/G/QQQ!&J (4",33C88!,3>? ? "'J &2//
0BCCwNN!&J NN"%/%8 &B%07A&0&:&O>O$.$6$A'%/%8'1'< *  
 
 
 
 NN>222NN"%+%6%=%G&7$+%/%8'1'< *  	 	 	 	 ~...W%%%%KW%%%%s   IL) )M r   c                 b    |dS 	 t          j        |          j        S # t          $ r |cY S w xY w)zA
        Normalize a charset string to a canonical form.
        N)codecslookupr4   LookupError)r_   r   s     r7   r   zMarkItDown._normalize_charset  sJ     ?4	=)).. 	 	 	NNN	s    ..)r+   N)!r?   r@   rA   rB   r	   boolra   rI   rJ   r   rQ   r   r   r
   r   r   r   r&   r   r   r   r   r   r   r   r   r%   r  PRIORITY_SPECIFIC_FILE_FORMATrD   r   r   r   rE   rF   r7   rH   rH   ]   sw       K K .2,0	)* )* )* tTz*)* dDj)	)* )* )* )*VZM ZM ZM ZMxL L L L0 -1	0 0 0c8,dH<=0 j)	0
 0 
!0 0 0 0l -1(,!#X #X #XCI#X j)	#X
 !#X c]#X #X 
!#X #X #X #XR -1(,!-X -X -X-X j)	-X
 !-X c]-X -X 
!-X -X -X -Xf -1(,"&
 
 

 j)	

 !
 3-
 
 
!
 
 
 
. -1(, ; ; ;; j)	;
 !; 
; ; 
!; ; ; ;B -1(,!FX FX FX#FX j)	FX
 !FX c]FX FX 
!FX FX FX FXP]
&]
=A*=M]
	 ]
 ]
 ]
 ]
~+1B +t + + + + 8	
 
 
$
 	

 

 
 
 
@c#c1;c	j	c c c cJ	#* 	t 	 	 	 	 	 	rF   rH   )Cr  rl   r   r   r~   r2   r   dataclassesr   importlib.metadatar   typingr   r   r   r   r	   r
   pathlibr   urllib.parser   warningsr   rQ   rV   r  r.  _stream_infor   
_uri_utilsr   r   
convertersr   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   _base_converterr%   r&   _exceptionsr'   r(   r)   r2  r   r*   rC   r8   r<   rH   rE   rF   r7   <module>r>     s"        				 				 



      				 ! ! ! ! ! ! + + + + + + = = = = = = = = = = = = = = = =       ! ! ! ! ! !              $ $ $ $ $ $ 8 8 8 8 8 8 8 8                                         , H G G G G G G G            	 
 $(%d3i
  ' ' 'uT49_-    ( 4%%%       &%r
 r
 r
 r
 r
 r
 r
 r
 r
 r
rF   