
    %jO                        d dl Z d dlZd dlZd dlmZmZ ddlmZmZ ddl	m
Z
 ddlmZmZ  ej        d          Zded	efd
ZdZ	 d dlZd dlZd dlZn# e$ r  e j                    ZY nw xY wddgZdgZddeee                  ded	efdZded	edz  fdZded	eeee                           fdZ G d de          ZdS )    N)BinaryIOAny   )DocumentConverterDocumentConverterResult)
StreamInfo)MissingDependencyExceptionMISSING_DEPENDENCY_MESSAGEz^\.\d+$textreturnc                    |                      d          }g }d}|t          |          k     r0||         }|                                }t                              |          r|dz   }|t          |          k     rL||                                         s2|dz  }|t          |          k     r||                                         2|t          |          k     r:||                                         }|                    | d|            |dz   }n5|                    |           |dz  }n|                    |           |dz  }|t          |          k     0d                    |          S )a  
    Post-process extracted text to merge MasterFormat-style partial numbering
    with the following text line.

    MasterFormat documents use partial numbering like:
        .1  The intent of this Request for Proposal...
        .2  Available information relative to...

    Some PDF extractors split these into separate lines:
        .1
        The intent of this Request for Proposal...

    This function merges them back together.
    
r       )splitlenstripPARTIAL_NUMBERING_PATTERNmatchappendjoin)r   linesresult_linesilinestrippedj	next_lines           l/home/rurouni/.hermes/hermes-agent/venv/lib/python3.11/site-packages/markitdown/converters/_pdf_converter.py_merge_partial_numbering_linesr       sf    JJtE L	A
c%jj..Qx::<< %**844 	AAc%jj..q)9)9.Q c%jj..q)9)9. 3u::~~!!HNN,,	##x$=$=)$=$=>>>E ##D)))Q%%%FA- c%jj..0 99\"""    zapplication/pdfzapplication/x-pdf.pdfTtableinclude_separatorc                    | sdS d | D             } d | D             } | sdS d t          |  D             dt          t                   dt          ffd|rl| ^}} |          g}|                    dd                    d	 D                       z   dz              |D ] }|                     |                     !nfd
| D             }d                    |          S )a  Convert a 2D list (rows/columns) into a nicely aligned Markdown table.

    Args:
        table: 2D list of cell values
        include_separator: If True, include header separator row (standard markdown).
                          If False, output simple pipe-separated rows.
     c                 &    g | ]}d  |D             S )c                     g | ]}||nd	S )Nr&    .0cells     r   
<listcomp>z1_to_markdown_table.<locals>.<listcomp>.<listcomp>Z   s!    ???4d&ddB???r!   r)   r+   rows     r   r-   z&_to_markdown_table.<locals>.<listcomp>Z   s'    QQQC??3???QQQr!   c                 D    g | ]}t          d  |D                       |S )c              3   >   K   | ]}|                                 V  d S Nr   r*   s     r   	<genexpr>z0_to_markdown_table.<locals>.<listcomp>.<genexpr>]   s*      (F(F$(F(F(F(F(F(Fr!   )anyr.   s     r   r-   z&_to_markdown_table.<locals>.<listcomp>]   s4    GGGSS(F(F#(F(F(F%F%FGSGGGr!   c                 @    g | ]}t          d  |D                       S )c              3   N   K   | ] }t          t          |                    V  !d S r2   )r   strr*   s     r   r4   z0_to_markdown_table.<locals>.<listcomp>.<genexpr>c   s.      55c#d))nn555555r!   max)r+   cols     r   r-   z&_to_markdown_table.<locals>.<listcomp>c   s/    MMM##5555555MMMr!   r/   r   c                 j    dd                     d t          |           D                       z   dz   S )N|c              3   `   K   | ])\  }}t          |                              |          V  *d S r2   )r8   ljust)r+   r,   widths      r   r4   z6_to_markdown_table.<locals>.fmt_row.<locals>.<genexpr>h   s8      VV+$s4yyu--VVVVVVr!   )r   zip)r/   
col_widthss    r   fmt_rowz#_to_markdown_table.<locals>.fmt_rowe   sA    hhVVS*AUAUVVVVVW	
r!   r=   c              3       K   | ]	}d |z  V  
dS -Nr)   r+   ws     r   r4   z%_to_markdown_table.<locals>.<genexpr>o   s&       = =Qq = = = = = =r!   c                 &    g | ]} |          S r)   r)   )r+   r/   rC   s     r   r-   z&_to_markdown_table.<locals>.<listcomp>s   s!    ,,,sggcll,,,r!   r   )rA   listr8   r   r   )r#   r$   headerrowsmdr/   rB   rC   s         @@r   _to_markdown_tablerN   N   sL     r RQ5QQQE HGEGGGE r NMeMMMJ
T#Y 
3 
 
 
 
 
 
  -gfoo
		# = =* = = ====CDDD 	$ 	$CIIggcll####	$ -,,,e,,,99R==r!   pagec                   6789 |                      ddd          }|sdS d}i }|D ]A}t          |d         |z            |z  }||vrg ||<   ||                             |           Bt          |                                          }t          | d          r| j        nd}g }|D ]*}t          ||         d	 
          }	|	s|	d         d         }
|	d         d         }||
z
  }d                    d |	D                       }d |	D             }g }t          |          D ](}|r||d         z
  dk    r|                    |           )||dz  k    ot          |          dk    }d}|	r<|	d         d         	                                }t                              |          rd}|                    ||	|||t          |          |d           ,g }|D ]1}|d         dk    r#|d         s|                    |d                    2|sdS |                                 g }t          t          |          dz
            D ]1}||dz            ||         z
  }|dk    r|                    |           2|rht          |          dk    rUt          |          }t          t          |          dz            }||         }t!          dt#          d|                    }nd}g 7|D ](}7r|7d         z
  |k    r7                    |           )t          7          dk    r7d         7d         z
  }|t          7          z  }|dk     rdS t          7          |d z  z  }|d!k    rdS t          d"|dz  z            }t!          d#|          }t          7          |k    rdS ndS |D ]}|d         rd|d$<   |d%         rd|d$<   t%                      } |d&         D ]L}|d         }!t'          7          D ]2\  }"}#t)          |!|#z
            d'k     r|                     |"            n3Mt          |           d(k    |d$<   g }$d}|t          |          k     r||         d$         rc|}%|t          |          k     r4||         d$         r&|dz  }|t          |          k     r||         d$         &|}&|$                    |%|&f           n|dz  }|t          |          k     t-          d) |$D                       }'t          |          dk    r|'t          |          z  d*k     rdS g }(t          7          8d+t.          d,t0          t2                   f78fd-})d}*|*t          |          k     r||*         }d}+|$D ]\  },}-|*|,k    r|,|-f}+ n|+r]|+\  },}-g 9t          |,|-          D ](}. |)||.                   }/9                    |/           )9r9fd.t          8          D             6d/ 6D             69d         }0d0d1                    6fd2t'          |0          D                       z   d3z   }1|(                    |1           d0d1                    6fd4t          8          D                       z   d3z   }2|(                    |2           9dd         D ]K}3d0d1                    6fd5t'          |3          D                       z   d3z   }4|(                    |4           L|-}*n@d}5|$D ]\  },}-|,|*cxk     r|-k     rn d}5 n|5s|(                    |d                    |*dz  }*|*t          |          k     d6                    |(          S )7a  
    Extract form-style content from a PDF page by analyzing word positions.
    This handles borderless forms/tables where words are aligned in columns.

    Returns markdown with proper table formatting:
    - Tables have pipe-separated columns with header separator rows
    - Non-table content is rendered as plain text

    Returns None if the page doesn't appear to be a form-style document,
    indicating that pdfminer should be used instead for better text spacing.
    T   keep_blank_charsx_tolerancey_toleranceN   topr@   id  c                     | d         S Nx0r)   rH   s    r   <lambda>z2_extract_form_content_from_words.<locals>.<lambda>   s
    1T7 r!   keyr   rZ   x1r   c              3   &   K   | ]}|d          V  dS )r   Nr)   rG   s     r   r4   z3_extract_form_content_from_words.<locals>.<genexpr>   s&       > >q6 > > > > > >r!   c                     g | ]
}|d          S )rZ   r)   rG   s     r   r-   z4_extract_form_content_from_words.<locals>.<listcomp>   s    2221qw222r!   2   g?<   Fr   )y_keywordsr   x_groupsis_paragraphnum_columnshas_partial_numberingri   rh   rg   r   gffffff?   #      H   
         is_table_rowrj   rf   (   r   c              3   &   K   | ]\  }}||z
  V  d S r2   r)   )r+   startends      r   r4   z3_extract_form_content_from_words.<locals>.<genexpr>)  s*      GG:5#3;GGGGGGr!   g?infor   c                    d t                    D             }| d         D ]i}|d         }dz
  }t          dz
            D ]}|dz            }||dz
  k     r|} n||         r||xx         d|d         z   z  cc<   ^|d         ||<   j|S )Nc                     g | ]}d S )r&   r)   )r+   _s     r   r-   zK_extract_form_content_from_words.<locals>.extract_cells.<locals>.<listcomp>3  s    8881B888r!   rf   rZ   r   rp   r   r   )range)	rw   cellswordword_xassigned_colcol_idxcol_endglobal_columnsnum_colss	          r   extract_cellsz7_extract_form_content_from_words.<locals>.extract_cells2  s    88h888M 	3 	3D$ZF#a<L A..  (15GbL((#*LE ) \" 3l###sT&\'99####&*6ll##r!   c                 H    g | ]t          fd D                       S )c              3   B   K   | ]}t          |                   V  d S r2   )r   )r+   r/   r;   s     r   r4   z>_extract_form_content_from_words.<locals>.<listcomp>.<genexpr>Z  s-      <<#CH<<<<<<r!   r9   )r+   r;   
table_datas    @r   r-   z4_extract_form_content_from_words.<locals>.<listcomp>Y  sD       ADC<<<<<<<<<  r!   c                 .    g | ]}t          |d           S )rQ   r9   rG   s     r   r-   z4_extract_form_content_from_words.<locals>.<listcomp>]  s     <<<Ac!Qii<<<r!   z| z | c              3   T   K   | ]"\  }}|                     |                   V  #d S r2   r?   r+   r   r,   rB   s      r   r4   z3_extract_form_content_from_words.<locals>.<genexpr>c  sH       ! !6=a

:a=11! ! ! ! ! !r!   z |c              3   .   K   | ]}d |         z  V  dS rE   r)   )r+   r   rB   s     r   r4   z3_extract_form_content_from_words.<locals>.<genexpr>m  s,       N Nz!}!4 N N N N N Nr!   c              3   T   K   | ]"\  }}|                     |                   V  #d S r2   r   r   s      r   r4   z3_extract_form_content_from_words.<locals>.<genexpr>v  sH       % %:A!TDJJz!}55% % % % % %r!   r   )extract_wordsroundr   sortedkeyshasattrr@   r   r   r   r   r   extendsortr{   intr:   minset	enumerateabsaddsumdictrJ   r8   ):rO   rf   rU   	rows_by_yr}   re   sorted_y_keys
page_widthrow_info	row_wordsfirst_x0last_x1
line_widthcombined_textx_positionsrg   xrh   rj   
first_wordall_table_x_positionsrw   gapsr   gapsorted_gapspercentile_70_idxadaptive_tolerancecontent_widthavg_col_widthcolumns_per_inchadaptive_max_columnsaligned_columnsr~   r   col_xtable_regions	start_idxend_idxtotal_table_rowsr   r   idxtable_regionru   rv   	table_idxr|   rK   
header_str	separatorr/   row_strin_tablerB   r   r   r   s:                                                         @@@@r    _extract_form_content_from_wordsr   x   s	    !QRSSE t K)+I & &d5kK/00;>	!!!Ie%%%%% 9>>++,,M&tW55>3J H &
 &
9U+1B1BCCC	 	Q<%B-%x'
 > >I > > >>> 32	222 "$$ 	# 	#A #q8B</"44""" "J$55Q#m:L:Lr:Q !& 	-"1f-3355J(..z:: -(,%"%$ ,"8}})> 
	
 
	
 
	
 
	
 *, ; ;!##D,@#!((j)9:::  t     D3,--122  #AE*-B1-EE77KK  
 D		QTllK 0 04 788():; !S-?%@%@AA   #%N" % % 	%^B%7!7:L!L!L!!!$$$ >Q&r*^A->>%N(;(;; 24 ~..-"2DE b  4  #2c)9#:;;"2';<<~!5554 6 t  9 9 	#(D  '( 	#(D  %(EEM 	 	D$ZF"+N";";  v~&&++#''000E ,
  #?33q8^ ,.M	A
c(mm

A;~& 	Ic(mm##N(C#Q c(mm##N(C#G  )W!56666FA c(mm

 GGGGGGG
8}}q-H=CCt !L>""HD T#Y       $ C
H

} ' 	 	JE3e|| %s|   :	%JE3*,J"5#.. ) )	%hy&9::!!%((((  #1   HMh  
 =<<<<
 $Ajj ! ! ! !AJ6ARAR! ! !   	  ##J/// jj N N N NeHoo N N NNNO 
 ##I... &abb> 1 1C** % % % %ENs^^% % %   	  !''0000CC H+  
s3$$$$$$$$$#HE %  2##DL1111HCI H

L 99\"""r!   c                    |                      ddd          }|sg S d}i }|D ]A}t          |d         |z            |z  }||vrg ||<   ||                             |           Bt          |                                          }g }|                                D ]"}|D ]}|                    |d                    #|sg S |                                 d}	g }
|D ](}|
r||
d         z
  |	k    r|
                    |           )t          |
          dk     st          |
          d	k    rg S g }|D ]}t          ||         d
           }dgt          |
          z  }|D ]w}d}t          d          }t          |
          D ]'\  }}t          |d         |z
            }||k     r|}|}(||         r||xx         d|d         z   z  cc<   l|d         ||<   xt          d |D                       }|dk    r|                    |           t          |          dk     rg S d}d}|D ]J}|D ]E}|                                r/|dz  }t          |                                          dk    r|dz  }FK|dk    r||z  dk    rg S |gS )a  
    Extract tables from a PDF page by analyzing word positions.
    This handles borderless tables where words are aligned in columns.

    This function is designed for structured tabular data (like invoices),
    not for multi-column text layouts in scientific documents.
    TrQ   rR   rV   rW   rZ   rp   r_   ro   c                     | d         S rY   r)   r[   s    r   r\   z,_extract_tables_from_words.<locals>.<lambda>  s
    ag r!   r]   r&   r   infr   r   c              3   B   K   | ]}|                                 d V  dS )r   Nr3   r*   s     r   r4   z-_extract_tables_from_words.<locals>.<genexpr>  s/      ??d$**,,???????r!   r   r   rm   g333333?)r   r   r   r   r   valuesr   r   floatr   r   r   r   )rO   rf   rU   r   r}   re   r   all_x_positionswords_in_rowx_tolerance_colcolumn_startsr   
table_rowsrow_databest_colmin_distr   r   dist	non_emptylong_cell_counttotal_cell_countr/   r,   s                           r   _extract_tables_from_wordsr     sv    !QRSSE 	 K)+I & &d5kK/00;>	!!!Ie%%%%% 9>>++,,M O!((** / /  	/ 	/D""4:....	/  	 O!#M $ $ 	$M"$5 5 G G  ### =A]!3!3b!8!8	 J ( (i.4E4EFFF 4#m,,,  	2 	2DHU||H%m44 ! !54:-..(??#H H! 2"""cDL&88""""%)&\"" ??h?????	>>h''' :	 O ) ) 	) 	)Dzz|| ) A% tzz||$$r))#q(O	) !2B BS H H	<r!   c                   B    e Zd ZdZdedededefdZdededede	fdZ
dS )	PdfConverterz
    Converts PDFs to Markdown.
    Supports extracting tables into aligned Markdown format (via pdfplumber).
    Falls back to pdfminer if pdfplumber is missing or fails.
    file_streamstream_infokwargsr   c                     |j         pd                                }|j        pd                                }|t          v rdS t          D ]}|                    |          r dS dS )Nr&   TF)mimetypelower	extensionACCEPTED_FILE_EXTENSIONSACCEPTED_MIME_TYPE_PREFIXES
startswith)selfr   r   r   r   r   prefixs          r   acceptszPdfConverter.accepts  s      (.B5577 *0b7799	00041 	 	F""6** tt ur!   c                     t           `t          t          j        t	          |           j        dd                    t           d                             t           d                   t          |t          j	                  sJ g }t          j
        |                                          }	 d}d}t          j        |          5 }|j        D ]}	t          |	          }
|
W|dz  }|	                                }|r;|                                r'|                    |                                           j|dz  }|
                                r|                    |
           	 d d d            n# 1 swxY w Y   ||k    r;|dk    r5|                    d           t(          j                            |          }n'd                    |                                          }nD# t.          $ r7 |                    d           t(          j                            |          }Y nw xY w|s4|                    d           t(          j                            |          }t1          |          }t3          |          S )	Nr"   pdf)	converterr   featurer   r   r   z

)markdown)_dependency_exc_infor	   r
   formattype__name__with_traceback
isinstanceioIOBaseBytesIOread
pdfplumberopenpagesr   extract_textr   r   seekpdfminer
high_levelr   	Exceptionr    r   )r   r   r   r   markdown_chunks	pdf_bytes
form_pagesplain_pagesr   rO   page_contentr   r   s                r   convertzPdfConverter.convert  s     +,*1"4jj1$!    (*99$Q'  +ry11111%' J{//1122	!	CJK++ AsI A AD#CD#I#IL $+#q(#0022 ADJJLL A+224::<<@@@"a
'--// A+22<@@@AA A A A A A A A A A A A A A A$ Z''K!OOq!!!#.;;IFF ";;77==?? 	C 	C 	CNN1*77	BBHHH	C  	CNN1*77	BBH 2(;;&9999s8   -G' B!E3'G' 3E77G' :E7;A+G' '>H('H(N)r   
__module____qualname____doc__r   r   r   boolr   r   r   r)   r!   r   r   r     s            	
 
   $C:C:  C: 	C:
 
!C: C: C: C: C: C:r!   r   )T) sysr   retypingr   r   _base_converterr   r   _stream_infor   _exceptionsr	   r
   compiler   r8   r    r   r   pdfminer.high_levelr   ImportErrorexc_infor   r   rJ   r  rN   r   r   r   r)   r!   r   <module>r     s   



 				 				                 H H H H H H H H % % % % % % P P P P P P P P 'BJz22 +# +# +# +# +# +#^  *OOO * * *'3<>>*
  
 #8 ' 'd49o '$ 'RU ' ' ' 'TS#3 S#3: S# S# S# S#l^S ^T$tCy/-B ^ ^ ^ ^B\: \: \: \: \:$ \: \: \: \: \:s   A A*)A*