
    %ju!                     F   d dl Z d dlZd dlZd dlZd dlmZmZmZmZm	Z	 d dl
mZmZmZ ddlmZmZ ddlmZ 	 d dlZ ej                    5   ej        de           d d	lmZ ddd           n# 1 swxY w Y   d
Zn# e$ r dZY nw xY wddgZddgZ G d de          ZdS )    N)AnyBinaryIODictListUnion)parse_qsurlparseunquote   )DocumentConverterDocumentConverterResult)
StreamInfoignore)category)YouTubeTranscriptApiTFz	text/htmlzapplication/xhtmlz.htmlz.htmc                       e Zd ZdZdedededefdZdededede	fdZ
	 dd	eeef         d
ee         deedf         deedf         fdZdededeedf         fdZddZdS )YouTubeConverterzSHandle YouTube specially, focusing on the video title, description, and transcript.file_streamstream_infokwargsreturnc                 ~   |j         pd}|j        pd                                }|j        pd                                }t	          |          }|                    dd                              dd          }|                    d          sdS |t          v rdS t          D ]}|                    |          r dS dS )	zK
        Make sure we're dealing with HTML content *from* YouTube.
         z\??z\==zhttps://www.youtube.com/watch?FT)	urlmimetypelower	extensionr
   replace
startswithACCEPTED_FILE_EXTENSIONSACCEPTED_MIME_TYPE_PREFIXES)selfr   r   r   r   r   r   prefixs           p/home/rurouni/.hermes/hermes-agent/venv/lib/python3.11/site-packages/markitdown/converters/_youtube_converter.pyacceptszYouTubeConverter.accepts(   s     o#(.B5577 *0b7799	cllkk%%%--eS99~~>?? 	500041 	 	F""6** tt u    c                 4	   |j         dn|j         }t          j        |d|          }i }|j        r|j        j        r|j        j        |d<    |dg          D ]|}t          |t          j                  s|j        D ]W}|dv rQt          |	                    |d                    }	t          |	                    dd                    }
|	r|
r|
||	<    nX}	  |d	g          D ]}t          |t          j                  s|j        s%|j        }
d
|
v rt          j        d|
          }|rzt          j        |                    d                    }|                     |d          }|r;t          |t                     r&t          |	                    dd                    |d<    nn)# t"          $ r}t%          d|            Y d }~nd }~ww xY wd}|                     |g d          }t          |t                    sJ |r	|d| dz  }d}|                     |dg          }|r	|d| dz  }|                     |dg          }|r	|d| dz  }|                     |dg          }|r	|d| dz  }t)          |          dk    r	|d| dz  }|                     |ddg          }|r	|d| dz  }t*          rt-                      d}t/          |j                  }t3          |j                  }d|v rS|d         d         rDt          |d         d                                                 }dg}|D ]}|                    |j                    	 |	                    d |          |                     fd!d"d#$          }|rd%                    d& |D                       }n# t"          $ r}t)          |          dk    rt%          d'|            n_|                     |          !                    d                   "                                }d%                    d( |D                       }Y d }~nd }~ww xY w|r	|d)| dz  }|r|n|j        r|j        j        nd}t          |t                    sJ tG          ||*          S )+Nzutf-8zhtml.parser)from_encodingtitlemeta)itemproppropertynamer   contentscriptytInitialDatazvar ytInitialData = ({.*?});   attributedDescriptionBodyTextdescriptionzError extracting description: z
# YouTube
)r+   zog:titler/   z
## 
interactionCountz- **Views:** keywordsz- **Keywords:** durationz- **Runtime:** r   z
### Video Metadata
zog:descriptionz
### Description
venyoutube_transcript_languagesc                  2                                    S )N)	languages)fetch)video_idr<   ytt_apis   r&   <lambda>z*YouTubeConverter.convert.<locals>.<lambda>   s     $0L !. ! ! r(      r   )retriesdelay c                     g | ]	}|j         
S  text.0parts     r&   
<listcomp>z,YouTubeConverter.convert.<locals>.<listcomp>   s    >>>4TY>>>r(   zError fetching transcript: c                     g | ]	}|j         
S rH   rI   rK   s     r&   rN   z,YouTubeConverter.convert.<locals>.<listcomp>   s    3U3U3U$DI3U3U3Ur(   z
### Transcript
)markdownr+   )$charsetbs4BeautifulSoupr+   string
isinstanceTagattrsstrgetresearchjsonloadsgroup_findKeydict	Exceptionprint_getlenIS_YOUTUBE_TRANSCRIPT_CAPABLEr   r	   r   r   querylistappendlanguage_code_retry_operationjoinfind_transcript	translater?   r   ) r$   r   r   r   encodingsoupmetadatar,   akeyr0   r1   matchdataattrdescewebpage_textr+   statsviewsr8   runtimer5   transcript_text
parsed_urlparamstranscript_listr>   
transcriptr@   r<   rA   s                                 @@@r&   convertzYouTubeConverter.convertF   s    *1977{?R m8TTT $&: 	2$*+ 	2 $
 1HWD&NN 
	 
	DdCG,, Z  888dhhq"oo..C!$((9b"9"9::G 0w 0(/E 9	$z**  !&#'22 }  -"g--I&EwOOE W#z%++a..99#'==7V#W#W# W
8T(B(B W69(,,yRT:U:U6V6VH]3E .  	 	 	6166777DDDD	
 %		($A$A$ABB%%%%%% 	.-E----L		(%7$899 	/.U....E99X
|44 	544444E))Hzl33 	32w2222Eu::>>>U>>>>Lii=:J*KLL 	BA+AAAAL( *	I*,,G O!+/22Jj.//Ff}}Q}vc{1~.."),,x"8"8!F	"1  J$$Z%=>>>W39::6	4 40 "&!6!6      !" "7 " "J " *-((>>:>>>+ + ! W W W9~~**?A??@@@@ ,;;IFF&Y'CA'FGG"UWW #
 +.((3U3U*3U3U3U*V*VW  I H_ H H HHM$*%LTZ%6%6"%%%%%%&!
 
 
 	
s3   CF* *
G4GGAN2 2
Q<BQQNrp   keysdefaultc                 ,    |D ]}||v r
||         c S |S )z<Get first non-empty value from metadata matching given keys.rH   )r$   rp   r   r   ks        r&   rc   zYouTubeConverter._get   s6      	# 	#AH}}{""" r(   r\   rr   c                     t          |t                    r"|D ]}|                     ||          }||c S nVt          |t                    rA |j                    D ]1\  }}||k    r
||         c S |                     ||          x}r|c S 2dS )zBRecursively search for a key in nested dictionary/list structures.N)rU   rg   r_   r`   items)r$   r\   rr   elmretr   r:   results           r&   r_   zYouTubeConverter._findKey   s    dD!! 
	"  mmC--?JJJ # d## 	""
 " "1887NNN!]]1c2226 "!MMM"tr(   rC   r   c                     d}||k     rb	  |            S # t           $ rD}t          d|dz    d|            ||dz
  k     rt          j        |           |dz  }Y d}~nd}~ww xY w||k     bt          d| d          )z"Retries the operation if it fails.r   zAttempt r3   z	 failed: NzOperation failed after z
 attempts.)ra   rb   timesleep)r$   	operationrD   rE   attemptrv   s         r&   rj   z!YouTubeConverter._retry_operation   s     y{{"   :1::q::;;;Wq[((Ju%%%1	  E'EEEFFFs   	 
A":AA")N)rC   r   )__name__
__module____qualname____doc__r   r   r   boolr'   r   r   r   rX   r   r   rc   r_   rj   rH   r(   r&   r   r   %   sE       ]]   	
 
   <

  
 	

 
!
 
 
 
J %)	
 
sCx.
 3i
 sDy!	

 
sDy	
 
 
 
S s uS$Y/?    G G G G G Gr(   r   )r\   r   rZ   rR   typingr   r   r   r   r   urllib.parser   r	   r
   _base_converterr   r   _stream_infor   warningscatch_warningsfilterwarningsSyntaxWarningyoutube_transcript_apir   re   ModuleNotFoundErrorr#   r"   r   rH   r(   r&   <module>r      s     				 



 3 3 3 3 3 3 3 3 3 3 3 3 3 3 4 4 4 4 4 4 4 4 4 4 H H H H H H H H % % % % % %*OOO	 	 	"	" @ @=AAAA??????@ @ @ @ @ @ @ @ @ @ @ @ @ @ @
 %)!! * * *$)!!!*
   
 IG IG IG IG IG( IG IG IG IG IGs5   A> A0$A> 0A44A> 7A48A> >BB