
    	D,j                      *   d dl Z d dlZd dlZd dlmZmZmZ d dlmZ de dej	         Z
d Zd Zd Z G d	 d
          Zd Zd Zedk    rG eej                  dk     r ed            ej        d            eej        d                    dS dS )    N)__version__urlsutils)dumpzAider/z +c                      	 ddl m}  d}n# t          $ r d}Y nw xY w	  |             5 }|j                                         d}d d d            n# 1 swxY w Y   n# t
          $ r d}Y nw xY w||fS )Nr   sync_playwrightTF)playwright.sync_apir	   ImportErrorchromiumlaunch	Exception)r	   has_pipphas_chromiums       [/home/rurouni/.local/share/uv/tools/aider-chat/lib/python3.11/site-packages/aider/scrape.py	check_envr      s    777777   _ 	 !JL	  	  	  	  	  	  	  	  	  	  	  	  	  	  	      L  s@    
A AA AA AA A+*A+c                  ,    t                      \  } }| o|S N)r   )r   r   s     r   has_playwrightr   #   s    %KKG\#|#    c                 d   t                      \  }}|r|rdS t          j        dg          }d}t          j        g|                                z   }d}|s|d                    |          dz   z  }|s|d                    |          dz   z  }d| dt          j         d	}| 	                    |           | 
                    d
d          sd S |s0t          j        |          \  }}|s|                     |           d S t          j        |          \  }}|s|                     |           d S dS )NTzaider-chat[playwright]z*-m playwright install --with-deps chromium  
z0For the best web scraping, install Playwright:

z
See z for more info.
zInstall playwright?y)default)r   r   get_pip_installsys
executablesplitjoinr   enable_playwrighttool_outputconfirm_askrun_install
tool_error)	ior   r   pip_cmdchromium_cmdcmdstextsuccessoutputs	            r   install_playwrightr/   (   ss   %KKG\ < t#%=$>??G?LN#l&8&8&:&::LD )!!D(( .&&--  
  D NN4>>/>==  +G44 	MM&!!!F'55OGV 
f4r   c                   F    e Zd ZdZdZdZddZd Zd Zd Z	d Z
d	 Zd
 ZdS )ScraperNFTc                 N    |r|| _         nt          | _         || _        || _        dS )z
        `print_error` - a function to call to print error/debug info.
        `verify_ssl` - if False, disable SSL certificate verification when scraping.
        N)print_errorprintplaywright_available
verify_ssl)selfr3   r5   r6   s       r   __init__zScraper.__init__U   s2    
  	%*D$D$8!$r   c                 \   | j         r|                     |          \  }}n|                     |          \  }}|s|                     d|            dS |r|                    d          s|>|                     |          r)|                                  |                     |          }|S )z
        Scrape a url and turn it into readable markdown if it's HTML.
        If it's plain text or non-HTML, return it as-is.

        `url` - the URL to scrape.
        z Failed to retrieve content from Nz	text/html)r5   scrape_with_playwrightscrape_with_httpxr3   
startswithlooks_like_html
try_pandochtml_to_markdown)r7   urlcontent	mime_types       r   scrapezScraper.scrapeb   s     $ 	=!%!<!<S!A!AGYY!%!7!7!<!<GY 	EEEFFF4  	5)..{;; 	5$"6"6w"?"?OO++G44Gr   c                 p    t          t                    rg d}t          fd|D                       S dS )z7
        Check if the content looks like HTML.
        )z<!DOCTYPE\s+htmlz<htmlz<headz<bodyz<divz<p>z
<a\s+href=c              3   X   K   | ]$}t          j        |t           j                  V  %d S r   )research
IGNORECASE).0patternrA   s     r   	<genexpr>z*Scraper.looks_like_html.<locals>.<genexpr>   s3      __gry'2=AA______r   F)
isinstancestrany)r7   rA   html_patternss    ` r   r=   zScraper.looks_like_html|   sT     gs## 	`  M ____Q^______ur   c           
          dd l }ddlm} ddlm} ddlm}  |            5 }	 |j                                        }nL# t          $ r?}d| _        | 	                    t          |                     Y d }~d d d            dS d }~ww xY w	 |                    | j                   }	|	                                }
|
                    d          }|                    d	d
          }|                    dd
          }|dt           z   z  }|
                    d|i           d }	 |
                    |dd          }nt# |$ r t'          d|            d }Y nY|$ rR}| 	                    d| dt          |                      Y d }~|                                 d d d            dS d }~ww xY w	 |
                                }d }|r2|                    d          }|r|                    d          d         }n;# |$ r3}| 	                    dt          |                      d }d }Y d }~nd }~ww xY w|                                 n# |                                 w xY w	 d d d            n# 1 swxY w Y   ||fS )Nr   )Error)TimeoutErrorr   FNN)ignore_https_errorsznavigator.userAgentHeadlessr   headlessr   
User-Agentnetworkidlei  )
wait_untiltimeoutz.Page didn't quiesce, scraping content anyway: zError navigating to z: content-type;zError retrieving page content: )
playwrightr
   rQ   rR   r	   r   r   r   r5   r3   rM   new_contextr6   new_pageevaluatereplaceaider_user_agentset_extra_http_headersgotor4   closerA   header_valuer!   )r7   r@   r]   PlaywrightErrorPlaywrightTimeoutErrorr	   r   browserecontextpage
user_agentresponserA   rB   content_types                   r   r:   zScraper.scrape_with_playwright   s   @@@@@@NNNNNN777777_ )	 !"*++-- " " ",1)  Q(((!zzz)	  )	  )	  )	  )	  )	  )	  )	 "
! !--doBU-VV''))!]]+@AA
'//
B??
'//
B??
c$444
++\:,FGGG&#yyPTyUUHH- $ $ $P3PPQQQ#HHH& & & &$$%KC%K%K3q66%K%KLLL%::: S)	  )	  )	  )	  )	  )	  )	  )	 2&
%"llnnG $I C'/'<'<^'L'L' C(4(:(:3(?(?(BI& % % %$$%Os1vv%O%OPPP"G $IIIIII%
 S)	  )	  )	  )	  )	  )	  )	  )	  )	  )	  )	  )	  )	  )	  )	 V 	!!s   I1=I1
B)B0I1BI1
BI"D;:I;F,IF,(F'II1'F,,I0A
G;:I;H3 )H.)I.H33I6I1I!!I11I58I5c                    dd l }ddt           di}	 |                    || j        d          5 }|                    |          }|                                 |j        |j                            dd                              d	          d         fcd d d            S # 1 swxY w Y   nY# |j	        $ r"}| 
                    d
|            Y d }~n2d }~wt          $ r"}| 
                    d|            Y d }~nd }~ww xY wdS )Nr   rW   zMozilla./5.0 ()T)headersverifyfollow_redirectsr[   r   r\   zHTTP error occurred: zAn error occurred: rS   )httpxrb   Clientr6   getraise_for_statusr,   rr   r!   	HTTPErrorr3   r   )r7   r@   ru   rr   clientrn   http_errerrs           r   r;   zScraper.scrape_with_httpx   s   !E2B!E!E!EF
	:$    ]!::c??))+++}h&6&:&:>2&N&N&T&TUX&Y&YZ[&\\] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ]  	A 	A 	A?X??@@@@@@@@ 	: 	: 	:838899999999	:zsG   B- A$B!B- !B%%B- (B%)B- -
D7CD!C>>Dc                    | j         rd S 	 t          j                     d| _         d S # t          $ r Y nw xY w	 t          j        d           n0# t
          $ r#}|                     d|            Y d }~d S d }~ww xY wd| _         d S )NT)delete_installerzUnable to install pandoc: )pandoc_availablepypandocget_pandoc_versionOSErrordownload_pandocr   r3   )r7   r|   s     r   r>   zScraper.try_pandoc   s      	F	')))$(D!F 	 	 	D		$d;;;;; 	 	 	?#??@@@FFFFF	 !%s#   ' 
44A 
A;A66A;c                 R   ddl m}  ||d          }t          |          }t          |          }| j        s|S 	 t          j        |dd          }n# t          $ r |cY S w xY wt          j	        dd|          }t          j	        d	d
|          }t          j	        dd|          }|S )Nr   )BeautifulSoupzhtml.parsermarkdownhtml)formatz</div>z      z<div>z     z\n\s*\nz

)
bs4r   slimdown_htmlrM   r   r   convert_textr   rF   sub)r7   page_sourcer   soupmds        r   r?   zScraper.html_to_markdown   s    %%%%%%}[-88T""$ii$ 		&{JvNNNBB 	 	 		 VIx,,VHgr**VJ++	s   A A"!A")NNT)__name__
__module____qualname__r   r5   playwright_instructions_shownr8   rC   r=   r:   r;   r>   r?    r   r   r1   r1   O   s        $)!% % % %  4  &1" 1" 1"f  "% % %&    r   r1   c                    |                      d          D ]}|                                 | j        r| j                                         |                      d           D ]}|                                 |                      d           D ]}|                                 |                      d          D ]:}t          |j                  D ]#}|dk    r|j                            |d            $;| S )Nsvgc                 0    | o|                      d          S Nzdata:r<   xs    r   <lambda>zslimdown_html.<locals>.<lambda>  s    A,G!,,w2G2G r   )hrefc                 0    | o|                      d          S r   r   r   s    r   r   zslimdown_html.<locals>.<lambda>  s    1+Fg1F1F r   )srcTr   )find_all	decomposeimglistattrspop)r   r   tagattrs       r   r   r      s   }}U##  x }}"G"G}HH  }}!F!F}GG  }}T"" * *OO 	* 	*Dv~~	dD)))	* Kr   c                     t          t                                }|                    |           }t          |           d S )N)r5   )r1   r   rC   r4   )r@   scraperrA   s      r   mainr     s8    >+;+;<<<GnnS!!G	'NNNNNr   __main__   zUsage: python playw.py <URL>   )rF   r   r   aiderr   r   r   
aider.dumpr   websiterb   r   r   r/   r1   r   r   r   lenargvr4   exitr   r   r   <module>r      sM   
			 



  * * * * * * * * * *      9K994<99 ! ! !$$ $ $
$ $ $Nk k k k k k k k\  *   z
s38}}q,---D!	 r   