o
    vOAjw%  ã                   @   sˆ   d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZ d dl	m
Z
 g d¢Zg d¢Zdd„ Zdd
d„Zdd„ Zddd„Zdd„ ZdS )é    N)ÚurlparseÚ
urlunparse)Úasync_playwright)ÚBeautifulSoup)	zmc.yandex.ruzgoogle-analytics.comzyandex.ru/adszdzen.ruzprivacy-cs.mail.ruzr.mradx.net/imgsÚpixelÚ1x1ztransparent.gif)ÚimageÚmediaÚfontÚ
stylesheetc                 C   sP   | sdS |   d¡rd|  } n	|   d¡sd|  } t| ƒ}t|j|j|jdddfƒS )NÚ ú/zhttps://ok.ruÚhttpzhttps://ok.ru/)Ú
startswithr   r   ÚschemeÚnetlocÚpath)ÚhrefÚparsed© r   úH/var/www/www-root/data/www/avtoreputaciya.ru/cron/ok_parser/ok_parser.pyÚ	clean_url$   s
   r   é   c                    sž   g }| j ddd�D ]?}t|ƒ|kr n6| d¡‰ d | dg ¡¡}ˆ s$q	d| ¡ v r+q	t‡ fdd	„d
D ƒƒr7q	dˆ v s?dˆ v rHˆ |vrH| ˆ ¡ q	|d |… S )NÚimgT©Úsrcr   ú ÚclassÚavatarc                 3   s   � | ]	}|ˆ   ¡ v V  qd S ©N)Úlower)Ú.0Úxr   r   r   Ú	<genexpr>J   ó   € z!extract_photos.<locals>.<genexpr>)r   ÚiconÚspriter   r   Útransparentzokcdn.ruzmycdn.me)Úfind_allÚlenÚgetÚjoinr    ÚanyÚappend)ÚblockÚ
max_photosÚphotosr   Úclassesr   r   r   Úextract_photos4   s"   

€r2   c           	   
   Ã   sò   �zZ| j |ddd�I dH  |  d¡I dH  |  ¡ I dH }t|dƒ}|jddd	�}|rY|jd
d�}dD ]#}| ¡  |¡rU| d¡}t	|ƒdkrQd 
|dd… ¡ ¡ } |W S q2|W S W dS  tyx } ztj d|› d�¡ W Y d}~dS d}~ww )u]   ÐžÑ‚ÐºÑ€Ñ‹Ð²Ð°ÐµÑ‚ Ð¿Ð¾Ñ�Ñ‚ Ð¿Ð¾ Ð¿Ñ€Ñ�Ð¼Ð¾Ð¹ Ñ�Ñ�Ñ‹Ð»ÐºÐµ Ð¸ Ð²Ð¾Ð·Ð²Ñ€Ð°Ñ‰Ð°ÐµÑ‚ Ñ‚ÐµÐºÑ�Ñ‚Údomcontentloadedi˜:  ©Ú
wait_untilÚtimeoutNéÐ  úhtml.parserÚdivzmedia-text_cnt_tx)Úclass_T©Ústrip)u   Ð¿Ñ€Ð¾Ð´Ð°Ð¶Ð°u   Ð¿Ñ€Ð¾Ð´Ð°Ð¼u
   ÐºÑƒÐ¿Ð»ÑŽu
   Ð¾Ð±Ð¼ÐµÐ½Ú.é   zError getting full text: Ú
)ÚgotoÚwait_for_timeoutÚcontentr   ÚfindÚget_textr    r   Úsplitr)   r+   r<   Ú	ExceptionÚsysÚstderrÚwrite)	ÚpageÚpost_urlÚhtmlÚsoupÚtext_elÚtextÚprefixÚ	sentencesÚer   r   r   Úget_full_text_from_postZ   s0   €

ô€ürS   é   c           !      ƒ   s^  �d‰d}i }t ƒ 4 I d H �šs}|jjdg d¢d�I d H }dddd	œd
dœ}	|r.d|i|	d< |jd?i |	¤ŽI d H }
|
 d¡I d H  |
 ¡ I d H }‡fdd„}dd„ }| d|¡ | d|¡I d H  zU|j| ddd�I d H  | 	d¡I d H  d}|dk rµ| 
d¡I d H }||krˆn-|j dd¡I d H  | 	d¡I d H  |d7 }|r±| 
d|› d�¡I d H }|r±d}n|dk s{W n tyÒ } ztj d|› d �¡ W Y d }~nd }~ww | ¡ I d H }t|d!ƒ}| d"¡}g }|d |… D ]?}| d#d$¡‰ |rþˆ |krþd} n.| d%¡}|�rd&| ¡ v �rqì| d'¡}|�r+| d(¡�r+t|d( ƒ}| ˆ ||d)œ¡ qì|D ]B}zt||d* ƒI d H }|�rKt|ƒd+k�rK|||d, < W �q. t�yp } ztj d-|d, › d.|› d �¡ W Y d }~�q.d }~ww | ¡ I d H  W d   ƒI d H  n1 I d H �s‰w   Y  g }|D ]ƒ}|d, ‰ |d/ }t‡ fd0d1„|D ƒƒ�rª�q’ˆ d$d$|d* g d2œ}ˆ |v �rÀ|ˆ  |d3< n*| d4¡}|�sÍ| d5¡}|�rê|jdd6�}d7|v �ræ|| d7¡d8 d … }||d3< | d9¡} | �rú| jdd6�|d:< t||ƒ|d;< |d3 �rt|d3 ƒd+k�r| |¡ �q’|d |… |�r$|d d, nd tˆd< d=ƒ|d>œS )@Nr   FT)z--no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlledz--disable-dev-shm-usage)ÚheadlessÚargsz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36i€  i8  )ÚwidthÚheightzru-RU)Ú
user_agentÚviewportÚlocaleÚserverÚproxyzFObject.defineProperty(navigator, 'webdriver', {get: () => undefined});c                 “   s0   �zt | j dd¡ƒ}ˆ |7 ‰ W d S    Y d S )Nzcontent-lengthr   )ÚintÚheadersr*   )ÚresponseÚcl)Útraffic_bytesr   r   Úcount_trafficÂ   s   €z"parse_group.<locals>.count_trafficc                 Ó   sd   �| j j}| j j}|tv r|  ¡ I d H  d S tD ]}||v r(|  ¡ I d H   d S q|  ¡ I d H  d S r   )ÚrequestÚurlÚresource_typeÚBLOCKED_RESOURCE_TYPESÚabortÚBLOCKED_PATTERNSÚ	continue_)Úroutere   ÚrtypeÚpatternr   r   r   Úroute_handlerÒ   s   €üz"parse_group.<locals>.route_handlerr`   z**/*r3   i0u  r4   i¸  é   zr() => {

                    return document.querySelectorAll('div.feed[data-feed-id]').length;

                }r7   r>   zß() => {

                        const posts = document.querySelectorAll('div.feed[data-feed-id]');

                        for (let post of posts) {

                            if (post.getAttribute('data-feed-id') === 'zª') {

                                return true;

                            }

                        }

                        return false;

                    }zNavigation error: r?   r8   zdiv.feed[data-feed-id]zdata-feed-idr   zdiv.feed-captionu   Ð—Ð°ÐºÑ€ÐµÐ¿Ð»ÐµÐ½Ð¾za[href*="/topic/"]r   )Úfeed_idre   r.   re   é
   rp   zError processing post z: r.   c                 3   s   � | ]	}|d  ˆ kV  qdS )rp   Nr   )r!   Úp)rp   r   r   r#   ’  r$   zparse_group.<locals>.<genexpr>)rp   rO   Ú	timestampre   r0   rO   zdiv.media-text_cnt_txzdiv.media-text_cntr;   u   ÐšÐ¾Ð¼Ð¼ÐµÐ½Ñ‚Ð¸Ñ€Ð¾Ð²Ð°Ñ‚ÑŒé   Útimers   r0   i   r   )ÚpostsÚlatest_post_idÚ
traffic_mbÚstopped_by_last_idr   ) r   ÚchromiumÚlaunchÚnew_contextÚadd_init_scriptÚnew_pageÚonrk   r@   rA   ÚevaluateÚmouseÚwheelrF   rG   rH   rI   rB   r   Úselectr*   Ú
select_onerD   r   r-   rS   r)   Úcloser,   rC   r2   Úround)!re   Ú	max_postsr/   Úlast_post_idÚ	proxy_urlÚfound_last_postÚ
full_textsrr   ÚbrowserÚctx_argsÚcontextrJ   rc   rn   ÚscrollsÚcurrent_postsÚfoundrR   rL   rM   Úpost_blocksÚposts_to_processr.   ÚcaptionÚlinkrK   Ú	post_infoÚ	full_textÚ	all_postsÚpostrN   rO   Útime_elr   )rp   rb   r   Úparse_group”   sà   €
ú
úÐ€4 €þ



ú€€,€þ �*› kö





€
ør›   c                  C   s:   t  ¡ } | jddd� | jdtdd� | jdtdd� d S )	Nz--urlT)Úrequiredz--max-postsrT   )ÚtypeÚdefaultz--max-photosr   )ÚargparseÚArgumentParserÚadd_argumentr^   )Úparserr   r   r   Úmainê  s   r£   )r   )rT   r   NN)ÚasynciorŸ   ÚjsonrG   Úurllib.parser   r   Úplaywright.async_apir   Úbs4r   ri   rg   r   r2   rS   r›   r£   r   r   r   r   Ú<module>   s    

&
:  X