o
    $F5jã6  ã                   @   s†   d dl Z d dlmZ d dlmZ d dlmZmZmZmZ d dl	Z	d dl
Z
d dlZd dlmZ ejejd� e e¡ZG dd„ dƒZdS )	é    N)ÚBeautifulSoup)Údatetime)ÚListÚDictÚOptionalÚTuple)Ú	UserAgent)Úlevelc                   @   sæ   e Zd Zd"dedefdd„Zd#ded	ed
edeee	e
  ee f fdd„Zdedede	e
 fdd„Zdedee
 fdd„Zde
fdd„Zde	e
 fdd„Zdedefdd„Zdedefdd„Zdededee fdd„Zd d!„ ZdS )$ÚVKParserNÚcookiesÚ
user_agentc              	   C   sR   t  ¡ | _tƒ | _| jj |p| jjddddddœ¡ |r$| jj |¡ d| _	d S )Nz?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8z#ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3zgzip, deflate, brz
keep-aliveÚ1)z
User-AgentÚAcceptzAccept-LanguagezAccept-EncodingÚ
ConnectionzUpgrade-Insecure-Requestsúhttps://m.vk.com)
ÚrequestsÚSessionÚsessionr   ÚuaÚheadersÚupdateÚrandomr   Úbase_url)Úselfr   r   © r   úC/var/www/www-root/data/www/avtoreputaciya.ru/vk_parser/vk_parser.pyÚ__init__   s   

ú	
zVKParser.__init__r   é
   Úgroup_idÚoffsetÚcountÚreturnc              
   C   s|  z€|  d¡s|  d¡r|}n&| ¡ s|  d¡r/|dd…  ¡ r/|  d¡s)d|› �nd|› �}nd|› �}| j› d|› �}d	||d
œ}t d|› d|› d|› �¡ | jj||dd�}| ¡  d|jv rnt 	d|› d�¡ W dS |  
|j|¡}|  |j|¡}	||	fW S  tjjy  }
 zt d|› d|
› �¡ W Y d}
~
dS d}
~
w ty½ }
 zt d|› d|
› �¡ W Y d}
~
dS d}
~
ww )u«   
        ÐŸÐ¾Ð»ÑƒÑ‡Ð¸Ñ‚ÑŒ Ð·Ð°Ð¿Ð¸Ñ�Ð¸ Ñ�Ð¾ Ñ�Ñ‚ÐµÐ½Ñ‹ Ð³Ñ€ÑƒÐ¿Ð¿Ñ‹
        Ð’Ð¾Ð·Ð²Ñ€Ð°Ñ‰Ð°ÐµÑ‚: (Ñ�Ð¿Ð¸Ñ�Ð¾Ðº Ð¿Ð¾Ñ�Ñ‚Ð¾Ð², next_from Ð´Ð»Ñ� Ð¿Ð°Ð³Ð¸Ð½Ð°Ñ†Ð¸Ð¸)
        ÚpublicÚclubú-é   Nzwall-Úwallú/r   )Úownr   r    u   Ð—Ð°Ð¿Ñ€Ð¾Ñ�: z?offset=z&count=é   )ÚparamsÚtimeoutuN   Ð”Ð¾Ñ�Ñ‚ÑƒÐ¿ Ðº Ð·Ð°Ð¿Ñ€Ð°ÑˆÐ¸Ð²Ð°ÐµÐ¼Ð¾Ð¹ Ñ�Ñ‚Ñ€Ð°Ð½Ð¸Ñ†Ðµ Ð¾Ð³Ñ€Ð°Ð½Ð¸Ñ‡ÐµÐ½u$   âš ï¸� Ð”Ð¾Ñ�Ñ‚ÑƒÐ¿ Ðº Ð³Ñ€ÑƒÐ¿Ð¿Ðµ u    Ð¾Ð³Ñ€Ð°Ð½Ð¸Ñ‡ÐµÐ½©NNu'   â�Œ ÐžÑˆÐ¸Ð±ÐºÐ° Ð·Ð°Ð¿Ñ€Ð¾Ñ�Ð° Ð´Ð»Ñ� z: u5   â�Œ Ð�ÐµÐ¿Ñ€ÐµÐ´Ð²Ð¸Ð´ÐµÐ½Ð½Ð°Ñ� Ð¾ÑˆÐ¸Ð±ÐºÐ° Ð´Ð»Ñ� )Ú
startswithÚisdigitr   ÚloggerÚinfor   ÚgetÚraise_for_statusÚtextÚwarningÚ_parse_wall_htmlÚ_get_next_offsetr   Ú
exceptionsÚRequestExceptionÚerrorÚ	Exception)r   r   r   r    Úwall_idÚurlr*   ÚresponseÚpostsÚnext_offsetÚer   r   r   Úget_group_wall!   s:   " 
ý

€€þzVKParser.get_group_wallÚhtmlc           	      C   s°   t |dƒ}g }|jdt d¡d�}|s| dddi¡}|D ]+}z|  ||¡}|r-| |¡ W q tyH } zt d|› �¡ W Y d}~qd}~ww t 	d	t
|ƒ› d
|› �¡ |S )u?   Ð Ð°Ð·Ð¾Ð±Ñ€Ð°Ñ‚ÑŒ HTML Ñ�Ñ‚ÐµÐ½Ñ‹ Ð¸ Ð¸Ð·Ð²Ð»ÐµÑ‡ÑŒ Ð¿Ð¾Ñ�Ñ‚Ñ‹úhtml.parserÚdivzwall_item|post|feed_row©Úclass_údata-post-idTõ*   ÐžÑˆÐ¸Ð±ÐºÐ° Ð¿Ð°Ñ€Ñ�Ð¸Ð½Ð³Ð° Ð¿Ð¾Ñ�Ñ‚Ð°: Nu   âœ… Ð�Ð°Ð¹Ð´ÐµÐ½Ð¾ u"    Ð¿Ð¾Ñ�Ñ‚Ð¾Ð² Ð´Ð»Ñ� Ð³Ñ€ÑƒÐ¿Ð¿Ñ‹ )r   Úfind_allÚreÚcompileÚ_parse_single_postÚappendr:   r/   Údebugr0   Úlen)	r   rB   r   Úsoupr>   Úpost_elementsÚ	post_elemÚ	post_datar@   r   r   r   r5   V   s$   

€€þzVKParser._parse_wall_htmlc                 C   sÄ  zÆ|  d¡}|s#|jdt d¡d�}|r |  dd¡ d¡d }nW d	S |jd
t d¡d�}|r5|jdd�nd}|jdt d¡d�}|rI|jdd�nd}d	}	|ra|  d¡ra|  d¡ dd¡ dd¡}	|jdt d¡d�}
|
rs|
jdd�nd}|r||  |¡nt 	¡ }|  
|¡}|  |¡}| d¡s—d|› d|› �nd|› d|› �}t|ƒ||	|||  |¡||  dd¡|  dd¡|  dd¡|  dd¡|t|ƒdœW S  tyá } zt d|› �¡ W Y d	}~d	S d	}~ww )u$   Ð Ð°Ð·Ð¾Ð±Ñ€Ð°Ñ‚ÑŒ Ð¾Ð´Ð¸Ð½ Ð¿Ð¾Ñ�Ñ‚rG   Úaú/wall-?\d+_\d+©ÚhrefrW   Ú Ú_éÿÿÿÿNrD   zwall_post_text|post_text|textrE   T©Ústripzauthor|nameu   Ð�ÐµÐ¸Ð·Ð²ÐµÑ�Ñ‚Ð½Ð¾r'   ÚidÚspanz	date|timer$   zhttps://vk.com/wall-zhttps://vk.com/wallÚlikesr   ÚrepostsÚviewsÚcomments)Úpost_idÚpost_urlÚ	author_idÚauthor_namer3   Ú
text_cleanÚdater_   r`   ra   rb   ÚattachmentsÚraw_htmlrH   )r1   ÚfindrJ   rK   ÚsplitÚget_textÚreplaceÚ_parse_dater   ÚnowÚ_parse_statsÚ_parse_attachmentsr-   ÚstrÚ_clean_textr:   r/   rN   )r   rR   r   rc   ÚlinkÚ	text_elemr3   Úauthor_elemrf   re   Ú	date_elemÚdate_strrh   Ústatsri   rd   r@   r   r   r   rL   p   sN   


*



ó€þzVKParser._parse_single_postc           
      C   sÂ   dddddœ}|  ¡ }t d|¡}|rt| d¡ƒ|d< t d|¡}|r-t| d¡ƒ|d< t d|¡}|r>t| d¡ƒ|d	< |jd
t d¡d�}|r_|  ¡ }t d|¡}	|	r_t|	 d¡ƒ|d< |S )u.   ÐŸÐ°Ñ€Ñ�Ð¸Ð½Ð³ Ñ�Ñ‚Ð°Ñ‚Ð¸Ñ�Ñ‚Ð¸ÐºÐ¸ Ð¿Ð¾Ñ�Ñ‚Ð°r   )r_   r`   ra   rb   u   â�¤\s*(\d+)r%   r_   u   ðŸ”„\s*(\d+)r`   u   ðŸ‘�\s*(\d+)ra   rT   rU   rV   u   (\d+)\s*ÐºÐ¾Ð¼Ð¼rb   )rm   rJ   ÚsearchÚintÚgrouprk   rK   )
r   rR   rz   r3   Ú
like_matchÚrepost_matchÚviews_matchÚcomment_elemÚcomment_textÚcomment_matchr   r   r   rq   ª   s$   zVKParser._parse_statsc           
   	   C   sæ   g }|j dt d¡d�}|D ]}| d¡r%| d| d¡| dd¡dœ¡ q|j d	t d
¡d�}|D ]}| d| d¡|jdd�dœ¡ q2|j d	t d¡d�}|D ]}| dd¡}	|	 d¡sp|	 d¡sp| d|	|jdd�dœ¡ qQ|S )uD   ÐŸÐ°Ñ€Ñ�Ð¸Ð½Ð³ Ð²Ð»Ð¾Ð¶ÐµÐ½Ð¸Ð¹ (Ñ„Ð¾Ñ‚Ð¾, Ð²Ð¸Ð´ÐµÐ¾, Ñ�Ñ�Ñ‹Ð»ÐºÐ¸)Úimgz\.(jpg|png|gif))Úsrcr…   ÚphotoÚaltrX   )Útyper<   r‡   rT   z/videorV   ÚvideorW   Tr[   )rˆ   r<   r3   z
^https?://zhttps://vk.comr   ru   )rI   rJ   rK   r1   rM   rm   r-   )
r   rR   ri   Úphoto_elemsr„   Úvideo_elemsr‰   Ú
link_elemsru   rW   r   r   r   rr   Ì   s8   

ý€

ý
ý€zVKParser._parse_attachmentsry   c                 C   sF  z�|  ¡ }d| ¡ v rt ¡ jddddd�W S d| ¡ v r.t ¡ tdd� jddddd�W S dddd	d
ddddddddœ}t d|¡}|r_| ¡ \}}}| 	| ¡ d¡}tt
|ƒ|t
|ƒƒW S t d|¡}|r}| ¡ \}}	t ¡ jt
|ƒt
|	ƒddd�W S t ¡ W S  ty¢ }
 zt d|› d|
› �¡ t ¡ W  Y d}
~
S d}
~
ww )u)   ÐŸÐ°Ñ€Ñ�Ð¸Ð½Ð³ Ð´Ð°Ñ‚Ñ‹ Ð¸Ð· Ñ�Ñ‚Ñ€Ð¾ÐºÐ¸u   Ñ�ÐµÐ³Ð¾Ð´Ð½Ñ�r   )ÚhourÚminuteÚsecondÚmicrosecondu
   Ð²Ñ‡ÐµÑ€Ð°r%   )Údaysé   é   é   é   é   é   é   é	   r   é   é   )u   Ñ�Ð½Ð²u   Ñ„ÐµÐ²u   Ð¼Ð°Ñ€u   Ð°Ð¿Ñ€u   Ð¼Ð°Ñ�u   Ð¸ÑŽÐ½u   Ð¸ÑŽÐ»u   Ð°Ð²Ð³u   Ñ�ÐµÐ½u   Ð¾ÐºÑ‚u   Ð½Ð¾Ñ�u   Ð´ÐµÐºu   (\d+)\s+([Ð°-Ñ�]{3})\s+(\d{4})z(\d{1,2}):(\d{2})u(   ÐžÑˆÐ¸Ð±ÐºÐ° Ð¿Ð°Ñ€Ñ�Ð¸Ð½Ð³Ð° Ð´Ð°Ñ‚Ñ‹ 'z': N)r\   Úlowerr   rp   rn   Ú	timedeltarJ   ÚmatchÚgroupsr1   r|   r:   r/   rN   )r   ry   Úmonthsrž   ÚdayÚ
month_nameÚyearÚmonthr�   rŽ   r@   r   r   r   ro   ñ   s0   "þ 
€þzVKParser._parse_dater3   c                 C   s$   t  dd|¡}t  dd|¡}| ¡ S )uN   ÐžÑ‡Ð¸Ñ�Ñ‚ÐºÐ° Ñ‚ÐµÐºÑ�Ñ‚Ð° Ð¾Ñ‚ Ð»Ð¸ÑˆÐ½Ð¸Ñ… Ñ�Ð¸Ð¼Ð²Ð¾Ð»Ð¾Ð² Ð¸ Ñ�Ð¼Ð¾Ð´Ð·Ð¸zhttps?://\S+rX   z\s+ú )rJ   Úsubr\   )r   r3   r   r   r   rt     s   zVKParser._clean_textÚcurrent_offsetc           	      C   s®   t |dƒ}|jdt d¡d�}|r(| d¡r(| d¡}t d|¡}|r(| d¡S |jdt d	¡d
�}|rU|jdt d¡d�}|rU| d¡rUt d| d¡¡}|rU| d¡S dS )uB   ÐŸÐ¾Ð»ÑƒÑ‡Ð¸Ñ‚ÑŒ offset Ð´Ð»Ñ� Ñ�Ð»ÐµÐ´ÑƒÑŽÑ‰ÐµÐ¹ Ñ�Ñ‚Ñ€Ð°Ð½Ð¸Ñ†Ñ‹rC   rT   u;   ÐŸÐ¾ÐºÐ°Ð·Ð°Ñ‚ÑŒ ÐµÑ‰Ñ‘|Ð—Ð°Ð³Ñ€ÑƒÐ·Ð¸Ñ‚ÑŒ ÐµÑ‰Ñ‘|Show more)ÚstringrW   z[?&]offset=(\d+)r%   rD   zpagination|pagesrE   u   Â»|Ð²Ð¿ÐµÑ€ÐµÐ´|nextN)r   rk   rJ   rK   r1   r{   r}   )	r   rB   r§   rP   Ú	show_morerW   rž   Ú
paginationÚ	next_linkr   r   r   r6   !  s   



zVKParser._get_next_offsetc                 C   s   | j  ¡  dS )u   Ð—Ð°ÐºÑ€Ñ‹Ñ‚ÑŒ Ñ�ÐµÑ�Ñ�Ð¸ÑŽN)r   Úclose)r   r   r   r   r¬   8  s   zVKParser.closer,   )r   r   )Ú__name__Ú
__module__Ú__qualname__Údictrs   r   r|   r   r   r   r   rA   r5   rL   rq   rr   r   ro   rt   r6   r¬   r   r   r   r   r
      s    05:"%&
r
   )r   Úbs4r   r   Útypingr   r   r   r   rJ   ÚtimeÚloggingÚfake_useragentr   ÚbasicConfigÚINFOÚ	getLoggerr­   r/   r
   r   r   r   r   Ú<module>   s    
