Файловый менеджер - Редактировать - /opt/imh-python/lib/python3.9/site-packages/parso/python/__pycache__/tokenize.cpython-39.pyc
Ðазад
a e�h�d � @ s� U d Z ddlmZ ddlZddlZddlZddlmZ ddl m Z m Z m Z m Z mZmZmZmZ ddlmZ ddlmZmZmZ dZejZejZejZej Z ej Z ej Z ej Z ej Z ej!Z!ej"Z"ej#Z#ej$Z$ej%Z%G d d � d e �Z&e�'d �Z(i Z)eee&f e*d < d d�dd�Z+dd� Z,d d d�dd�Z-dd� Z.dd� Z/dZ0e.de0 d �Z1e.de0 d �Z2e.d �Z3e.d �Z4d d!� Z5G d"d#� d#e �Z6G d$d%� d%e6�Z7G d&d'� d'�Z8d(d)� Z9d*d+� Z:d,d-�e;ee e<e<f e e7 d.�d/d0�Z=d1d2� Z>dd,d3d4�e e; eee< e e<e<f e e7 d5�d6d7�Z?d8d9� Z@eAd:k�r~ejBd; ZCeDeC��ZEeE�F� ZGW d � n1 �sT0 Y e=eGed<�d=�D ]ZHeIeH� �qndS )>a� This tokenizer has been copied from the ``tokenize.py`` standard library tokenizer. The reason was simple: The standard library tokenizer fails if the indentation is not right. To make it possible to do error recovery the tokenizer needed to be rewritten. Basically this is a stripped down version of the standard library module, so you can read the documentation there. Additionally we included some speed and memory optimizations here. � )�absolute_importN)�BOM_UTF8)� NamedTuple�Tuple�Iterator�Iterable�List�Dict�Pattern�Set)�PythonTokenTypes)� split_lines�PythonVersionInfo�parse_version_stringu c @ sb e Zd ZU eed< ee ed< ee ed< eeef ed< eed< eeef ed< ee ed< dS ) �TokenCollection� pseudo_token� single_quoted� triple_quoted�endpats� whitespace�fstring_pattern_map�always_break_tokensN) �__name__� __module__� __qualname__r �__annotations__r �strr r � r r ��/root/rpmbuild/BUILDROOT/imh-python39-modules-3.9.7-92.el8.x86_64/opt/imh-python/lib/python3.9/site-packages/parso/python/tokenize.pyr + s r zutf-8�_token_collection_cacheF��capturec O s* |rJ �d}| s|d7 }|d� |� d S )N�(z?:�|�))�join)r! �choices�kwargs�startr r r �group: s r) c G s t | � d S )N�?)r) )r& r r r �maybeC s r+ ��include_fstring� only_fstringc C sr dd� }g d�}dh}| r<ddg}|r2|}t � }qF||7 }n |rFt � S |D ]"}t�|�D ]}|�||�� qXqJ|S )Nc s s* t jdd� | D �� D ]}d�|�V qd S )Nc S s g | ]}||� � f�qS r )�upper)�.0�cr r r � <listcomp>J � zI_all_string_prefixes.<locals>.different_case_versions.<locals>.<listcomp>� )� _itertools�productr% )�prefix�sr r r �different_case_versionsI s z5_all_string_prefixes.<locals>.different_case_versions)�b�r�u�brr4 �f�fr)�setr5 � permutations�update)r- r. r9 Zvalid_string_prefixes�resultr> r7 �tr r r �_all_string_prefixesH s rE c C s t �| t j�S �N)�re�compile�UNICODE)�exprr r r �_compilee s rK c C s> zt t| � W S ty8 t| � t t| �< }| Y S 0 d S rF )r �tuple�KeyError�_create_token_collection)� version_inforC r r r �_get_token_collectioni s �rP z"[A-Za-z0-9\-]+(?: [A-Za-z0-9\-]+)*z(?:\{\{|\}\}|\\N\{z*\}|\\(?:\r\n?|\n)|\\[^\r\nN]|[^{}\r\n\\])+z\}|\\[^N]|[^{}\\])+z (?:\\(?:\r\n?|\n)|[^{}\r\n])+z[^{}]+c - C s� d}t |�}d}dt d }d}d}d}d}t||||�} d } td d �t| � } d | } t| | �} td | d �}t|| | �}t� }t|� }ttdd�� }tddd�}t|� }d}d}d}d}t|d |d �}tddddd d d �}d }g d �} | d!k�r| �d"d#� n | �d"d$� t| � } t||| �} t|d% td&d'� |d( td)d'� �} ||g} d*}!|�rp| �|t|!� � td+g| �R � }"t|dd,�t|"|| | |dd,� }#i }$|D ]F}%t |�|$|%d& < t |�|$|%d) < t |�|$|%d < t |�|$|%d < �q�t� }&t� }'i }(|D ]:})d-D ]}*|&�|)|* � �q d.D ]}*|'�|)|* � �q&�q|D ] })|!D ]}*|*|(|)|* < �qL�qDd/}+t |#�},t |,|&|'|$||(t|+��S )0Nz[ \f\t]*z #[^\r\n]*u ([A-Za-z_0-9-z]+)z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z (?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]T)r- r, z(?:\\.|[^'\\])*'z(?:\\.|[^"\\])*"z(?:\\.|'(?!'')|[^'\\])*'''z(?:\\.|"(?!"")|[^"\\])*"""�'''�"""z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@`|^!=<>]=?�~z[][(){}])z\.\.\.z\r\n?z\nz[;.,@])� � r z:=?�:z '[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*�'z\\(?:\r\n?|\n)z "[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*�")rX rW rR rQ z\\(?:\r\n?|\n)|\Zr )rX rW )rR rQ )�;�import�class�def�try�except�finally�while�with�return�continue�break�del�pass�global�assert�nonlocal) rK � MAX_UNICODEr) r+ rE �insert�appendr@ �addr )-rO � Whitespacer �Comment�Name� Hexnumber� Binnumber� Octnumber� Decnumber� Intnumber�Exponent� Pointfloat�Expfloat� Floatnumber� Imagnumber�NumberZpossible_prefixes� StringPrefixZStringPrefixWithFZfstring_prefixesZ FStringStart�Single�Double�Single3�Double3�Triple�OperatorZBracketZ special_args�Special�Funny�ContStrZpseudo_extra_poolZ all_quotes� PseudoExtras� PseudoTokenr �_prefixr r r rD �quoteZALWAYS_BREAK_TOKENSZpseudo_token_compiledr r r rN ~ s� �� � ��� � �rN c @ sP e Zd ZU eed< eed< eeef ed< eed< eeeef d�dd��Z dS ) �Token�type�string� start_posr7 )rb c C sR t | j�}t|�dkr0| jd t|� d dfS | jd | jd t| j� fS d S �N� r )r r� �lenr� )�self�linesr r r �end_pos� s z Token.end_posN) r r r r r r r �int�propertyr� r r r r r� � s r� c @ s e Zd Zdd� ZdS )� PythonTokenc C s d| j | jjd� S )Nz6TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r))r� )�_replacer� �name�r� r r r �__repr__� s �zPythonToken.__repr__N)r r r r� r r r r r� � s r� c @ s< e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd S )� FStringNodec C s" || _ d| _d| _d | _d| _d S )Nr r4 )r� �parentheses_count�previous_lines�last_string_start_pos�format_spec_count)r� r� r r r �__init__ s zFStringNode.__init__c C s | j d7 _ d S )Nr� )r� �r� � characterr r r �open_parentheses s z FStringNode.open_parenthesesc C s" | j d8 _ | j dkr d| _d S r� �r� r� r� r r r �close_parentheses s z FStringNode.close_parenthesesc C s t | j�dkS )NrT )r� r� r� r r r �allow_multiline s zFStringNode.allow_multilinec C s | j | jkS rF r� r� r r r � is_in_expr s zFStringNode.is_in_exprc C s | � � o| jS rF )r� r� r� r r r �is_in_format_spec s z FStringNode.is_in_format_specN) r r r r� r� r� r� r� r� r r r r r� � s r� c C s� t | �D ]�\}}|�� }t|�t|� }|�|j�rtt|j||| f||d |� d�} d}|jrhJ �| |d �= | dt|j�| f S qd |dfS )N�r7 r4 r )� enumerate�lstripr� � startswithr� r� � FSTRING_ENDr� ) � fstring_stackr� Zline_nr�column�additional_prefixZfstring_stack_index�nodeZlstripped_stringZ len_lstrip�tokenr r r �_close_fstring_if_necessary s � r� c C s� |d }|� � }|�� r(|r"t}q6t}n|r2t}nt}|�||�}|d u rT|j|fS |jsd||f|_|� d�} |D ]6} | | j �| �} | d urr| � d�d t | j � � } qr|} | t | �7 } | � d�s�| � d�r�| j| 7 _d} n |j| } | | fS )N���r � � r4 ) r� r� � fstring_format_spec_multi_line� fstring_format_spec_single_line�fstring_string_multi_line�fstring_string_single_line�matchr� r� r) r� r� �endswith) r r� �line�lnum�pos�tosr� �regexr� r� �fstring_stack_node� end_match�new_posr r r �_find_fstring_string. s4 r� )r� r )r� )�coderO r� rb c C s t | dd�}t|||d�S )z0Generate tokens from a the source code (string).T)�keepends)rO r� )r �tokenize_lines)r� rO r� r� r r r �tokenizeV s r� c s � fdd�}|S )zL A small helper function to help debug the tokenize_lines function. c ? s& � | i |��D ]}t |� |V qd S rF )�print)�argsr' r� ��funcr r �wrapperb s z _print_tokens.<locals>.wrapperr )r� r� r r� r � _print_tokens^ s r� T)�indentsr� �is_first_token)r� rO r� r� rb c . # s � ��fdd�}t |�\}}}} } } } d} � du r8dg� d}d}d}d}d}d}|d d �g }| D �] }�d7 �d}t|�}|r�|�t�r�t}|dd� }t|�}d |d | }|d }||d 7 }d }|�r$|�|�}|�r|�d�}tt||d|� ||�V d}d}n|| }|| }qd||k rd|�r�|d }|�� �s�t | ||�|�\} }| �r|tt | |j dd �V d|_ �q$||k�r�qd||d� } t || �||�\} }} || 7 }| du�r�| V �q$|�rP|} |D ]l}!|!j}"| |" �||�}#|#du�r�|#�d�}$t|$�t|"� | t| �k �r�|d|� |$dt|"� � } �q�|�| |�}%n |�||�}%|%�r�||%�d� }d}|%�d �\}&}�|&f�|%�d �}'|'dk�r�|�s�J �|}qd|'d }(n(| �||�})||)�� }(|)�� }&�|&f�|�rR|(dv�rR|(dk�s|%du �rRd }| dk�rR|�sR|&}*|*� d k�rDttd�d�V � �|*� ||*�E dH |%�s�| �||�})|�r�| dk�r�|�s�||)�� �E dH |)�� }d }tt|| �|f||)�d� �V d}|d7 }�q$|(|v �s�|(dk�r|'dk�r|'dk�rtt|'�|�V �q$|%�d�du�r�|'| v �rl|�s,| �rlg |dd�< d} t�d|d|&� �}+|+du�rl||+�� �E dH |'�� �r�tt|'�|�V nt|'�|�E dH �q$|(dv �r�tdd� |D ���r�|�� |�s�| dk�r�|�s�tt|'�|�V n||' }d}�q$|(dk�r\|'� d��s |'� d��r"J �|�rP|d �� �rPtt|(�|�V |&d }n||' }�q$|'|v �r�| |' }|�||�}|�r�|�d�}||&|� }'tt|'�|�V n�}||&d� }|}qd�q$|(|v �s�|'dd � |v �s�|'dd� |v �rX|'d dv �rD�|&f}| � |(��p.| � |'d ��p.| � |'d �}||&d� }|}qdntt|'�|�V �q$|'| v �r�|�t | |' �� tt |'�|�V n�|(dk�r�||&d� dv �r�||||&d� 7 }qdn�|'dv �r�|�r�|d � |'� n| d7 } nz|'d v �r|�r|d �!|'� n| �r^| d8 } nJ|'�d ��r^|�r^|d j"|d j# dk�r^|d j#d7 _#d }'|&d }tt$|'�|�V �q$qd|�r�tt|||�V |� d��s�|� d��r�d}|�r�|d }|j �r�tt |j |j dd �V �|f},� dd� D ] }-� �%� tt&d|,d�V �q�tt'd|,|�V dS ) a) A heavily modified Python standard library tokenizer. Additionally to the default information, yields also the prefix of each token. This idea comes from lib2to3. The prefix contains all information that is irrelevant for the parser like newlines in parentheses or comments. c 3 sT | � d k rP| � d kr6t td�| fd�V | � d< qP� �� t td�d�V q d S )Nr� ���r4 )r� � ERROR_DEDENT�pop�DEDENT)r( �r� r� �sposr r �dedent_if_necessaryz s z+tokenize_lines.<locals>.dedent_if_necessaryr N� 0123456789r4 Tr� �^Fr� r� � z #�\�.z...rT z [ \f\t]*$z c s s | ]}|� � V qd S rF )r� )r0 r> r r r � <genexpr> r3 z!tokenize_lines.<locals>.<genexpr>�#r� r� )z\ z\ z\ z([{z)]}rV )(rP r� r� �BOM_UTF8_STRINGr� �endr� �STRINGr� r� �FSTRING_STRINGr� r� r� r� r) �span�INDENTrl � ERRORTOKEN�NUMBERrG � isidentifier�NAME�_split_illegal_unicode_name�any�clear�NEWLINEr� �getr� � FSTRING_STARTr� r� r� r� �OPr� r� � ENDMARKER).r� rO r� r� r� r� r r r r r r r Z paren_levelZmax_�numchars�contstr�new_liner7 r� r� r� r� �endprog�endmatchZ contstr_start�contliner� r� �restZfstring_end_tokenZ quote_lengthZ string_liner� r� r� Zend_match_string� pseudomatchr( r� �initialr� Z indent_start�mr� �indentr r� r r� k s� � � � � " $ � ��� �� � ����r� c # s� � ���fdd�}d� d�|�t | �D ]�\}}�rn|�� rd|� V |� d�d�|d |d | f�q�� |7 � q&� | }|�� r�|� q&� r�|� V d�|d |d | f�|� d�q&� r�|� V d S )Nc s t �r tnt� ���S rF )r� r� r� r ��foundZ is_illegalr� r7 r r � create_token� s z1_split_illegal_unicode_name.<locals>.create_tokenr4 Fr r� T)r� r� )r� r� r7 r� �i�charZ new_foundr r� r r� � s0 r� �__main__r� z3.10)rO )J�__doc__� __future__r �sysrG � itertoolsr5 �codecsr �typingr r r r r r r r Zparso.python.tokenr Z parso.utilsr r r rj r� r� r� r� r� r� r� r� r� r� r� r� r� r �decoder� r r r) r+ rE rK rP Zunicode_character_namer� r� r� r� rN r� r� r� r� r� r r� r� r� r� r� r �argv�path�openr> �readr� r� r� r r r r �<module> s� ( �� �l )�� � � ! (
| ver. 1.1 | |
.
| PHP 8.3.30 | Ð“ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ Ñтраницы: 0 |
proxy
|
phpinfo
|
ÐаÑтройка