Your IP : 52.15.53.164


Current Path : /lib64/python3.6/lib2to3/pgen2/__pycache__/
Upload File :
Current File : //lib64/python3.6/lib2to3/pgen2/__pycache__/tokenize.cpython-36.opt-2.pyc

3


 \NX�=@s�dZdZddlZddlZddlmZmZddlTddlm	Z	dd	�e
e	�D�d
ddgZ[	yeWne
k
rzeZYnXd
d�Zdd�Zdd�ZdZdZeede�ee�ZdZdZdZdZedd�Zeeeee�ZdZedd�ee�ZdeZeee�Zed ed!�Zeeee�Z d"Z!d#Z"d$Z#d%Z$d&Z%ee%d'e%d(�Z&ee%d)e%d*�Z'ed+d,d-d.d/d0d1d2d3�	Z(d4Z)ed5d6�Z*ee(e)e*�Z+ee e+e'e�Z,ee,Z-ee%d7ed8d�e%d9ed:d��Z.edee&�Z/eee/e e+e.e�Z0e1e2ej3e-e0e#e$f��\Z4Z5Z6Z7ej3e!�ej3e"�e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7ddddddddd;�4Z8iZ9xd�D]Z:e:e9e:<�q�WiZ;xd�D]Z:e:e;e:<�q�Wd�Z<Gd�d��d�e=�Z>Gd�d��d�e=�Z?d�d��Z@e@fd�d
�ZAd�d��ZBGd�d��d��ZCej3d�ejD�ZEej3d�ejD�ZFd�d��ZGd�d��ZHd�d�ZId�d�ZJeKd�k�r�ddlLZLeMeLjN�dk�rxeAeOeLjNd�jP�neAeLjQjP�dS)�zKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro�N)�BOM_UTF8�lookup)�*�)�tokencCsg|]}|ddkr|�qS)r�_�)�.0�xrr�./usr/lib64/python3.6/lib2to3/pgen2/tokenize.py�
<listcomp>%sr�tokenize�generate_tokens�
untokenizecGsddj|�dS)N�(�|�))�join)�choicesrrr�group0srcGst|�dS)Nr)r)rrrr�any1srcGst|�dS)N�?)r)rrrr�maybe2srz[ \f\t]*z	#[^\r\n]*z\\\r?\nz[a-zA-Z_]\w*z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z\d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z#(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?�'''�"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�~z[][(){}]z\r?\nz[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�")4rrz'''z"""zr'''zr"""zu'''zu"""zb'''zb"""zf'''zf"""zur'''zur"""zbr'''zbr"""zrb'''zrb"""zR'''zR"""zU'''zU"""zB'''zB"""zF'''zF"""zuR'''zuR"""zUr'''zUr"""zUR'''zUR"""zbR'''zbR"""zBr'''zBr"""zBR'''zBR"""zrB'''zrB"""zRb'''zRb"""zRB'''zRB"""�r�R�u�U�f�F�b�B�r'''�r"""�R'''�R"""�u'''�u"""�U'''�U"""�b'''�b"""�B'''�B"""�f'''�f"""�F'''�F"""�ur'''�ur"""�Ur'''�Ur"""�uR'''�uR"""�UR'''�UR"""�br'''�br"""�Br'''�Br"""�bR'''�bR"""�BR'''�BR"""�rb'''�rb"""�Rb'''�Rb"""�rB'''�rB"""�RB'''�RB"""�r'�r"�R'�R"�u'�u"�U'�U"�b'�b"�B'�B"�f'�f"�F'�F"�ur'�ur"�Ur'�Ur"�uR'�uR"�UR'�UR"�br'�br"�Br'�Br"�bR'�bR"�BR'�BR"�rb'�rb"�Rb'�Rb"�rB'�rB"�RB'�RB"�c@seZdZdS)�
TokenErrorN)�__name__�
__module__�__qualname__rrrrrw�srwc@seZdZdS)�StopTokenizingN)rxryrzrrrrr{�sr{c		Cs4|\}}|\}}td||||t|t|�f�dS)Nz%d,%d-%d,%d:	%s	%s)�print�tok_name�repr)	�typerZxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolrrr�
printtoken�sr�cCs(yt||�Wntk
r"YnXdS)N)�
tokenize_loopr{)�readline�
tokeneaterrrrr
�s
cCsxt|�D]}||�q
WdS)N)r)r�r�Z
token_inforrrr��sr�c@s,eZdZdd�Zdd�Zdd�Zdd�Zd	S)
�UntokenizercCsg|_d|_d|_dS)Nrr)�tokens�prev_row�prev_col)�selfrrr�__init__�szUntokenizer.__init__cCs*|\}}||j}|r&|jjd|�dS)N� )r�r��append)r��start�row�col�
col_offsetrrr�add_whitespace�s
zUntokenizer.add_whitespacecCs�xv|D]n}t|�dkr$|j||�P|\}}}}}|j|�|jj|�|\|_|_|ttfkr|jd7_d|_qWdj	|j�S)N�rr�)
�len�compatr�r�r�r�r��NEWLINE�NLr)r��iterable�t�tok_typerr��endr�rrrr�s


zUntokenizer.untokenizec	Cs�d}g}|jj}|\}}|ttfkr,|d7}|ttfkr<d}x�|D]�}|dd�\}}|ttttfkrn|d7}|tkr�|j|�qBn>|t	kr�|j
�qBn*|ttfkr�d}n|r�|r�||d�d}||�qBWdS)NFr�Tr�r���)r�r��NAME�NUMBERr�r��ASYNC�AWAIT�INDENT�DEDENT�pop)	r�rr��	startline�indents�toks_append�toknum�tokval�tokrrrr��s0

zUntokenizer.compatN)rxryrzr�r�rr�rrrrr��sr�z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$)cCsH|dd�j�jdd�}|dks*|jd�r.dS|dks@|jd
�rDdS|S)N�r�-zutf-8zutf-8-�latin-1�
iso-8859-1�iso-latin-1�latin-1-�iso-8859-1-�iso-latin-1-)r�r�r�)r�r�r�)�lower�replace�
startswith)�orig_enc�encrrr�_get_normal_names
r�cs�d�d}d}�fdd�}�fdd�}|�}|jt�rHd�|dd�}d	}|sT|gfS||�}|rj||gfStj|�s~||gfS|�}|s�||gfS||�}|r�|||gfS|||gfS)
NFzutf-8cs"y��Stk
rt�SXdS)N)�
StopIteration�bytesr)r�rr�read_or_stop sz%detect_encoding.<locals>.read_or_stopcs�y|jd�}Wntk
r"dSXtj|�}|s6dSt|jd��}yt|�}Wn tk
rptd|��YnX�r�|j	dkr�td��|d7}|S)N�asciirzunknown encoding: zutf-8zencoding problem: utf-8z-sig)
�decode�UnicodeDecodeError�	cookie_re�matchr�rr�LookupError�SyntaxError�name)r��line_stringr��encoding�codec)�	bom_foundrr�find_cookie&s"

z$detect_encoding.<locals>.find_cookieT�z	utf-8-sig)r�r�blank_rer�)r�r��defaultr�r��first�secondr)r�r�r�detect_encodings0




r�cCst�}|j|�S)N)r�r)r��utrrrrTsc!cs�d}}}tjdd}}d\}}d}dg}	d}
d}d}d}
�x�y
|�}Wntk
rdd}YnX|d}dt|�}}|�rF|s�td|��|j|�}|r�|jd�}}t||d|�|||f||fVd \}}d}nd|�r0|d!d�d	k�r0|d"d�dk�r0t||||t|�f|fVd}d}qBn||}||}qB�nF|dk�rt|�rt|�s`Pd}xf||k�r�||dk�r�|d}n6||d
k�r�|t	dt	}n||dk�r�d}nP|d}�qfW||k�r�P|
�r�|
Vd}
||dk�r�||dk�rh||d�j
d�}|t|�}t|||f||t|�f|fVt||d�||f|t|�f|fVqBttf||dk||d�||f|t|�f|fVqB||	d#k�r�|	j
|�t|d|�|df||f|fVxt||	d$k�rJ||	k�rtdd|||f��|	dd%�}	|�r.||	d&k�r.d}d}
d}td||f||f|fV�q�W|�r�|
�r�||	d'k�r�d}d}
d}n|�s�td|df��d}�x�||k�r8tj||�}|�r|jd�\}}||f||f|}}}|||�||}}||k�s|dk�r|dk�rt||||fV�q4|dk�rft}|dk�r8t}n
|�rBd}
|
�rR|
Vd}
|||||fV�q4|dk�r�|
�r�|
Vd}
t||||fV�q4|tk�rt|}|j||�}|�r�|jd�}|||�}|
�r�|
Vd}
t||||f|fVn||f}||d�}|}P�q4|tk�s@|dd�tk�s@|dd
�tk�r�|d(dk�r�||f}t|�pxt|d�pxt|d}||d�d}}|}Pn |
�r�|
Vd}
t||||fV�q4||k�r�|d)k�r�|�r�|dk�r�tnt||||fV�q�t||||f}|dk�r|
�r|}
�q�|dk�rx|
�rx|
dtk�rx|
ddk�rxd}|	d*}t|
d|
d|
d
|
dfVd}
|
�r�|
Vd}
|Vnz|dk�r�|
�r�|
Vd}
t||||f|fVd}nF|dk�r�|d}n|dk�r�|d}|
�r�|
Vd}
t||||fVn(t||||f||df|fV|d}�q�WqBW|
�rN|
Vd}
x.|	dd�D]} td|df|dfdfV�q\Wtd|df|dfdfVdS)+Nrr�
0123456789r�FrzEOF in multi-line stringr�z\
r�z\
r��	�z#
�#z
z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�.T�
�async�await�def��\z([{z)]})r�r)r�r������r�r�r�r�r�r�)r�r�r�)�stringZ
ascii_lettersr�r�rwr�r��STRING�
ERRORTOKEN�tabsize�rstrip�COMMENTr�r�r��IndentationErrorr��
pseudoprog�spanr�r��
triple_quoted�endprogs�
single_quotedr�r�r��OP�	ENDMARKER)!r��lnum�parenlev�	continuedZ	namechars�numchars�contstr�needcont�contliner��stashed�	async_def�async_def_indent�async_def_nlr��pos�max�strstart�endprog�endmatchr��column�
comment_token�nl_pos�pseudomatchr��spos�eposr�initial�newliner��indentrrrrisp



*


 














�__main__)*rrr&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGrHrIrJrKrLrM)*rrrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r]r^r_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrrsrtru)R�
__author__�__credits__r��re�codecsrrZlib2to3.pgen2.tokenr�r�dir�__all__r��	NameError�strrrr�
Whitespace�Comment�Ignore�Name�	Binnumber�	Hexnumber�	Octnumber�	Decnumber�	Intnumber�Exponent�
Pointfloat�Expfloat�Floatnumber�
Imagnumber�Number�Single�Double�Single3�Double3Z
_litprefix�Triple�String�Operator�Bracket�Special�Funny�
PlainToken�Token�ContStr�PseudoExtras�PseudoToken�list�map�compileZ	tokenprogr�Zsingle3progZdouble3progr�r�r�r�r��	Exceptionrwr{r�r
r�r��ASCIIr�r�r�r�rrrx�sysr��argv�openr��stdinrrrr�<module>s�







8Ic