# Validates email addres to ensure it has @ and . characters

def validate_email(email):
    while True: # Loops until the the email is correct.
        if email.find('@') != -1 and email.find('.') != -1: # checks to make sure @ and . are in the email
            if email.find('@') < email.find('.'): # checks to make sure that @ is before . in an email
                return True
            else:
                print('\nThe @ must come before . in the email\n')
                email = reenter_email(email)
        else:
            print('\nMissing @ or . in the email\n')
            email = reenter_email(email)

def reenter_email(email):
    return input(f'Enter the email address and press ENTER: ')

emailTest = ['abdabc.abc', 'abd@abcabc', 'abc.abc@abc', 'abd@abc.abc']

for email in emailTest:
    validate_email(email)


import logging
from typing import List
from typing import Dict

import attr
from thoth.storages import GraphDatabase
from thoth.python import DigestsFetcherBase

_LOGGER = logging.getLogger(__name__)

@attr.s(slots=True)
class GraphDigestsFetcher(DigestsFetcherBase):  # type: ignore
    """Fetch digests from the graph database."""

    graph = attr.ib(type=GraphDatabase)

    @graph.default
    def _graph_default(self) -> GraphDatabase:
        """Get default graph instance if no explicitly provided."""
        graph = GraphDatabase()
        graph.connect()
        return graph

    def fetch_digests(self, package_name: str, package_version: str) -> Dict[str, List[Dict[str, str]]]:
        """Fetch digests for the given package in specified version, consider only enabled indexes."""
        _LOGGER.debug(
            "Querying graph database for digests for package %r in version %r",
            package_name,
            package_version,
        )

        result = {}
        for index_url in self.graph.get_python_package_index_urls_all(enabled=True):
            query_result = self.graph.get_python_package_hashes_sha256(
                package_name, package_version, index_url, distinct=True
            )
            result[index_url] = [{"sha256": digest} for digest in query_result]

        return result


from collections import deque

def solver():
    case = 1
    while True:
        PC = [int(x) for x in input().split(" ")]
        P = PC[0]
        C = PC[1]
        if P == 0 and C == 0:
            break
        q = deque()
        for i in range(1, min(P, C) + 1):
            q.append(i)
        print("Case {0}:".format(case))
        case += 1
        for i in range(C):
            v = input().split(" ")
            if v[0] == "N":
                value = q.popleft()
                print(value)
                q.append(value)
            else:
                k = int(v[1])
                value = k
                if k in q:
                    q.remove(value)
                q.appendleft(value)

if __name__ == '__main__':
    solver()


#!/usr/local/bin/python
# -*- coding: utf-8 -*-

import requests
import json
# import base64
import datetime
import numpy as np

from robo.Util import util

URL = 'http://127.0.0.1:8000/api/v1/article/'

def post_news(info):
    date_now = datetime.datetime.now()
    if(info['date'][0] > date_now):
        date = info['date'][0]
    else:
        date = date_now

    str_date = date.strftime("%Y-%m-%d %H:%M:%S")

    news = info['body'][0]
    reduced_news = util.get_reduced_news(news)
    content = reduced_news + '...'

    if(info['thumb'] == '0'):
        img = ''
    else:
        img = info['thumb']

    payload = {
            "title": info['title'][0],
            "slug": info['slug'][0],
            "body": content,
            "date": str_date,
            "thumb": img,
            "link": info['link'][0],
            "author": str(1),
            "categories": info['categories']
        }

    r = requests.request("POST", URL, data=payload)
    print('POST = ' + str(r))
    print(r.text)


from __future__ import division
import os
import math
#import json
import random
#import pprint
#import scipy.misc
import numpy as np
from time import gmtime, strftime
#from osgeo import gdal
import glob
#from skimage.transform import resize
#from sklearn import preprocessing as pre
#import matplotlib.pyplot as plt
import cv2
import pathlib
#from sklearn.feature_extraction.image import extract_patches_2d
#from skimage.util import view_as_windows
import sys
import pickle
# Local
import deb
import argparse
from sklearn.preprocessing import StandardScaler
from skimage.util import view_as_windows

from PIL import Image
from osgeo import gdal

#Input configuration
parser = argparse.ArgumentParser(description='')
parser.add_argument('-pl','--patch_len', dest='patch_len', type=int, default=5, help='# timesteps used to train')
parser.add_argument('-ps','--patch_step', dest='patch_step', type=int, default=0, help='Debug')
parser.add_argument('--band_n', dest='band_n', type=int, default=6, help='Debug')
parser.add_argument('--path', dest='path', default="../../data/AP1_Para/", help='Data path')
parser.add_argument('--class_n', dest='class_n', type=int, default=2, help='Class number')
parser.add_argument('-bs','--balance_samples_per_class', dest='balance_samples_per_class',type=int,default=None, help="Class number. 'local' or 'remote'")
parser.add_argument('-ttmn','--train_test_mask_name', dest='train_test_mask_name',default="TrainTestMask.tif", help="Class number. 'local' or 'remote'")
parser.add_argument('-psv','--patches_save', dest='patches_save',default=True, help="Patches npy store")

a = parser.parse_args()
np.set_printoptions(suppress=True)

#class Patches(object):
#	def __init__(self,a):
#		self.a=a
#		deb.prints(self.a)
#	def load_image():
#	def train_test_split():
#	def normalize():

#def load_image():
def print_min_max_avg(in_):
	print(np.min(in_),np.max(in_),np.average(in_))

def normalize(im, mask):
	h,w,channels=im.shape
	im_flat=np.reshape(im,(h*w,channels))
	mask_flat=np.reshape(mask,-1)

	deb.prints(im_flat.shape)
	deb.prints(mask_flat.shape)
	train_flat=im_flat[mask_flat==255,:]

	deb.prints(train_flat.shape)

	print_min_max_avg(train_flat)

	scaler=StandardScaler()
	scaler.fit(train_flat)
	train_norm_flat=scaler.transform(train_flat)

	print_min_max_avg(train_norm_flat)

	im_norm_flat=scaler.transform(im_flat)
	im_norm=np.reshape(im_norm_flat,(h,w,channels))
	deb.prints(im_norm.shape)

	print("FINISHED NORMALIZING, RESULT:")
	print_min_max_avg(im_norm)
	return im_norm

def patches_extract(im,patch_len,band_n,step,debug=1):
	deb.prints(band_n)
	if band_n != -1:
		window_shape=(patch_len,patch_len,band_n)
	else:
		window_shape=(patch_len,patch_len)
	if debug:
		deb.prints(window_shape)
		deb.prints(im.shape)
	return np.squeeze(view_as_windows(im, window_shape, step=step))

def im_reconstruct_from_patches(patches,a):
	patches=patches.reshape((a.patches_info['rows'], \
		a.patches_info['cols']) + patches.shape[1:])
	out=np.zeros(a.im_shape[:-1])
	for row in range(a.patches_info['rows']):
		for col in range(a.patches_info['cols']):
			out[row*a.patch_step:(row+1)*a.patch_step, \
				col*a.patch_step:(col+1)*a.patch_step]= \
				patches[row,col]
	return out

if __name__ == '__main__':
	path={}
	path['raster']=a.path+'L8_224-66_ROI_clip.tif'
	path['label']=a.path+'labels.tif'
	path['train_test_mask']=a.path+'TrainTestMask.png'
	# Read image
	im = gdal.Open(path['raster'])
	im = np.array(im.ReadAsArray())
	im = np.transpose(im, (1, 2, 0))
	im = im[0:-1,0:-1,:] # Eliminate non used pixels
	im = im.astype(np.float32)
	deb.prints(im.shape)
	deb.prints(im.dtype)

	a.band_n=im.shape[2]
	deb.prints(a.band_n)
	masks={}
	masks['train_test']=cv2.imread(path['train_test_mask'],0).astype(np.uint8)
	masks['label']=cv2.imread(path['label'],-1).astype(np.uint8)
	masks['label'][masks['label']==2]=1 # Only use 2 classes
	cv2.imwrite("label_original.png",masks['label']*255)
	deb.prints(masks['train_test'].shape)
	deb.prints(masks['label'].shape)

	# ============ Normalize ========================
	im = normalize(im, masks['train_test'])
	deb.prints(im.shape)

	# =========== Extract patches  ===================
	patches={}
	patches['im'] = patches_extract(im,a.patch_len,a.band_n,a.patch_step)
	patches['label'] = patches_extract(masks['label'],a.patch_len,-1,a.patch_step)
	patches['train_test'] = patches_extract(masks['train_test'],a.patch_len,-1,a.patch_step)

	a.im_shape=im.shape
	a.patches_info={}
	a.patches_info['rows']=patches['label'].shape[0]
	a.patches_info['cols']=patches['label'].shape[1]

	deb.prints(patches['label'].shape)
	deb.prints(patches['im'].shape)

	#print()
	patches['im']=patches['im'].reshape((a.patches_info['rows']*a.patches_info['cols'],)+patches['im'].shape[2:])
	patches['label']=patches['label'].reshape((a.patches_info['rows']*a.patches_info['cols'],)+patches['label'].shape[2:])
	patches['train_test']=patches['train_test'].reshape((a.patches_info['rows']*a.patches_info['cols'],)+patches['train_test'].shape[2:])

	# ====== Reconstruct label. Just for assertion here. =====
	label_reconstruct = im_reconstruct_from_patches(patches['label'],a)
	deb.prints(patches['im'].shape)
	deb.prints(label_reconstruct.shape)
	deb.prints(np.unique(label_reconstruct))
	cv2.imwrite("label_reconstruct.png",label_reconstruct*255)

	id_train_test=np.zeros(patches['im'].shape[0])
	# ====== Train test split
	for idx in range(patches['im'].shape[0]):
		if np.all(patches['train_test'][idx]>=1):
			id_train_test[idx]=True
		else:
			id_train_test[idx]=False

	names={"train":{},"test":{}}
	names['train']['im']='train_ims.npy'

	patches_im_name="patches_im.npy"
	patches_label_name="patches_label.npy"
	id_train_test_name="id_train_test.npy"

	np.save(patches_im_name,patches['im'])
	np.save(patches_label_name,patches['label'])
	np.save(id_train_test_name,id_train_test)

	deb.prints(id_train_test.shape)
	deb.prints(patches['im'][id_train_test==True].shape)
	deb.prints(patches['im'][id_train_test==False].shape)

	# ============== Store patches into npy

	# Extract
	# Train test split
	#patches=Patches(a)
	#patches.load_image()
	#patches.train_test_split()
	#patches.normalize()
	#patches.extract()


from datetime import datetime
from models.musics import Musics
from db.crud import CRUDBase

class MusicUpdateService:
    def __init__(self, music_id: dict, data: dict, permission: dict):
        self.data = data
        self.music_id = music_id
        self.permission = permission
        self.music_crud_base = CRUDBase(model=Musics)
        self.status = False

    def start(self) -> None:
        self._run()

    def _prep_data(self) -> None:
        self.data["updated_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    def _run(self) -> None:
        self._prep_data()
        self._run_update_pipeline()

    def _run_update_pipeline(self) -> None:
        role = int(self.permission.get("role", 0))

        if role == 1 or role == 2:
            self._update()
            return

        if role == 3 and self._verify_user():
            self._update()
            return

    def _verify_user(self) -> bool:
        owner = self.permission.get("id", 0)
        music = self.music_crud_base.filter(
            {
                "id": self.music_id,
                "user_id": owner,
            }
        )
        return True if len(music) == 1 else False

    def _update(self) -> None:
        self.music_crud_base.update(id=self.music_id, data=self.data)
        self.status = True if len(self.music_crud_base.errors) == 0 else False


class Node:
    def __init__(self,data):
        self.data=data
        self.ref=None
class CLinkedList:
    def __init__(self):
        self.head=None
    def print_cll(self):
        if self.head is None:
            print('Linked list is empty')
            return
        if self.head.ref==self.head:
            print(self.head.data)
            return
        print(self.head.data)
        n=self.head.ref
        while n is not self.head:

            print(n.data)
            n=n.ref
    def add_begin(self,data):
        new_node=Node(data)
        if self.head is None:
            self.head=new_node
            new_node.ref=self.head
            return
        n=self.head
        new_node.ref=self.head
        while n.ref is not self.head:
            n=n.ref
        n.ref=new_node
        self.head=new_node
        print('addded')
    def add_after(self,x,data):
        if self.head is None:
            print('Linked list is empty what to delete')
            return
        n=self.head
        while n.ref is not self.head:
            if n.data==x:
                break
            n=n.ref
        if n.ref is self.head:
            print('the value is not found in LL')
            return
        new_node=Node(data)
        new_node.ref=n.ref
        n.ref=new_node
    def add_before(self,x,data):
        if self.head is None:
            print('Linked list is empty what to delete')
            return
        n=self.head
        while n.ref is not self.head:
            if n.ref.data==x:
                break
            n=n.ref
        if n.ref is self.head:
            print('the value is not found in LL')
            return
        new_node=Node(data)
        new_node.ref=n.ref
        n.ref=new_node

ll=CLinkedList()
'''
ll.add_begin(2)
ll.add_begin(4)
ll.add_begin(30)
ll.add_after(4,10)'''
ll.add_before(10,30)
ll.print_cll()


#!/usr/bin/python
# Author:       F Massimo, implemented as function by K Cassou
# Date:         2018-02-10, 2020-02-12, 2020-06-21
# Purpose:      set of function for LPA simulation with SMILEI
# Source:       Python 3 (python2)
#####################################################################

### loading system module
from __future__ import (division, print_function, absolute_import, unicode_literals)
import os, sys
import warnings

# for execution on local SERA0X computers unit
host = os.popen('hostname -f').read()[:-1]
host_list = ['sera01.lal.in2p3.fr','sera03.lal.in2p3.fr','sera03.lal.in2p3.fr','sera04.lal.in2p3.fr']
if host in host_list:
    sys.path.append('/silver/PALLAS/simulations/smilei/postprocess/')

# set the current path for happi download path
module_filename = os.path.basename(__file__)
module_filepath = __file__
module_dirpath = module_filepath[:-len(module_filename)]
os.chdir(module_dirpath)

#print(module_dirpath)

# happi import KC : to be modified to enough to install happi, wrong path
try :
    import happi
except ImportError :
    try:
        os.system('svn checkout https://github.com/SmileiPIC/Smilei.git/trunk/happi/')
        print('last version of happi has been downloaded with in:', os.path.abspath(module_dirpath))
    except IOError:
        print('download the happi module from github: https://github.com/SmileiPIC/Smilei.git and copy it in `lpa2` module directory')
    pass

# modules import
import statsmodels.api              # necessary to be located before statsmodels.stats import.
import statsmodels.stats as stats
import numpy as np
import scipy.constants as sc
from scipy.optimize import curve_fit
from scipy.signal import find_peaks, peak_widths
import matplotlib.pyplot as plt

#################### Inputs ##############################################

species_name   = "electronfromion"
lambda0        = 0.8e-6 # Wavelength of the laser

default_directory = os.path.abspath(os.getcwd())
home_directory     = os.path.expanduser("~")

# used to apply a filter in energy (m_e c^2 units, or Lorentz factor)
E_min          = 0.
E_max          = 500

chunk_size     = 100000000  #Chunck of particles treated simultaneously

horiz_axis_conversion_factor = 0.512 # to convert from Smilei units to MeV
hist_conversion_factor       = 1.    # if equal to 1, the charge is in pC

################# Fundamental Physical constants ########################
eps0   = sc.epsilon_0;  # Electric permittivity of vacuum, F/m
mu0    = sc.mu_0;       # Magnetic permittivity of vacuum, kg.m.A-2s-2
e      = sc.e;          # Elementary charge, C
EeV    = sc.eV;         # 1 eV = 1.6e-19 Joules
c      = sc.c;          # Lightspeed, m/s
me     = sc.m_e;        # Electron mass, kg
mp     = sc.m_p;        # Proton mass, kg
h      = sc.h;          # Planck's constant, J.s
hbar   = sc.hbar

########## Physical constants ##########################################
omega0 = 2*np.pi*c/lambda0          #
onel   = lambda0/ (2*np.pi)         #
ncrit  = eps0*me*omega0**2/e**2;    # critical density (m^-3, not cm^-3)

######### useful functions ####################################

def lin_interp(x, y, i, half):
    return x[i] + (x[i+1] - x[i]) * ((half - y[i]) / (y[i+1] - y[i]))

def half_max_x(x, y):
    half = max(y)/2.0
    signs = np.sign(np.add(y, -half))
    zero_crossings = (signs[0:-2] != signs[1:-1])
    zero_crossings_i = np.where(zero_crossings)[0]
    return [lin_interp(x, y, zero_crossings_i[0], half),lin_interp(x, y, zero_crossings_i[1], half)]

def fwhm(x,y):
    hmx = half_max_x(x,y)
    return hmx[1]-hmx[0]

def gaussian(x, amp, xcenter, width):
    return amp * np.exp(-(x-xcenter)**2 / width**2)

def weighted_median(data, weights):
    """
    Args:
      data (list or numpy.array): data
      weights (list or numpy.array): weights
    """
    data, weights = np.array(data).squeeze(), np.array(weights).squeeze()
    s_data, s_weights = map(np.array, zip(*sorted(zip(data, weights))))
    midpoint = 0.5 * sum(s_weights)
    if any(weights > midpoint):
        w_median = (data[weights == np.max(weights)])[0]
    else:
        cs_weights = np.cumsum(s_weights)
        idx = np.where(cs_weights <= midpoint)[0][-1]
        if cs_weights[idx] == midpoint:
            w_median = np.mean(s_data[idx:idx+2])
        else:
            w_median = s_data[idx+1]
    return w_median

def mad(data, axis=None):
    """
    Compute *Median Absolute Deviation* of an array along given axis.
    """
    # Median along given axis, but *keeping* the reduced axis so that
    # result can still broadcast against a.
    med = np.median(data, axis=axis, keepdims=True)
    mad = np.median(np.absolute(data - med), axis=axis)  # MAD along given axis

    return mad

def weighted_std(data, weights):
    """
    Compute *weighted_standard Deviation* of an array along given axis.
    """
    d = stats.weightstats.DescrStatsW(data,weights)

    return d.std

def weighted_mean(data, weights):
    """
    Compute *weighted_mean* of an array along given axis.
    """
    d = stats.weightstats.DescrStatsW(data,weights)

    return d.mean

def weighted_mad(data, weights):
    """
    Compute *weighted_Median Absolute Deviation* of an array along given axis.
    """
    wmed = weighted_median(data,weights)
    wmad = weighted_median(np.absolute(data - wmed),weights)  # MAD along given axis

    return wmad

########## load data with happi ##############################

def loadData(directory=default_directory):
    """loading data in the simulation directory and return an object pointing to the various
    files, see smilei website"""
    S = happi.Open(directory, show = False,verbose = False)
    return S

######### extract laser var ##############################

def getMaxinMovingWindow(S,var="Env_E_abs"):
    """ return the max of var on axis (r=0) for all timestep available
    S : is the simulation output object return by happi.Open()
    var : check namelist ["Env_E_abs]
    return a numpy array - var.max() and the timestep vector [0:iteration_max]
    """
    # read all timestep Available
    ts = S.Probe(0,var).getTimesteps()
    data = np.max(S.Prove(0,var).getData(),axis=1)
    varmax = np.stack((ts,data),axis=0)
    return varmax

def getLasera0(S,timeStep,var='Env_E_abs'):
    """ return the max of var on axis (r=0) for the timestep
    S : is the simulation output object return by happi.Open()
    timeStep : timestep smilei unit
    var : check namelist ["Env_E_abs]
    return a numpy array - var.max() and the timestep vector [0:iteration_max]
    """
    return np.max(S.Probe(0,var,timeStep).getData()[0])

def getLaserWaist(S,timeStep,var='Env_E_abs'):
    """ return the laser waist of Env or field `var` at the iteration
    S : is the simulation output object return by happi.Open()
    timestep : simulation timestep
    var : check namelist ["Env_E_abs" or laser field] to be updated for AM geometry
    return the waist evaluated with Gaussian fit in code units (lamda_0/2pi)
    """
    temp = S.Probe(1,var,timeStep).getData()[0]
    x_max,y_max = np.unravel_index(np.argmax(temp),temp.shape)
    init_vals = [np.max(temp),y_max, 1.0]
    a_val = temp[x_max,:]
    y_val = np.arange(0,temp.shape[1],1)
    # gaussian fit
    best_vals, covar = curve_fit(gaussian, y_val, a_val, p0=init_vals)
    return best_vals[2]

def getLaserPulselength(S,timeStep,var='Env_E_abs'):
    """ return the laser pulse length of Env or field `var` at the iteration
    S : is the simulation output object return by happi.Open()
    timestep : simulation timestep
    var : check namelist ["Env_E_abs" or laser field] to be updated for AM geometry
    return the pulse length FWHM evaluated with Gaussian fit in code units (lamda_0/2pi)
    """
    temp = S.Probe(1,var,timeStep).getData()[0]
    x_max,y_max = np.unravel_index(np.argmax(temp),temp.shape)
    init_vals = [np.max(temp),x_max, 1.0]
    a_val = temp[:,y_max]
    x_val = np.arange(0,temp.shape[0],1)
    # gaussian fit slightly underestimate FWHM value
    best_vals, _ = curve_fit(gaussian, x_val, a_val, p0=init_vals)
    return best_vals[2]*2*np.sqrt(2*np.log(2))

######### extract plasma profile ############################

def plasmaProfile(S):
    """ return the electon plasma density  profile
    S : is the simulation output object return by happi.Open()
    return the numpy array - plasProfile (x,ne) e-/m^3
    """
    nc = S.namelist.ncrit
    ne = np.array(S.namelist.xh_values)*nc
    plasProfile = np.array((S.namelist.xh_points,ne))
    return plasProfile

def dopantProfile(S):
    """ return the electon dopan density profile
    S : is the simulation output object return by happi.Open()
    return the numpy array - plasProfile (x,nN2) N2/m^3
    """
    nc = S.namelist.ncrit
    nd = np.array(S.namelist.xd_values)*nc
    dopProfile = np.array((S.namelist.xd_points,nd))
    return dopProfile

######### extract beam parameter for one iteration ###########

def getBeamParam(S,iteration,species_name="electronfromion",sort = False, E_min=50,E_max=520,chunk_size=100000000,print_flag=True,save_flag=False):
    """return beams paramater for the species_name of the Smilei simulation data
    iteration : timestep
    S : is the simulation output object return by happi.Open()
    species_name :  [electronfromion], electron
    E_min :         [0] energy filter min
    E_max :         [400] energy filter max
    printflag :     [True] print output on screen.
    saveflag :      [False] True to save the data in an csv file
     """
    ########## Read data from Track Particles Diag ############
    track_part = S.TrackParticles(species = species_name, sort = sort, chunksize=chunk_size)
    #print("Available timesteps = ",track_part.getAvailableTimesteps())
    dt_adim    = S.namelist.dt
    for particle_chunk in track_part.iterParticles(iteration, chunksize=chunk_size):
        # Read data
        #if print_flag==True:
        #    print(particle_chunk.keys())
        px           = particle_chunk["px"]
        py           = particle_chunk["py"]
        pz           = particle_chunk["pz"]
        x            = particle_chunk["x"]
        y            = particle_chunk["y"]
        z            = particle_chunk["z"]
        w            = particle_chunk["w"]
        p            = np.sqrt((px**2+py**2+pz**2))                # momentum
        E            = np.sqrt((1.+p**2))
        Nparticles   = np.size(w)
        if print_flag == True:                                  # Number of particles read
            print("Read ",Nparticles," particles from the file")
        total_weight = w.sum()
        Q            = total_weight* e * ncrit * onel**3 * 10**(12) # Total charge in pC
        if print_flag == True:
            print("Total charge before filter in energy= ",Q," pC")
        # Apply a filter on energy
        filter       = np.intersect1d( np.where( E > E_min )[0] ,  np.where( E < E_max )[0] )
        x            = x[filter]
        y            = y[filter]
        z            = z[filter]
        px           = px[filter]
        py           = py[filter]
        pz           = pz[filter]
        E            = E[filter]
        w            = w[filter]
        p            = p[filter]
        total_weight = w.sum()
        Q            = total_weight* e * ncrit * onel**3 * 10**(12) # Total charge in pC
        if print_flag == True:
            print("Total charge after filter in Energy = ",Q," pC")
            print("Filter energy limits: ",E_min,", ",E_max," (m_e c^2)")
        if total_weight > 0:
            #Compute mean values
            x_moy    = (x    *w).sum() / total_weight
            y_moy    = (y    *w).sum() / total_weight
            z_moy    = (z    *w).sum() / total_weight
            #px_moy   = (px   *w).sum() / total_weight
            py_moy   = (py   *w).sum() / total_weight
            pz_moy   = (pz   *w).sum() / total_weight
            # p_moy    = (p    *w).sum() / total_weight
            #Place center of mass at the center of the coordinates
            x  -= x_moy
            y  -= y_moy
            z  -= z_moy
            #px -= px_moy
            py -= py_moy
            pz -= pz_moy
            # Compute properties of the bunch
            x2_moy   = (x**2 *w).sum() / total_weight
            y2_moy   = (y**2 *w).sum() / total_weight
            z2_moy   = (z**2 *w).sum() / total_weight
            #px2_moy  = (px**2*w).sum() / total_weight
            py2_moy  = (py**2*w).sum() / total_weight
            pz2_moy  = (pz**2*w).sum() / total_weight
            ypy_moy  = (y*py *w).sum() / total_weight
            zpz_moy  = (z*pz *w).sum() / total_weight
            py2ovpx2 = (py**2/px**2*w).sum()/total_weight #divergence y squared
            pz2ovpx2 = (pz**2/px**2*w).sum()/total_weight

            # normalized rms emittances based on Floettmann K. "Some basic features of the beam emittance" PRSTA, vol. 6, 3 (2003)
            # https://link.aps.org/doi/10.1103/PhysRevSTAB.6.034202

            emittancey = ( py2_moy*y2_moy - ypy_moy**2 )
            emittancez = ( pz2_moy*z2_moy - zpz_moy**2 )
            if emittancey > 0:
                emittancey = np.sqrt(emittancey) * onel * 1e6 # [um]
            else:
                emittancey = 0.
            if emittancez > 0:
                emittancez = np.sqrt(emittancez) * onel * 1e6 # [um]
            else:
                emittancez = 0.

            rmssize_longitudinal = 2*np.sqrt(x2_moy) * onel * 1e6 # [micron]
            rmssize_y =            2*np.sqrt(y2_moy) * onel * 1e6 # [micron]
            rmssize_z =            2*np.sqrt(z2_moy) * onel * 1e6 # [micron]
            divergence_rms = np.sqrt( py2ovpx2 + pz2ovpx2 )

            # Statistic on energy distribution of particules
            E_mean = weighted_mean(E,w)*0.512
            E_med = weighted_median(E,w)*0.512
            dE_wrms = weighted_std(E,w)/weighted_mean(E,w)*100
            dE_rms = np.std(E)/weighted_mean(E,w)*100
            dE_mad = weighted_mad(E,w)/weighted_median(E,w)*100

            # print beam parameter
            if print_flag == True:
                print("")
                print("--------------------------------------------")
                print("")
                print(" Read \t\t\t\t\t\t", np.size(E)," particles")
                print( "[0] Iteration = \t\t\t\t", iteration)
                print( "[1] Simulation time = \t\t\t\t\t", iteration*dt_adim*onel/c*1e15," fs")
                print( "[2] E_mean = \t\t\t\t\t", E_mean," MeV")
                print( "[3] E_med = \t\t\t\t\t", E_med, "MeV")
                print( "[4] DeltaE_rms / E_mean = \t\t\t", dE_rms , " %.")
                print( "[5] E_mad /E_med  = \t\t\t\t", dE_mad, " %.")
                print( "[6] Total charge = \t\t\t\t", Q, " pC.")
                print( "[7] Emittance_y = \t\t\t\t", emittancey," mm-mrad")
                print( "[8] Emittance_z = \t\t\t\t", emittancez," mm-mrad")
                print( "[9] size_x = \t\t\t\t", rmssize_longitudinal,"um (RMS)")
                print( "[10] divergence_rms = \t\t\t\t", divergence_rms*1e3,"mrad")
                print( "")
                print( "--------------------------------------------")
                print( "")

            # beam paramater list for iteration timestep

            if Q > 0.0 :
                # [0] timestep
                # [1] time [fs]
                # [2] weighted mean energy   [MeV]
                # [3] weighted median value   [MeV]
                # [4] weighted RMS energy spread   [%]
                # [5] RMS energy spread   [%]
                # [6] MAD energy spread [%]
                # [7] charge [pC]
                # [8] normalized RMS y-emittance [um]
                # [9] normalized RMS z-emittance [um]
                # [10] bunch RMS length [um]
                # [11] bunch RMS sigy [um]
                # [12] bunch RMS sigz [um]
                # [13] RMS divergence [mrad]
                beamparam_dict = {"iteration":iteration,
                "time": iteration*dt_adim*onel/c*1e15,
                "energy_wmean": E_mean,
                "energy_wmedian": E_med,
                "energy_wrms": dE_wrms,
                "energy_rms": dE_rms,
                "energy_wmad": dE_mad,
                "charge": Q,
                "emittance_y": emittancey,
                "emittance_z": emittancez,
                "size_x_rms": rmssize_longitudinal,
                "size_y_rms" : rmssize_y,
                "size_z_rms" : rmssize_z,
                "divergence_rms": divergence_rms*1e3}
            else:
                print('no data in the filtered energy range')
                beamparam_dict = {"iteration":iteration,
                "time": iteration*dt_adim*onel/c*1e15,
                "energy_wmean": np.nan,
                "energy_wmedian": np.nan,
                "energy_wrms": np.nan,
                "energy_rms": np.nan,
                "energy_wmad": np.nan,
                "charge": np.nan,
                "emittance_y": np.nan,
                "emittance_z": np.nan,
                "size_x_rms": np.nan,
                "size_y_rms" : np.nan,
                "size_z_rms" : np.nan,
                "divergence_rms": np.nan}

            # save beam parameter in a file
            if save_flag == True:
                print( "data saved in npy file")
                filename = 'smilei-beamparam'+str(iteration)+'.npy'
                filepath = home_directory+'/'+filename
                np.save(filepath,beamparam_dict)
            return beamparam_dict

def getPartAvailableSteps(S,species_name="electronfromion",sort = False, chunk_size=10000000):
    """return available timesteps for the trackParticles"""
    return S.TrackParticles(species = species_name, sort = False, chunksize=chunk_size).getAvailableTimesteps()

def getBeamCharge(S,iteration,species_name="electronfromion",sort = False, E_min=10,E_max=520,chunk_size=10000000,print_flag=True):
    """return beam charge for the species_name of the Smilei simulation data at the timestep iteration
    iteration : timestep
    S : is the simulation output object return by happi.Open()
    species_name :  [electronfromion], electron
    E_min :         [10] energy filter min
    E_max :         [520] energy filter max
    printflag :     [True] print output on screen.
    Q : charge []
     """
    ########## Read data from Track Particles Diag ############
    track_part = S.TrackParticles(species = species_name, sort = sort, chunksize=chunk_size)
    test_part =  S.TrackParticles(species = species_name, timesteps=iteration, sort = sort, chunksize=chunk_size).getData()
    #print("Available timesteps = ",track_part.getAvailableTimesteps())
    px = 0.
    py = 0.
    pz = 0.
    w  = 0.
    if iteration is None:
        Q = 0.
        if print_flag == True:
            print("Iteration or timeStep is None type, return Q=", Q)
    elif test_part[int(iteration)]['w'].sum() < 0.1:
        Q = 0.
        if print_flag == True:
            print("no enough particles return Q=", Q)
    else:
        for particle_chunk in track_part.iterParticles(iteration, chunksize=chunk_size):
            # Read data
            px           += particle_chunk["px"]
            py           += particle_chunk["py"]
            pz           += particle_chunk["pz"]
            w            += particle_chunk["w"]
        p            = np.sqrt(px**2+py**2+pz**2)
        E            = np.sqrt(1.+p**2)
        Nparticles   = np.size(w)
        if Nparticles < 1.:
            Q= 0.
        if print_flag == True:                                  # Number of particles read
            print("Read ",Nparticles," particles from the file")
        total_weight = w.sum()
        Q            = total_weight* e * ncrit * onel**3 * 10**(12) # Total charge in pC
        if print_flag == True:
            print("Total charge before filter in energy= ",Q," pC")
            print("Filter energy limits: ",E_min,", ",E_max," (m_e c^2)")
        # Apply a filter on energy
        filter       = np.intersect1d( np.where( E > E_min )[0] ,  np.where( E < E_max )[0] )
        w            = w[filter]
        total_weight = w.sum()
        Q            = total_weight* e * ncrit * onel**3 * 10**(12) # Total charge in pC
        if print_flag==True:
            print("Total charge after filter in energy= ",Q," pC")
    return Q

def getInjectionTime(S,ts,probeVar='Rho_electronfromion',threshold = 5e-3,print_flag = False):
    """ return the injection timestep and longitudinal coordinate of the injection.
    The injection is defined by a threshold on the `electron_from_ion` density
    S : is the simulation output object return by happi.Open()
    ts : timestep vector [numpy array]
    threshold : value of e- from ionisation max density on axis -n_ei/ncrit  [smilei units]
    t : index of ts at which injection occcurs
    ti : injection timestep
    xi : injection longitudinal position [m]
    """
    dls = S.namelist.lambda_0/(2*np.pi)
    for t in range(len(ts)):
        rhoei = S.Probe(0,probeVar,ts[t]).getData()[0]
        if np.abs(rhoei.min())> threshold:
            ti = ts[t]
            xi = ts[t]*dls
            if print_flag == True :
                print('index:', t)
                print('injection time:',ti,'timestep')
                print('injection x:',xi,'mm')
            break
        else :
            ti = None
            xi = None
    return t,ti,xi

def getSpectrum(S,iteration_to_plot,species_name= "electronfromion",horiz_axis_name= "E", E_min=50, E_max = 640,plot_flag = False, print_flag = False, nbins_horiz = 200, normalized = False):
    """ return spectrum plot or data for a given timesteps
    S : smilei output data
    iteration_to_plot : timestep
    species_name : [electronfromion], electron
    horiz_axis_name : [E] can be px, p or E
    E_min : [50] min value considered in histogram for the horiz axis, in code units
    E_max : [640] max value considered in histogram for the horiz axis, in code units
    option : plot_flag, print_flag,
    nbins_horiz : binning energy histogram [200]
    normalized : normalization of the histogram [False]
    return spectrum data as numpy arrays  (horizontal axis (E, or p)), dQd(E,or p), Epeak, dQdE_max, Ewidth
    """

    #  horizontal axis limits (m_e c^2 units, or Lorentz factor)
    horiz_axis_min = E_min  # Max value considered in histogram for the horiz axis, in code units
    horiz_axis_max = E_max   # Min value considered in histogram for the horiz axis, in code units
    horiz_axis_conversion_factor = 0.512 # to convert from Smilei units to MeV
    hist_conversion_factor       = 1.    # if equal to 1, the charge is in pC
    energy_axis = np.zeros((1,nbins_horiz)) # initialization to avoid unbondedlocalerror.
    specData = np.zeros((1,nbins_horiz))
    Ewidth = 0.0
    Epeak = 0.0
    dQdE_max = 0.0

    ########## Read data from Track Particles Diag #####################
    sort = False
    chunk_size=100000000
    track_part = S.TrackParticles(species = species_name, sort = sort, chunksize=chunk_size)
    #print("Available timesteps = ",track_part.getAvailableTimesteps())

    for particle_chunk in track_part.iterParticles(iteration_to_plot, chunksize=chunk_size):
        # Read data
        #if print_flag==True:
        #    print(particle_chunk.keys())
        px           = particle_chunk["px"]
        py           = particle_chunk["py"]
        pz           = particle_chunk["pz"]
        x            = particle_chunk["x"]
        y            = particle_chunk["y"]
        z            = particle_chunk["z"]
        w            = particle_chunk["w"]
        p            = np.sqrt((px**2+py**2+pz**2))                # momentum
        E            = np.sqrt((1.+p**2))
        Nparticles   = np.size(w)                                    # Number of particles read
        if print_flag == True:
            print("Read ",Nparticles," particles from the file")
        total_weight = w.sum()
        Q            = total_weight* e * ncrit * onel**3 * 10**(12) # Total charge in pC
        if print_flag == True:
            print("Total charge before filter in energy= ",Q," pC")
        # Apply a filter on energy
        filter       = np.intersect1d( np.where( E > E_min )[0] ,  np.where( E < E_max )[0] )
        x            = x[filter]
        y            = y[filter]
        z            = z[filter]
        px           = px[filter]
        py           = py[filter]
        pz           = pz[filter]
        E            = E[filter]
        w            = w[filter]
        p            = p[filter]
        total_weight = w.sum()
        Q            = total_weight* e * ncrit * onel**3 * 10**(12) # Total charge in pC
        if print_flag == True:
            print("Total charge after filter in Energy = ",Q," pC")
            print("Filter energy limits: ",E_min,", ",E_max," (m_e c^2)")

        if Q > 0.0 :

            # Compute 1D histogram
            possible_axes_names =["x","y","z","px","py","pz","E"]
            axes                =[x,y,z,px,py,pz,E]

            if horiz_axis_name in possible_axes_names:
                horiz_axis = axes[possible_axes_names.index(horiz_axis_name)]
            else:
                print("Error, invalid axis")
                exit(0)

            hist1D, horiz_edges = np.histogram(horiz_axis, \
                                        bins=nbins_horiz, \
                                        range=[horiz_axis_min,horiz_axis_max], weights=w)
            #print(np.shape(horiz_edges))
            dhoriz_axis                    = abs(horiz_edges[1]-horiz_edges[0]) # bin size

            # histogram: integrated in dhoriz_axis and gives the total charge
            histogram_spectrum = hist1D*hist_conversion_factor/dhoriz_axis/horiz_axis_conversion_factor*e * ncrit * onel**3  * 10**(12)
            if normalized == True:
                histogram_spectrum = histogram_spectrum / histogram_spectrum[:].max()

            # horizontal axis
            horiz_edges = horiz_edges[0:-1]
            binx = dhoriz_axis*horiz_axis_conversion_factor
            horiz_edges = horiz_edges + 0.5*binx
            energy_axis = horiz_edges*horiz_axis_conversion_factor

            # Preparation for Plot
            if normalized == True:
                plot_title   = "Normalized histogram"
            else:
                plot_title   = 'dQ/d'+horiz_axis_name+" (pC/MeV)"

            #print np.shape(histogram_spectrum)
            #
            if print_flag == True:
                print('Bins size: dx = ',binx)

            histogram_spectrum[histogram_spectrum==0.]=float(np.nan)

            #if print_flag==True:
            #    print(len(energy_axis))

            specData = np.array((histogram_spectrum))

            # Plot
            if plot_flag == True:
                fig = plt.figure()
                fig.set_facecolor('w')

                plt.xlabel(horiz_axis_name+" (MeV)")
                plt.title(plot_title)

                #extnt = np.array([horiz_axis.min()*horiz_axis_conversion_factor, \
                #            horiz_axis.max()*horiz_axis_conversion_factor ])
                #print("Values extension for ",horiz_axis_name," (all particles):")
                #print(extnt)

                #extnt = np.array([horiz_axis_min*horiz_axis_conversion_factor, \
                #            horiz_axis_max*horiz_axis_conversion_factor])
                #print( "Values extension for ",horiz_axis_name," (particles included in the chosen horiz axis limits):")
                #print(extnt)

                plt.plot(energy_axis,histogram_spectrum)
                plt.xlim([horiz_axis_min*horiz_axis_conversion_factor,horiz_axis_max*horiz_axis_conversion_factor])

                #plt.savefig(home_directory+"/E_Spectrum.png",format='png')
                plt.show()

            # compute the full width half maximum using scipy.signal.findpeaks
            try :
                with warnings.catch_warnings():
                    warnings.filterwarnings('ignore', r'All-NaN slice encountered')
                    prom = (np.nanmax(specData)-np.nanmin(specData))*0.66 #factor might be adjusted
                    p , _  = find_peaks(specData,prominence=prom)
                if len(p)==0 :
                    Epeak = 0
                    Ewidth = 0
                    dQdE_max = 0
                else :
                    Epeak = energy_axis[p[0]]
                    dQdE_max = specData[p[0]]
                    Ewidth = binx*peak_widths(specData, p, rel_height=0.5)[0][0]
            except ValueError :
                Epeak = np.nan
                Ewidth = np.nan
                dQdE_max = np.nan

            if print_flag == True:
                print( "")
                print( "--------------------------------------------")
                print( "")
                print("beam Peak energy: \t",Epeak,"MeV")
                print("beam FWHM energy: \t",Ewidth,"MeV")
                print( "")
                print( "--------------------------------------------")
                print( "")
        # no charge in the energy range  Q = 0.
        else :
            energy_axis = np.nan*np.zeros((nbins_horiz))
            specData = np.nan*np.zeros((nbins_horiz))
            Epeak  = np.nan
            dQdE_max = np.nan
            Ewidth = np.nan

    return energy_axis, specData, Epeak, dQdE_max, Ewidth

def getPartParam(S,iteration,species_name="electronfromion",sort= False,chunk_size=100000000,E_min=25, E_max = 520,print_flag = True):
    """return x,y,z,px,py,pz,E,w,p for all particle at timesteps iteration within the filter"""
    track_part = S.TrackParticles(species = species_name,sort = sort,  chunksize=chunk_size)
    #print("Available timesteps = ",track_part.getAvailableTimesteps())

    for particle_chunk in track_part.iterParticles(iteration, chunksize=chunk_size):
        # Read data
        #if print_flag==True:
        #	print(particle_chunk.keys())
        px           = particle_chunk["px"]
        py           = particle_chunk["py"]
        pz           = particle_chunk["pz"]
        x            = particle_chunk["x"]
        y            = particle_chunk["y"]
        z            = particle_chunk["z"]
        w            = particle_chunk["w"]
        p            = np.sqrt((px**2+py**2+pz**2))                # momentum
        E            = np.sqrt((1.+p**2))
        Nparticles   = np.size(w)
        if print_flag==True:                                  # Number of particles read
            print("Read ",Nparticles," particles from the file")
        total_weight = w.sum()
        Q            = total_weight* e * ncrit * onel**3 * 10**(12) # Total charge in pC
        if print_flag==True:
            print("Total charge before filter in energy= ",Q," pC")
            print("Filter energy limits: ",E_min,", ",E_max," (m_e c^2)")
        # Apply a filter on energy
        filter       = np.intersect1d( np.where( E > E_min )[0] ,  np.where( E < E_max )[0] )
        x            = x[filter]
        y            = y[filter]
        z            = z[filter]
        px           = px[filter]
        py           = py[filter]
        pz           = pz[filter]
        E            = E[filter]
        w            = w[filter]
        p            = p[filter]
        total_weight = w.sum()
        Q            = total_weight* e * ncrit * onel**3 * 10**(12) # Total charge in pC
        if print_flag==True:
            print("Total charge after filter in energy= ",Q," pC")
    return np.array([x,y,z,px,py,pz,E,w,p])

def getPSxrms(S,iteration,species_name="electronfromion",sort= False,chunk_size=100000000,E_min=25, E_max = 520,print_flag = True):
    """return x,px for all particle at timesteps iteration within the filter"""
    track_part = S.TrackParticles(species = species_name,sort = sort,  chunksize=chunk_size)
    #print("Available timesteps = ",track_part.getAvailableTimesteps())

    for particle_chunk in track_part.iterParticles(iteration, chunksize=chunk_size):
        # Read data
        #if print_flag==True:
        #	print(particle_chunk.keys())
        px           = particle_chunk["px"]
        py           = particle_chunk["py"]
        pz           = particle_chunk["pz"]
        x            = particle_chunk["x"]
        w            = particle_chunk["w"]
        p            = np.sqrt((px**2+py**2+pz**2))                # momentum
        E            = np.sqrt((1.+p**2))
        Nparticles   = np.size(w)
        if print_flag==True:                                  # Number of particles read
            print("Read ",Nparticles," particles from the file")

        # Apply a filter on energy
        filter       = np.intersect1d( np.where( E > E_min )[0] ,  np.where( E < E_max )[0] )
        x            = x[filter]
        px           = px[filter]
        w            = w[filter]
        Nparticles   = np.size(w)
        if print_flag==True:                                  # Number of particles read
            print("After filtering",Nparticles," particles")

    return np.array([x,px,w])


from lib import Chamber, SHAPES

# --- Constants ---
INPUT_FILE = "../input/input.txt"
LIMIT_1 = 2022
LIMIT_2 = 1000000000000

# --- Runs the simulations ---
def sim(flows, limit):
    count = 0
    rocks = 1
    gcd = len(SHAPES) * len(flows)
    chamber = Chamber()
    chamber.add_shape(SHAPES[0])

    # --- Variables for tracking a pattern ---
    prev_height = 0
    prev_rocks = 0
    delta_height = 0
    delta_rocks = 0
    simulated = 0

    # --- Iterate until the limit is reached ---
    while True:
        if count > 0 and count % gcd == 0:
            height = chamber.height()
            dh = height - prev_height
            dr = rocks - prev_rocks

            # --- Pattern found ---
            if dh == delta_height and dr == delta_rocks:
                rate = (limit - rocks) // dr
                simulated = dh * rate + 1
                rocks = limit - ((limit - rocks) % dr)
            prev_height = height
            prev_rocks = rocks
            delta_height = dh
            delta_rocks = dr

        # --- Normal simulation ---
        chamber.move(flows[count % len(flows)])
        if chamber.move_down() == False:
            if rocks > limit:
                return chamber.height() + simulated
            chamber.add_shape(SHAPES[rocks % len(SHAPES)])
            rocks += 1
        count += 1

# --- Read and parse the input file ---
with open(INPUT_FILE) as file:
    flows = file.read().strip()

height = sim(flows, LIMIT_1)
print(f"1. The tower is {height:,d} units tall")

height = sim(flows, LIMIT_2)
print(f"2. The tower is {height:,d} units tall")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals, print_function

import ast
import inspect
import io
import os
import re
import sys
import textwrap
import token
from typing import List
import unittest
from time import time

import astroid
import six
from asttokens import util, ASTTokens

from . import tools

try:
  from astroid.nodes.utils import Position as AstroidPosition
except Exception:
  AstroidPosition = ()

class TestMarkTokens(unittest.TestCase):
  maxDiff = None

  # We use the same test cases to test both nodes produced by the built-in `ast` module, and by
  # the `astroid` library. The latter derives TestAstroid class from TestMarkTokens. For checks
  # that differ between them, .is_astroid_test allows to distinguish.
  is_astroid_test = False
  module = ast

  def create_mark_checker(self, source, verify=True):
    atok = self.create_asttokens(source)
    checker = tools.MarkChecker(atok)

    # The last token should always be an ENDMARKER
    # None of the nodes should contain that token
    assert atok.tokens[-1].type == token.ENDMARKER
    if atok.text:  # except for empty files
      for node in checker.all_nodes:
        assert node.last_token.type != token.ENDMARKER

    if verify:
      checker.verify_all_nodes(self)
    return checker

  @staticmethod
  def create_asttokens(source):
    return ASTTokens(source, parse=True)

  def print_timing(self):
    # Print the timing of mark_tokens(). This doesn't normally run as a unittest, but if you'd like
    # to see timings, e.g. while optimizing the implementation, run this to see them:
    #
    #     nosetests -m print_timing -s tests.test_mark_tokens tests.test_astroid
    #
    # pylint: disable=no-self-use
    import timeit
    print("mark_tokens", sorted(timeit.repeat(
      setup=textwrap.dedent(
        '''
        import ast, asttokens
        source = "foo(bar(1 + 2), 'hello' + ', ' + 'world')"
        atok = asttokens.ASTTokens(source)
        tree = ast.parse(source)
        '''),
      stmt='atok.mark_tokens(tree)',
      repeat=3,
      number=1000)))

  def test_mark_tokens_simple(self):
    source = tools.read_fixture('astroid', 'module.py')
    m = self.create_mark_checker(source)

    # Line 14 is: [indent 4] MY_DICT[key] = val
    self.assertEqual(m.view_nodes_at(14, 4), {
      "Name:MY_DICT",
      "Subscript:MY_DICT[key]",
      "Assign:MY_DICT[key] = val"
    })

    # Line 35 is: [indent 12] raise XXXError()
    self.assertEqual(m.view_nodes_at(35, 12), {'Raise:raise XXXError()'})
    self.assertEqual(m.view_nodes_at(35, 18), {'Call:XXXError()', 'Name:XXXError'})

    # Line 53 is: [indent 12] autre = [a for (a, b) in MY_DICT if b]
    self.assertEqual(m.view_nodes_at(53, 20), {'ListComp:[a for (a, b) in MY_DICT if b]'})
    self.assertEqual(m.view_nodes_at(53, 21), {'Name:a'})
    if self.is_astroid_test:
      self.assertEqual(m.view_nodes_at(53, 23), {'Comprehension:for (a, b) in MY_DICT if b'})
    else:
      self.assertEqual(m.view_nodes_at(53, 23), {'comprehension:for (a, b) in MY_DICT if b'})

    # Line 59 is: [indent 12] global_access(local, val=autre)
    self.assertEqual(m.view_node_types_at(59, 12), {'Name', 'Call', 'Expr'})
    self.assertEqual(m.view_nodes_at(59, 26), {'Name:local'})
    if self.is_astroid_test:
      self.assertEqual(m.view_nodes_at(59, 33), {'Keyword:val=autre'})
    else:
      self.assertEqual(m.view_nodes_at(59, 33), {'keyword:val=autre'})
    self.assertEqual(m.view_nodes_at(59, 37), {'Name:autre'})

  def test_mark_tokens_multiline(self):
    source = (
"""(    # line1
a,      # line2
b +     # line3
  c +   # line4
  d     # line5
)""")
    m = self.create_mark_checker(source)

    self.assertIn('Name:a', m.view_nodes_at(2, 0))
    self.assertEqual(m.view_nodes_at(3, 0),  {
      'Name:b',
      'BinOp:b +     # line3\n  c',
      'BinOp:b +     # line3\n  c +   # line4\n  d',
    })

    all_text = {m.atok.get_text(node) for node in m.all_nodes}
    self.assertEqual(all_text, {
      source,
      'a', 'b', 'c', 'd',
      # All other expressions preserve newlines and comments but are parenthesized.
      'b +     # line3\n  c',
      'b +     # line3\n  c +   # line4\n  d',
    })
    self.assertIn('Tuple:' + source, m.view_nodes_at(1, 0))

  def verify_fixture_file(self, path):
    source = tools.read_fixture(path)
    m = self.create_mark_checker(source, verify=False)
    tested_nodes = m.verify_all_nodes(self)

    exp_index = (0 if six.PY2 else 1) + (3 if self.is_astroid_test else 0)
    # For ast on Python 3.9, slices are expressions, we handle them and test them.
    if not self.is_astroid_test and issubclass(ast.Slice, ast.expr):
      exp_index += 1
    exp_tested_nodes = self.expect_tested_nodes[path][exp_index]
    self.assertEqual(tested_nodes, exp_tested_nodes)

  # There is not too much need to verify these counts. The main reason is: if we find that some
  # change reduces the count by a lot, it's a red flag that the test is now covering fewer nodes.
  expect_tested_nodes = {
    #                                   AST                  | Astroid
    #                                   Py2   Py3  Py3+slice | Py2   Py3
    'astroid/__init__.py':            ( 4,    4,   4,          4,    4,   ),
    'astroid/absimport.py':           ( 4,    3,   3,          4,    3,   ),
    'astroid/all.py':                 ( 21,   23,  23,         21,   23,  ),
    'astroid/clientmodule_test.py':   ( 75,   67,  67,         69,   69,  ),
    'astroid/descriptor_crash.py':    ( 30,   28,  28,         30,   30,  ),
    'astroid/email.py':               ( 3,    3,   3,          1,    1,   ),
    'astroid/format.py':              ( 64,   61,  61,         62,   62,  ),
    'astroid/module.py':              ( 185,  174, 174,        171,  171, ),
    'astroid/module2.py':             ( 248,  253, 255,        240,  253, ),
    'astroid/noendingnewline.py':     ( 57,   59,  59,         57,   63,  ),
    'astroid/notall.py':              ( 15,   17,  17,         15,   17,  ),
    'astroid/recursion.py':           ( 6,    6,   6,          4,    4,   ),
    'astroid/suppliermodule_test.py': ( 20,   17,  17,         18,   18,  ),
  }

  # This set of methods runs verifications for the variety of syntax constructs used in the
  # fixture test files.
  # pylint: disable=multiple-statements
  def test_fixture1(self): self.verify_fixture_file('astroid/__init__.py')
  def test_fixture2(self): self.verify_fixture_file('astroid/absimport.py')
  def test_fixture3(self): self.verify_fixture_file('astroid/all.py')
  def test_fixture4(self): self.verify_fixture_file('astroid/clientmodule_test.py')
  def test_fixture5(self): self.verify_fixture_file('astroid/descriptor_crash.py')
  def test_fixture6(self): self.verify_fixture_file('astroid/email.py')
  def test_fixture7(self): self.verify_fixture_file('astroid/format.py')
  def test_fixture8(self): self.verify_fixture_file('astroid/module.py')
  def test_fixture9(self): self.verify_fixture_file('astroid/module2.py')
  def test_fixture10(self): self.verify_fixture_file('astroid/noendingnewline.py')
  def test_fixture11(self): self.verify_fixture_file('astroid/notall.py')
  def test_fixture12(self): self.verify_fixture_file('astroid/recursion.py')
  def test_fixture13(self): self.verify_fixture_file('astroid/suppliermodule_test.py')

  def test_deep_recursion(self):
    # This testcase has 1050 strings joined with '+', which causes naive recursions to fail with
    # 'maximum recursion depth exceeded' error. We actually handle it just fine, but we can't use
    # to_source() on it because it chokes on recursion depth. So we test individual nodes.
    source = tools.read_fixture('astroid/joined_strings.py')

    if self.is_astroid_test:
      if getattr(astroid, '__version__', '1') >= '2':
        # Astroid 2 no longer supports this; see
        # https://github.com/PyCQA/astroid/issues/557#issuecomment-396004274
        self.skipTest('astroid-2.0 does not support this')

      # Astroid < 2 does support this with optimize_ast set to True
      astroid.MANAGER.optimize_ast = True
      try:
        m = self.create_mark_checker(source, verify=False)
      finally:
        astroid.MANAGER.optimize_ast = False

      self.assertEqual(len(m.all_nodes), 4)     # This is the result of astroid's optimization
      self.assertEqual(m.view_node_types_at(1, 0), {'Module', 'Assign', 'AssignName'})
      const = next(n for n in m.all_nodes if isinstance(n, astroid.nodes.Const))
      # TODO: Astroid's optimization makes it impossible to get the right start-end information
      # for the combined node. So this test fails. To avoid it, don't set 'optimize_ast=True'. To
      # fix it, astroid would probably need to record the info from the nodes it's combining. Or
      # astroid could avoid the need for the optimization by using an explicit stack like we do.
      #self.assertEqual(m.atok.get_text_range(const), (5, len(source) - 1))
    else:
      m = self.create_mark_checker(source, verify=False)
      self.assertEqual(len(m.all_nodes), 2104)
      self.assertEqual(m.view_node(m.all_nodes[-1]),
                       "Constant:'F1akOFFiRIgPHTZksKBAgMCLGTdGNIAAQgKfDAcgZbj0odOnUA8GBAA7'")
      self.assertEqual(m.view_node(m.all_nodes[-2]),
                       "Constant:'Ii0uLDAxLzI0Mh44U0gxMDI5JkM0JjU3NDY6Kjc5Njo7OUE8Ozw+Oz89QTxA'")
      self.assertEqual(m.view_node(m.all_nodes[1053]),
                       "Constant:'R0lGODlhigJnAef/AAABAAEEAAkCAAMGAg0GBAYJBQoMCBMODQ4QDRITEBkS'")
      self.assertEqual(m.view_node(m.all_nodes[1052]),
                       "BinOp:'R0lGODlhigJnAef/AAABAAEEAAkCAAMGAg0GBAYJBQoMCBMODQ4QDRITEBkS'\r\n" +
                       "     +'CxsSEhkWDhYYFQ0aJhkaGBweGyccGh8hHiIkIiMmGTEiHhQoPSYoJSkqKDcp'")

      binop = next(n for n in m.all_nodes if n.__class__.__name__ == 'BinOp')
      self.assertTrue(m.atok.get_text(binop).startswith("'R0l"))
      self.assertTrue(m.atok.get_text(binop).endswith("AA7'"))

    assign = next(n for n in m.all_nodes if n.__class__.__name__ == 'Assign')
    self.assertTrue(m.atok.get_text(assign).startswith("x = ("))
    self.assertTrue(m.atok.get_text(assign).endswith(")"))

  def test_slices(self):
    # Make sure we don't fail on parsing slices of the form `foo[4:]`.
    source = "(foo.Area_Code, str(foo.Phone)[:3], str(foo.Phone)[3:], foo[:], bar[::2, :], bar2[:, ::2], [a[:]][::-1])"
    m = self.create_mark_checker(source)
    self.assertIn("Tuple:" + source, m.view_nodes_at(1, 0))
    self.assertEqual(m.view_nodes_at(1, 1),
                     { "Attribute:foo.Area_Code", "Name:foo" })
    self.assertEqual(m.view_nodes_at(1, 16),
                     { "Subscript:str(foo.Phone)[:3]", "Call:str(foo.Phone)", "Name:str"})
    self.assertEqual(m.view_nodes_at(1, 36),
                     { "Subscript:str(foo.Phone)[3:]", "Call:str(foo.Phone)", "Name:str"})
    # Slice and ExtSlice nodes are wrong, and in particular placed with parents. They are not very
    # important, so we skip them here.
    self.assertEqual({n for n in m.view_nodes_at(1, 56) if 'Slice:' not in n},
                     { "Subscript:foo[:]", "Name:foo" })
    self.assertEqual({n for n in m.view_nodes_at(1, 64) if 'Slice:' not in n and 'Tuple:' not in n},
                     { "Subscript:bar[::2, :]", "Name:bar" })

  def test_adjacent_strings(self):
    source = """
foo = 'x y z' \\
'''a b c''' "u v w"
bar = ('x y z'   # comment2
       'a b c'   # comment3
       'u v w'
      )
"""
    m = self.create_mark_checker(source)
    node_name = 'Const' if self.is_astroid_test else 'Constant'
    self.assertEqual(m.view_nodes_at(2, 6), {
      node_name + ":'x y z' \\\n'''a b c''' \"u v w\""
    })
    self.assertEqual(m.view_nodes_at(4, 7), {
      node_name + ":'x y z'   # comment2\n       'a b c'   # comment3\n       'u v w'"
    })

  def test_print_function(self):
    # This testcase imports print as function (using from __future__). Check that we can parse it.
    # verify_all_nodes doesn't work on Python 2 because the print() call parsed in isolation
    # is viewed as a Print node since it doesn't see the future import
    source = tools.read_fixture('astroid/nonregr.py')
    m = self.create_mark_checker(source, verify=six.PY3)

    # Line 16 is: [indent 8] print(v.get('yo'))
    self.assertEqual(m.view_nodes_at(16, 8),
                     { "Call:print(v.get('yo'))", "Expr:print(v.get('yo'))", "Name:print" })
    self.assertEqual(m.view_nodes_at(16, 14), {"Call:v.get('yo')", "Attribute:v.get", "Name:v"})

  # To make sure we can handle various hard cases, we include tests for issues reported for a
  # similar project here: https://bitbucket.org/plas/thonny

  if not six.PY2:
    def test_nonascii(self):
      # Test of https://bitbucket.org/plas/thonny/issues/162/weird-range-marker-crash-with-non-ascii
      # Only on PY3 because Py2 doesn't support unicode identifiers.
      for source in (
        "℘·2=1+a℘·b+a℘·2b",  # example from https://github.com/python/cpython/issues/68382
        "sünnikuupäev=str((18+int(isikukood[0:1])-1)//2)+isikukood[1:3]",
        "sünnikuupaev=str((18+int(isikukood[0:1])-1)//2)+isikukood[1:3]"):
        m = self.create_mark_checker(source)
        self.assertEqual(m.view_nodes_at(1, 0), {
          "Module:%s" % source,
          "Assign:%s" % source,
          "%s:%s" % ("AssignName" if self.is_astroid_test else "Name", source.split("=")[0]),
        })

  if sys.version_info[0:2] >= (3, 6):
    # f-strings are only supported in Python36. We don't handle them fully, for a couple of
    # reasons: parsed AST nodes are not annotated with correct line and col_offset (see
    # https://bugs.python.org/issue29051), and there are confusingly two levels of tokenizing.
    # Meanwhile, we only parse to the level of JoinedStr, and verify that.
    def test_fstrings(self):
      for source in (
        '(f"He said his name is {name!r}.",)',
        "f'{function(kwarg=24)}'",
        'a = f"""result: {value:{width}.{precision}}"""',
        """[f"abc {a['x']} def"]""",
        "def t():\n  return f'{function(kwarg=24)}'"):
        self.create_mark_checker(source)

    def test_adjacent_joined_strings(self):
        source = """
foo = f'x y z' \\
f'''a b c''' f"u v w"
bar = ('x y z'   # comment2
       'a b c'   # comment3
       f'u v w'
      )
"""
        m = self.create_mark_checker(source)
        self.assertEqual(m.view_nodes_at(2, 6), {
            "JoinedStr:f'x y z' \\\nf'''a b c''' f\"u v w\""
        })
        self.assertEqual(m.view_nodes_at(4, 7), {
            "JoinedStr:'x y z'   # comment2\n       'a b c'   # comment3\n       f'u v w'"
        })

  def test_splat(self):
    # See https://bitbucket.org/plas/thonny/issues/151/debugger-crashes-when-encountering-a-splat
    source = textwrap.dedent("""
      arr = [1,2,3,4,5]
      def print_all(a, b, c, d, e):
          print(a, b, c, d ,e)
      print_all(*arr)
    """)
    m = self.create_mark_checker(source)
    self.assertEqual(m.view_nodes_at(5, 0),
        { "Expr:print_all(*arr)", "Call:print_all(*arr)", "Name:print_all" })
    if not six.PY2 or self.is_astroid_test:
      self.assertEqual(m.view_nodes_at(5, 10), { "Starred:*arr" })
    self.assertEqual(m.view_nodes_at(5, 11), { "Name:arr" })

  def test_paren_attr(self):
    # See https://bitbucket.org/plas/thonny/issues/123/attribute-access-on-parenthesized
    source = "(x).foo()"
    m = self.create_mark_checker(source)
    self.assertEqual(m.view_nodes_at(1, 1), {"Name:x"})
    self.assertEqual(m.view_nodes_at(1, 0),
                     {"Module:(x).foo()", "Expr:(x).foo()", "Call:(x).foo()", "Attribute:(x).foo"})

  def test_conditional_expr(self):
    # See https://bitbucket.org/plas/thonny/issues/108/ast-marker-crashes-with-conditional
    source = "a = True if True else False\nprint(a)"
    m = self.create_mark_checker(source)
    name_a = 'AssignName:a' if self.is_astroid_test else 'Name:a'
    const_true = ('Const:True' if self.is_astroid_test else
                  'Name:True' if six.PY2 else
                  'Constant:True')
    self.assertEqual(m.view_nodes_at(1, 0),
                     {name_a, "Assign:a = True if True else False", "Module:" + source})
    self.assertEqual(m.view_nodes_at(1, 4),
                     {const_true, 'IfExp:True if True else False'})
    if six.PY2:
      self.assertEqual(m.view_nodes_at(2, 0), {"Print:print(a)"})
    else:
      self.assertEqual(m.view_nodes_at(2, 0), {"Name:print", "Call:print(a)", "Expr:print(a)"})

  def test_calling_lambdas(self):
    # See https://bitbucket.org/plas/thonny/issues/96/calling-lambdas-crash-the-debugger
    source = "y = (lambda x: x + 1)(2)"
    m = self.create_mark_checker(source)
    self.assertEqual(m.view_nodes_at(1, 4), {'Call:(lambda x: x + 1)(2)'})
    self.assertEqual(m.view_nodes_at(1, 15), {'BinOp:x + 1', 'Name:x'})
    if self.is_astroid_test:
      self.assertEqual(m.view_nodes_at(1, 0), {'AssignName:y', 'Assign:'+source, 'Module:'+source})
    else:
      self.assertEqual(m.view_nodes_at(1, 0), {'Name:y', 'Assign:' + source, 'Module:' + source})

  def test_comprehensions(self):
    # See https://bitbucket.org/plas/thonny/issues/8/range-marker-doesnt-work-correctly-with
    for source in (
      "[(key, val) for key, val in ast.iter_fields(node)]",
      "((key, val) for key, val in ast.iter_fields(node))",
      "{(key, val) for key, val in ast.iter_fields(node)}",
      "{key: val for key, val in ast.iter_fields(node)}",
      "[[c for c in key] for key, val in ast.iter_fields(node)]"):
      self.create_mark_checker(source)

  def test_trailing_commas(self):
    # Make sure we handle trailing commas on comma-separated structures (e.g. tuples, sets, etc.)
    for source in (
      "(a,b,)",
      "[c,d,]",
      "{e,f,}",
      "{h:1,i:2,}"):
      self.create_mark_checker(source)

  def test_tuples(self):
    def get_tuples(code):
      m = self.create_mark_checker(code)
      return [m.atok.get_text(n) for n in m.all_nodes if n.__class__.__name__ == "Tuple"]

    self.assertEqual(get_tuples("a,"), ["a,"])
    self.assertEqual(get_tuples("(a,)"), ["(a,)"])
    self.assertEqual(get_tuples("(a),"), ["(a),"])
    self.assertEqual(get_tuples("((a),)"), ["((a),)"])
    self.assertEqual(get_tuples("(a,),"), ["(a,),", "(a,)"])
    self.assertEqual(get_tuples("((a,),)"), ["((a,),)", "(a,)"])
    self.assertEqual(get_tuples("()"), ["()"])
    self.assertEqual(get_tuples("(),"), ["(),", "()"])
    self.assertEqual(get_tuples("((),)"), ["((),)", "()"])
    self.assertEqual(get_tuples("((),(a,))"), ["((),(a,))", "()", "(a,)"])
    self.assertEqual(get_tuples("((),(a,),)"), ["((),(a,),)", "()", "(a,)"])
    self.assertEqual(get_tuples("((),(a,),),"), ["((),(a,),),", "((),(a,),)", "()", "(a,)"])
    self.assertEqual(get_tuples('((foo, bar),)'), ['((foo, bar),)', '(foo, bar)'])
    self.assertEqual(get_tuples('(foo, bar),'), ['(foo, bar),', '(foo, bar)'])
    self.assertEqual(get_tuples('def foo(a=()): ((x, (y,)),) = ((), (a,),),'), [
      '()', '((x, (y,)),)', '(x, (y,))', '(y,)', '((), (a,),),', '((), (a,),)', '()', '(a,)'])
    self.assertEqual(get_tuples('def foo(a=()): ((x, (y,)),) = [(), [a,],],'), [
      '()', '((x, (y,)),)', '(x, (y,))', '(y,)', '[(), [a,],],', '()'])

  def test_dict_order(self):
    # Make sure we iterate over dict keys/values in source order.
    # See https://github.com/gristlabs/asttokens/issues/31
    source = 'f({1: (2), 3: 4}, object())'
    self.create_mark_checker(source)

  def test_del_dict(self):
    # See https://bitbucket.org/plas/thonny/issues/24/try-del-from-dictionary-in-debugging-mode
    source = "x = {4:5}\ndel x[4]"
    m = self.create_mark_checker(source)
    self.assertEqual(m.view_nodes_at(1, 4), {'Dict:{4:5}'})
    if self.is_astroid_test:
      self.assertEqual(m.view_nodes_at(1, 5), {'Const:4'})
    else:
      self.assertEqual(m.view_nodes_at(1, 5), {'Constant:4'})
    self.assertEqual(m.view_nodes_at(2, 0), {'Delete:del x[4]'})
    self.assertEqual(m.view_nodes_at(2, 4), {'Name:x', 'Subscript:x[4]'})

  if not six.PY2:
    def test_bad_tokenless_types(self):
      # Cases where _get_text_positions_tokenless is incorrect in 3.8.
      source = textwrap.dedent("""
        def foo(*, name: str):  # keyword-only argument with type annotation
          pass

        f(*(x))  # ast.Starred with parentheses
      """)
      self.create_mark_checker(source)

    def test_return_annotation(self):
      # See https://bitbucket.org/plas/thonny/issues/9/range-marker-crashes-on-function-return
      source = textwrap.dedent("""
        def liida_arvud(x: int, y: int) -> int:
          return x + y
      """)
      m = self.create_mark_checker(source)
      self.assertEqual(m.view_nodes_at(2, 0),
        {'FunctionDef:def liida_arvud(x: int, y: int) -> int:\n  return x + y'})
      if self.is_astroid_test:
        self.assertEqual(m.view_nodes_at(2, 16),   {'Arguments:x: int, y: int', 'AssignName:x'})
      else:
        self.assertEqual(m.view_nodes_at(2, 16),   {'arguments:x: int, y: int', 'arg:x: int'})
      self.assertEqual(m.view_nodes_at(2, 19),   {'Name:int'})
      self.assertEqual(m.view_nodes_at(2, 35),   {'Name:int'})
      self.assertEqual(m.view_nodes_at(3, 2),    {'Return:return x + y'})

  def test_keyword_arg_only(self):
    # See https://bitbucket.org/plas/thonny/issues/52/range-marker-fails-with-ridastrip-split
    source = "f(x=1)\ng(a=(x),b=[y])"
    m = self.create_mark_checker(source)
    self.assertEqual(m.view_nodes_at(1, 0),
                     {'Name:f', 'Call:f(x=1)', 'Expr:f(x=1)', 'Module:' + source})
    self.assertEqual(m.view_nodes_at(2, 0),
                     {'Name:g', 'Call:g(a=(x),b=[y])', 'Expr:g(a=(x),b=[y])'})
    self.assertEqual(m.view_nodes_at(2, 11), {'Name:y'})
    if self.is_astroid_test:
      self.assertEqual(m.view_nodes_at(1, 2), {'Keyword:x=1'})
      self.assertEqual(m.view_nodes_at(1, 4), {'Const:1'})
      self.assertEqual(m.view_nodes_at(2, 2), {'Keyword:a=(x)'})
      self.assertEqual(m.view_nodes_at(2, 8), {'Keyword:b=[y]'})
    else:
      self.assertEqual(m.view_nodes_at(1, 2), {'keyword:x=1'})
      self.assertEqual(m.view_nodes_at(1, 4), {'Constant:1'})
      self.assertEqual(m.view_nodes_at(2, 2), {'keyword:a=(x)'})
      self.assertEqual(m.view_nodes_at(2, 8), {'keyword:b=[y]'})

  def test_decorators(self):
    # See https://bitbucket.org/plas/thonny/issues/49/range-marker-fails-with-decorators
    source = textwrap.dedent("""
      @deco1
      def f():
        pass
      @deco2(a=1)
      def g(x):
        pass

      @deco3()
      def g(x):
        pass

      @deco4
      class C:
        pass
    """)
    m = self.create_mark_checker(source)
    # The `arguments` node has bogus positions here (and whenever there are no arguments). We
    # don't let that break our test because it's unclear if it matters to anything anyway.
    self.assertIn('FunctionDef:@deco1\ndef f():\n  pass', m.view_nodes_at(2, 0))
    self.assertEqual(m.view_nodes_at(2, 1), {'Name:deco1'})
    if self.is_astroid_test:
      self.assertEqual(m.view_nodes_at(5, 0), {
        'FunctionDef:@deco2(a=1)\ndef g(x):\n  pass',
        'Decorators:@deco2(a=1)'
      })
    else:
      self.assertEqual(m.view_nodes_at(5, 0), {'FunctionDef:@deco2(a=1)\ndef g(x):\n  pass'})
    self.assertEqual(m.view_nodes_at(5, 1), {'Name:deco2', 'Call:deco2(a=1)'})

    self.assertEqual(m.view_nodes_at(9, 1), {'Name:deco3', 'Call:deco3()'})

  def test_with(self):
    source = "with foo: pass"
    m = self.create_mark_checker(source)
    self.assertEqual(m.view_node_types_at(1, 0), {"Module", "With"})
    self.assertEqual(m.view_nodes_at(1, 0), {
      "Module:with foo: pass",
      "With:with foo: pass",
    })

    source = textwrap.dedent(
      '''
      def f(x):
        with A() as a:
          log(a)
          with B() as b, C() as c: log(b, c)
        log(x)
      ''')
    # verification fails on Python2 which turns `with X, Y` turns into `with X: with Y`.
    m = self.create_mark_checker(source, verify=six.PY3)
    self.assertEqual(m.view_nodes_at(5, 4), {
      'With:with B() as b, C() as c: log(b, c)'
    })
    self.assertEqual(m.view_nodes_at(3, 2), {
      'With:  with A() as a:\n    log(a)\n    with B() as b, C() as c: log(b, c)'
    })
    with_nodes = [n for n in m.all_nodes if n.__class__.__name__ == 'With']
    self.assertEqual({m.view_node(n) for n in with_nodes}, {
      'With:with B() as b, C() as c: log(b, c)',
      'With:  with A() as a:\n    log(a)\n    with B() as b, C() as c: log(b, c)',
    })

  def test_one_line_if_elif(self):
    source = """
if 1: a
elif 2: b
    """
    self.create_mark_checker(source)

  def test_statements_with_semicolons(self):
    source = """
a; b; c(
  17
); d # comment1; comment2
if 2: a; b; # comment3
if a:
  if b: c; d  # comment4
    """
    m = self.create_mark_checker(source)
    self.assertEqual(
      [m.atok.get_text(n) for n in m.all_nodes if util.is_stmt(n)],
      ['a', 'b', 'c(\n  17\n)', 'd', 'if 2: a; b', 'a', 'b',
       'if a:\n  if b: c; d', 'if b: c; d', 'c', 'd'])

  def test_complex_numbers(self):
    source = """
1
-1
j  # not a complex number, just a name
1j
-1j
1+2j
3-4j
1j-1j-1j-1j
    """
    self.create_mark_checker(source)

  def test_parens_around_func(self):
    source = textwrap.dedent(
      '''
      foo()
      (foo)()
      (lambda: 0)()
      (lambda: ())()
      (foo)((1))
      (lambda: ())((2))
      x = (obj.attribute.get_callback() or default_callback)()
      ''')
    m = self.create_mark_checker(source)
    self.assertEqual(m.view_nodes_at(2, 0), {"Name:foo", "Expr:foo()", "Call:foo()"})
    self.assertEqual(m.view_nodes_at(3, 1), {"Name:foo"})
    self.assertEqual(m.view_nodes_at(3, 0), {"Expr:(foo)()", "Call:(foo)()"})
    self.assertEqual(m.view_nodes_at(4, 0), {"Expr:(lambda: 0)()", "Call:(lambda: 0)()"})
    self.assertEqual(m.view_nodes_at(5, 0), {"Expr:(lambda: ())()", "Call:(lambda: ())()"})
    self.assertEqual(m.view_nodes_at(6, 0), {"Expr:(foo)((1))", "Call:(foo)((1))"})
    self.assertEqual(m.view_nodes_at(7, 0), {"Expr:(lambda: ())((2))", "Call:(lambda: ())((2))"})
    self.assertEqual(m.view_nodes_at(8, 4),
                     {"Call:(obj.attribute.get_callback() or default_callback)()"})
    self.assertIn('BoolOp:obj.attribute.get_callback() or default_callback', m.view_nodes_at(8, 5))

  def test_complex_slice_and_parens(self):
    source = 'f((x)[:, 0])'
    self.create_mark_checker(source)

  if six.PY3:
    def test_sys_modules(self):
      """
      Verify all nodes on source files obtained from sys.modules.
      This can take a long time as there are many modules,
      so it only tests all modules if the environment variable
      ASTTOKENS_SLOW_TESTS has been set.
      """
      from .test_astroid import AstroidTreeException

      modules = list(sys.modules.values())
      if not os.environ.get('ASTTOKENS_SLOW_TESTS'):
        modules = modules[:20]

      start = time()
      for module in modules:
        # Don't let this test (which runs twice) take longer than 13 minutes
        # to avoid the travis build time limit of 30 minutes
        if time() - start > 13 * 60:
          break

        try:
          filename = inspect.getsourcefile(module)
        except Exception:  # some modules raise weird errors
          continue

        if not filename:
          continue

        filename = os.path.abspath(filename)
        print(filename)
        try:
          with io.open(filename) as f:
            source = f.read()
        except OSError:
          continue

        if self.is_astroid_test and (
            # Astroid fails with a syntax error if a type comment is on its own line
            re.search(r'^\s*# type: ', source, re.MULTILINE)
        ):
          print('Skipping', filename)
          continue

        try:
          self.create_mark_checker(source)
        except AstroidTreeException:
          # Astroid sometimes fails with errors like:
          #     AttributeError: 'TreeRebuilder' object has no attribute 'visit_typealias'
          # See https://github.com/gristlabs/asttokens/actions/runs/6015907789/job/16318767911?pr=110
          # Should be fixed in the next astroid release:
          #     https://github.com/pylint-dev/pylint/issues/8782#issuecomment-1669967220
          # Note that this exception is raised before asttokens is even involved,
          # it's purely an astroid bug that we can safely ignore.
          continue

  if six.PY3:
    def test_dict_merge(self):
      self.create_mark_checker("{**{}}")

    def test_async_def(self):
      self.create_mark_checker("""
async def foo():
  pass

@decorator
async def foo():
  pass
""")

    def test_async_for_and_with(self):
      # Can't verify all nodes because in < 3.7
      # async for/with outside of a function is invalid syntax
      m = self.create_mark_checker("""
async def foo():
  async for x in y: pass
  async with x as y: pass
  """, verify=False)
      assert m.view_nodes_at(3, 2) == {"AsyncFor:async for x in y: pass"}
      assert m.view_nodes_at(4, 2) == {"AsyncWith:async with x as y: pass"}

    def test_await(self):
      # Can't verify all nodes because in astroid
      # await outside of an async function is invalid syntax
      m = self.create_mark_checker("""
async def foo():
  await bar
  """, verify=False)
      assert m.view_nodes_at(3, 2) == {"Await:await bar", "Expr:await bar"}

  if sys.version_info >= (3, 8):
    def test_assignment_expressions(self):
      # From https://www.python.org/dev/peps/pep-0572/
      self.create_mark_checker("""
# Handle a matched regex
if (match := pattern.search(data)) is not None:
    # Do something with match
    pass

# A loop that can't be trivially rewritten using 2-arg iter()
while chunk := file.read(8192):
   process(chunk)

# Reuse a value that's expensive to compute
[y := f(x), y**2, y**3]

# Share a subexpression between a comprehension filter clause and its output
filtered_data = [y for x in data if (y := f(x)) is not None]

y0 = (y1 := f(x))  # Valid, though discouraged

foo(x=(y := f(x)))  # Valid, though probably confusing

def foo(answer=(p := 42)):  # Valid, though not great style
    ...

def foo(answer: (p := 42) = 5):  # Valid, but probably never useful
    ...

lambda: (x := 1) # Valid, but unlikely to be useful

(x := lambda: 1) # Valid

lambda line: (m := re.match(pattern, line)) and m.group(1) # Valid

if any((comment := line).startswith('#') for line in lines):
    print("First comment:", comment)

if all((nonblank := line).strip() == '' for line in lines):
    print("All lines are blank")

partial_sums = [total := total + v for v in values]
""")

  if sys.version_info >= (3, 10):
    def test_match_case(self):
      m = self.create_mark_checker("""
if 0:
  match x:
    case ast.BinOp():
      if z:
        pass
    case cls(a,b) if y:
      pass
    case _:
      match y:
        case 1:
          pass
        case _:
          pass
""")
      self.assertEqual(m.view_nodes_at(10, 6), {
        'Match:'
        '      match y:\n'
        '        case 1:\n'
        '          pass\n'
        '        case _:\n'
        '          pass',
      })

  def parse_snippet(self, text, node):
    """
    Returns the parsed AST tree for the given text, handling issues with indentation and newlines
    when text is really an extracted part of larger code.
    """
    # If text is indented, it's a statement, and we need to put in a scope for indents to be valid
    # (using textwrap.dedent is insufficient because some lines may not indented, e.g. comments or
    # multiline strings). If text is an expression but has newlines, we parenthesize it to make it
    # parsable.
    # For expressions and statements, we add a dummy statement '_' before it because if it's just a
    # string contained in an astroid.Const or astroid.Expr it will end up in the doc attribute and be
    # a pain to extract for comparison
    # For starred expressions, e.g. `*args`, we wrap it in a function call to make it parsable.
    # For slices, e.g. `x:`, we wrap it in an indexing expression to make it parsable.
    indented = re.match(r'^[ \t]+\S', text)
    if indented:
      return self.module.parse('def dummy():\n' + text).body[0].body[0]
    if util.is_starred(node):
      return self.module.parse('f(' + text + ')').body[0].value.args[0]
    if util.is_slice(node):
      return self.module.parse('a[' + text + ']').body[0].value.slice
    if util.is_expr(node):
      return self.module.parse('_\n(' + text + ')').body[1].value
    if util.is_module(node):
      return self.module.parse(text)
    return self.module.parse('_\n' + text).body[1]

  def test_assert_nodes_equal(self):
    """
    Checks that assert_nodes_equal actually fails when given different nodes
    """

    def check(s1, s2):
      n1 = self.module.parse(s1)
      n2 = self.module.parse(s2)
      with self.assertRaises(AssertionError):
        self.assert_nodes_equal(n1, n2)

    check('a', 'b')
    check('a*b', 'a+b')
    check('a*b', 'b*a')
    check('(a and b) or c', 'a and (b or c)')
    check('a = 1', 'a = 2')
    check('a = 1', 'a += 1')
    check('a *= 1', 'a += 1')
    check('[a for a in []]', '[a for a in ()]')
    check("for x in y: pass", "for x in y: fail")
    check("1", "1.0")
    check("foo(a, b, *d, c=2, **e)",
          "foo(a, b, *d, c=2.0, **e)")
    check("foo(a, b, *d, c=2, **e)",
          "foo(a, b, *d, c=2)")
    check('def foo():\n    """xxx"""\n    None',
          'def foo():\n    """xx"""\n    None')

  nodes_classes = ast.AST
  context_classes = [ast.expr_context] # type: List[util.AstNode]
  iter_fields = staticmethod(ast.iter_fields)

  def assert_nodes_equal(self, t1, t2):
    # Ignore the context of each node which can change when parsing
    # substrings of source code. We just want equal structure and contents.
    for context_classes_group in self.context_classes:
      if isinstance(t1, context_classes_group):
        self.assertIsInstance(t2, context_classes_group)
        break
    else:
      self.assertEqual(type(t1), type(t2))

    if isinstance(t1, AstroidPosition):
      # Ignore the lineno/col_offset etc. from astroid
      return

    if isinstance(t1, (list, tuple)):
      self.assertEqual(len(t1), len(t2))
      for vc1, vc2 in zip(t1, t2):
        self.assert_nodes_equal(vc1, vc2)
    elif isinstance(t1, self.nodes_classes):
      self.assert_nodes_equal(
        list(self.iter_fields(t1)),
        list(self.iter_fields(t2)),
      )
    else:
      # Weird bug in astroid that collapses spaces in docstrings sometimes maybe
      if self.is_astroid_test and isinstance(t1, six.string_types):
        t1 = re.sub(r'^ +$', '', t1, flags=re.MULTILINE)
        t2 = re.sub(r'^ +$', '', t2, flags=re.MULTILINE)

      self.assertEqual(t1, t2)