import pandas as pd
import streamlit as st
from wordcloud import WordCloud
import matplotlib.pyplot as plt
import string

import ds_ref_data as data

st.title("Data Science Resources")
st.sidebar.subheader("Quick Links:")
links = st.sidebar.container()
with links:
    st.sidebar.write("Link to Pay Comparisons: [Payscale](https://www.payscale.com/research/US/Job=Data_Scientist/Salary)")
    st.sidebar.write("Link to Training Resources: [Learn DS Online](https://www.simplilearn.com/resources-to-learn-data-science-online-article)")
    st.sidebar.write("Link to Job Comparison: [Different DS Job Types](https://builtin.com/data-science/data-science-jobs)")

tab1, tab2, tab3 = st.tabs(["Overview", "Job Types", "Job Req Analysis"])
with tab1:
   st.subheader("Quick Overview of Data Science:")
   url = "https://www.youtube.com/embed/xvEKQefqQ7A"
   st.video(url)
   st.text("What is the roadmap to becoming a Data Scientist?")
   st.image("images/modern_ds.webp")
   st.image('images/ds_roadmap.png')
   st.markdown("""#### What is the difference between AI/ML/DS?
   Definitions:

    - AI: François Chollet, creator of Keras, described it as “the effort to automate intellectual tasks normally performed by humans”.
    - ML: a pioneer of ML, Arthur Samuel, defined it as a “field of study that gives computers the ability to learn without being explicitly programmed”.
         -  Includes single layer NN
    - DL:  a subset of ML that is inspired by how human brains work
         -  Multi-layer NN, CNN, RNN

        """)
   st.image('images/venn1.jpg')
   st.image('images/diagram_2.png')
   st.markdown("""#### What does a Data Scientist do?
    Drawing useful conclusions from data using computation

        - Data Wrangling and Preprocessing
            -  Gather, clean, normalize data

        - Exploration
            -  Identifying patterns in information
            -  Uses visualizations

        - Inference
            -  Quantifying whether those patterns are reliable
            -  Uses randomization

        - Prediction
            -  Making informed guesses
            -  Uses machine learning

        - Deploying
            -  Make model results/features available to stakeholders/other teams

        - Data Storytelling
            -  Share results/findings from model/analyses with stakeholders/other teams""")
   st.image('images/ds_process_2.jpeg')
   st.image('images/data-prep-approach-time.png')

with tab2:
    st.subheader("Differences in DS Jobs")
    url2='https://youtu.be/VrdnBxx8BBI'
    st.video(url2)
    st.caption("For different types of jobs in Data Science, here are some skill focus separations. \
        Keep in mind that every company is different and can align their DS roles slightly differently. \
            This is meant as a guide, not a steadfast rule.")
    st.image("images/ds_skills_comparison.webp")
    st.caption("Within a role, there are ways you can distinguish yourself. Are you interested in focusing in one area\
        and becoming a subject matter expert? Are you very curious and want to explore lots of different areas and want \
            to become more of a generalist? While all of these are good options, it is helpful in interviews to know \
                yourself, so you can market your strengths")
    st.image("images/types_of_ds.jpg")

with tab3:
    st.subheader("Job Req Analysis")
    st.caption("The data used for this study is obtained by aggregated data from [ai-jobs.net](https://ai-jobs.net), under this [license](https://creativecommons.org/publicdomain/zero/1.0/).")

    ## WordCloud
    data_df = data.gather_salary_data()
    text =str(data_df["job_title"].values)
    clean_text = text.translate(str.maketrans('','',string.punctuation))

    wordcloud = WordCloud(width=480, height=480, margin=0).generate(clean_text)

    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    plt.margins(x=0,y=0)

    st.caption("Word Frequencies of Job Titles in 2020-2022")
    st.pyplot(fig=plt, clear_figure=True)

    highest_2022 = (data_df[data_df.work_year == 2022].groupby(["job_title", "experience_level", "company_size", "company_location"]).max()['salary_in_usd'].reset_index()
                                        ).sort_values(["salary_in_usd"], ascending=False).head(10)

    st.caption("Top 2022 Job Titles by Salaries")
    fig = plt.barh(highest_2022['job_title'],
            highest_2022["salary_in_usd"]
            )
    plt.xlabel("Salaries")
    st.pyplot(fig=plt, clear_figure=True)

    average_earn = data_df.groupby(["job_title"]).mean()["salary_in_usd"].reset_index().sort_values(["salary_in_usd"], ascending=False).head(10)
    average_earn["salary_in_usd"]  = average_earn["salary_in_usd"].round(decimals=2)

    st.caption("Average Earnings by Job Title")
    fig = plt.barh(average_earn['job_title'],
            average_earn["salary_in_usd"]
            )
    plt.xlabel('Salaries')
    st.pyplot(fig=plt, clear_figure=True)

    xp_level = (data_df.groupby("experience_level").mean()["salary_in_usd"].reset_index()
                .sort_values("salary_in_usd", ascending=False)
            .round(decimals=2))

    st.caption("Average Earnings by Experience Level")

    fig = plt.barh(xp_level['experience_level'],
        xp_level["salary_in_usd"]
        )
    plt.xlabel('Salaries')
    st.pyplot(fig=plt, clear_figure=True)

    org_size = (data_df.groupby("company_size")
                        .mean()['salary_in_usd']
                            .reset_index()
                        .sort_values("salary_in_usd", ascending=False)
                    .round(decimals=2))
    st.caption("Average Earnings by Company Size")

    fig = plt.barh(org_size['company_size'],
        org_size["salary_in_usd"]
        )
    plt.xlabel('Salaries')
    st.pyplot(fig=plt, clear_figure=True)
## future idea: bring in live pay, state, and title graphs


# parse KGIP and KGIQ
# Author : Sankar Ramaiah
# 12-7-21 - comines price and qty files from vendor download and gathers relevant data for next steps
# 12-8-21 - added aditional logic to compute unitcost(price+shipping)
# 12-8-21 - carry forward unit price in addition to Unit cost computed for pricing pack 2s later
import csv
import io
i=0
minQty=5
skipCount=0
shipPct=0.25
DefShipFee=10.95  # use this unless 25% of product cost is higher
markupPct=0.40
header=['Sku','Title','Quantity','Unit Cost','VendorUPC','Brand','Description','Item Weight','Item Price']
#
#p = open('bbdPrc.csv', 'w')  # once items we want is located in our SKU LIST
#with open('bbdQtyPrc.txt', 'r') as f:   # sku,price and qty (0/1)
#     readerV = csv.reader(f, delimiter='\t')
     #BDVendorList = f.read()
     #for qlist in BDVendorList:
         #print(qlist[0],qlist[1],qlist[2])
         #print(BDVendorList[0][1])
     #print(BDVendorList)
#
with open('KGIQ.csv', 'r') as rf:
     reader = csv.reader(rf, delimiter=',')
     with open('KGIPQ.csv', 'w') as wf:  # combine price and qty in one file output
          writer = csv.writer(wf, delimiter=',')
          writer.writerow(header)
          #writer = csv.writer(wf, delimiter='|')
          #desired_column = [6]
          for line in reader:  # qty file top loop
              sku=line[0]
              itemT=line[1]
              qtyIn=line[2]  # vendor qty
              itemL=line[5]
              itemW=line[6]
              itemH=line[7]
              itemWt=line[8]  # in case we redo walmart and need this data collect it now
              itemUpc=line[13]
              brandN=line[14]
              itemM=line[17] # material
              itemDesc=line[22]
              #validate qty
              try:
                  int(qtyIn)
              except ValueError:
                  skipCount +=1
                  print(sku,"invalid qty data..skiping..",qtyIn)
                  continue  # skip invalid data
              # skip qty 0 rows
              #qty=int(qtyIn)
              #if qty <= minQty:
              #    print(sku,"qty low .. skipping",qty)
#
              found=0
              #print('sku= ',line[0])
              with open('KGIP.csv', 'r', encoding="ISO-8859-1") as f:   # qty file from vendor
                readerP = csv.reader(f, delimiter=',')
                for line in readerP:
                    skuIn=line[0]
                    if sku == skuIn:
                        priceIn=line[2]  # vendor wholesale price
                        #qtyIn=line[2]
                        #print('matched',skuIn,priceIn,qtyIn)
                        found=1
              #print(sku,'foundflag = ',found)
              if found == 1:
                  #validate price
                  try:
                     float(priceIn)
                  except ValueError:
                     skipCount +=1
                     print(sku,"invalid price data..skiping..",priceIn)
                     continue  # skip invalid data
                  #
                  # compute unit cost (vendor price + shipping cost to us) no markup
                  price=float(priceIn) #vendor price
                  shipFee=DefShipFee # unless next calc exceeds
                  shipFeeP=(price*shipPct) # ship fee pct 25%
                  if shipFeeP > shipFee:
                      shipFee=shipFeeP
                  priceX=(price+shipFee)
                  unitCost = round(priceX,2) # single unit cost with pricing included
                  unitPrice = round(price,2) # single unit price without any ship fee added
                  addDesc=" Item Dimenions LxWxH in inches = "+itemL+"X"+itemW+"X"+itemH+" Material: "+itemM
                  ##print(sku, addDesc)
                  lines=[sku,itemT,qtyIn,unitCost,itemUpc,brandN,itemDesc+addDesc,itemWt,unitPrice]  # customized output cols
                  writer.writerow(lines)
                  #priceData=sku+","+priceIn+"\n"
                  #p.write(priceData)
#
              #   continue
              #upcColumn = list(line[i] for i in desired_column)
              #itemNColumn = line[0]    # sku
              #line[4] = "***NULLIFIED-Not Used***"
              #if len(itemNColumn) == 0:
              #if not upcColumn.strip():
              #line[6] = "NA"
              #for i in range(0,35):
              #  if not line[i].strip():
              #       #print(line[i], 'has NO value - empty String?')
              #        line[i] = "NA"
              #  #else
              #skuIn=line[0]
              #line[0]='BD-'+skuIn
              #print(line[0])
              ##print(line[35])
              #shipWt=float(line[35])
              #print(shipWt*shipFeePerLb)
              #shipCost=(shipWt*shipFeePerLb)
              #totalShipCost=shipCost+shipFeePerItem+shipFeePerOrder
              #print('total ship fee for sku= ',skuIn, totalShipCost)
              #Sprice=float(line[11])   ## sale price in vendor
              #print(Sprice)
              #pctOfprice = (Sprice*discountPct) #price discount
              #ourPrice = (Sprice-pctOfprice)+totalShipCost
              #print('ourPrice',ourPrice)
              #markupPrice = (ourPrice*markupPct)+ourPrice
              #line[10] = round(markupPrice,2) # here we use the street price col to store our computed price
              #print('markuprice',line[10])
              #print('OurPrice for sku= ',skuIn, line[10])
              ## write out needed cols only
              ##print(line[0],line[2],line[6],line[8],line[9],line[10],line[14],line[15],line[35])
              #### output variables defined
              #skuOut=line[0]
              #itemName=line[2]
              #desc1=line[6]
              #desc2=line[7] # details formated
              #condition=line[8]
              #status=line[9] # in-stock or out-of-stock
              #price=line[10]
              #img1=line[14] #main image
              #img2=line[15] # additional 1
              #img3=line[16] # additional 2
              #shipWtout=shipWt
              ##custom cols needed for uploading to marketplaces
              #pIdtype='GTIN'
              ###pId='PlaceHolder'
              #pId=line[9]   # this place holder for GTIN for upload, we store the inventory status here for monitoring!
              #brandN='Generic'
              ##lines = [line[0],line[2],line[6],line[8],line[9],line[10],line[14],line[15],line[35]]
              ##lines = [skuOut,pIdtype,pId,itemName,brandN,price,shipWtout,desc1+desc2,img1,img2,img3]
              #writer.writelines(lines)
rf.close()
wf.close()
#p.close()


import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_iris

if __name__ == '__main__':

    # Download the data of Iris Fischer flowers
    # The Fischer iris dataset contains three balanced classes of 50 observations,
    # each of which indicates the type of flower - iris bristly (Iris setosa),
    # iris virginica and iris multicolored (Iris versicolor).
    iris = load_iris()

    # Create a feature matrix
    features = iris.data

    # Create a vector of goals
    target = iris.target

    # Delete the first 40 observations
    features = features[40:,:]
    target = target[40:]

    # Create a binary vector of targets indicating whether the class is 0
    target = np.where((target == 0), 0, 1)

    # Take a look at the unbalanced vector of goals
    print(target)

    weights = {0: .9, 1: 0.1}

    RandomForestClassifier(class_weight=weights)
    RandomForestClassifier(bootstrap=True, class_weight={0: 0.9, 1: 0.1},
                           criterion='gini', max_depth=None,
                           max_features='auto',
                           max_leaf_nodes=None, min_impurity_decrease=0.0,
                           min_impurity_split=None, min_samples_leaf=l,
                           min_samples_split=2, min_weight_fraction_leaf=0.0,
                           n_estimators=10, n_jobs=l, oob_score=False,
                           random_state=None, verbose=0, warm_start=False)


import copy
#Q.1- Reverse the whole list using list methods.
l=[1,2,3,4,5,6]
print("list is:",l)
l.reverse()
print("reversed list is",l)

#Q.2- Print all the uppercase letters from a string.
s=input("enter the string in which you want to check uppercase letters")
def upper_characters(s):
    ans=""
    for x in s:
        if(x.isupper()==True):
            ans +=x
    return ans
c=upper_characters(s)

print("the entered string was :",s)
print("it contains these uppercase letters",c)

#QUESTION: 3  Split and Store the Values After TypeCasting
l=input()#enter no 45,46,47
a=l.split(',')
l2=[]
for x in a:
    b=int(x)
    l2.append(b)
print(l2)

#Q.4- Check whether a string is palindromic or not.
a=input("enter any string or number to be checked as pailnfrom ")
b=len(a)
flag=0
i=0
while(i!=b):
    if(a[i]!=a[b-i-1]):
        flag=1
        break
    else:
         flag=0
    i=i+1
if(flag):
    print("not a pailndrom")
else:
    print("pailndrom")

#Q.5- Make a deepcopy of a list and write the difference between shallow copy and deep copy.
    # Python code to demonstrate copy operations
l1= [1, 2, [3,5], 4]
l2= copy.deepcopy(l1)
print(l1)
l2[2][0] = 7
print(l2)
# Change is NOT reflected in original list l1
# as it is a deep copy
print(l1)

#diff between deep and shallow copy:
#1. In case of deep copy, a copy of object is copied in other object. It means that any changes made to a copy of object do not reflect in the original object.
#  In case of shallow copy, a reference of object is copied in other object. It means that any changes made to a copy of object do reflect in the original object.
#2.In python, this is implemented using “deepcopy()” function.
#   In python, this is implemented using “copy()” function.
#3.*******************************************************
#l1= [1, 2, [3,5], 4]
#l2= copy.deepcopy(l1)
#print(l1)
#l2[2][0] = 7
#print(l2)
# Change is NOT reflected in original list l1 # as it is a deep copy
#print(l1)
#output:
#1 2 [3, 5] 4
#1 2 [7, 5] 4
#1 2 [3, 5] 4
#**********************************
#li1 = [1, 2, [3,5], 4]/////eg of shallow cpy
#li2 = copy.copy(li1)
#print(li1)
#li2[2][0] = 7
#print(li1)
#output
#1 2 [3, 5] 4
#1 2 [7, 5] 4


import os
import json
import traceback
from kubernetes import client, config
from kubernetes.client.api.custom_objects_api import CustomObjectsApi

def process_cr_object(cr_obj: dict):
    """ This method processes a CR object """
    operation_type = cr_obj.get("type", None)

    crd_object = cr_obj.get("object", None)
    crd_object_metadata = crd_object.get("metadata", None) if crd_object else None

    object_kind = crd_object.get("kind", None) if crd_object else None
    object_name = crd_object_metadata.get("name", None) if crd_object_metadata else None
    object_uid = crd_object_metadata.get("uid", None) if crd_object_metadata else None
    object_resourceversion = crd_object_metadata.get("resourceVersion", None) if crd_object_metadata else None

    print("DETECTED: "+str(operation_type)+", "+str(object_kind)+", "+str(object_name)+", "+str(object_uid)+", "+str(object_resourceversion))
    return object_resourceversion

def event_loop(CRD_GROUP: str, CRD_VERSION: str, CRD_PLURAL: str, CRD_SCOPE: str, CRD_NAMESPACE: str):
    """ This method always watches for the CRD """
    v1=client.CustomObjectsApi()
    resource_version = None

    while(True):
        custom_objects = None
        if CRD_SCOPE == "Cluster":
            custom_objects = v1.list_cluster_custom_object(CRD_GROUP, CRD_VERSION, CRD_PLURAL, timeout_seconds=5, resource_version=resource_version, watch=True)
        elif CRD_SCOPE == "Namespaced":
            custom_objects = v1.list_namespaced_custom_object(CRD_GROUP, CRD_VERSION, CRD_NAMESPACE, CRD_PLURAL, timeout_seconds=5, resource_version=resource_version, watch=True)

        # The list() API returns objects as string of there are multiple CRs detcted
        # The API returns an object as a dictionary if only 1 CR is detected
        returned_objects_type = str(type(custom_objects))

        if "str" in returned_objects_type and str(custom_objects):
            cr_objects_list = str(custom_objects).split("\n")
            for cr_str in cr_objects_list:
                if not cr_str:
                    continue
                cr_obj = json.loads(cr_str)
                resource_version = process_cr_object(cr_obj)
        elif "dict" in returned_objects_type:
            resource_version = process_cr_object(custom_objects)

def main():
    """ This is the main method """
    CRD_GROUP = os.environ.get("CRD_GROUP", None)
    CRD_PLURAL = os.environ.get("CRD_PLURAL", None)
    CRD_SINGULAR = os.environ.get("CRD_SINGULAR", None)
    CRD_VERSION = os.environ.get("CRD_VERSION", None)
    CRD_SCOPE = os.environ.get("CRD_SCOPE", None)
    CRD_NAMESPACE = os.environ.get("CRD_NAMESPACE", "")
    try:
        if CRD_GROUP and CRD_PLURAL and CRD_SINGULAR and CRD_VERSION and CRD_SCOPE:
            if CRD_SCOPE == "Cluster":
                event_loop(CRD_GROUP, CRD_VERSION, CRD_PLURAL, CRD_SCOPE, CRD_NAMESPACE)
            elif CRD_SCOPE == "Namespaced":
                if CRD_NAMESPACE:
                    event_loop(CRD_GROUP, CRD_VERSION, CRD_PLURAL, CRD_SCOPE, CRD_NAMESPACE)
                else:
                    raise Exception("For Namespaced CRDs, Namespace is required. Please pass it as environment variable")
            else:
                raise Exception("CRD Scope has to be either Cluster OR Namespaced. Please provide appropriate choice")
        else:
            raise Exception("CRD Group OR Plural OR Singular OR Version OR Scope is undefined. Please provide them as environment variables")
    except Exception as ex:
        print("Caught exception : "+str(ex))
        print(str(traceback.format_exc()))

if __name__ == "__main__":
    in_cluster = os.environ.get("KUBERNETES_SERVICE_HOST", None)
    if in_cluster:
        config.load_incluster_config()
    else:
        config.load_kube_config()
    main()


from re import T
import numpy as np
import matplotlib.pyplot as plt
from collections import defaultdict
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import confusion_matrix, precision_score, roc_auc_score, \
                            recall_score, f1_score, classification_report

dataPath = 'ml-1m/ratings.dat'
nUsers = 6040   # from the users.dat file TODO: read file for no. of entries
nMovies = 3706  # from the movies.dat file

def printHead(myString):
    if type(myString) is str:
        print(f'\n \u001b[4m\u001b[33m{myString}\u001b[0m')

def load_rating_data(dataPath, nUsers, nMovies):
    '''
    Load rating data from file and also return the number of ratings for each
    movie and movieId index mapping

    @param dataPath: path of the rating datafile
    @param nUsers: number of users
    @param nMovies: number of movies that have ratings
    @return: rating data in the numpy array of [user, movie];
             movieNRating, {movieId: number or ratings};
             movieIdMapping, {movieId: column index in rating data}
    '''
    # data is 2D array mapping users to movies and ratings
    # unranked data is given a zero
    data = np.zeros([nUsers, nMovies], dtype = np.float32)
    movieIdMapping = {}
    movieNRating = defaultdict(int)

    with open(dataPath, 'r') as file:
        for line in file.readlines()[1:]:
            # read line
            userId, movieId, rating, _ = line.split('::')
            # parse data
            userId = int(userId) - 1
            rating = int(rating)
            # add movie to mapping
            if movieId not in movieIdMapping:
                movieIdMapping[movieId] = len(movieIdMapping)
            # add data to array
            data[userId, movieIdMapping[movieId]] = rating
            if rating > 0:
                movieNRating[movieId] += 1
    return data, movieNRating, movieIdMapping

def display_distribution(data):
    '''
    Prints the distribution of ratings from zero (unrated) to 5

    @param data: rating data in the numpy array of [user, movie];
    '''
    values, counts = np.unique(data, return_counts = True)
    printHead('Data Distribution')
    for value, count in zip(values, counts):
        print(f' rating {int(value)}:{count:-8d}')
    print('')

if __name__ == '__main__':
    recommended = 3
    data, movieNRating, movieIdMapping = load_rating_data(dataPath, nUsers, nMovies)

    display_distribution(data)

    movieIdMost, nRatingsMost = sorted(movieNRating.items(),
                                       key = lambda d: d[1], reverse = True)[0]
    print(f'Movie ID {movieIdMost} has {nRatingsMost} ratings!')
    # Build X-Y dataset
    xRaw = np.delete(data, movieIdMapping[movieIdMost], axis = 1)
    yRaw = data[:, movieIdMapping[movieIdMost]]
    # Discard samples without a rating for "Most" Movie
    X = xRaw[yRaw > 0]
    Y = yRaw[yRaw > 0]
    print(f'Shape of X: {X.shape}')
    print(f'Shape of Y: {Y.shape}')
    display_distribution(Y)
    # Check for recommendation
    Y[Y <= recommended] = 0
    Y[Y > recommended] = 1
    nPos = (Y == 1).sum()
    nNeg = (Y == 0).sum()
    print(f'There are {nPos} positive samples and {nNeg} negative samples')

    # Train sklearn model
    xTrain, xTest, yTrain, yTest = train_test_split(X, Y, test_size=0.2, random_state=42)
    print(f'Training: {len(yTrain)}\tTest: {len(yTest)}')

    # Train multinomial Naive Bayes
    clf = MultinomialNB(alpha=1.0, fit_prior=True)
    clf.fit(xTrain, yTrain)

    predProb = clf.predict_proba(xTest)
    prediction = clf.predict(xTest)
    print(f'Predicted probability:\n{predProb[:10]}')
    print(f'Prediction:\n{prediction[:10]}')
    print(f'Accuracy: {clf.score(xTest,yTest)*100:0.1f}%')

    # Compute confusion matrix
    printHead('Confusion Matrix')
    print(confusion_matrix(yTest, prediction, labels=[0, 1]))
    print(f'\nprecision: {precision_score(yTest, prediction, pos_label=1):0.2f}')
    print(f'recall: {recall_score(yTest, prediction, pos_label=1):0.2f};  a.k.a TPR')
    print(f'f1 (neg) score: {f1_score(yTest, prediction, pos_label=0):0.2f}')
    print(f'f1 (pos) score: {f1_score(yTest, prediction, pos_label=1):0.2f}')

    printHead('Classification Report')
    report = classification_report(yTest, prediction)
    print(report)

    # Messing with "Receiver Operating Characteristic (ROC)"
    posProb = predProb[:,1]
    thresholds = np.arange(0.0, 1.1, 0.05)
    truePos, falsePos = [0]*len(thresholds), [0]*len(thresholds)
    for pred, y in zip(posProb, yTest):
        for i, threshold in enumerate(thresholds):
            if pred >= threshold:
                # if truth and predict are both '1'
                if y == 1:
                    truePos[i] += 1
                # if truth is '0' and predict is '1'
                else:
                    falsePos[i] += 1
            else:
                break

    nPosTest = (yTest == 1).sum()
    nNegTest = (yTest == 0).sum()
    truePosRate = [tp / nPosTest for tp in truePos]
    falsePosRate = [fp / nNegTest for fp in falsePos]
    # Calculate Area Under the Curve (AUC) score
    print(f'AUC Score: {roc_auc_score(yTest, posProb):0.2f}')

    plt.figure()
    lw = 2
    plt.plot(falsePosRate, truePosRate,
             color = 'darkorange', lw=lw)
    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver Operating Characteristic')
    plt.legend(loc='lower right')
    plt.show()


from tkinter import *
import random

class GUI:
    def __init__(self, window: None) -> None:

        """
        This function is used to create the GUI for the rock, paper, scissors game to be played.
        :param window: Used to create the locked dimensions for the GUI.
        """
        self.window = window

        self.frame_game = Frame(self.window)
        self.label_game = Label(self.frame_game, text='Welcome to Rock/Paper/Scissors!')
        self.label_game.pack(padx=5, side='left')
        self.frame_game.pack(anchor='w', pady=5)

        self.frame_game2 = Frame(self.window)
        self.label_game2 = Label(self.frame_game2, text='Select an option to face against the computer!')
        self.label_game2.pack(padx=5, side='left')
        self.frame_game2.pack(anchor='w', pady=5)

        self.frame_status = Frame(self.window)
        self.label_status = Label(self.frame_status, text='Choose:')
        self.radio = IntVar()
        self.radio.set(-1)
        self.radio_paper = Radiobutton(self.frame_status, text='Paper', variable=self.radio, value=0)
        self.radio_rock = Radiobutton(self.frame_status, text='Rock', variable=self.radio, value=1)
        self.radio_scissor = Radiobutton(self.frame_status, text='Scissor', variable=self.radio, value=2)
        self.label_status.pack(padx=5, side='left')
        self.radio_paper.pack(side='left')
        self.radio_rock.pack(side='left')
        self.radio_scissor.pack(side='left')
        self.frame_status.pack()
        self.frame_status.pack(anchor='w', pady=5)

        self.frame_results = Frame(self.window)
        self.label_results = Label(self.frame_results, text='')
        self.label_results.pack(padx=5, side='left')
        self.frame_results.pack(anchor='w', pady=5)

        self.frame_save = Frame(self.window)
        self.button_save = Button(self.frame_save, text='SAVE', command=self.clicked)
        self.button_save.pack()
        self.frame_save.pack(pady=10)

    def clicked(self) -> None:
        """
        This function is utilized when the 'SAVE' button is clicked in the GUI.
        This function gathers necessary information from the user's input, creates a randomized computer choice,
        and calls the game function.
        After this is complete and there are results from the game function, the results are printed on the GUI
        and the interface is cleaned off for the next round.
        :return: This function returns no data.
        """
        status = self.radio.get()
        if status == 0:
            user = 'paper'
        elif status == 1:
            user = 'rock'
        elif status == 2:
            user = 'scissor'
        else:
            self.label_results.destroy()
            self.label_results = Label(self.frame_results, text='Please select a choice before clicking SAVE.')
            self.label_results.pack(padx=5, side='left')

        computer = ['rock', 'paper', 'scissor']
        comp = random.choice(computer)
        results = self.game(user, comp)
        self.label_results.destroy()
        self.label_results = Label(self.frame_results, text='Computer is {}. You are {}. You {}'.format(comp, user, results))
        self.label_results.pack(padx=5, side='left')
        self.radio.set(-1)

    def game(self, user: str, comp: str) -> str:
        """
        This function is for the rock, paper, scissors game.
        It takes the user's input plus the computer's randomized choice and determines the winner.
        Paper beats rock, rock beats scissors, and scissors beats paper.
        :param user: User's input
        :param comp: Computer's randomized choice
        :return: The results of the game in terms of the user, win, tie, or lose
        """
        if comp == 'rock':
            if user == 'rock':
                return 'tie'
            elif user == 'paper':
                return 'win'
            else:
                return 'lose'
        elif comp == 'paper':
            if user == 'rock':
                return 'lose'
            elif user == 'paper':
                return 'tie'
            else:
                return 'win'
        else:
            if user == 'rock':
                return 'win'
            elif user == 'paper':
                return 'lose'
            else:
                return 'tie'


"""Tests for pandas_window_example_gen component."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import os

from mlp.components import transform_with_graph as component
from google.protobuf import json_format

from tfx.types import channel_utils
from tfx.types import standard_artifacts
from tfx.proto import pusher_pb2

class ComponentTest(tf.test.TestCase):
  def testConstruct(self):
    output_data_dir = os.path.join(
      os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()), self._testMethodName)
    examples_channel = channel_utils.as_channel([standard_artifacts.Examples()])
    schema_channel = channel_utils.as_channel([standard_artifacts.Schema()])
    transform_graph_channel = channel_utils.as_channel([standard_artifacts.TransformGraph()])
    component_instance = component.TransformWithGraph(
      examples=examples_channel,
      schema=schema_channel,
      transform_graph=transform_graph_channel,
    )
    self.assertEqual(
      'Schema',
      component_instance.inputs['schema'].type_name
    )
    self.assertEqual(
      'Examples',
      component_instance.inputs['examples'].type_name
    )
    self.assertEqual(
      'TransformGraph',
      component_instance.inputs['transform_graph'].type_name
    )
    self.assertEqual(
      'Examples',
      component_instance.outputs['transformed_examples'].type_name
    )

if __name__ == '__main__':
  tf.test.main()


import os
import sys

from flask import Flask, render_template, jsonify
from flask import request

from preprocessing import csvprocessor as csvproc
from shared.config import Config

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + '/')

app = Flask(__name__)

########################################################################################################################
# ROUTES
########################################################################################################################

@app.route('/')
def start():
    return render_template('index.html')

@app.route('/datasets')
def datasets():
    ls = []
    for filename in os.listdir(Config.PATH_DATA):
        if not filename.startswith('_') and filename.endswith(".csv"):
            ls.append(os.path.splitext(filename)[0])
    return jsonify(ls)

@app.route('/partynames')
def partynames():
    blocks = csvproc.get_blockDataSet(getParam("filename"))
    parties = csvproc.get_partyNames(blocks)
    return jsonify(parties)

@app.route('/election-results')
def electionResults():
    summary = csvproc.election_summary(getParam("filename"))
    return jsonify(summary)

@app.route('/compactness')
def compactness():
    measure = getParam('measure')
    fileName = getParam("filename")

    result = csvproc.compactness(fileName, measure)
    return jsonify(result)

def getParam(name):
    value = request.args.get(name, None)
    if not value:
        raise Exception("The parameter '{0}' was not provided.".format(name))
    return value

if __name__ == '__main__':
    app.debug = True
    app.run()


import argparse
import logging
import os
from nanoloop_mobile_sample_tools import commands

def get_parser() -> argparse.Namespace:
    """Get the CLI parser.

    :return argparse.Namespace:
    """
    parser = argparse.ArgumentParser(
        description="Nanoloop Mobile Sample Tools CLI"
    )

    parser.add_argument(
        "--debug",
        dest="debug",
        nargs='?',
        const=logging.DEBUG,
        default=logging.INFO,
        help="Set logging level to DEBUG, default INFO.",
    )

    parser.add_argument(
        "audio_inputs",
        nargs='+',
        help="Audio files to process in order.",
    )
    parser.add_argument(
        "--concatenate",
        dest="concatenate",
        action="store_true",
        help="Concatenate the audio inputs to the output. Deafult 'False'.",
    )
    parser.add_argument(
        "--mono",
        dest="mono",
        nargs='?',
        const='left',
        type=str,
        default='left',
        choices=['left', 'right'],
        help="Make audio mono. Default 'left'. Options; 'left' and 'right'",
    )
    parser.add_argument(
        "--sample-rate",
        dest="sample_rate",
        nargs='?',
        const=44100.0,
        type=float,
        default=44100.0,
        help="Sample rate for audio. Default '44100.0'",
    )
    parser.add_argument(
        "--bit-rate",
        dest="bit_rate",
        nargs='?',
        const=16,
        type=int,
        default=16,
        choices=[8, 16],
        help="Bit rate for audio. Default '16'. Options; '8' or '16' bit.",
    )
    parser.add_argument(
        "--speed-multiplier",
        dest="speed_multiplier",
        nargs='?',
        const=1.0,
        type=float,
        default=1.0,
        help="Speed multiplier for audio. Default '1.0' i.e. no change",
    )
    parser.add_argument(
        "--compress",
        dest="compress",
        nargs='?',
        type=str,
        choices=['soft', 'hard'],
        help="Compress the audio signal. Default 'None'. Options; 'soft' and 'hard'",
    )
    parser.add_argument(
        "--normalize",
        dest="normalize",
        action="store_true",
        help="Normalize the audio output. Deafult 'False'.",
    )
    parser.add_argument(
        "--reverse",
        dest="reverse",
        action="store_true",
        help="Reverse the audio output. Deafult 'False'.",
    )
    parser.add_argument(
        "--audio-output",
        dest="audio_output",
        nargs='?',
        const="output.wav",
        type=str,
        default="output.wav",
        help="Audio output filename. Default 'output.wav'. Audio filenames appended.",
    )
    return parser

def main():
    """Run the main CLI."""
    parser = get_parser()
    args = parser.parse_args()

    logging.basicConfig(level=args.debug)

    processed_audio_arrays = commands.process(
        args.audio_inputs,
        sample_rate=args.sample_rate,
        speed_multiplier=args.speed_multiplier,
        concatenate=args.concatenate,
        mono=args.mono,
        compress=args.compress,
        normalize=args.normalize,
        reverse=args.reverse,
    )

    for processed_audio_array, audio_input in zip(processed_audio_arrays, args.audio_inputs):

        _, output_filename = os.path.split(args.audio_output)
        prefix, _ = os.path.splitext(output_filename)

        _, input_filename = os.path.split(audio_input)
        suffix, _ = os.path.splitext(input_filename)

        audio_output = "{prefix}_{suffix}.wav".format(prefix=prefix, suffix=suffix)

        if len(processed_audio_arrays) == 1 and args.concatenate:
            audio_output = args.audio_output

        commands.save(
            processed_audio_array,
            sample_rate=args.sample_rate,
            bit_rate=args.bit_rate,
            audio_output=audio_output
        )


import random
import pandas as pd
import boto3
import io
import numpy as np
import unicodedata
import csv

s3 = boto3.client('s3')

def bonusFunc(pvCsv,handicap,averageScore,header,userId):
  # 走ってる
  print('userId2開始',userId)

  br = 0
  possibilitiesAndValueArray = []
  result = []
  # 一回叩くだけ
  test = []
  l = [float(x) for x in list(str(averageScore))]
  averageScore2 = float(float(averageScore)-l[-1])
  for row in pvCsv:
      # 一発目だけ走ってる
      print('userId2途中',userId)
      testArray = []
      for i,v in enumerate(row):
          if i == 0:
            testArray.append(v)
          else:
            testArray.append(float(v.split('"')[0]))
      possibilitiesAndValueArray.append(testArray)
  for possibilitiesAndValue in possibilitiesAndValueArray:
    # 一発目だけ走ってる
    print('userId2',userId)

    print('possibilitiesAndValueCompute',possibilitiesAndValueCompute(header,possibilitiesAndValue,averageScore2))

    if possibilitiesAndValueCompute(header,possibilitiesAndValue,averageScore2) == 1:
    # if 1 == 1:
      # 走ってる
      print('1です')
      print('possibilitiesAndValueArray',possibilitiesAndValueArray)
      # 走ってない
      br += possibilitiesAndValue[1]
      # 一発目だけ走ってる
      print(f'br:{userId}',br)
    else:
      # 走ってる
      print('0です')

  print(f'bonus result{userId}',float(br)+float(handicap))
  return float(br)+float(handicap)

def possibilitiesAndValueCompute(header,row,score):

  array = []
  obj = {}
  value = 0
  keyArray = []
  for i,h in enumerate(header):
    if i == 0:
      continue
    elif i == 1:
      obj['value'] = row[i]
    else:
      key = float(h.replace('"',''))
      keyArray.append(key)
      obj[key] = row[i]
  if score<keyArray[0]:
    score = keyArray[0]
  elif score>keyArray[-1]:
    score = keyArray[-1]
  value = random.choices([0,1],k=1,weights=[1-obj[score],obj[score]])[0]
  return value

def totalPossibilitiesAndValueCompute(header,row):

  array = []
  obj = {}
  value = 0
  for i,h in enumerate(header):
    if i == 0:
      continue
    elif i == 1:
      obj['value'] = row[i]
    else:
      key = float(h.replace('"',''))
      obj['p'] = row[i]

  # value = random.choices([0,1],k=1,weights=[1-obj[score],obj[score]])[0]
  return obj

def computeNetScore(obj):
  pvCsv = obj['pvCsv']
  handicap = float(obj['handicap'])
  averageScore = obj['averageScore']
  header = obj['header']
  min = obj['min']
  max = obj['max']
  totalCourse = obj['totalCourse']
  rangeArray = obj['rangeArray']
  slopeRating = obj['slopeRating']
  value2Array = obj['value2Array']
  average = (min+max)/2
  hiddenCourse = get_integral_value_combination(totalCourse, 48)
  npData = [round(p) for p in np.random.normal(average, 1, 18)]
  grossScore = sum(npData)+72
  hiddenCourseScore = 0
  t = 0
  isBonus = 0
  userId = obj['userId']
  for i,course in enumerate(totalCourse):
    t +=course+npData[i]
    if len(hiddenCourse) !=0 and hiddenCourse[0] == course:
      hiddenCourseScore+=course+npData[i]
      hiddenCourse.remove(course)
      if isBonus ==0:
        bonusResult = bonusFunc(pvCsv,handicap,averageScore,header,userId)
        # ちゃんと計算されてる
        print('bonusResult',bonusResult)
        handicap += bonusResult
        isBonus +=1

  print('after userId',userId)
  print('after handicap',handicap)

  growth = 0
  par = 72

  growth = 0
  # growth =  averageScore-grossScore

  # growthObj = value2Array[0][0]
  # l = [float(x) for x in list(str(growth[i]))]
  # growth = float(growth-l)
  # growthKeys = l.keys()
  # print('handicap',handicap)
  # if growthKeys[-1]>growth:
  #   growth = growthKeys[-1]
  # try:
  #   growthIndex = growthKeys.index(growth)
  #   growth = growthObj[growthKeys[growthIndex]]
  # except:
  #   print('no')

  # print(f'min:{min}',f'max:{max}')

  netScore = grossScore - (hiddenCourseScore*1.5-par)*0.8+ handicap -(slopeRating-55)/10+growth
  print('grossScore',grossScore)
  print('netScore',netScore)

  return {
    'netScore':netScore,
    'grossScore':grossScore,
    'hiddenCourseScore':hiddenCourseScore,
    'handicap':handicap,
    'growth':growth

  }

def get_integral_value_combination(value, target):
    def a(idx, l, r, t):
        if t == sum(l): r.append(l)
        elif t < sum(l): return
        for u in range(idx, len(value)):
            a((u + 1), l + [value[u]], r, t)
        return r
    return a(0, [], [], target)[0]

def courseLayoutDF(clubName,corseName):
  bucketName = 's3-genders'

  courseLayout = unicodedata.normalize('NFKD', f'total-layout2/{clubName}{corseName}詳細.csv')
  d = s3.get_object(Bucket=bucketName, Key=courseLayout)

  df = pd.read_csv(io.BytesIO(d['Body'].read()))
  cols = [2,3,5,6,8,9]
  dropDF = df.drop(df.columns[cols],axis=1)
  afterDF = ''
  try:
    afterDF = dropDF[dropDF["HOLE：1"].str.contains("PAR：", na=False)]
  except:
    afterDF = dropDF[dropDF["HOLE：10"].str.contains("PAR：", na=False)]

  afterArray = afterDF.values
  resultArray = []
  for v in range(len(afterDF.values)):
    for i in range(len(afterArray[v])-1):
      resultArray.append(int(afterArray[v][i+1].split('：')[1]))
  return list(resultArray)

def mainFunc(handicap,rangeArray,pvCsv,value2Array,header2,mainBody):
  st = io.StringIO()
  st.write(mainBody)
  st.seek(0)
  csv_f =csv.reader(st)
  header = csv_f.__next__()  # ヘッダーの読み込み
  content = [row for row in csv_f]  # 各年のデータを要素とするリスト

  rArray  = rangeArray[0]
  genderArray = []
  userIdArray = [x+1 for x in range(1000)]
  ages = [0,1,2,3,4,5,6]
  maleProbability =  [1.1, 2.2, 10.0, 18.9, 27.8, 28.9, 11.1]
  femaleProbability = [0.01,0.02,0.17,0.25,0.26,0.24,0.05]

  averageScores = [0,1,2,3,4,5,6]

  averageScoreProbability = [0.036,0.263,0.434,0.199,0.048,0.019,0.001]

  for i in range(1000):
    if (i+1)%10 == 0:
      genderArray.append(1)
    else:
      genderArray.append(0)

  ages = [0,1,2,3,4,5,6]
  maleProbability =  [1.1, 2.2, 10.0, 18.9, 27.8, 28.9, 11.1]
  femaleProbability = [0.01,
                      0.02,
                      0.17,
                      0.25,
                      0.26,
                      0.24,
                      0.05]

  averageScores = [0,1,2,3,4,5,6]

  averageScoreProbability = [0.036,0.263,0.434,0.199,0.048,0.019,0.001]

  for i in range(len(maleProbability)):
    maleProbability[i]=round(maleProbability[i]/100,4)

  par = 72

  totalGrossScoreArray = []
  averageScoreArray = []

  userIdArray = [x+1 for x in range(1000)]
  ageArray = []

  for i in range(1000):
    if genderArray[i] == 0:
      p = random.choices(ages,k=1,weights=maleProbability)[0]
      ageArray.append(p)
      if p<6:
        aroundAverageScore = random.choices(averageScores,k=1,weights=averageScoreProbability)[0]
        averageScore = 72+aroundAverageScore*10+random.randint(0,9)
        averageScoreArray.append(averageScore)
      else:
        aroundAverageScore = random.choices(averageScores,k=1,weights=averageScoreProbability)[0]
        averageScore = random.randint(131,150)
        averageScoreArray.append(averageScore)
    else:
      p = random.choices(ages,k=1,weights=femaleProbability)[0]
      ageArray.append(p)
      if p<6:
        aroundAverageScore = random.choices(averageScores,k=1,weights=averageScoreProbability)[0]
        averageScore = 72+aroundAverageScore*10+random.randint(0,9)
        averageScoreArray.append(averageScore)
      else:
        aroundAverageScore = random.choices(averageScores,k=1,weights=averageScoreProbability)[0]
        averageScore = random.randint(131,150)
        averageScoreArray.append(averageScore)

  userDF = pd.DataFrame({
          'userId':userIdArray,
          'gender':genderArray,
          'age':ageArray,
          'averageScore':averageScoreArray,
  })
  gettingDf  = pd.DataFrame({
              'userId':[],
          'gender':[],
          'age':[],
          'averageScore':[],
          'grossScore':[],
          'hiddenScore':[],
          'netScore':[],
          'handicap':[],
          'prevNet':[],
          'corseName':[],
          'courseRating':[],
          'corseType':[],
          'slopeRating':[],
          'weather':[]
  })

  dfArray = []
  df2Array = []

  rank = [x+1 for x in range(1000)]
  geddingArray = []

  for g in range(1000):
    if g+1<=100:
      geddingArray.append(1)
    elif (g+1)%5 ==0 and (g+1)!=1000:
      geddingArray.append(1)
    elif g+1 == 999:
      geddingArray.append(1)
    else:
      geddingArray.append(0)

  bucketName = 's3-genders'
  maleObj = s3.get_object(Bucket=bucketName, Key='final-male.csv')
  maleDf = pd.read_csv(io.BytesIO(maleObj['Body'].read()))

  femaleObj = s3.get_object(Bucket=bucketName, Key='final-female.csv')
  femaleDf = pd.read_csv(io.BytesIO(femaleObj['Body'].read()))

  maleDf = maleDf.query('Par == 72')
  femaleDf = femaleDf.query('Par == 72')

  genderArray = []

  for i in range(1000):
    if (i+1)%10 == 0:
      genderArray.append(1)
    else:
      genderArray.append(0)

  ages = [0,1,2,3,4,5,6]
  maleProbability =  [1.1, 2.2, 10.0, 18.9, 27.8, 28.9, 11.1]
  femaleProbability = [0.01,0.02,0.17,0.25,0.26,0.24,0.05]

  averageScores = [0,1,2,3,4,5,6]

  averageScoreProbability = [0.036,0.263,0.434,0.199,0.048,0.019,0.001]

  for i in range(len(maleProbability)):
    maleProbability[i]=round(maleProbability[i]/100,4)

  par = 72

  totalGrossScoreArray = []
  averageScoreArray = []

  userIdArray = [x+1 for x in range(1000)]
  ageArray = []

  for i in range(1000):
    if genderArray[i] == 0:
      p = random.choices(ages,k=1,weights=maleProbability)[0]
      ageArray.append(p)
      if p<6:
        aroundAverageScore = random.choices(averageScores,k=1,weights=averageScoreProbability)[0]
        averageScore = 72+aroundAverageScore*10+random.randint(0,9)
        averageScoreArray.append(averageScore)
      else:
        aroundAverageScore = random.choices(averageScores,k=1,weights=averageScoreProbability)[0]
        averageScore = random.randint(131,150)
        averageScoreArray.append(averageScore)
    else:
      p = random.choices(ages,k=1,weights=femaleProbability)[0]
      ageArray.append(p)
      if p<6:
        aroundAverageScore = random.choices(averageScores,k=1,weights=averageScoreProbability)[0]
        averageScore = 72+aroundAverageScore*10+random.randint(0,9)
        averageScoreArray.append(averageScore)
      else:
        aroundAverageScore = random.choices(averageScores,k=1,weights=averageScoreProbability)[0]
        averageScore = random.randint(131,150)
        averageScoreArray.append(averageScore)

  userDF = pd.DataFrame({
          'userId':userIdArray,
          'gender':genderArray,
          'age':ageArray,
          'averageScore':averageScoreArray,
  })
  gettingDf  = pd.DataFrame({
              'userId':[],
          'gender':[],
          'age':[],
          'averageScore':[],
          'grossScore':[],
          'hiddenScore':[],
          'netScore':[],
          'handicap':[],
          'prevNet':[],
          'corseName':[],
          'courseRating':[],
          'corseType':[],
          'slopeRating':[],
          'weather':[]
  })

  dfArray = []
  df2Array = []

  rank = [x+1 for x in range(1000)]
  geddingArray = []

  for g in range(1000):
    if g+1<=100:
      geddingArray.append(1)
    elif (g+1)%5 ==0 and (g+1)!=1000:
      geddingArray.append(1)
    elif g+1 == 999:
      geddingArray.append(1)
    else:
      geddingArray.append(0)
  for _ in range(1):
      userIdArray2 = []
      genderArray2 = []
      ageArray2 = []
      averageScoreArray2 = []
      hiddenScoreArray = []
      grossScoreArray = []
      netScoreArray = []
      course = []
      clubNameArray = []
      corse1NameArray = []
      corse2NameArray = []
      courseRatingArray = []
      slopeRatingArray = []
      corseTypeArray = []
      handicapArray = []
      prevNetArray = []
      weatherArray = []
      slopeRateWeigthArray = []
      weatherIndexWeigthArray = []
      growthArray = []
      for i in range(len(userIdArray)):
        weatherIndex = -1*random.choices([0,1,2],k=1,weights=[0.7,0.2,1])[0]
        weatherArray.append(weatherIndex)
        # slopeRateWeigthArray.append(slopeRateWeigth)
        # weatherIndexWeigthArray.append(weatherIndexWeigth)
        userIdArray2.append(userIdArray[i])
        genderArray2.append(genderArray[i])
        ageArray2.append(ageArray[i])
        averageScoreArray2.append(averageScoreArray[i])
        obj = {}
        # text = [row for row in reader]
        # print('text',text)
        if genderArray[i] == 0:
          n = np.random.randint(0,len(maleDf))
          clubName = maleDf.iloc[n]['Unnamed: 0.1.1.1']
          clubNameArray.append(clubName)
          corse1Name = maleDf.iloc[n]['コース１']
          corse1NameArray.append(corse1Name)
          corse2Name = maleDf.iloc[n]['コース2']
          corse2NameArray.append(corse2Name)
          courseRating = maleDf.iloc[n]['CourseRating']
          courseRatingArray.append(courseRating)
          slopeRating  = int(maleDf.iloc[n]['SlopeRating'])
          slopeRatingArray.append(slopeRating)
          corseType = maleDf.iloc[n]['コース１.2']
          corseTypeArray.append(corseType)
          corse1Name = courseLayoutDF(clubName,corse1Name)
          corse2Name = courseLayoutDF(clubName,corse2Name)
          totalCourse = corse2Name+corse1Name

          l = [float(x) for x in list(str(averageScoreArray[i]))]
          averageScore2 = float(averageScoreArray[i] - l[-1])
          if averageScore2<70:
            averageScore2 = 70
          elif averageScore2>140:
            averageScore2 = 140
          testObj = {}

          min = 0
          max = 0
          try:
            min = rArray[averageScore2][0]
            max = rArray[averageScore2][1]
          except:
            print('no')
          obj['min'] = min
          obj['max'] = max
          obj['totalCourse'] = totalCourse
          obj['averageScore'] = averageScoreArray[i]
          obj['rangeArray'] = rangeArray
          obj['pvCsv'] = content
          obj['handicap'] = handicap
          obj['slopeRating'] = slopeRating
          obj['value2Array'] = value2Array
          obj['header'] = header2
          obj['userId'] = i
          result = computeNetScore(obj)
          result['userId'] = i
          netScoreArray.append(result['netScore'])
          grossScoreArray.append(result['grossScore'])
          hiddenScoreArray.append(result['hiddenCourseScore'])
          handicapArray.append(result['handicap'])
          # growthArray.append(result['growth'])
        else:
          n = np.random.randint(0,len(maleDf))
          clubName = maleDf.iloc[n]['Unnamed: 0.1.1.1']
          clubNameArray.append(clubName)
          corse1Name = maleDf.iloc[n]['コース１']
          corse1NameArray.append(corse1Name)
          corse2Name = maleDf.iloc[n]['コース2']
          corse2NameArray.append(corse2Name)
          courseRating = maleDf.iloc[n]['CourseRating']
          courseRatingArray.append(courseRating)
          slopeRating  = int(maleDf.iloc[n]['SlopeRating'])
          slopeRatingArray.append(slopeRating)
          corseType = maleDf.iloc[n]['コース１.2']
          corseTypeArray.append(corseType)

          corse1Name = courseLayoutDF(clubName,corse1Name)
          corse2Name = courseLayoutDF(clubName,corse2Name)
          totalCourse = corse2Name+corse1Name

          l = [float(x) for x in list(str(averageScoreArray[i]))]
          averageScore2 = float(averageScoreArray[i] - l[-1])
          if averageScore2<70:
            averageScore2 = 70
          elif averageScore2>140:
            averageScore2 = 140
          testObj = {}

          min = 0
          max = 0
          try:
            min = rArray[averageScore2][0]
            max = rArray[averageScore2][1]
          except:
            print('no')
          obj['min'] = min
          obj['max'] = max
          obj['totalCourse'] = totalCourse
          obj['averageScore'] = averageScoreArray[i]
          obj['rangeArray'] = rangeArray
          obj['pvCsv'] = content
          obj['handicap'] = handicap
          obj['slopeRating'] = slopeRating
          obj['value2Array'] = value2Array
          obj['header'] = header2
          obj['userId'] = i
          print('before userId',i)
          result = computeNetScore(obj)
          print('female result',result)
          netScoreArray.append(result['netScore'])
          grossScoreArray.append(result['grossScore'])
          hiddenScoreArray.append(result['hiddenCourseScore'])
          handicapArray.append(result['handicap'])
          # growthArray.append(result['growth'])
      ageArray.append(p)
      userDF2 = pd.DataFrame({
                'userId':userIdArray2,
                'gender':genderArray2,
                'age':ageArray2,
                'averageScore':averageScoreArray2,
                'grossScore':grossScoreArray,
                'hiddenScore':hiddenScoreArray,
                'netScore':netScoreArray,
                'handicap':handicapArray,
                'clubName':clubNameArray,
                'corse1':corse1NameArray,
                'corse2':corse2NameArray,
                'courseRating':courseRatingArray,
                'corseType':corseTypeArray,
                'slopeRating':slopeRatingArray,
                'weather':weatherArray,
                # 'growth':growthArray,
                # 'slopeRateWeigth':slopeRateWeigthArray,
                # 'weatherIndexWeigth':weatherIndexWeigthArray

                })

      userDF3 = userDF2.sort_values("netScore")
      userDF3['rank'] = rank
      userDF3['getting'] = geddingArray
      df2Array.append(userDF3)

  totalDF = pd.concat(df2Array)
  return totalDF

  # df = totalDF.to_csv("/content/drive/MyDrive/golf/total6.csv")


import math

def factorial(n):
    sum_factorial = 1
    for i in range(1, n + 1):
        sum_factorial *= i
    return sum_factorial

def bernouli(n):
    if n <= 0:
        return 1
    else:
        s = 0
        for k in range(1, n + 1):
            s += bink(n + 1, k + 1) * bernouli(n - k)
        return -1 / (n + 1) * s

# Биномиальный коэффициент Ньютона
def bink(n, k):
    return factorial(n) / (factorial(k) * factorial(n - k))

# alpha = 1.1
# j = 1
# n = math.pow(2, 1)
# z_exact = 10.5844484649508098
#
# z_n = -1 / (alpha - 1) * math.pow(n, -(alpha - 1)) + 0.5 * math.pow(n, -alpha)
# for i in range(1, j + 1):
#     divisior = 1
#     for k in range(1, i + 1):
#         divisior *= (alpha + 2 * k - 2)
#     divisior = -(divisior * bernouli(2 * i) * math.pow(n, -(alpha + 2 * i - 1))) / factorial(2 * i)
#     z_n += divisior
# z_n = z_exact - z_n
# print(z_n)

# j - степень при n
# z_exact - массив z_n  вычисленных при предыдущем шаге
# возращает массив погрешностей текущий результат минус предыдущий
def extrapolation(j, z_exact):
    alpha = 1.1
    n = []
    zn_zexact = []
    z_n1 = []
    z_n_temp = [10.5844484649508098 - 1 / (alpha - 1) * math.pow(1, -(alpha - 1)) + 0.5 * math.pow(1, -alpha)]
    for degree in range(1, j + 1):
        n.append(math.pow(2, degree))
        z_n = 1 / (alpha - 1) * math.pow(n[-1], -(alpha - 1)) - 0.5 * math.pow(n[-1], -alpha)
        for i in range(1, degree + 1):
            divisior = 1
            for k in range(1, i + 1):
                divisior *= (alpha + 2 * k - 2)
            divisior = (divisior * bernouli(2 * i) * math.pow(n[-1], -(alpha + 2 * i - 1))) / factorial(2 * i)
            z_n += divisior
        z_n = z_exact[degree] - z_n
        z_n1.append(z_n + (z_n - z_n_temp[-1]) / (math.pow(2, alpha) - 1))
        xer = z_n - z_n1[-1]
        xer2 = z_n1[-1] - z_exact[degree]
        z_n_temp.append(z_n)
        zn_zexact.append(z_n - z_exact[degree])
    return z_n_temp

def extrapolation2(j, z_exact):
    alpha = 1.1
    n = []
    zn_zexact = []
    z_n_temp = [z_exact[0] - 1 / (alpha - 1) * math.pow(1, -(alpha - 1)) + 0.5 * math.pow(1, -alpha)]
    for degree in range(1, j + 1):
        n.append(math.pow(2, degree))
        z_n = 1 / (alpha - 1) * math.pow(n[-1], -(alpha - 1)) - 0.5 * math.pow(n[-1], -alpha)
        for i in range(1, degree + 1):
            divisior = 1
            for k in range(1, i + 1):
                divisior *= (alpha + 2 * k - 2)
            divisior = (divisior * bernouli(2 * i) * math.pow(n[-1], -(alpha + 2 * i - 1))) / factorial(2 * i)
            z_n += divisior
        z_n = z_exact[degree] - z_n
        z = z_n + (z_n - z_n_temp[-1]) / (math.pow(2, degree) - 1)
        z_n_temp.append(z_n)
        zn_zexact.append(z_exact[degree] - z)
    return zn_zexact

z_exact = [10.5844484649508098 for i in range(15)]
z_temp = extrapolation(4, z_exact)
print(z_temp)
z_exact2 = extrapolation2(4, z_temp)
print(z_exact2)


import sys
# sys.path.append("/code/python/check_mt")
import pathlib
sys.path.append(f"{pathlib.Path(__file__).parent.parent.absolute()}")

import tracemalloc
import socket, requests, time, datetime, threading
import concurrent.futures
from includes.RootLogger import rootLogger
from includes.RedisWrapperClass import RedisWrapperClass
import includes.Database as db
#
from includes.CampaignThread import CS_CREATED, CS_PROCESSING, CS_PAUSED, CS_HIBERNATED, CS_COMPLETED, CS_DELETED
from webmails.WebMailClass import display
if display.is_started:
	rootLogger.info("Stopping X with display")
	display.stop()

import traceback
import random
import ssl
import re

import sys
import string
import os

from dotenv import load_dotenv
from os.path import join, dirname
import redis
import json
import zlib

def my_crc32(my_str):
	return hex(zlib.crc32(my_str.encode('utf-8')) % (1<<32))

def rule_processor(record):
	r_name = f"r_{my_crc32(record.group + record.rule)}"
	ret = {
		'regex': f"(?P<{r_name}>{record.rule})",
		'rule': record.rule,
		'group': record.group,
		'r_name': r_name,
	}
	return ret

def processCampaignAccount(campaign, account, rule_regex=None, my_redis=None):
	from includes.SMTPThread import RS_NEW, RS_PROCESSING, RS_SUCCESS, RS_FAILED, RS_SKIPPED, RS_BLACKLIST

	if my_redis is None:
		my_redis = RedisWrapperClass()

	# raise Exception('pizdec')

	# get all contacts for account
	contacts = db.getAccountAddressBook( [account.id] )
	good_contacts = [item for item in contacts if not rule_regex.search( item.address )]
	fail_contacts = [item for item in contacts if rule_regex.search( item.address )]

	# save failed contacts immedeately
	fail_records = []
	for contact in fail_contacts:
		save_item = {
			'campaign_id': campaign.id,
			'mail_account_addressbook_id': contact.id,
			'mail_account_id': account.id,
			'spam_base_id': None,
			'spam_base_record_id': None,
			'address': contact.address,
			'name': contact.name,
			'company': contact.company,
			'rest': contact.rest,
			'record_status': RS_BLACKLIST,
			'record_status_txt': f'Blacklisted due to regex:{rule}',
			'dump_subject': '',
			'dump_body': '',
			'dump_headers': '',
		}
		fail_records.append( save_item )

	db.putRecordsToRedis( campaign=campaign, mail_account=account, records=fail_records, record_status=RS_BLACKLIST, record_status_txt='skipped via regexp', my_redis=my_redis )

	good_records = []
	# for normal contacts get whole
	for contact in good_contacts:
		conversation = db.getAddressBookContactConversation( account, contact, max_days=1000 )
		rootLogger.info(f"[processCampaignAccount({account.id})][{contact.id}]:{conversation}")

		dump_subject = next( (item.subject for item in conversation), '')
		dump_body = ''.join( [item.body for item in conversation] )
		dump_headers = json.dumps(next( (item.headers for item in conversation), {}))

		save_item = {
			'campaign_id': campaign.id, # $campaign_id,
			'mail_account_addressbook_id': contact.id, # property_exists($item , 'id') ? $item->id : null,
			'mail_account_id': account.id, # $mail_account_id,
			'spam_base_id': None, # property_exists($item , 'spam_base_id') ? $item->spam_base_id : null,
			'spam_base_record_id': None, # property_exists($item , 'spam_base_record_id') ? $item->spam_base_record_id : null,
			'address': contact.address, # $item->address,
			'name': contact.name, # $item->name,
			'company': contact.company, # $item->company,
			'rest': contact.rest, # $item->rest,
			'record_status': RS_NEW, # $record_status,
			'record_status_txt': '', # empty($item->record_status_txt) ? $record_status_txt : $item->record_status_txt,
			'dump_subject': dump_subject, # empty($item->dump_subject) ? '' : $item->dump_subject,
			'dump_body': dump_body, # empty($item->dump_body) ? '' : $item->dump_body,
			'dump_headers': dump_headers, # empty($item->dump_headers) ? '' : $item->dump_headers,
		}

		good_records.append( save_item )

	db.putRecordsToRedis( campaign=campaign, mail_account=account, records=fail_records, record_status=RS_NEW, record_status_txt='', my_redis=my_redis )

if __name__ == "__main__":

	all_campaigns = db.getCampaigns(status=None, name='universalio')
	# rootLogger.info(f"all_campaigns:{all_campaigns}")
	try:
		first_one = next( all_campaigns )
	except Exception as e:
		# first_one = all_campaigns
		rootLogger.info("no campaign, create new")
		my_campaign = db.createCampaign( status=CS_PAUSED, title='universalio', subject='Take a look at document attached' )

		all_campaigns = db.getCampaigns( status=None, name='universalio' )
		first_one = next( all_campaigns )

	rootLogger.info(f"First campaign:{first_one.id}")

	db.appendAccountsToCampaign(first_one.id)

	# # get all addresses - no reason as we might process per account in multithread
	# addresses = db.getCampaignAccountsAddresses( first_one.id )

	# generate filter for addresses
	rules = db.getCampaignScanRules( first_one.filters )
	rules_processed = [rule_processor(item) for item in rules]
	# rootLogger.info(f"rules_processed:{rules_processed}")

	if len( rules_processed ):
		rule_regex = "/" + "|".join( [item['regex'] for item in rules_processed] ) + "/i"
	else:
		rule_regex = "/^\!skip-me\!/"

	# rootLogger.info(f"rule_regex:{rule_regex}")
	rule_regex = re.compile(rule_regex)

	accounts = db.getCampaignAccounts(first_one.id, iterable_ret=False)

	# rootLogger.info(f"campaign accounts:{accounts}")

	my_redis = RedisWrapperClass()

	with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
		future_to_url = {executor.submit(processCampaignAccount, first_one, account, rule_regex, my_redis): account for account in accounts}
		# iterate through all
		for future in concurrent.futures.as_completed(future_to_url):
			account = future_to_url[future]

			if future.exception():
				formatted = traceback.format_tb(future.exception().__traceback__)
				new_line = "\n"
				rootLogger.error(f"Account[{account.id}] process:{future.exception()}")
				rootLogger.error(f"Account[{account.id}] process:{new_line}{''.join(formatted)}")
			else:
				rootLogger.info(f"Account[{account.id}] process: ok")

	# filter addresses
	# filtered_addresses = [item for item in addresses if not rule_regex.search( item.address )]
	# bad_addresses = [item for item in addresses if rule_regex.search( item.address )]
	# rootLogger.info(f"First campaign addresses:{[item.address for item in addresses]}")
	# rootLogger.info(f"First campaign addresses:{[item.address for item in filtered_addresses]}")
	# rootLogger.info(f"First campaign addresses filtered:{len(filtered_addresses)}")


# https://www.acmicpc.net/problem/11723

import sys
input = sys.stdin.readline

n = int(input())
store = 0
for _ in range(n):
    op = input().rstrip()

    if op == "all":
        store = 2 ** 20-1
        continue
    elif op == "empty":
        store = 0
        continue
    op, num = op.split()
    num = int(num)
    if op == "add":
        store = store | (2 ** (num - 1))
    elif op == "remove":
        store = store & ((2 ** 20 - 1) ^ (2 ** (num - 1)))
    elif op == "check":
        if store & (2 ** (num - 1)):
            print(1)
        else: print(0)
    elif op == "toggle":
        store = store ^ (2 ** (num - 1))


class Solution(object):
    def slidingPuzzle(self, board):
        """
        :type board: List[List[int]]
        :rtype: int
        """
        ss = ''.join([str(i) for i in (board[0] + board[1])])
        sp = [[1, 3], [0, 2, 4], [1, 5], [0, 4], [1, 3, 5], [2, 4]]
        seen, step = set(), 0
        cl, nl = [ss], []
        while cl:
            for cur in cl:
                if cur == '123450':
                    return step
                ind = cur.index('0')
                for pos in sp[ind]:
                    nsl = [ch for ch in cur]
                    nsl[ind], nsl[pos] = nsl[pos], nsl[ind]
                    next_str = ''.join(nsl)
                    if next_str not in seen:
                        seen.add(next_str)
                        nl.append(next_str)
            cl, nl = nl, []
            step += 1
        return -1


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.

import functools
import glob
import logging
import os
import re
import shutil
import stat
import sys
import tarfile
import tempfile
from collections import defaultdict

import mozfile
import mozpack.path as mozpath
import requests

from mozbuild.base import MozbuildObject
from mozbuild.vendor.rewrite_mozbuild import (
    MozBuildRewriteException,
    add_file_to_moz_build_file,
    remove_file_from_moz_build_file,
)

DEFAULT_EXCLUDE_FILES = [".git*", ".git*/**"]
DEFAULT_KEEP_FILES = ["**/moz.build", "**/moz.yaml"]
DEFAULT_INCLUDE_FILES = []

def throwe():
    raise Exception

def _replace_in_file(file, pattern, replacement, regex=False):
    with open(file) as f:
        contents = f.read()

    if regex:
        newcontents = re.sub(pattern, replacement, contents)
    else:
        newcontents = contents.replace(pattern, replacement)

    if newcontents == contents:
        raise Exception(
            "Could not find '%s' in %s to %sreplace with '%s'"
            % (pattern, file, "regex-" if regex else "", replacement)
        )

    with open(file, "w") as f:
        f.write(newcontents)

def list_of_paths_to_readable_string(paths):
    # From https://stackoverflow.com/a/41578071
    dic = defaultdict(list)
    for item in paths:
        if os.path.isdir(item):  # To check path is a directory
            _ = dic[item]  # will set default value as empty list
        else:
            path, file = os.path.split(item)
            dic[path].append(file)

    final_string = "["
    for key, val in dic.items():
        if len(val) == 0:
            final_string += key + ", "
        elif len(val) < 3:
            final_string += ", ".join([os.path.join(key, v) for v in val]) + ", "
        elif len(val) < 10:
            final_string += "%s items in %s: %s and %s, " % (
                len(val),
                key,
                ", ".join(val[0:-1]),
                val[-1],
            )
        else:
            final_string += "%s (omitted) items in %s, " % (len(val), key)

    if final_string[-2:] == ", ":
        final_string = final_string[:-2]

    return final_string + "]"

class VendorManifest(MozbuildObject):
    def should_perform_step(self, step):
        return step not in self.manifest["vendoring"].get("skip-vendoring-steps", [])

    def vendor(
        self,
        command_context,
        yaml_file,
        manifest,
        revision,
        ignore_modified,
        check_for_update,
        force,
        add_to_exports,
        patch_mode,
    ):
        self.manifest = manifest
        self.yaml_file = yaml_file
        self._extract_directory = throwe
        self.logInfo = functools.partial(self.log, logging.INFO, "vendor")
        if "vendor-directory" not in self.manifest["vendoring"]:
            self.manifest["vendoring"]["vendor-directory"] = os.path.dirname(
                self.yaml_file
            )

        # ==========================================================
        # If we're only patching; do that
        if "patches" in self.manifest["vendoring"] and patch_mode == "only":
            self.import_local_patches(
                self.manifest["vendoring"]["patches"],
                os.path.dirname(self.yaml_file),
                self.manifest["vendoring"]["vendor-directory"],
            )
            return

        # ==========================================================
        self.source_host = self.get_source_host()

        ref_type = self.manifest["vendoring"].get("tracking", "commit")
        flavor = self.manifest["vendoring"].get("flavor", "regular")
        # Individiual files are special

        if revision == "tip":
            # This case allows us to force-update a tag-tracking library to master
            new_revision, timestamp = self.source_host.upstream_commit("HEAD")
        elif ref_type == "tag":
            new_revision, timestamp = self.source_host.upstream_tag(revision)
        else:
            new_revision, timestamp = self.source_host.upstream_commit(revision)

        self.logInfo(
            {"ref_type": ref_type, "ref": new_revision, "timestamp": timestamp},
            "Latest {ref_type} is {ref} from {timestamp}",
        )

        # ==========================================================
        if not force and self.manifest["origin"]["revision"] == new_revision:
            # We're up to date, don't do anything
            self.logInfo({}, "Latest upstream matches in-tree.")
            return
        elif flavor != "individual-file" and check_for_update:
            # Only print the new revision to stdout
            print("%s %s" % (new_revision, timestamp))
            return

        # ==========================================================
        if flavor == "regular":
            self.process_regular(
                new_revision, timestamp, ignore_modified, add_to_exports
            )
        elif flavor == "individual-files":
            self.process_individual(new_revision, timestamp, ignore_modified)
        elif flavor == "rust":
            self.process_rust(
                command_context,
                self.manifest["origin"]["revision"],
                new_revision,
                timestamp,
                ignore_modified,
            )
        else:
            raise Exception("Unknown flavor")

    def process_rust(
        self, command_context, old_revision, new_revision, timestamp, ignore_modified
    ):
        # First update the Cargo.toml
        cargo_file = os.path.join(os.path.dirname(self.yaml_file), "Cargo.toml")
        try:
            _replace_in_file(cargo_file, old_revision, new_revision)
        except Exception:
            # If we can't find it the first time, try again with a short hash
            _replace_in_file(cargo_file, old_revision[:8], new_revision)

        # Then call ./mach vendor rust
        from mozbuild.vendor.vendor_rust import VendorRust

        vendor_command = command_context._spawn(VendorRust)
        vendor_command.vendor(
            ignore_modified=True, build_peers_said_large_imports_were_ok=False
        )

        self.update_yaml(new_revision, timestamp)

    def process_individual(self, new_revision, timestamp, ignore_modified):
        # This design is used because there is no github API to query
        # for the last commit that modified a file; nor a way to get file
        # blame.  So really all we can do is just download and replace the
        # files and see if they changed...

        def download_and_write_file(url, destination):
            self.logInfo(
                {"local_file": destination, "url": url},
                "Downloading {local_file} from {url}...",
            )

            with mozfile.NamedTemporaryFile() as tmpfile:
                try:
                    req = requests.get(url, stream=True)
                    for data in req.iter_content(4096):
                        tmpfile.write(data)
                    tmpfile.seek(0)

                    shutil.copy2(tmpfile.name, destination)
                except Exception as e:
                    raise (e)

        # Only one of these loops will have content, so just do them both
        for f in self.manifest["vendoring"].get("individual-files", []):
            url = self.source_host.upstream_path_to_file(new_revision, f["upstream"])
            destination = self.get_full_path(f["destination"])
            download_and_write_file(url, destination)

        for f in self.manifest["vendoring"].get("individual-files-list", []):
            url = self.source_host.upstream_path_to_file(
                new_revision,
                self.manifest["vendoring"]["individual-files-default-upstream"] + f,
            )
            destination = self.get_full_path(
                self.manifest["vendoring"]["individual-files-default-destination"] + f
            )
            download_and_write_file(url, destination)

        self.spurious_check(new_revision, ignore_modified)

        self.logInfo({}, "Checking for update actions")
        self.update_files(new_revision)

        self.update_yaml(new_revision, timestamp)

        self.logInfo({"rev": new_revision}, "Updated to '{rev}'.")

        if "patches" in self.manifest["vendoring"]:
            # Remind the user
            self.log(
                logging.CRITICAL,
                "vendor",
                {},
                "Patches present in manifest!!! Please run "
                "'./mach vendor --patch-mode only' after commiting changes.",
            )

    def process_regular(self, new_revision, timestamp, ignore_modified, add_to_exports):

        if self.should_perform_step("fetch"):
            self.fetch_and_unpack(new_revision)
        else:
            self.logInfo({}, "Skipping fetching upstream source.")

        self.logInfo({}, "Checking for update actions")
        self.update_files(new_revision)

        if self.should_perform_step("hg-add"):
            self.logInfo({}, "Registering changes with version control.")
            self.repository.add_remove_files(
                self.manifest["vendoring"]["vendor-directory"],
                os.path.dirname(self.yaml_file),
            )
        else:
            self.logInfo({}, "Skipping registering changes.")

        if self.should_perform_step("spurious-check"):
            self.logInfo({}, "Checking for a spurious update.")
            self.spurious_check(new_revision, ignore_modified)
        else:
            self.logInfo({}, "Skipping the spurious update check.")

        if self.should_perform_step("update-moz-yaml"):
            self.logInfo({}, "Updating moz.yaml.")
            self.update_yaml(new_revision, timestamp)
        else:
            self.logInfo({}, "Skipping updating the moz.yaml file.")

        if self.should_perform_step("update-moz-build"):
            self.logInfo({}, "Updating moz.build files")
            self.update_moz_build(
                self.manifest["vendoring"]["vendor-directory"],
                os.path.dirname(self.yaml_file),
                add_to_exports,
            )
        else:
            self.logInfo({}, "Skipping update of moz.build files")

        self.logInfo({"rev": new_revision}, "Updated to '{rev}'.")

        if "patches" in self.manifest["vendoring"]:
            # Remind the user
            self.log(
                logging.CRITICAL,
                "vendor",
                {},
                "Patches present in manifest!!! Please run "
                "'./mach vendor --patch-mode only' after commiting changes.",
            )

    def get_source_host(self):
        if self.manifest["vendoring"]["source-hosting"] == "gitlab":
            from mozbuild.vendor.host_gitlab import GitLabHost

            return GitLabHost(self.manifest)
        elif self.manifest["vendoring"]["source-hosting"] == "github":
            from mozbuild.vendor.host_github import GitHubHost

            return GitHubHost(self.manifest)
        elif self.manifest["vendoring"]["source-hosting"] == "googlesource":
            from mozbuild.vendor.host_googlesource import GoogleSourceHost

            return GoogleSourceHost(self.manifest)
        elif self.manifest["vendoring"]["source-hosting"] == "angle":
            from mozbuild.vendor.host_angle import AngleHost

            return AngleHost(self.manifest)
        elif self.manifest["vendoring"]["source-hosting"] == "codeberg":
            from mozbuild.vendor.host_codeberg import CodebergHost

            return CodebergHost(self.manifest)
        else:
            raise Exception(
                "Unknown source host: " + self.manifest["vendoring"]["source-hosting"]
            )

    def get_full_path(self, path, support_cwd=False):
        if support_cwd and path[0:5] == "{cwd}":
            path = path.replace("{cwd}", ".")
        elif "{tmpextractdir}" in path:
            # _extract_directory() will throw an exception if it is invalid to use it
            path = path.replace("{tmpextractdir}", self._extract_directory())
        elif "{yaml_dir}" in path:
            path = path.replace("{yaml_dir}", os.path.dirname(self.yaml_file))
        elif "{vendor_dir}" in path:
            path = path.replace(
                "{vendor_dir}", self.manifest["vendoring"]["vendor-directory"]
            )
        else:
            path = mozpath.join(self.manifest["vendoring"]["vendor-directory"], path)
        return os.path.abspath(path)

    def convert_patterns_to_paths(self, directory, patterns):
        # glob.iglob uses shell-style wildcards for path name completion.
        # "recursive=True" enables the double asterisk "**" wildcard which matches
        # for nested directories as well as the directory we're searching in.
        paths = []
        for pattern in patterns:
            pattern_full_path = mozpath.join(directory, pattern)
            # If pattern is a directory recursively add contents of directory
            if os.path.isdir(pattern_full_path):
                # Append double asterisk to the end to make glob.iglob recursively match
                # contents of directory
                paths.extend(
                    glob.iglob(mozpath.join(pattern_full_path, "**"), recursive=True)
                )
            # Otherwise pattern is a file or wildcard expression so add it without altering it
            else:
                paths.extend(glob.iglob(pattern_full_path, recursive=True))
        # Remove folder names from list of paths in order to avoid prematurely
        # truncating directories elsewhere
        # Sort the final list to ensure we preserve 01_, 02_ ordering for e.g. *.patch globs
        final_paths = sorted(
            [mozpath.normsep(path) for path in paths if not os.path.isdir(path)]
        )
        return final_paths

    def fetch_and_unpack(self, revision):
        """Fetch and unpack upstream source"""

        def validate_tar_member(member, path):
            def is_within_directory(directory, target):
                real_directory = os.path.realpath(directory)
                real_target = os.path.realpath(target)
                prefix = os.path.commonprefix([real_directory, real_target])
                return prefix == real_directory

            member_path = os.path.join(path, member.name)
            if not is_within_directory(path, member_path):
                raise Exception("Attempted path traversal in tar file: " + member.name)
            if member.issym():
                link_path = os.path.join(os.path.dirname(member_path), member.linkname)
                if not is_within_directory(path, link_path):
                    raise Exception(
                        "Attempted link path traversal in tar file: " + member.name
                    )
            if member.mode & (stat.S_ISUID | stat.S_ISGID):
                raise Exception(
                    "Attempted setuid or setgid in tar file: " + member.name
                )

        def safe_extract(tar, path=".", *, numeric_owner=False):
            def _files(tar, path):
                for member in tar:
                    validate_tar_member(member, path)
                    yield member

            tar.extractall(path, members=_files(tar, path), numeric_owner=numeric_owner)

        url = self.source_host.upstream_snapshot(revision)
        self.logInfo({"url": url}, "Fetching code archive from {url}")

        with mozfile.NamedTemporaryFile() as tmptarfile:
            tmpextractdir = tempfile.TemporaryDirectory()
            try:
                req = requests.get(url, stream=True)
                for data in req.iter_content(4096):
                    tmptarfile.write(data)
                tmptarfile.seek(0)

                vendor_dir = mozpath.normsep(
                    self.manifest["vendoring"]["vendor-directory"]
                )
                if self.should_perform_step("keep"):
                    self.logInfo({}, "Retaining wanted in-tree files.")
                    to_keep = self.convert_patterns_to_paths(
                        vendor_dir,
                        self.manifest["vendoring"].get("keep", [])
                        + DEFAULT_KEEP_FILES
                        + self.manifest["vendoring"].get("patches", []),
                    )
                else:
                    self.logInfo({}, "Skipping retention of in-tree files.")
                    to_keep = []

                self.logInfo({"vd": vendor_dir}, "Cleaning {vd} to import changes.")
                # We use double asterisk wildcard here to get complete list of recursive contents
                for file in self.convert_patterns_to_paths(vendor_dir, ["**"]):
                    file = mozpath.normsep(file)
                    if file not in to_keep:
                        mozfile.remove(file)

                self.logInfo({"vd": vendor_dir}, "Unpacking upstream files for {vd}.")
                with tarfile.open(tmptarfile.name) as tar:

                    safe_extract(tar, tmpextractdir.name)

                    def get_first_dir(p):
                        halves = os.path.split(p)
                        return get_first_dir(halves[0]) if halves[0] else halves[1]

                    one_prefix = get_first_dir(tar.getnames()[0])
                    has_prefix = all(
                        map(lambda name: name.startswith(one_prefix), tar.getnames())
                    )

                # GitLab puts everything down a directory; move it up.
                if has_prefix:
                    tardir = mozpath.join(tmpextractdir.name, one_prefix)
                    mozfile.copy_contents(tardir, tmpextractdir.name)
                    mozfile.remove(tardir)

                if self.should_perform_step("include"):
                    self.logInfo({}, "Retaining wanted files from upstream changes.")
                    to_include = self.convert_patterns_to_paths(
                        tmpextractdir.name,
                        self.manifest["vendoring"].get("include", [])
                        + DEFAULT_INCLUDE_FILES,
                    )
                else:
                    self.logInfo({}, "Skipping retention of included files.")
                    to_include = []

                if self.should_perform_step("exclude"):
                    self.logInfo({}, "Removing excluded files from upstream changes.")
                    to_exclude = self.convert_patterns_to_paths(
                        tmpextractdir.name,
                        self.manifest["vendoring"].get("exclude", [])
                        + DEFAULT_EXCLUDE_FILES,
                    )
                else:
                    self.logInfo({}, "Skipping removing excluded files.")
                    to_exclude = []

                to_exclude = list(set(to_exclude) - set(to_include))
                if to_exclude:
                    self.logInfo(
                        {"files": list_of_paths_to_readable_string(to_exclude)},
                        "Removing: {files}",
                    )
                    for exclusion in to_exclude:
                        mozfile.remove(exclusion)

                # Clear out empty directories
                # removeEmpty() won't remove directories containing only empty directories
                #   so just keep callign it as long as it's doing something
                def removeEmpty(tmpextractdir):
                    removed = False
                    folders = list(os.walk(tmpextractdir))[1:]
                    for folder in folders:
                        if not folder[2]:
                            try:
                                os.rmdir(folder[0])
                                removed = True
                            except Exception:
                                pass
                    return removed

                while removeEmpty(tmpextractdir.name):
                    pass

                # Then copy over the directories
                if self.should_perform_step("move-contents"):
                    self.logInfo({"d": vendor_dir}, "Copying to {d}.")
                    mozfile.copy_contents(tmpextractdir.name, vendor_dir)
                else:
                    self.logInfo({}, "Skipping copying contents into tree.")
                    self._extract_directory = lambda: tmpextractdir.name
            except Exception as e:
                tmpextractdir.cleanup()
                raise e

    def update_yaml(self, revision, timestamp):
        with open(self.yaml_file) as f:
            yaml = f.readlines()

        replaced = 0
        replacements = [
            ["  release:", " %s (%s)." % (revision, timestamp)],
            ["  revision:", " %s" % (revision)],
        ]

        for i in range(0, len(yaml)):
            l = yaml[i]

            for r in replacements:
                if r[0] in l:
                    print("Found " + l)
                    replaced += 1
                    yaml[i] = re.sub(r[0] + " [v\.a-f0-9]+.*$", r[0] + r[1], yaml[i])

        assert len(replacements) == replaced

        with open(self.yaml_file, "wb") as f:
            f.write(("".join(yaml)).encode("utf-8"))

    def spurious_check(self, revision, ignore_modified):
        changed_files = set(
            [
                os.path.abspath(f)
                for f in self.repository.get_changed_files(mode="staged")
            ]
        )
        generated_files = set(
            [
                self.get_full_path(f)
                for f in self.manifest["vendoring"].get("generated", [])
            ]
        )
        changed_files = set(changed_files) - generated_files
        if not changed_files:
            self.logInfo({"r": revision}, "Upstream {r} hasn't modified files locally.")
            # We almost certainly won't be here if ignore_modified was passed, because a modified
            # local file will show up as a changed_file, but we'll be safe anyway.
            if not ignore_modified and generated_files:
                for g in generated_files:
                    self.repository.clean_directory(g)
            elif generated_files:
                self.log(
                    logging.CRITICAL,
                    "vendor",
                    {"files": generated_files},
                    "Because you passed --ignore-modified we are not cleaning your"
                    + " working directory, but the following files were probably"
                    + " spuriously edited and can be reverted: {files}",
                )
            sys.exit(-2)

        self.logInfo(
            {"rev": revision, "num": len(changed_files)},
            "Version '{rev}' has changed {num} files.",
        )

    def update_files(self, revision):
        if "update-actions" not in self.manifest["vendoring"]:
            return

        for update in self.manifest["vendoring"]["update-actions"]:
            if update["action"] == "copy-file":
                src = self.get_full_path(update["from"])
                dst = self.get_full_path(update["to"])

                self.logInfo(
                    {"s": src, "d": dst}, "action: copy-file src: {s} dst: {d}"
                )

                with open(src) as f:
                    contents = f.read()
                with open(dst, "w") as f:
                    f.write(contents)
            elif update["action"] == "move-file":
                src = self.get_full_path(update["from"])
                dst = self.get_full_path(update["to"])

                self.logInfo(
                    {"s": src, "d": dst}, "action: move-file src: {s} dst: {d}"
                )

                shutil.move(src, dst)
            elif update["action"] == "move-dir":
                src = self.get_full_path(update["from"])
                dst = self.get_full_path(update["to"])

                self.logInfo(
                    {"src": src, "dst": dst}, "action: move-dir src: {src} dst: {dst}"
                )

                if not os.path.isdir(src):
                    raise Exception(
                        "Cannot move from a source directory %s that is not a directory"
                        % src
                    )
                os.makedirs(dst, exist_ok=True)

                def copy_tree(src, dst):
                    names = os.listdir(src)
                    os.makedirs(dst, exist_ok=True)

                    for name in names:
                        srcname = os.path.join(src, name)
                        dstname = os.path.join(dst, name)

                        if os.path.isdir(srcname):
                            copy_tree(srcname, dstname)
                        else:
                            shutil.copy2(srcname, dstname)

                copy_tree(src, dst)
                shutil.rmtree(src)

            elif update["action"] in ["replace-in-file", "replace-in-file-regex"]:
                file = self.get_full_path(update["file"])

                self.logInfo({"file": file}, "action: replace-in-file file: {file}")

                replacement = update["with"].replace("{revision}", revision)
                _replace_in_file(
                    file,
                    update["pattern"],
                    replacement,
                    regex=update["action"] == "replace-in-file-regex",
                )
            elif update["action"] == "delete-path":
                path = self.get_full_path(update["path"])
                self.logInfo({"path": path}, "action: delete-path path: {path}")
                mozfile.remove(path)
            elif update["action"] in ["run-script", "run-command"]:
                if update["action"] == "run-script":
                    command = self.get_full_path(update["script"], support_cwd=True)
                else:
                    command = update["command"]

                run_dir = self.get_full_path(update["cwd"], support_cwd=True)

                args = []
                for a in update.get("args", []):
                    if a == "{revision}":
                        args.append(revision)
                    elif any(
                        s in a
                        for s in [
                            "{cwd}",
                            "{vendor_dir}",
                            "{yaml_dir}",
                            "{tmpextractdir}",
                        ]
                    ):
                        args.append(self.get_full_path(a, support_cwd=True))
                    else:
                        args.append(a)

                self.logInfo(
                    {
                        "command": command,
                        "run_dir": run_dir,
                        "args": args,
                        "type": update["action"],
                    },
                    "action: {type} command: {command} working dir: {run_dir} args: {args}",
                )
                extra_env = (
                    {"GECKO_PATH": os.getcwd()}
                    if "GECKO_PATH" not in os.environ
                    else {}
                )
                # We also add a signal to scripts that they are running under mach vendor
                extra_env["MACH_VENDOR"] = "1"
                self.run_process(
                    args=[command] + args,
                    cwd=run_dir,
                    log_name=command,
                    require_unix_environment=True,
                    append_env=extra_env,
                )
            else:
                assert False, "Unknown action supplied (how did this pass validation?)"

    def update_moz_build(self, vendoring_dir, moz_yaml_dir, add_to_exports):
        if vendoring_dir == moz_yaml_dir:
            vendoring_dir = moz_yaml_dir = None

        # If you edit this (especially for header files) you should double check
        # rewrite_mozbuild.py around 'assignment_type'
        source_suffixes = [".cc", ".c", ".cpp", ".S", ".asm"]
        header_suffixes = [".h", ".hpp"]

        files_removed = self.repository.get_changed_files(diff_filter="D")
        files_added = self.repository.get_changed_files(diff_filter="A")

        # Filter the files added to just source files we track in moz.build files.
        files_added = [
            f for f in files_added if any([f.endswith(s) for s in source_suffixes])
        ]
        header_files_to_add = [
            f for f in files_added if any([f.endswith(s) for s in header_suffixes])
        ]
        if add_to_exports:
            files_added += header_files_to_add
        elif header_files_to_add:
            self.log(
                logging.WARNING,
                "header_files_warning",
                {},
                (
                    "We found %s header files in the update, pass --add-to-exports if you want"
                    + " to attempt to include them in EXPORTS blocks: %s"
                )
                % (len(header_files_to_add), header_files_to_add),
            )

        self.logInfo(
            {"added": len(files_added), "removed": len(files_removed)},
            "Found {added} files added and {removed} files removed.",
        )

        should_abort = False
        for f in files_added:
            try:
                add_file_to_moz_build_file(f, moz_yaml_dir, vendoring_dir)
            except MozBuildRewriteException:
                self.log(
                    logging.ERROR,
                    "vendor",
                    {},
                    "Could not add %s to the appropriate moz.build file" % f,
                )
                should_abort = True

        for f in files_removed:
            try:
                remove_file_from_moz_build_file(f, moz_yaml_dir, vendoring_dir)
            except MozBuildRewriteException:
                self.log(
                    logging.ERROR,
                    "vendor",
                    {},
                    "Could not remove %s from the appropriate moz.build file" % f,
                )
                should_abort = True

        if should_abort:
            self.log(
                logging.ERROR,
                "vendor",
                {},
                "This is a deficiency in ./mach vendor . "
                + "Please review the affected files before committing.",
            )
            # Exit with -1 to distinguish this from the Exception case of exiting with 1
            sys.exit(-1)

    def import_local_patches(self, patches, yaml_dir, vendor_dir):
        self.logInfo({}, "Importing local patches...")
        for patch in self.convert_patterns_to_paths(yaml_dir, patches):
            script = [
                "patch",
                "-p1",
                "--directory",
                vendor_dir,
                "--input",
                os.path.abspath(patch),
                "--no-backup-if-mismatch",
            ]
            self.run_process(
                args=script,
                log_name=script,
            )