Week over week + exclude org members + cumulated

app.py

@@ -27,9 +27,24 @@ def _range(e):
     global val
     e['range'] = val
     val += 1
+
+    current_date = datetime.strptime(e['dates'], "%Y-%m-%dT%H:%M:%SZ")
+    first_date = datetime.fromtimestamp(1)
+    week = abs(current_date - first_date).days // 7
+    e['week'] = week
+
     return e
 
 
+def _ignore_org_members(e):
+    global val
+    e['range_non_org'] = val
+
+    if e['type']['authorAssociation'] != 'MEMBER':
+        val += 1
+
+    return e
+
 stars = {}
 for k, v in datasets['stars'].items():
     stars[k] = v.map(_range)
@@ -39,6 +54,8 @@ issues = {}
 for k, v in datasets['issues'].items():
     issues[k] = v.map(_range)
     val = 0
+    issues[k] = issues[k].map(_ignore_org_members)
+    val = 0
 
 datasets['stars'] = DatasetDict(**stars)
 datasets['issues'] = DatasetDict(**issues)
@@ -69,6 +86,17 @@ def running_mean(x, N, total_length=-1):
     return np.pad(cumsum[N:] - cumsum[:-N], (to_pad, 0)) / float(N)
 
 
+def parse_name_and_options(path):
+    url = urlparse(path)
+    query = parse_qs(url.query)
+    library_names = query.get("input", None)[0]
+    library_names = library_names.split(',')
+    options = query.get("options", None)[0]
+    options = options.split(',')
+
+    return library_names, options
+
+
 class RequestHandler(SimpleHTTPRequestHandler):
     def do_GET(self):
         print(self.path)
@@ -86,28 +114,16 @@ class RequestHandler(SimpleHTTPRequestHandler):
                 if len(v) < len(dataset_with_most_splits):
                     warnings.extend(f"The {k} dataset does not contain all splits. Missing: {dataset_with_most_splits - v}")
 
-            self.send_response(200)
-            self.send_header("Content-Type", "application/json")
-            self.end_headers()
-
             # TODO: Send and display warnings
             dataset_with_most_splits = list(dataset_with_most_splits)
             dataset_with_most_splits.sort()
-            self.wfile.write(json.dumps(list(dataset_with_most_splits)).encode("utf-8"))
 
-            return SimpleHTTPRequestHandler
+            return self.response(list(dataset_with_most_splits))
 
         if self.path.startswith("/retrievePipInstalls"):
-            url = urlparse(self.path)
-            query = parse_qs(url.query)
-            library_names = query.get("input", None)[0]
-            library_names = library_names.split(',')
-
-            if 'Cumulated' in library_names:
-                dataset_keys = {k: set(v.keys()) for k, v in datasets.items()}
-                dataset_with_most_splits = max([d for d in dataset_keys.values()], key=len)
-                library_names = list(dataset_with_most_splits)
+            library_names, options = parse_name_and_options(self.path)
 
+            if '1' in options:
                 returned_values = {}
                 for library_name in library_names:
                     for i in datasets['pip'][library_name]:
@@ -136,31 +152,29 @@ class RequestHandler(SimpleHTTPRequestHandler):
             output = {l: [k[l] for k in returned_values.values()] for l in library_names}
             output['day'] = list(returned_values.keys())
 
-            self.send_response(200)
-            self.send_header("Content-Type", "application/json")
-            self.end_headers()
-
-            self.wfile.write(json.dumps(output).encode("utf-8"))
-
-            return SimpleHTTPRequestHandler
+            return self.response(output)
 
         if self.path.startswith("/retrieveStars"):
-            url = urlparse(self.path)
-            query = parse_qs(url.query)
-            library_names = query.get("input", None)[0]
-            library_names = library_names.split(',')
-
+            library_names, options = parse_name_and_options(self.path)
             returned_values = {}
             dataset_dict = datasets['stars']
+            week_over_week = '1' in options
 
             for library_name in library_names:
                 dataset = dataset_dict[library_name]
 
+                last_value = 0
+                last_week = dataset[0]['week']
                 for i in dataset:
+                    if week_over_week and last_week == i['week']:
+                        continue
                     if i['dates'] in returned_values:
-                        returned_values[i['dates']][library_name] = i['range']
+                        returned_values[i['dates']][library_name] = i['range'] - last_value
                     else:
-                        returned_values[i['dates']] = {library_name: i['range']}
+                        returned_values[i['dates']] = {library_name: i['range'] - last_value}
+
+                    last_value = i['range'] if week_over_week else 0
+                    last_week = i['week']
 
             returned_values = collections.OrderedDict(sorted(returned_values.items()))
             returned_values = link_values(library_names, returned_values)
@@ -168,33 +182,40 @@ class RequestHandler(SimpleHTTPRequestHandler):
             output['day'] = list(returned_values.keys())[::-1]
 
             # Trim down to a smaller number of points.
-            output = {k: [v for i, v in enumerate(value) if i % int(len(value) / 100) == 0] for k, value in output.items()}
-
-            self.send_response(200)
-            self.send_header("Content-Type", "application/json")
-            self.end_headers()
+            if len(output) > 100:
+                output = {
+                    k: [v for i, v in enumerate(value) if i % int(len(value) / 100) == 0] for k, value in output.items()
+                }
 
-            self.wfile.write(json.dumps(output).encode("utf-8"))
+            return self.response(output)
 
-            return SimpleHTTPRequestHandler
 
         if self.path.startswith("/retrieveIssues"):
-            url = urlparse(self.path)
-            query = parse_qs(url.query)
-            library_names = query.get("input", None)[0]
-            library_names = library_names.split(',')
+            library_names, options = parse_name_and_options(self.path)
+
+            exclude_org_members = '1' in options
+            week_over_week = '2' in options
 
             returned_values = {}
             dataset_dict = datasets['issues']
+            range_id = 'range' if not exclude_org_members else 'range_non_org'
 
             for library_name in library_names:
                 dataset = dataset_dict[library_name]
 
+                last_value = 0
+                last_week = dataset[0]['week']
                 for i in dataset:
+                    if week_over_week and last_week == i['week']:
+                        continue
+
                     if i['dates'] in returned_values:
-                        returned_values[i['dates']][library_name] = i['range']
+                        returned_values[i['dates']][library_name] = i[range_id] - last_value
                     else:
-                        returned_values[i['dates']] = {library_name: i['range']}
+                        returned_values[i['dates']] = {library_name: i[range_id] - last_value}
+
+                    last_value = i[range_id] if week_over_week else 0
+                    last_week = i['week']
 
             returned_values = collections.OrderedDict(sorted(returned_values.items()))
             returned_values = link_values(library_names, returned_values)
@@ -202,17 +223,23 @@ class RequestHandler(SimpleHTTPRequestHandler):
             output['day'] = list(returned_values.keys())[::-1]
 
             # Trim down to a smaller number of points.
-            output = {k: [v for i, v in enumerate(value) if i % int(len(value) / 100) == 0] for k, value in output.items()}
+            if len(output) > 100:
+                output = {
+                    k: [v for i, v in enumerate(value) if i % int(len(value) / 100) == 0] for k, value in output.items()
+                }
 
-            self.send_response(200)
-            self.send_header("Content-Type", "application/json")
-            self.end_headers()
+            return self.response(output)
 
-            self.wfile.write(json.dumps(output).encode("utf-8"))
+        return SimpleHTTPRequestHandler.do_GET(self)
 
-            return SimpleHTTPRequestHandler
+    def response(self, output):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
 
-        return SimpleHTTPRequestHandler.do_GET(self)
+        self.wfile.write(json.dumps(output).encode("utf-8"))
+
+        return SimpleHTTPRequestHandler
 
 
 server = ThreadingHTTPServer(("", 7860), RequestHandler)

index.js

@@ -13,6 +13,61 @@ const load = () => {
     return l0
 }
 
+const getCheckedOptions = () => {
+    const options = Array.from(document.querySelectorAll('.option-div'))
+        .map(e => Array.from(e.children)
+        .filter(e => e.nodeName == 'DIV'))
+        .filter(e => e.length)
+        .flat()
+        .map(e => e.id)
+        .filter(e => document.querySelector(`#${e}-checkbox`).checked)
+
+    const optionsDict = {}
+    for (let option of options) {
+        const key = option.split('-option-')[0]
+        const value = option.split('-option-')[1]
+
+        if (key in optionsDict)
+            optionsDict[key].push(value)
+        else
+            optionsDict[key] = [value]
+    }
+
+    console.log('dict', optionsDict)
+
+    return optionsDict;
+}
+
+const addOption = (category, optionName) => {
+    /* Options for the issue div */
+    const issueDiv = document.getElementById(`${category}Div`);
+    const div = document.createElement('div')
+
+    let found = false;
+    let optionNumber = 0;
+    while (!found && ++optionNumber < 100) {
+        let previousOption = document.getElementById(`${category}-option-${optionNumber}`);
+        console.log(previousOption)
+        found = previousOption === null;
+    }
+
+    div.id = `${category}-option-${optionNumber}`;
+    issueDiv.appendChild(div);
+
+    const checkBox = document.createElement('input');
+    checkBox.type = 'checkbox'
+    checkBox.id = `${category}-option-${optionNumber}-checkbox`
+
+    const checkBoxLabel = document.createElement('label');
+    const labelSpan = document.createElement('span')
+    labelSpan.textContent = optionName;
+    checkBoxLabel.appendChild(checkBox)
+    checkBoxLabel.appendChild(labelSpan)
+    div.appendChild(checkBoxLabel)
+
+    return optionNumber
+}
+
 let charts = [];
 
 const createButton = (title, libraries, methods) => {
@@ -21,16 +76,20 @@ const createButton = (title, libraries, methods) => {
     button.onclick = async () => {
         document.getElementById('pip-graph').innerHTML = ''
         document.getElementById('star-graph').innerHTML = ''
+        document.getElementById('issue-graph').innerHTML = ''
         const e = load()
         document.body.appendChild(e)
         const selectedLibraries = libraries.filter(e => document.querySelector(`#${e}Checkbox`).checked);
+
+        const relevantOptions = getCheckedOptions();
+
         if (charts.length !== 0) {
             for (const chart of charts) {
                 chart.destroy()
             }
         }
         for (const method of methods()) {
-            charts.push(await method(selectedLibraries))
+            charts.push(await method(selectedLibraries, relevantOptions))
         }
         document.body.removeChild(e)
     };
@@ -75,8 +134,11 @@ const initialize = async () => {
         librarySelector.appendChild(div)
     }
 
-    for (const element of ['pip', 'stars', 'issues']) {
+    for (const element of ['pip', 'stars', 'issue']) {
         const div = document.createElement('div');
+        div.classList.add('option-div')
+        div.id = `${element}Div`;
+
         const checkBox = document.createElement('input');
         checkBox.type = 'checkbox'
         checkBox.id = `${element}CheckboxGraph`;
@@ -91,8 +153,13 @@ const initialize = async () => {
         graphSelector.appendChild(div)
     }
 
+    addOption('pip', "Cumulated");
+    addOption('issue', "Exclude org members");
+    addOption('issue', "Week over week");
+    addOption('stars', "Week over week");
+
     const fetchButton = createButton('Fetch', inferJson, () => {
-        const graphNames = ['pip', 'stars', 'issues'].filter(e => document.querySelector(`#${e}CheckboxGraph`).checked);
+        const graphNames = ['pip', 'stars', 'issue'].filter(e => document.querySelector(`#${e}CheckboxGraph`).checked);
         const graphs = []
 
         if (graphNames.includes('pip'))
@@ -101,7 +168,7 @@ const initialize = async () => {
         if (graphNames.includes('stars'))
             graphs.push(retrieveStars)
 
-        if (graphNames.includes('issues'))
+        if (graphNames.includes('issue'))
             graphs.push(retrieveIssues)
 
         return graphs
@@ -109,8 +176,9 @@ const initialize = async () => {
     selectorSubmit.appendChild(fetchButton);
 };
 
-const retrievePipInstalls = async (libraryNames) => {
-    const inferResponse = await fetch(`retrievePipInstalls?input=${libraryNames}`);
+const retrievePipInstalls = async (libraryNames, options) => {
+    const relevantOptions = options['pip']
+    const inferResponse = await fetch(`retrievePipInstalls?input=${libraryNames}&options=${relevantOptions}`);
     const inferJson = await inferResponse.json();
     const colors = ['#FF0000', '#00FF00', '#0000FF', '#FF00FF', '#FFFF00', '#0000FF', '#F090F0', '#90F0F0', '#F0F090']
 
@@ -159,8 +227,9 @@ const retrievePipInstalls = async (libraryNames) => {
     return myChart;
 };
 
-const retrieveStars = async (libraryNames) => {
-    const inferResponse = await fetch(`retrieveStars?input=${libraryNames}`);
+const retrieveStars = async (libraryNames, options) => {
+    const relevantOptions = options['stars']
+    const inferResponse = await fetch(`retrieveStars?input=${libraryNames}&options=${relevantOptions}`);
     const inferJson = await inferResponse.json();
     const colors = ['#FF0000', '#00FF00', '#0000FF', '#FF00FF', '#FFFF00', '#0000FF', '#F090F0', '#90F0F0', '#F0F090']
 
@@ -209,8 +278,9 @@ const retrieveStars = async (libraryNames) => {
     return myChart;
 };
 
-const retrieveIssues = async (libraryNames) => {
-    const inferResponse = await fetch(`retrieveIssues?input=${libraryNames}`);
+const retrieveIssues = async (libraryNames, options) => {
+    const relevantOptions = options['issue']
+    const inferResponse = await fetch(`retrieveIssues?input=${libraryNames}&options=${relevantOptions}`);
     const inferJson = await inferResponse.json();
     const colors = ['#FF0000', '#00FF00', '#0000FF', '#FF00FF', '#FFFF00', '#0000FF', '#F090F0', '#90F0F0', '#F0F090']
 
@@ -252,7 +322,7 @@ const retrieveIssues = async (libraryNames) => {
             plugins: {
                 title: {
                     display: true,
-                    text: 'Number of issues, PRs, and comments on these'
+                    text: 'Number of issue, PRs, and comments on these'
                 }
             }
         }

style.css

@@ -39,6 +39,19 @@ button:hover {
     width: 100%;
 }
 
+.option-div {
+    border-radius: 3px;
+    border-color: rgb(180, 180, 200);
+    border-style: solid;
+    border-width: 2px 4px 4px 2px;
+    margin: 14px 0;
+    padding: 5px;
+}
+
+.option-div > div {
+    margin-left: 20px;
+}
+
 .submit {
     margin-bottom: 50px;
 }