<file_sep>import sys
import requests as r
import urllib.parse as purl
import json

f = open('../scopus.key')
keys = [ln.strip() for ln in f if len(ln) > 5]
key = keys[0]
f.close()

f = open(sys.argv[1])

journalFields = [('dc:title','title'),
                 ('dc:publisher','publisher')]

def formatJournal(jrnl):
    res = {}
    if ('serial-metadata-response' not in jrnl or 
        'entry' not in jrnl['serial-metadata-response'] or
        len(jrnl['serial-metadata-response']['entry']) == 0): 
        return res
    jrnl = jrnl['serial-metadata-response']['entry'][0]
    for jsId, oId in journalFields:
        res[oId] = jrnl[jsId]
    if 'subject-area' in jrnl and len(jrnl['subject-area']) > 0:
        res['subject'] = jrnl['subject-area'][0]['@abbrev']
        res['subjects'] = "; ".join([sb['$'] for sb in jrnl['subject-area']])

    if 'SNIPList' in jrnl and 'SNIP' in jrnl['SNIPList']:
        res['SNIP'] = jrnl['SNIPList']["SNIP"][0]["$"]
        res['SNIPyear'] = jrnl['SNIPList']["SNIP"][0]["@year"]

    if 'SJRList' in jrnl and 'SJR' in jrnl['SJRList']:
        res['SJR'] = jrnl['SJRList']["SJR"][0]["$"]
        res['SJRyear'] = jrnl['SJRList']["SJR"][0]["@year"]

    if 'IPPList' in jrnl and 'IPP' in jrnl['IPPList']:
        res['IPP'] = jrnl['IPPList']["IPP"][0]["$"]
        res['IPPyear'] = jrnl['IPPList']["IPP"][0]["@year"]
    return res

outp = []
baseUrl = "http://api.elsevier.com/content/serial/title/issn/"
for i,ln in enumerate(f):
    ln = ln.strip().replace("\"","")
    url = purl.urlencode({'apiKey':key,
                          'httpAccept':'application/json'})

    resUrl = "{}{}?{}".format(baseUrl,ln,url)
    m = r.get(resUrl)
    itm = formatJournal(m.json())
    itm['issn'] = ln
    outp.append(itm)

f = open('journalData.json','w')
json.dump(outp,f)
f.close()
<file_sep>#!/usr/bin/env python3
from ScopusScrapus import *
from ScopusScrapus.utils import formatScopusEntry
import json

f = open('../scopus.key')
keys = [ln.strip() for ln in f if len(ln) > 5]
f.close()

params = {'query':'TITLE-ABS-KEY(arctic)'}

ppList = []
qExceeded = False
for year in range(2016,1945,-1):
    params['date'] = year
    ssq = StartScopusSearch(keys,params,delay=10)
    for pp in ssq:
        try:
            ppList.append(formatScopusEntry(pp))
        except Exception as e:
            if 'QuotaExceeded' not in str(e):
                raise
            qExceeded = True
            print("Quota Exceeded. Made it to",year)
            break
    if qExceeded: break



outf = open("firstPapers_1.json",'w')
json.dump(ppList,outf)
outf.close()
<file_sep>## Dataset of Arctic-related papers
The dataset contains two main graphs:
* A bipartite graph of authors and papers
* A **tripartite** graph of authors, papers, and institutions. This is the main graph. To operate over 'pairs' of nodesets, it is only necessary to remove nodes of category 'paper', 'inst' or 'author'.

The code to ceate the datasets is in `scratch/`. Their functions are the following:
* `scratch/scrapePapers.py` - It scrapes all papers containing the word 'arctic' in their title, keywords or abstract, every year back to 1945.
* `scratch/scrapeIssn.py` - Takes in a list of ISSNs (for instance in `data/issn.txt`) and downloads data about their publication
* `scratch/makeAuthorPaperGraph.py` - It takes in data from the two previous scripts (check first few lines), and outputs a bipartite graph of authors and papers
* `scratch/makeTripartite.py` - It takes in data from the two previous scripts (check first few lines), and outputs a tripartite graph that includes papers, authors, and institutions.

### Notes on what the data files are
* `data/acia.csv` is a list of author ids that we want to mark in the graph.
* `data/paperData.json` is the list of paper information as scraped from the SCOPUS database.
* `data/scopusRefined.json` is a refined version of `data/paperData.json`. The paper data is spread into several
dictionaries, due to how openrefine works.

### Graph conventions
Some data from Scopus was 'corrupted', so there will be a few papers without publication, author or affiliation information.

Node ids capture what entity the node represents by their first letter: an author("a"), paper("p") or institution("i"). For instance, node "a7563051" would represent an author, node "i7563957" would represent an institution, and node "p67865" would represent a paper.

Also, in `tripartite.graphml`, the nodes contain a `category` field which has a value of `'author'`,`'paper'`, or `'inst'`, for each entity.

## Stats for tripartite graph and derivative graphs

50189 papers. 78901 authors. 13150 institutions.

The author-publication graph has 129090 nodes, 7916 connected components.

Their sizes: [95637, 295, 102, 101, 96, 77, 64, 55, 47, 46], etc.


50639 authors have ONE publication.

11646 authors have TWO publications.

5196 authors have THREE publications.

11420 authors have MORE THAN THREE publications.


8491 papers have ONE author.

10458 papers have TWO authors.

9397 papers have THREE authors.

21843 papers have MORE THAN THREE authors.


The author-institution graph has 92051 nodes, 7493 connected components.

Their sizes: [73729, 67, 53, 47, 45, 38, 36, 35, 35, 29], etc.


58527 authors have ONE institutions.

12546 authors have TWO instututions.

3370 authors have THREE institutions.

4458 authors have MORE THAN THREE institutions.



6850 institutions have ONE author.

1942 institutions have TWO authors.

1022 institutions have THREE authors.

3336 institutions have MORE THAN THREE authors.
<file_sep>import networkx as nx
import json

f = open('scratch/firstPapers.json')
papers = json.load(f)
f.close()

f = open('scratch/journalData.json')
journals = json.load(f)
f.close()

journalDic = {j['issn']:j for j in journals}

G = nx.Graph()


seenAuths = set()
for i,pp in enumerate(papers):
    addPp = {}
    for k,e in pp.items():
        if e is None: continue
        addPp[k] = e
    pp = addPp
    if 'pubissn' in pp and pp['pubissn'] in journalDic:
        for k,e in journalDic[pp['pubissn']].items():
            if e is None: continue
            pp['pub_'+k] = e
    else:
        if 'pubissn' not in pp: continue
        print("Publication with issn {} unavailable or publication of paper \"{}\" unavailable."
              .format(str(pp.get('pubissn')),pp['title']))

    ppId = 'p{}'.format(i+1)
    authors = pp.get('authors')
    if 'authors' in pp: del pp['authors']

    affiliations = pp.get('affiliations')
    if 'affiliations' in pp: del pp['affiliations']
    G.add_node(ppId,pp)
    if authors is None: continue
    for j, auth in enumerate(authors):
        athId = 'a{}'.format(auth['id'])
        newAuth = {}
        for k,e in auth.items():
            if e is None: continue
            newAuth[k] = e
        auth = newAuth
        if 'affiliations' in auth:
            auth['affiliations'] = ', '.join(auth['affiliations'])
        if athId not in seenAuths:
            seenAuths.add(athId)
            G.add_node(athId,auth)
        G.add_edge(ppId,athId)

nx.write_graphml(G,'author_paper.graphml')
<file_sep>import ipdb
import networkx as nx
import json
import csv
import itertools

from utilities import (PaperGenerator,
                       OpenRefinePaperGenerator,
                       NamesIdDictionary)


def load_journals(file_name='data/journalData.json'):
  with open(file_name) as f:
    journals = json.load(f)
  journal_dictionary = {j['issn']:j for j in journals}
  return journal_dictionary

def load_important_authors(file_name='data/acia.csv'):
  with open(file_name) as f:
    ids = [a[0] for a in list(csv.reader(f))]
  return ids

acia_auths = set(load_important_authors())


journalDic = load_journals()
#papers = PaperGenerator()
papers = OpenRefinePaperGenerator()

def different_institutions(x, y):
  return (x['names'] != y['names'] and
          all([x[k] != y[k] for k in ['country', 'city']
              if k in y and k in x and None not in [x[k], y[k]]] or [False]))

institution_dictionary = NamesIdDictionary(
    name_field=lambda x: '{} {}'.format(x.get('names', ''), x.get('country', '')),
    differentiator=different_institutions)

for pp in papers:
  if 'affiliations' not in pp: continue
  for j, aff in enumerate(pp['affiliations']):
    try:
      institution_dictionary.add(aff)
    except KeyError:
      continue

G = nx.Graph()

def fill_in_publication_data(pp, pub):
    for k,e in pub.items():
        if e is None: continue
        pp['pub_'+k] = e

seenAuths = set()
seenAffs = set()
def add_node(G, id, node):
  if id == 'i0':
    print("Inserting i0!")
    ipdb.set_trace()
  G.add_node(id, node)

def add_edge(G, id1, id2):
  if id1[0] == id2[0]:
    print("Intra-category edge!")
    ipdb.set_trace()
  G.add_edge(id1, id2)

for i, pp in enumerate(papers):
    addPp = {}

    # First we clean the paper info from None fields
    for k,e in pp.items():
        if e is None: continue
        addPp[k] = e
    pp = addPp

    # The we check that the paper has an ISSN (we ignore books), and that the
    # publication is one that we know.
    if 'pubissn' in pp and pp['pubissn'] in journalDic:
        fill_in_publication_data(pp, journalDic[pp['pubissn']])
    else:
        if 'pubissn' not in pp: continue
        journalDic[pp['pubissn']] = {'issn':pp['pubissn']}
        fill_in_publication_data(pp, journalDic[pp['pubissn']])
        print("Publication with issn {} unavailable or publication of paper \"{}\" unavailable."
              .format(str(pp.get('pubissn')),pp['title']))

    ppId = 'p{}'.format(i+1)

    # Now we retrieve author information from a paper.
    authors = pp.get('authors')
    if 'authors' in pp: del pp['authors']
    if authors is None or len(authors) == 0:
        # If there are no authors in the paper, we ignore it
        continue

    # Now we retrieve affiliation information for a paper
    affiliations = pp.get('affiliations')
    if 'affiliations' in pp: del pp['affiliations']
    if affiliations is None or len(affiliations) == 0:
        # If there are no affiliations in the paper, we ignore it
        continue
    pp['category'] = 'paper'
    add_node(G, ppId,pp)

    if affiliations is not None:
      for j, aff in enumerate(affiliations):
        if not aff.get('id'): continue
        aff = institution_dictionary.get(aff['id'])
        if not aff: continue
        try:
          aff_ids = sorted(list(
              institution_dictionary.get_ids_from_id(aff['id'])))
        except TypeError:
          pass
        if len(aff_ids) > 1:
          print('Long ids. Good. {} ids'.format(str(len(aff_ids))))
        affId = 'i'+aff_ids[0]
        seenAffs.add(affId)
        aff['category'] = 'inst'
        add_node(G, affId, aff)
        add_edge(G, ppId, affId) # These might not be useful

    if authors is None: continue
    for j, auth in enumerate(authors):
        if 'id' not in auth:
            continue
        auth['acia'] = auth['id'] in acia_auths
        athId = 'a{}'.format(auth['id'])
        newAuth = {}
        for k,e in auth.items():
            if e is None: continue
            newAuth[k] = e
        auth = newAuth
        #authAffs = []
        #if 'affiliations' in auth:
        #    authAffs = auth['affiliations']
        #    auth['affiliations'] = ', '.join(auth['affiliations'])
        if athId not in seenAuths:
            seenAuths.add(athId)
            auth['category'] = 'author'
            add_node(G, athId,auth)
        add_edge(G, ppId,athId)
        #for aff in authAffs:
        if 'affiliations' in auth:
          affId = 'i'+auth['affiliations']
          add_edge(G, athId, affId)

nx.write_graphml(G,'tripartite.graphml')
<file_sep>from collections import defaultdict
import json


class NamesIdDictionary(object):

  def __init__(self, differentiator=lambda x, y: False,
               name_field='names', id_field='id'):
    """differentiator is a function that returns True if x and y is different.
    """
    # To pair ids to their institutions
    self._ids_to_items = {}

    # For when a single institution has different ids
    self._ids_to_ids_dict = {}

    # To pair institution names to their ids
    self._names_to_ids_dict = defaultdict(lambda : set())

    self._different = differentiator

    if isinstance(name_field, str):
      self._name_fld = lambda x: x.get(name_field)
    else: # name_field MUST BE A LAMBDA
      self._name_fld = name_field

    self._id_fld = id_field

  def get_ids_from_id(self, id):
    return self._ids_to_ids_dict.get(id)

  def get_id_from_name(self, name):
    return self._names_to_ids_dict.get(name)

  def get(self, id):
      return self._ids_to_items.get(id)

  def add(self, item):
    id = item.get(self._id_fld)
    name = self._name_fld(item)
    if not id:
        raise KeyError('ID can not be None.')

    if self._ids_to_items.get(id, None):
        # We have an item with the same ID
        # We need to check whether it's the same or not
        if self._different(item, self._ids_to_items[id]):
            print('Two items with equal ids are different: {}, {}'.format(
                    str(item), str(self._ids_to_items[id])))
            return

    self._ids_to_items[id] = item

    if not name:
        # If we don't know the name, then there's nothing to do
        return

    self._names_to_ids_dict[name].add(id)
    known_ids = self._names_to_ids_dict[name]

    for _id in known_ids:
        self._ids_to_ids_dict[_id] = known_ids


def clean_up_dict(d):
  return {k:v for k, v in d.items() if v is not None}

def contains_data(arg):
  if isinstance(arg, dict):
    values = [v for k, v in arg.items()]
  elif isinstance(arg, list):
    values = [v for v in arg]
  else:
    values = [arg]
  return any(values)


class PaperGenerator(object):

  def __init__(self, file_name='data/paperData.json'):
    self._file_name = file_name

  def next_paper(self):
    return next(self._iterator)

  def __iter__(self):
    self.papers = json.load(open(self._file_name, 'r'))
    self._iterator = iter(self.papers)
    return self

  def __next__(self):
    return self.next_paper()

  def next(self):
    return self.next_paper()


class OpenRefinePaperGenerator(PaperGenerator):

  def __init__(self, file_name='data/scopusRefined.json'):
    self._file_name = file_name

  def __iter__(self):
    self.json_list = json.load(open(self._file_name, 'r'))
    self._iterator = iter(self.json_list)
    self._next_paper_base = next(self._iterator)
    return self

  def format_output_paper(self, paper):
    paper['authors'] = [clean_up_dict(author)
                        for author in paper['authors']
                        if contains_data(author)]
    paper['affiliations'] = [clean_up_dict(institution)
                             for institution in paper['affiliations']
                             if contains_data(institution)]
    res = clean_up_dict(paper)
    return res

  def next_paper(self):
    paper_base = self._next_paper_base
    if not paper_base:
      raise StopIteration
    if not paper_base.get('pubName'):
      raise ValueError('The paper list has issues.')

    if paper_base.get('affiliations') is None:
      paper_base['affiliations'] = []
    if paper_base.get('authors') is None:
      paper_base['authors'] = []

    for e in self._iterator:
      if e.get('pubName'):
        self._next_paper_base = e
        return self.format_output_paper(paper_base)

      if e.get('authors'):
        paper_base['authors'] += e['authors']

      if e.get('affiliations'):
        paper_base['affiliations'] += e['affiliations']

    self._next_paper_base = None
    return self.format_output_paper(paper_base)
<file_sep>#!/usr/bin/env python
import sys

import networkx as nx

def section():
  print('')

G = nx.read_graphml(sys.argv[1])

paper_ids = [p for p in G.nodes() if 'p' in p]
author_ids = [p for p in G.nodes() if 'a' in p]
institution_ids = [p for p in G.nodes() if 'i' in p]

print('{} papers. {} authors. {} institutions.'.format(len(paper_ids),
                                                       len(author_ids),
                                                       len(institution_ids)))

# Removing all institutions
[G.remove_node(p) for p in institution_ids]

section()
connected_components_sizes = sorted(
    [len(a) for a in nx.connected_components(G)],
    key=lambda x: -x)
print(
    'The author-publication graph has {} nodes, {} connected components.'
    .format(len(G.nodes()), len(connected_components_sizes)))
print('Their sizes: {}, etc.'.format(connected_components_sizes[0:10]))

from collections import Counter
author_publications_distribution = Counter(G.degree(a) for a in author_ids)

section()
print('{} authors have ONE publication.'.format(
    author_publications_distribution[1]))
print('{} authors have TWO publications.'.format(
    author_publications_distribution[2]))
print('{} authors have THREE publications.'.format(
    author_publications_distribution[3]))
low_count = sum(author_publications_distribution[i] for i in range(1,4))
print('{} authors have MORE THAN THREE publications.'.format(
    len(author_ids) - low_count))

section()
publication_authors_distribution = Counter(G.degree(a) for a in paper_ids)
print('{} papers have ONE author.'.format(
    publication_authors_distribution[1]))
print('{} papers have TWO authors.'.format(
    publication_authors_distribution[2]))
print('{} papers have THREE authors.'.format(
    publication_authors_distribution[3]))
low_count = sum(publication_authors_distribution[i] for i in range(1,4))
print('{} papers have MORE THAN THREE authors.'.format(
    len(paper_ids) - low_count))

G = nx.read_graphml(sys.argv[1])
# Removing all papers
[G.remove_node(p) for p in paper_ids]

section()
connected_components_sizes = sorted(
    [len(a) for a in nx.connected_components(G)],
    key=lambda x: -x)
print(
    'The author-institution graph has {} nodes, {} connected components.'
    .format(len(G.nodes()), len(connected_components_sizes)))
print('Their sizes: {}, etc.'.format(connected_components_sizes[0:10]))



author_institutions_distribution = Counter(G.degree(a) for a in author_ids)
section()
print('{} authors have ONE institutions.'.format(
    author_institutions_distribution[1]))
print('{} authors have TWO instututions.'.format(
    author_institutions_distribution[2]))
print('{} authors have THREE institutions.'.format(
    author_institutions_distribution[3]))
low_count = sum(author_institutions_distribution[i] for i in range(1,4))
print('{} authors have MORE THAN THREE institutions.'.format(
    len(author_ids) - low_count))


institution_authors_distribution = Counter(G.degree(a) for a in institution_ids)
section()
print('{} institutions have ONE author.'.format(
    institution_authors_distribution[1]))
print('{} institutions have TWO authors.'.format(
    institution_authors_distribution[2]))
print('{} institutions have THREE authors.'.format(
    institution_authors_distribution[3]))
low_count = sum(institution_authors_distribution[i] for i in range(1,4))
print('{} institutions have MORE THAN THREE authors.'.format(
    len(institution_ids) - low_count))
<file_sep>""" This script is a sketch of the script that makes the country collaboration
graph for future reference. This was used on Jan 31 2017."""


country_edges = defaultdict(lambda : 0)

for p in all_papers:
  if len(p['affiliations']) < 2:
    continue
  countries = [a['country'] for a in p['affiliations'] if a.get('country')]
  if len(countries) < 2:
    continue
  print('Countries: {}'.format(countries))
  combos = combinations(countries, 2)
  for combo in combos:
    combo = sorted(combo)
    key = tuple(combo)
    country_edges[key] += 1

G = nx.Graph()
for k, v in country_edges.items():
  c1 = k[0]
  c2 = k[1]
  G.add_node(c1)
  G.add_node(c2)
  G.add_edge(c1, c2, weight=v)
